mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

afl 插装编译 测试用例生成 联动逻辑

Browse Source
This commit is contained in:
even 2025-06-24 15:27:28 +08:00
parent 5419ee5dce
commit 8bc10fa999
5 changed files with 40 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java
View File

@ -21,9 +21,11 @@ public class PipelineGlobalVariableConstant {
* */ * */
public static final String AFL_DOCKER_SLOT_COMPILE_PATH_KEY = "AFL_DOCKER_SLOT_COMPILE_PATH_KEY"; public static final String AFL_DOCKER_SLOT_COMPILE_PATH_KEY = "AFL_DOCKER_SLOT_COMPILE_PATH_KEY";
/** /**
* 测试用例生成，ai生成或者直接文件上传的路径，工作目录下 SEED * 测试用例生成，ai生成或者直接文件上传的文件夹名称：SEED
* */ * */
public static final String AFL_DOCKER_SEED_PATH="SEED"; public static final String AFL_DOCKER_SEED ="SEED";
public static final String AFL_DOCKER_SEED_PATH_KEY="AFL_DOCKER_SEED_PATH_KEY";
/** /**
* AFL输出目录 * AFL输出目录
* */ * */

17
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java
View File

@ -69,17 +69,13 @@ public class AFLSlotCompileWorker extends DockerWorker {
toBadEnding(); toBadEnding();
} }
// 工作目录下的相对路径,/test 作为根路径 // 工作目录下的相对路径,/test 作为根路径
String compilePath = PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_PREFIX
+ task.getPipelineId()
+ File.separator
+ PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_PREFIX+task.getId();
// 卷挂载目录+流水线目录 + task编译目录 // 卷挂载目录+流水线目录 + task编译目录
String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ compilePath; String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir;
// 上传目标文件 到指定资源服务器 // 上传目标文件 到指定资源服务器
try { try {
SftpUploadUtil.uploadFileViaSftp( SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost() machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",realPath,filePath,file.getName()); ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,realPath,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) { } catch (SftpUploadUtil.SftpUploadException e) {
append(context,"上传文件失败，请确认资源信息是否有误:"+JSON.toJSONString(machineInfo)); append(context,"上传文件失败，请确认资源信息是否有误:"+JSON.toJSONString(machineInfo));
toBadEnding(); toBadEnding();
@ -87,16 +83,15 @@ public class AFLSlotCompileWorker extends DockerWorker {
// 执行预设命令 ,进入目录 // 执行预设命令 ,进入目录
String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" + String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" +
"cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录 "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录
"mkdir -p "+compilePath+"\n"+ "mkdir -p "+workDir+"\n"+
"cd "+compilePath+"\n"; "cd "+workDir+"\n";
Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT); Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null; String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
allCommand += commandScript+"\nll -s"; allCommand += commandScript+"\nll -s";
log.info("AFL插装编译容器内执行的命令：{}",allCommand);
dockerRun(allCommand,dockerInfo,context); dockerRun(allCommand,dockerInfo,context);
// 更新全局变量 // 更新全局变量
// /test目录下编译文件与目标所在目录,当前pipelineId+taskId组合 // /test目录下当前流水线工作目录
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_KEY,compilePath);
// /test目录下当前流水线工作目录（没有task部分）
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir);
} }
} }

30
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java
View File

@ -5,6 +5,7 @@ import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.AFLConstant; import cd.casic.ci.process.engine.constant.AFLConstant;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker; import cd.casic.ci.process.engine.worker.base.DockerWorker;
import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.dataObject.task.PipTask;
@ -12,6 +13,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.io.File;
import java.util.Map; import java.util.Map;
@WorkAtom(taskType = "AFL") @WorkAtom(taskType = "AFL")
@ -23,23 +25,23 @@ public class AFLWorker extends DockerWorker {
public void execute(TaskRunContext context) { public void execute(TaskRunContext context) {
int statusCode = -1; int statusCode = -1;
Map<String, Object> localVariables = context.getLocalVariables(); Map<String, Object> localVariables = context.getLocalVariables();
Map<String, Object> globalVariables = context.getGlobalVariables();
if (context.getContextDef() instanceof PipTask taskDef) { if (context.getContextDef() instanceof PipTask taskDef) {
log.info(taskDef.getTaskName()); log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties(); Map<String, Object> taskProperties = taskDef.getTaskProperties();
//从taskProperties中获取资源id //从taskProperties中获取资源id
String resourceType = taskProperties.get("resourceType").toString(); String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString(); String resourceId = taskProperties.get("resourceId").toString();
// 待测试文件路径 String commandEnd = taskProperties.get("commandEnd").toString();
String binaryPath = taskProperties.get(AFLConstant.BINARY) instanceof String ? ((String) taskProperties.get(AFLConstant.BINARY)) : null; String imageName = taskProperties.get("imageName").toString();
String output = taskProperties.get(AFLConstant.OUTPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.OUTPUT)) : null; // 代码编译目录
String input = taskProperties.get(AFLConstant.INPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.INPUT)) : null; String workDir = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY);
String workDir = taskProperties.get(AFLConstant.WORK_DIR) instanceof String ? ((String) taskProperties.get(AFLConstant.WORK_DIR)) : null; // 入参目录
String seedPath = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY);
if (StringUtils.isEmpty(binaryPath) || if (StringUtils.isEmpty(seedPath) ||
StringUtils.isEmpty(output) ||
StringUtils.isEmpty(input) ||
StringUtils.isEmpty(workDir) || StringUtils.isEmpty(workDir) ||
StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceId) ||
StringUtils.isEmpty(commandEnd) ||
StringUtils.isEmpty(resourceType)) { StringUtils.isEmpty(resourceType)) {
// 缺少参数 // 缺少参数
toBadEnding(); toBadEnding();
@ -52,12 +54,16 @@ public class AFLWorker extends DockerWorker {
append(context,"当前机器不支持docker"); append(context,"当前机器不支持docker");
return; return;
} }
String commandScript = "docker run -v "+workDir+":/test -it aflplusplus/aflplusplus bash\n" + String output = workDir+PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT;
//
String volumeWorkDirPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX;
String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" +
"cd /test\n" + "cd /test\n" +
"afl-fuzz -i "+input+" -o "+output+" -t 3000 -Q "+binaryPath+" @@"; "mkdir -p"+output+"\n" +
"afl-fuzz -i "+seedPath+" -o "+output+" "+workDir+ File.separator+commandEnd;
try { try {
//将节点的配置信息反编译成对象 //将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript); log.info("AFL模糊测试执行脚本：{}",commandScript);
// 获取docker 暂时先写固定值 // 获取docker 暂时先写固定值
dockerRun(commandScript,resourceListByType.getDockerEndpointList().get(0),context); dockerRun(commandScript,resourceListByType.getDockerEndpointList().get(0),context);
} catch (Exception e) { } catch (Exception e) {

21
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java
View File

@ -33,8 +33,7 @@ public class TestCaseGenerationWorker extends DockerWorker {
Map<String, Object> globalVariables = context.getGlobalVariables(); Map<String, Object> globalVariables = context.getGlobalVariables();
String workDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null; String workDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null;
if (context.getContextDef() instanceof PipTask taskDef) { if (context.getContextDef() instanceof PipTask taskDef) {
String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_PREFIX+taskDef.getId()+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH; String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED;
log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties(); Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY); Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY);
String binaryName = taskProperties.get("binaryName").toString(); String binaryName = taskProperties.get("binaryName").toString();
@ -57,15 +56,15 @@ public class TestCaseGenerationWorker extends DockerWorker {
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器 //如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) { if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) {
// 容器内部test 目录, 获取编译后的文件 // 容器内部test 目录, 获取编译后的文件
String compileDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null; String runScript = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" // 目前测试用例ai生成只有这个路径能跑
+ "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n" +
String commandScript = "cd /test\n" +
"mkdir -p "+seedPath +"\n"+ "mkdir -p "+seedPath +"\n"+
"PYTHONPATH=/test/CaseGenerator/src python3 /test/CaseGenerator/src/CaseGenerator/main.py --work-dir "+compileDir+" --binary "+compileDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100"; "cd "+seedPath +"\n"+
"PYTHONPATH=CaseGenerator/src python3 CaseGenerator/src/CaseGenerator/main.py --work-dir "+workDir+" --binary "+workDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100";
//将节点的配置信息反编译成对象 //将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript); log.info("测试用例生成-AI生成，实际执行命令:{}" , runScript);
//获取机器 //获取机器
dockerRun(commandScript,dockerInfo,context); dockerRun(runScript,dockerInfo,context);
} else { } else {
// 文件上传 // 文件上传
String filePath = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE)) : null; String filePath = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE)) : null;
@ -74,9 +73,10 @@ public class TestCaseGenerationWorker extends DockerWorker {
toBadEnding(); toBadEnding();
} }
File file = new File(filePath); File file = new File(filePath);
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+workDir+File.separator+seedPath;
// 将文件上传到服务器的 目录底下 // 将文件上传到服务器的 目录底下
try { log.info("测试用例选用上传模式,种子文件路径：{},种子上传路径:{}",filePath,seedTarget);
try {
SftpUploadUtil.uploadFileViaSftp( SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost() machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,seedTarget,file.getName()); ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,seedTarget,file.getName());
@ -85,6 +85,7 @@ public class TestCaseGenerationWorker extends DockerWorker {
log.error("seed文件上传失败",e); log.error("seed文件上传失败",e);
} }
} }
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY,seedPath);
} }
} }
} }

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/DockerWorker.java
View File

@ -126,7 +126,7 @@ public abstract class DockerWorker extends BaseWorker{
log.warn("未知流类型: {}", frame.getStreamType()); log.warn("未知流类型: {}", frame.getStreamType());
} }
} }
}).awaitCompletion(60, TimeUnit.SECONDS); }).awaitCompletion();
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.error("执行异常",e); log.error("执行异常",e);
append(context,"执行异常,容器id："+containerId); append(context,"执行异常,容器id："+containerId);