diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java index fbc41613..55b45c1b 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java @@ -21,9 +21,11 @@ public class PipelineGlobalVariableConstant { * */ public static final String AFL_DOCKER_SLOT_COMPILE_PATH_KEY = "AFL_DOCKER_SLOT_COMPILE_PATH_KEY"; /** - * 测试用例生成，ai生成或者直接文件上传的路径，工作目录下 SEED + * 测试用例生成，ai生成或者直接文件上传的文件夹名称：SEED * */ - public static final String AFL_DOCKER_SEED_PATH="SEED"; + public static final String AFL_DOCKER_SEED ="SEED"; + public static final String AFL_DOCKER_SEED_PATH_KEY="AFL_DOCKER_SEED_PATH_KEY"; + /** * AFL输出目录 * */ diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java index ad75a7b6..53f0ec25 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java @@ -69,17 +69,13 @@ public class AFLSlotCompileWorker extends DockerWorker { toBadEnding(); } // 工作目录下的相对路径,/test 作为根路径 - String compilePath = PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_PREFIX - + task.getPipelineId() - + File.separator - + PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_PREFIX+task.getId(); // 卷挂载目录+流水线目录 + task编译目录 - String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ compilePath; + String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir; // 上传目标文件 到指定资源服务器 try { SftpUploadUtil.uploadFileViaSftp( machineInfo.getMachineHost() - ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",realPath,filePath,file.getName()); + ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,realPath,file.getName()); } catch (SftpUploadUtil.SftpUploadException e) { append(context,"上传文件失败，请确认资源信息是否有误:"+JSON.toJSONString(machineInfo)); toBadEnding(); @@ -87,16 +83,15 @@ public class AFLSlotCompileWorker extends DockerWorker { // 执行预设命令 ,进入目录 String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" + "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录 - "mkdir -p "+compilePath+"\n"+ - "cd "+compilePath+"\n"; + "mkdir -p "+workDir+"\n"+ + "cd "+workDir+"\n"; Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT); String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null; allCommand += commandScript+"\nll -s"; + log.info("AFL插装编译容器内执行的命令：{}",allCommand); dockerRun(allCommand,dockerInfo,context); // 更新全局变量 - // /test目录下编译文件与目标所在目录,当前pipelineId+taskId组合 - globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_KEY,compilePath); - // /test目录下当前流水线工作目录（没有task部分） + // /test目录下当前流水线工作目录 globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java index e29a28b3..1f8fe4d7 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java @@ -5,6 +5,7 @@ import cd.casic.ci.process.dto.req.resource.ResourceQueryReq; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.engine.constant.AFLConstant; import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; +import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.DockerWorker; import cd.casic.ci.process.process.dataObject.task.PipTask; @@ -12,6 +13,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; +import java.io.File; import java.util.Map; @WorkAtom(taskType = "AFL") @@ -23,23 +25,23 @@ public class AFLWorker extends DockerWorker { public void execute(TaskRunContext context) { int statusCode = -1; Map localVariables = context.getLocalVariables(); + Map globalVariables = context.getGlobalVariables(); if (context.getContextDef() instanceof PipTask taskDef) { log.info(taskDef.getTaskName()); Map taskProperties = taskDef.getTaskProperties(); //从taskProperties中获取资源id String resourceType = taskProperties.get("resourceType").toString(); String resourceId = taskProperties.get("resourceId").toString(); - // 待测试文件路径 - String binaryPath = taskProperties.get(AFLConstant.BINARY) instanceof String ? ((String) taskProperties.get(AFLConstant.BINARY)) : null; - String output = taskProperties.get(AFLConstant.OUTPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.OUTPUT)) : null; - String input = taskProperties.get(AFLConstant.INPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.INPUT)) : null; - String workDir = taskProperties.get(AFLConstant.WORK_DIR) instanceof String ? ((String) taskProperties.get(AFLConstant.WORK_DIR)) : null; - - if (StringUtils.isEmpty(binaryPath) || - StringUtils.isEmpty(output) || - StringUtils.isEmpty(input) || + String commandEnd = taskProperties.get("commandEnd").toString(); + String imageName = taskProperties.get("imageName").toString(); + // 代码编译目录 + String workDir = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY); + // 入参目录 + String seedPath = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY); + if (StringUtils.isEmpty(seedPath) || StringUtils.isEmpty(workDir) || StringUtils.isEmpty(resourceId) || + StringUtils.isEmpty(commandEnd) || StringUtils.isEmpty(resourceType)) { // 缺少参数 toBadEnding(); @@ -52,12 +54,16 @@ public class AFLWorker extends DockerWorker { append(context,"当前机器不支持docker"); return; } - String commandScript = "docker run -v "+workDir+":/test -it aflplusplus/aflplusplus bash\n" + + String output = workDir+PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT; + // + String volumeWorkDirPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX; + String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" + "cd /test\n" + - "afl-fuzz -i "+input+" -o "+output+" -t 3000 -Q "+binaryPath+" @@"; + "mkdir -p"+output+"\n" + + "afl-fuzz -i "+seedPath+" -o "+output+" "+workDir+ File.separator+commandEnd; try { //将节点的配置信息反编译成对象 - log.info("构建脚本" + commandScript); + log.info("AFL模糊测试执行脚本：{}",commandScript); // 获取docker 暂时先写固定值 dockerRun(commandScript,resourceListByType.getDockerEndpointList().get(0),context); } catch (Exception e) { diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java index 0aaa89f9..1be8f64f 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java @@ -33,8 +33,7 @@ public class TestCaseGenerationWorker extends DockerWorker { Map globalVariables = context.getGlobalVariables(); String workDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null; if (context.getContextDef() instanceof PipTask taskDef) { - String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_PREFIX+taskDef.getId()+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH; - log.info(taskDef.getTaskName()); + String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED; Map taskProperties = taskDef.getTaskProperties(); Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY); String binaryName = taskProperties.get("binaryName").toString(); @@ -57,15 +56,15 @@ public class TestCaseGenerationWorker extends DockerWorker { //如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器 if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) { // 容器内部test 目录, 获取编译后的文件 - String compileDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null; - - String commandScript = "cd /test\n" + + String runScript = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" // 目前测试用例ai生成只有这个路径能跑 + + "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n" + "mkdir -p "+seedPath +"\n"+ - "PYTHONPATH=/test/CaseGenerator/src python3 /test/CaseGenerator/src/CaseGenerator/main.py --work-dir "+compileDir+" --binary "+compileDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100"; + "cd "+seedPath +"\n"+ + "PYTHONPATH=CaseGenerator/src python3 CaseGenerator/src/CaseGenerator/main.py --work-dir "+workDir+" --binary "+workDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100"; //将节点的配置信息反编译成对象 - log.info("构建脚本" + commandScript); + log.info("测试用例生成-AI生成，实际执行命令:{}" , runScript); //获取机器 - dockerRun(commandScript,dockerInfo,context); + dockerRun(runScript,dockerInfo,context); } else { // 文件上传 String filePath = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE)) : null; @@ -74,9 +73,10 @@ public class TestCaseGenerationWorker extends DockerWorker { toBadEnding(); } File file = new File(filePath); - String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; + String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+workDir+File.separator+seedPath; // 将文件上传到服务器的 目录底下 - try { + log.info("测试用例选用上传模式,种子文件路径：{},种子上传路径:{}",filePath,seedTarget); + try { SftpUploadUtil.uploadFileViaSftp( machineInfo.getMachineHost() ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,seedTarget,file.getName()); @@ -85,6 +85,7 @@ public class TestCaseGenerationWorker extends DockerWorker { log.error("seed文件上传失败",e); } } + globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY,seedPath); } } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/DockerWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/DockerWorker.java index 63e134fb..e47648d7 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/DockerWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/DockerWorker.java @@ -126,7 +126,7 @@ public abstract class DockerWorker extends BaseWorker{ log.warn("未知流类型: {}", frame.getStreamType()); } } - }).awaitCompletion(60, TimeUnit.SECONDS); + }).awaitCompletion(); } catch (InterruptedException e) { log.error("执行异常",e); append(context,"执行异常,容器id："+containerId);