From f87d5c6c4e40ba092943f5b6dec930f193a15236 Mon Sep 17 00:00:00 2001 From: even <827656971@qq.com> Date: Thu, 26 Jun 2025 18:52:20 +0800 Subject: [PATCH 1/3] =?UTF-8?q?afl=20=E7=9B=B8=E5=85=B3=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../process/engine/worker/afl/AFLSlotCompileWorker.java | 9 ++++++--- .../cd/casic/ci/process/engine/worker/afl/AFLWorker.java | 9 ++++++--- .../service/variable/impl/VariableServiceImpl.java | 3 +++ ops-server/src/test/java/cd/casic/server/SftpTest.java | 4 ++++ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java index c455f240..db8e8e28 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java @@ -96,6 +96,7 @@ public class AFLSlotCompileWorker extends DockerWorker { String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" + "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录 "mkdir -p "+workDir+"\n"+ + "chmod -R o+rwx "+ workDir +"\n"+ "cd "+workDir+"\n"+handleZipFile(fileName); Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT); String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null; @@ -135,8 +136,9 @@ public class AFLSlotCompileWorker extends DockerWorker { private void putInCompileResult(TaskRunContext context,String path, PipResourceMachine machineInfo){ String host = machineInfo.getMachineHost(); - String user = machineInfo.getPassword(); - String password = machineInfo.getPassword(); + String user = machineInfo.getUsername(); + String password = CryptogramUtil.doDecrypt(machineInfo.getPassword()); + append(context,"AFL编译完毕"); List fileNameList = null; try { JSch jsch = new JSch(); @@ -149,7 +151,7 @@ public class AFLSlotCompileWorker extends DockerWorker { Vector files = sftpChannel.ls(path); fileNameList = new ArrayList<>(files.size()); for (ChannelSftp.LsEntry file : files) { - if (!file.getAttrs().isDir()) { + if (!file.getAttrs().isDir()&&!file.getFilename().contains(".")) { fileNameList.add(file.getFilename()); } } @@ -161,6 +163,7 @@ public class AFLSlotCompileWorker extends DockerWorker { if (!CollectionUtils.isEmpty(fileNameList)&&context!=null) { Map taskProperties = context.getGlobalVariables(); // 放入全局上下文中 + append(context,"AFL编译完毕:"+JSON.toJSONString(fileNameList)); taskProperties.put(AFLSlotCompileConstant.COMPILE_RESULT,fileNameList); } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java index 1a8dc196..f0b94c39 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java @@ -36,8 +36,9 @@ public class AFLWorker extends DockerWorker { //从taskProperties中获取资源id String resourceType = taskProperties.get("resourceType").toString(); String resourceId = taskProperties.get("resourceId").toString(); - String commandEnd = taskProperties.get("commandEnd").toString(); - String imageName = taskProperties.get("imageName").toString(); + String commandEnd = taskProperties.get("commandEnd") instanceof String ? ((String) taskProperties.get("commandEnd")) : null; + String imageName = taskProperties.get("imageName") instanceof String ? ((String) taskProperties.get("imageName")) : null; + String executableName = taskProperties.get("executableName") instanceof String ? ((String) taskProperties.get("executableName")) : null; // 代码编译目录 String workDir = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY); // 入参目录 @@ -45,6 +46,8 @@ public class AFLWorker extends DockerWorker { if (StringUtils.isEmpty(seedPath) || StringUtils.isEmpty(workDir) || StringUtils.isEmpty(resourceId) || + StringUtils.isEmpty(imageName) || + StringUtils.isEmpty(executableName) || StringUtils.isEmpty(commandEnd) || StringUtils.isEmpty(resourceType)) { // 缺少参数 @@ -71,7 +74,7 @@ public class AFLWorker extends DockerWorker { "cd " +PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+ "\n"+ cdSourceName(fileName) + "mkdir -p "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+"\n" + - "afl-fuzz -i "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+" "+commandEnd; + "afl-fuzz -i "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+" "+ executableName + " " +commandEnd; try { //将节点的配置信息反编译成对象 log.info("AFL模糊测试执行脚本：{}",commandScript); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java index eabce159..23bdb0d6 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java @@ -5,11 +5,13 @@ import cd.casic.ci.process.engine.manager.RunContextManager; import cd.casic.ci.process.engine.runContext.BaseRunContext; import cd.casic.ci.process.process.service.variable.VariableService; import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.util.Map; @Service +@Slf4j public class VariableServiceImpl implements VariableService { @Resource private RunContextManager contextManager; @@ -20,6 +22,7 @@ public class VariableServiceImpl implements VariableService { return ""; } Map globalVariables = context.getGlobalVariables(); + log.info("当前全局变量：{}",globalVariables); return globalVariables.get(varName); } } diff --git a/ops-server/src/test/java/cd/casic/server/SftpTest.java b/ops-server/src/test/java/cd/casic/server/SftpTest.java index cbba6dc9..48d37999 100644 --- a/ops-server/src/test/java/cd/casic/server/SftpTest.java +++ b/ops-server/src/test/java/cd/casic/server/SftpTest.java @@ -36,6 +36,10 @@ public class SftpTest { System.out.println(CryptogramUtil.doEncrypt("hnidc0327cn!@#xhh")); } @Test + public void ssh(){ + System.out.println(CryptogramUtil.doDecrypt("cf917c4ff8463a54df75120e2ff802e8")); + } + @Test public void getFileList(){ String host = "175.6.27.158"; String user = "ubuntu"; From 8157f29645698e1c4ec512047796d1933b0d0f46 Mon Sep 17 00:00:00 2001 From: even <827656971@qq.com> Date: Fri, 27 Jun 2025 14:35:53 +0800 Subject: [PATCH 2/3] =?UTF-8?q?AFL=E7=9B=B8=E5=85=B3=20worker=EF=BC=9A?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B=E7=94=9F=E6=88=90=E5=8E=BB?= =?UTF-8?q?=E6=8E=89=E8=A1=A8=E5=8D=95=E7=9A=84=E8=B5=84=E6=BA=90=E9=80=89?= =?UTF-8?q?=E9=A1=B9=EF=BC=8CAFL=E5=8E=BB=E6=8E=89=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E3=80=81=E9=95=9C=E5=83=8F=E8=A1=A8=E5=8D=95=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../process/engine/constant/AFLConstant.java | 11 ++-- .../constant/AFLSlotCompileConstant.java | 1 - ...ant.java => PipelineVariableConstant.java} | 19 +++---- .../constant/TestCaseGenerationConstant.java | 5 +- .../worker/afl/AFLSlotCompileWorker.java | 52 ++++++++++++------- .../process/engine/worker/afl/AFLWorker.java | 39 ++++++++------ .../worker/afl/TestCaseGenerationWorker.java | 40 +++++++------- .../engine/worker/base/BaseWorker.java | 41 ++++++++++++++- 8 files changed, 130 insertions(+), 78 deletions(-) rename modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/{PipelineGlobalVariableConstant.java => PipelineVariableConstant.java} (68%) diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java index 67fb898d..2f48a1a9 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java @@ -1,12 +1,7 @@ package cd.casic.ci.process.engine.constant; public class AFLConstant { - /** - * 脚本内容 - */ - public static final String COMMAND_SCRIPT ="buildScript"; - public static final String WORK_DIR = "workDir"; - public static final String BINARY="binary"; - public static final String OUTPUT = "output"; - public static final String INPUT="input"; + + public static final String COMMAND_END="commandEnd"; + public static final String EXECUTABLE_NAME ="executableName"; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLSlotCompileConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLSlotCompileConstant.java index 350cd8fc..6d158047 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLSlotCompileConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLSlotCompileConstant.java @@ -4,5 +4,4 @@ public class AFLSlotCompileConstant { public static final String MANAGER_ID = "managerId"; public static final String IMAGE_NAME = "imageName"; public static final String COMMAND_SCRIPT ="buildScript"; - public static final String COMPILE_RESULT ="compileResult"; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineVariableConstant.java similarity index 68% rename from modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java rename to modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineVariableConstant.java index 55b45c1b..42fb5ec4 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineVariableConstant.java @@ -1,6 +1,9 @@ package cd.casic.ci.process.engine.constant; - -public class PipelineGlobalVariableConstant { +/** + * key为后缀代表除了根据key获取，不直接使用值 + * 非key后缀的值都会在某地直接使用修改请谨慎 + * */ +public class PipelineVariableConstant { /** * AFL线基础工作目录，完整的工作目录为 基础工作目录 + /PIP_${pipelineId} * */ @@ -11,15 +14,6 @@ public class PipelineGlobalVariableConstant { * 真正用于绑定AFL docker卷挂载的目录的全局变量的key，值为 基础工作目录 + /PIP_${pipelineId} * */ public static final String AFL_DOCKER_WORK_DIR_KEY = "AFL_DOCKER_WORK_DIR_KEY"; - /** - * 目标上传到工作目录下文件夹的前缀（TASK_${taskId}） - * */ - public static final String AFL_DOCKER_SLOT_COMPILE_PATH_PREFIX = "TASK_"; - /** - * 执行插装编译节点以后输出目录，用于设置到全局变量给后续节点使用 - * - * */ - public static final String AFL_DOCKER_SLOT_COMPILE_PATH_KEY = "AFL_DOCKER_SLOT_COMPILE_PATH_KEY"; /** * 测试用例生成，ai生成或者直接文件上传的文件夹名称：SEED * */ @@ -30,4 +24,7 @@ public class PipelineGlobalVariableConstant { * AFL输出目录 * */ public static final String AFL_DOCKER_OUTPUT = "ai_afl"; + public static final String COMPILE_RESULT_KEY ="compileResult"; + public static final String AFL_RESOURCE_MANAGER_ID_KEY = "managerId"; + public static final String IMAGE_NAME = "IMAGE_NAME"; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/TestCaseGenerationConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/TestCaseGenerationConstant.java index 197115fa..f90a530c 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/TestCaseGenerationConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/TestCaseGenerationConstant.java @@ -4,13 +4,12 @@ public class TestCaseGenerationConstant { /** * 脚本内容key */ - public static final String COMMAND_SCRIPT ="buildScript"; public static final String CASE_TYPE_KEY ="caseType"; public static final String CASE_TYPE_AI ="AI"; public static final String CASE_TYPE_FILE ="FILE"; public static final String CASE_TYPE_ITEM = "TEST"; public static final String SEED_SOURCE = "filePath"; - public static final String SEED_TARGET = "outputDir"; public static final String ITEM_LIST = "itemList"; - public static final String MANAGER_ID = "managerId"; + public static final String BINARY_NAME="binaryName"; + } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java index db8e8e28..d159b896 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java @@ -5,20 +5,15 @@ import cd.casic.ci.process.dto.req.resource.ResourceQueryReq; import cd.casic.ci.process.dto.resp.resource.ResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.engine.constant.AFLSlotCompileConstant; -import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; -import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.DockerWorker; -import cd.casic.ci.process.process.dataObject.machine.MachineInfo; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine; import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.task.PipTask; -import cd.casic.ci.process.process.service.resource.ResourceManagerService; import cd.casic.ci.process.process.service.target.TargetVersionService; import cd.casic.ci.process.util.CryptogramUtil; import cd.casic.ci.process.util.SftpUploadUtil; -import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo; import com.alibaba.fastjson.JSON; import com.jcraft.jsch.*; @@ -28,10 +23,10 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Vector; +import java.util.*; + +import static cd.casic.ci.process.engine.constant.PipelineVariableConstant.*; + @WorkAtom(taskType = "AFL_SLOT_COMPILE") @Slf4j @@ -42,10 +37,10 @@ public class AFLSlotCompileWorker extends DockerWorker { public void execute(TaskRunContext context) { PipTask task = context.getContextDef() instanceof PipTask ? ((PipTask) context.getContextDef()) : null; // 每一个流水线不同目录互不干扰 - String workDir = PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_PREFIX + task.getPipelineId(); + String workDir = AFL_DOCKER_WORK_DIR_PREFIX + task.getPipelineId(); Map globalVariables = context.getGlobalVariables(); if (!globalVariables.containsKey(workDir)) { - globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); + globalVariables.put(AFL_DOCKER_WORK_DIR_KEY,workDir); } Map taskProperties = task.getTaskProperties(); String managerId = taskProperties.get(AFLSlotCompileConstant.MANAGER_ID) instanceof String ? ((String) taskProperties.get(AFLSlotCompileConstant.MANAGER_ID)) : null; @@ -80,7 +75,7 @@ public class AFLSlotCompileWorker extends DockerWorker { } // 工作目录下的相对路径,/test 作为根路径 // 卷挂载目录+流水线目录 + task编译目录 - String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir; + String realPath = AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir; // 上传目标文件 到指定资源服务器 try { append(context,"AFL编译，上传文件路径:"+realPath); @@ -93,8 +88,8 @@ public class AFLSlotCompileWorker extends DockerWorker { toBadEnding(); } // 执行预设命令 ,进入目录 - String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" + - "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录 + String allCommand = "docker run -v "+ AFL_VOLUME_WORK_DIR_PREFIX +":"+ AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" + + "cd "+ AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录 "mkdir -p "+workDir+"\n"+ "chmod -R o+rwx "+ workDir +"\n"+ "cd "+workDir+"\n"+handleZipFile(fileName); @@ -106,8 +101,27 @@ public class AFLSlotCompileWorker extends DockerWorker { dockerRun(allCommand,dockerInfo,context); // 更新全局变量 // /test目录下当前流水线工作目录 - globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); - putInCompileResult(context,realPath+File.separator+getSourceName(fileName),machineInfo); + globalVariables.put(AFL_DOCKER_WORK_DIR_KEY,workDir); + // 获取编译结果并存放 + List compileResult = getCompileResult(context, realPath + File.separator + getSourceName(fileName), machineInfo); + if (Objects.isNull(getGlobalVariable(context, COMPILE_RESULT_KEY))) { + // 如果全局变量不存在则放入全局变量中 + putGlobalVariable(context,COMPILE_RESULT_KEY,compileResult);; + } + // 放入最近一级作用域 + putSecondStageVariable(context,COMPILE_RESULT_KEY,compileResult); + // 放入 所选资源供测试用例生成，AFL模糊测试节点使用 + if (Objects.isNull(getGlobalVariable(context, AFL_RESOURCE_MANAGER_ID_KEY))) { + // 如果全局变量不存在则放入全局变量中 + putGlobalVariable(context, AFL_RESOURCE_MANAGER_ID_KEY,managerId);; + } + putSecondStageVariable(context,AFL_RESOURCE_MANAGER_ID_KEY,managerId); + // 放入镜像名称 + if (Objects.isNull(getGlobalVariable(context, IMAGE_NAME))) { + // 如果全局变量不存在则放入全局变量中 + putGlobalVariable(context, IMAGE_NAME,imageName);; + } + putSecondStageVariable(context,IMAGE_NAME,imageName); } private String handleZipFile(String fileName){ if (StringUtils.isEmpty(fileName)) { @@ -134,7 +148,7 @@ public class AFLSlotCompileWorker extends DockerWorker { return ""; } - private void putInCompileResult(TaskRunContext context,String path, PipResourceMachine machineInfo){ + private List getCompileResult(TaskRunContext context, String path, PipResourceMachine machineInfo){ String host = machineInfo.getMachineHost(); String user = machineInfo.getUsername(); String password = CryptogramUtil.doDecrypt(machineInfo.getPassword()); @@ -164,8 +178,10 @@ public class AFLSlotCompileWorker extends DockerWorker { Map taskProperties = context.getGlobalVariables(); // 放入全局上下文中 append(context,"AFL编译完毕:"+JSON.toJSONString(fileNameList)); - taskProperties.put(AFLSlotCompileConstant.COMPILE_RESULT,fileNameList); + + return fileNameList; } + return Collections.emptyList(); } public String getSourceName(String fileName){ int dotIndex = fileName.lastIndexOf("."); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java index f0b94c39..4019ddf0 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java @@ -2,10 +2,12 @@ package cd.casic.ci.process.engine.worker.afl; import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.dto.req.resource.ResourceQueryReq; +import cd.casic.ci.process.dto.resp.resource.ResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.engine.constant.AFLConstant; +import cd.casic.ci.process.engine.constant.AFLSlotCompileConstant; import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; -import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant; +import cd.casic.ci.process.engine.constant.PipelineVariableConstant; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.DockerWorker; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; @@ -20,6 +22,9 @@ import org.springframework.util.CollectionUtils; import java.io.File; import java.util.Map; +import static cd.casic.ci.process.engine.constant.AFLConstant.*; +import static cd.casic.ci.process.engine.constant.PipelineVariableConstant.*; + @WorkAtom(taskType = "AFL") @Slf4j public class AFLWorker extends DockerWorker { @@ -29,20 +34,25 @@ public class AFLWorker extends DockerWorker { public void execute(TaskRunContext context) { int statusCode = -1; Map localVariables = context.getLocalVariables(); - Map globalVariables = context.getGlobalVariables(); if (context.getContextDef() instanceof PipTask taskDef) { log.info(taskDef.getTaskName()); Map taskProperties = taskDef.getTaskProperties(); + String managerId = getVariableNearby(context, AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) getVariableNearby(context, AFL_RESOURCE_MANAGER_ID_KEY)) : null; + // ssh 上传目标文件 + ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId); + if (resourceById==null) { + append(context,"缺少资源信息，请添加编译节点"); + } //从taskProperties中获取资源id - String resourceType = taskProperties.get("resourceType").toString(); - String resourceId = taskProperties.get("resourceId").toString(); - String commandEnd = taskProperties.get("commandEnd") instanceof String ? ((String) taskProperties.get("commandEnd")) : null; - String imageName = taskProperties.get("imageName") instanceof String ? ((String) taskProperties.get("imageName")) : null; - String executableName = taskProperties.get("executableName") instanceof String ? ((String) taskProperties.get("executableName")) : null; + String resourceType = "docker"; + String resourceId = resourceById.getDockerId(); + String commandEnd = taskProperties.get(COMMAND_END) instanceof String ? ((String) taskProperties.get(COMMAND_END)) : null; + String imageName = getVariableNearby(context,IMAGE_NAME) instanceof String ? ((String) getVariableNearby(context,IMAGE_NAME)) : null; + String executableName = taskProperties.get(EXECUTABLE_NAME) instanceof String ? ((String) taskProperties.get(EXECUTABLE_NAME)) : null; // 代码编译目录 - String workDir = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY); + String workDir = (String)getGlobalVariable(context,AFL_DOCKER_WORK_DIR_KEY); // 入参目录 - String seedPath = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY); + String seedPath = (String)getGlobalVariable(context,AFL_DOCKER_SEED_PATH_KEY); if (StringUtils.isEmpty(seedPath) || StringUtils.isEmpty(workDir) || StringUtils.isEmpty(resourceId) || @@ -61,9 +71,8 @@ public class AFLWorker extends DockerWorker { append(context,"当前机器不支持docker"); return; } - String output = workDir +File.separator+ PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT; - // - String volumeWorkDirPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX; + String output = workDir +File.separator+ AFL_DOCKER_OUTPUT; + String volumeWorkDirPath = AFL_VOLUME_WORK_DIR_PREFIX; PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef(); TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId()); String fileName = targetVersion.getFileName(); @@ -71,10 +80,10 @@ public class AFLWorker extends DockerWorker { Object timeObj = taskProperties.get("runningTime"); Double timeHours = Double.valueOf(String.valueOf(timeObj)); String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" + - "cd " +PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+ "\n"+ + "cd " + AFL_DOCKER_BASE_DIR+File.separator+workDir+ "\n"+ cdSourceName(fileName) + - "mkdir -p "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+"\n" + - "afl-fuzz -i "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+" "+ executableName + " " +commandEnd; + "mkdir -p "+ AFL_DOCKER_BASE_DIR+File.separator+output+"\n" + + "afl-fuzz -i "+ AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+ AFL_DOCKER_BASE_DIR+File.separator+output+" "+ executableName + " " +commandEnd; try { //将节点的配置信息反编译成对象 log.info("AFL模糊测试执行脚本：{}",commandScript); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java index e16b9f2b..53cf34e7 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java @@ -3,16 +3,12 @@ package cd.casic.ci.process.engine.worker.afl; import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.dto.req.resource.ResourceQueryReq; -import cd.casic.ci.process.dto.req.testCase.TestCaseInfoReq; import cd.casic.ci.process.dto.resp.resource.ResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; -import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; -import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant; +import cd.casic.ci.process.engine.constant.PipelineVariableConstant; import cd.casic.ci.process.engine.constant.TestCaseGenerationConstant; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.DockerWorker; -import cd.casic.ci.process.engine.worker.base.SshWorker; -import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine; import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.dataObject.testCase.TestCaseInfo; @@ -24,13 +20,15 @@ import com.alibaba.fastjson.JSON; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; import java.io.File; import java.util.List; import java.util.Map; import java.util.Objects; +import static cd.casic.ci.process.engine.constant.PipelineVariableConstant.*; +import static cd.casic.ci.process.engine.constant.TestCaseGenerationConstant.*; + @WorkAtom(taskType = "TEST_CASE_GENERATION") @Slf4j public class TestCaseGenerationWorker extends DockerWorker { @@ -40,14 +38,14 @@ public class TestCaseGenerationWorker extends DockerWorker { public void execute(TaskRunContext context) { int statusCode = -1; Map globalVariables = context.getGlobalVariables(); - String workDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null; + String workDir = globalVariables.get(AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(AFL_DOCKER_WORK_DIR_KEY)) : null; if (context.getContextDef() instanceof PipTask taskDef) { - String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED; + String seedPath= workDir+File.separator+ AFL_DOCKER_SEED; Map taskProperties = taskDef.getTaskProperties(); - Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY); - String binaryName = taskProperties.get("binaryName") instanceof String ? ((String) taskProperties.get("binaryName")) : null; - String managerId = taskProperties.get("managerId") instanceof String ? ((String) taskProperties.get("managerId")) : null; - Object itemListObject = taskProperties.get(TestCaseGenerationConstant.ITEM_LIST); + Object caseType = taskProperties.get(CASE_TYPE_KEY); + String binaryName = taskProperties.get(BINARY_NAME) instanceof String ? ((String) taskProperties.get("binaryName")) : null; + String managerId = getVariableNearby(context, AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) getVariableNearby(context, AFL_RESOURCE_MANAGER_ID_KEY)) : null; + Object itemListObject = taskProperties.get(ITEM_LIST); List itemList = JSON.parseArray(JSON.toJSONString(itemListObject),String.class); ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId); String machineId = resourceById.getMachineId(); @@ -65,10 +63,10 @@ public class TestCaseGenerationWorker extends DockerWorker { PipResourceMachine machineInfo = machineQuery.getResourceMachineList().get(0); DockerEndpointDo dockerInfo = dockerQuery.getDockerEndpointList().get(0); //如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器 - if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) { + if (CASE_TYPE_AI.equals(caseType)) { // 容器内部test 目录, 获取编译后的文件 - String runScript = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" // 目前测试用例ai生成只有这个路径能跑 - + "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n" + + String runScript = "docker run -v "+ AFL_VOLUME_WORK_DIR_PREFIX +":"+ AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" // 目前测试用例ai生成只有这个路径能跑 + + "cd "+ AFL_DOCKER_BASE_DIR+"\n" + "mkdir -p "+seedPath +"\n"+ "cd "+seedPath +"\n"+ "PYTHONPATH=CaseGenerator/src python3 CaseGenerator/src/CaseGenerator/main.py --work-dir "+workDir+" --binary "+workDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100"; @@ -76,9 +74,9 @@ public class TestCaseGenerationWorker extends DockerWorker { log.info("测试用例生成-AI生成，实际执行命令:{}" , runScript); //获取机器 dockerRun(runScript,dockerInfo,context); - } else if (TestCaseGenerationConstant.CASE_TYPE_FILE.equals(caseType)){ + } else if (CASE_TYPE_FILE.equals(caseType)){ // 文件上传 - Object filePathObj = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE); + Object filePathObj = taskProperties.get(SEED_SOURCE); log.info("当前测试用例taskProperties:{}",taskProperties); if (Objects.isNull(filePathObj)) { append(context,"缺少文件"); @@ -87,7 +85,7 @@ public class TestCaseGenerationWorker extends DockerWorker { List filePathList = JSON.parseArray(JSON.toJSONString(filePathObj),String.class); for (String filePath : filePathList) { File file = new File(filePath); - String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; + String seedTarget = AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; // 将文件上传到服务器的 目录底下 log.info("测试用例选用上传模式,种子文件路径：{},种子上传路径:{}",filePath,seedTarget); append(context,"测试用例选用上传模式,种子文件路径："+filePath+",种子上传路径:"+seedPath); @@ -101,9 +99,9 @@ public class TestCaseGenerationWorker extends DockerWorker { } append(context,file.getName()+"上传文件成功！"); } - }else if (TestCaseGenerationConstant.CASE_TYPE_ITEM.equals(caseType)){ + }else if (CASE_TYPE_ITEM.equals(caseType)){ List testCaseInfos = testCaseInfoService.listByIds(itemList); - String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; + String seedTarget = AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; for (TestCaseInfo testCaseInfo : testCaseInfos) { String testFilePath = testCaseInfo.getTestFilePath(); if (StringUtils.isEmpty(testFilePath)) { @@ -122,7 +120,7 @@ public class TestCaseGenerationWorker extends DockerWorker { } } } - globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY,seedPath); + globalVariables.put(AFL_DOCKER_SEED_PATH_KEY,seedPath); } } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java index 9c6743c1..53cb5661 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java @@ -106,7 +106,46 @@ public abstract class BaseWorker implements Runnable{ } } public abstract void execute(TaskRunContext context); - public void toBadEnding(){ + protected void toBadEnding(){ throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),""); } + /** + * 往全局变量中保存变量，作用域为全局 + * */ + protected void putGlobalVariable(TaskRunContext context,String key,Object value){ + Map globalVariables = context.getGlobalVariables(); + globalVariables.put(key,value); + } + /** + * 往当前执行链路中保存变量，作用域为当前二级stage + * */ + protected void putSecondStageVariable(TaskRunContext context,String key,Object value){ + BaseRunContext stageContext = context.getParentContext(); + Map localVariables = stageContext.getLocalVariables(); + localVariables.put(key,value); + } + protected Object getGlobalVariable(TaskRunContext context,String key){ + Map globalVariables = context.getGlobalVariables(); + return globalVariables.get(key); + } + protected Object getSecondStageVariable(TaskRunContext context,String key){ + BaseRunContext stageContext = context.getParentContext(); + Map localVariables = stageContext.getLocalVariables(); + return localVariables.get(key); + } + /** + * 就近获取从secondStage 到pipeline 逐级获取获取到了直接返回 + * */ + protected Object getVariableNearby(TaskRunContext context,String key){ + BaseRunContext parentContext = context.getParentContext(); + Map localVariables = parentContext.getLocalVariables(); + Map globalVariables = context.getGlobalVariables(); + if (localVariables.containsKey(key)) { + return localVariables.get(key); + } + if (globalVariables.containsKey(key)) { + return globalVariables.get(key); + } + return null; + } } From 7160577a5688d1a3f6e730ad3dd2fae57c63c3a8 Mon Sep 17 00:00:00 2001 From: even <827656971@qq.com> Date: Fri, 27 Jun 2025 15:28:26 +0800 Subject: [PATCH 3/3] =?UTF-8?q?AFL=E7=9B=B8=E5=85=B3=20worker=EF=BC=9A?= =?UTF-8?q?=E9=AD=94=E6=B3=95=E5=80=BC=E6=8A=BD=E5=8F=96=E4=B8=BA=E9=9D=99?= =?UTF-8?q?=E6=80=81=E5=8F=98=E9=87=8F=20=E8=BF=90=E8=A1=8C=E6=97=B6?= =?UTF-8?q?=E5=85=A8=E5=B1=80=E5=8F=98=E9=87=8F=E4=BD=9C=E7=94=A8=E5=9F=9F?= =?UTF-8?q?=E8=AE=BF=E9=97=AE=E6=96=B9=E6=B3=95=E6=8F=90=E4=BE=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cd/casic/ci/api/VariableController.java | 5 +++++ .../ci/process/engine/constant/AFLConstant.java | 1 + .../ci/process/engine/worker/afl/AFLWorker.java | 6 +++--- .../process/service/variable/VariableService.java | 3 +++ .../service/variable/impl/VariableServiceImpl.java | 12 ++++++++++++ 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/VariableController.java b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/VariableController.java index 41e7df5e..8ac74f7c 100644 --- a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/VariableController.java +++ b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/VariableController.java @@ -14,6 +14,7 @@ import org.springframework.web.bind.annotation.*; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * @pi.protocol: http @@ -29,4 +30,8 @@ public class VariableController { Object variable = variableService.getPipelineContextGlobalVariable(pipelineId, varName); return CommonResult.success(variable); } + @GetMapping("/getPipelineContextGlobalVariables/{pipelineId}") + public CommonResult> getPipelineContextGlobalVariables(@PathVariable("pipelineId") String pipelineId){ + return CommonResult.success(variableService.getPipelineContextGlobalVariables(pipelineId)); + } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java index 2f48a1a9..91894e83 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java @@ -4,4 +4,5 @@ public class AFLConstant { public static final String COMMAND_END="commandEnd"; public static final String EXECUTABLE_NAME ="executableName"; + public static final String RUNNING_TIME ="runningTime"; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java index 4019ddf0..7f7f5ab8 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java @@ -77,13 +77,13 @@ public class AFLWorker extends DockerWorker { TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId()); String fileName = targetVersion.getFileName(); append(context,"AFL 输出路径："+output); - Object timeObj = taskProperties.get("runningTime"); + Object timeObj = taskProperties.get(RUNNING_TIME); Double timeHours = Double.valueOf(String.valueOf(timeObj)); - String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" + + String commandScript = "docker run -v "+volumeWorkDirPath+":"+AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" + "cd " + AFL_DOCKER_BASE_DIR+File.separator+workDir+ "\n"+ cdSourceName(fileName) + "mkdir -p "+ AFL_DOCKER_BASE_DIR+File.separator+output+"\n" + - "afl-fuzz -i "+ AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+ AFL_DOCKER_BASE_DIR+File.separator+output+" "+ executableName + " " +commandEnd; + "afl-fuzz -i "+ AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+ AFL_DOCKER_BASE_DIR+File.separator+output+" ./"+ executableName + " " +commandEnd; try { //将节点的配置信息反编译成对象 log.info("AFL模糊测试执行脚本：{}",commandScript); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/VariableService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/VariableService.java index 48293818..37934d6f 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/VariableService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/VariableService.java @@ -1,5 +1,8 @@ package cd.casic.ci.process.process.service.variable; +import java.util.Map; + public interface VariableService { Object getPipelineContextGlobalVariable(String pipelineId,String varName); + Map getPipelineContextGlobalVariables(String pipelineId); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java index 23bdb0d6..0b1ff04b 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java @@ -8,6 +8,7 @@ import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import java.util.HashMap; import java.util.Map; @Service @@ -25,4 +26,15 @@ public class VariableServiceImpl implements VariableService { log.info("当前全局变量：{}",globalVariables); return globalVariables.get(varName); } + + @Override + public Map getPipelineContextGlobalVariables(String pipelineId) { + BaseRunContext context = contextManager.getContext(pipelineId); + if (context== null) { + return new HashMap<>(); + } + Map globalVariables = context.getGlobalVariables(); + log.info("当前全局变量：{}",globalVariables); + return globalVariables; + } }