Merge remote-tracking branch 'origin/master'

This commit is contained in:
HopeLi 2025-06-27 16:41:44 +08:00
commit 3685c0a312
12 changed files with 168 additions and 82 deletions

View File

@ -14,6 +14,7 @@ import org.springframework.web.bind.annotation.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* @pi.protocol: http * @pi.protocol: http
@ -29,4 +30,8 @@ public class VariableController {
Object variable = variableService.getPipelineContextGlobalVariable(pipelineId, varName); Object variable = variableService.getPipelineContextGlobalVariable(pipelineId, varName);
return CommonResult.success(variable); return CommonResult.success(variable);
} }
@GetMapping("/getPipelineContextGlobalVariables/{pipelineId}")
public CommonResult<Map<String,Object>> getPipelineContextGlobalVariables(@PathVariable("pipelineId") String pipelineId){
return CommonResult.success(variableService.getPipelineContextGlobalVariables(pipelineId));
}
} }

View File

@ -1,12 +1,8 @@
package cd.casic.ci.process.engine.constant; package cd.casic.ci.process.engine.constant;
public class AFLConstant { public class AFLConstant {
/**
* 脚本内容 public static final String COMMAND_END="commandEnd";
*/ public static final String EXECUTABLE_NAME ="executableName";
public static final String COMMAND_SCRIPT ="buildScript"; public static final String RUNNING_TIME ="runningTime";
public static final String WORK_DIR = "workDir";
public static final String BINARY="binary";
public static final String OUTPUT = "output";
public static final String INPUT="input";
} }

View File

@ -4,5 +4,4 @@ public class AFLSlotCompileConstant {
public static final String MANAGER_ID = "managerId"; public static final String MANAGER_ID = "managerId";
public static final String IMAGE_NAME = "imageName"; public static final String IMAGE_NAME = "imageName";
public static final String COMMAND_SCRIPT ="buildScript"; public static final String COMMAND_SCRIPT ="buildScript";
public static final String COMPILE_RESULT ="compileResult";
} }

View File

@ -1,6 +1,9 @@
package cd.casic.ci.process.engine.constant; package cd.casic.ci.process.engine.constant;
/**
public class PipelineGlobalVariableConstant { * key为后缀代表除了根据key获取不直接使用值
* 非key后缀的值都会在某地直接使用修改请谨慎
* */
public class PipelineVariableConstant {
/** /**
* AFL线基础工作目录完整的工作目录为 基础工作目录 + /PIP_${pipelineId} * AFL线基础工作目录完整的工作目录为 基础工作目录 + /PIP_${pipelineId}
* */ * */
@ -11,15 +14,6 @@ public class PipelineGlobalVariableConstant {
* 真正用于绑定AFL docker卷挂载的目录的全局变量的key值为 基础工作目录 + /PIP_${pipelineId} * 真正用于绑定AFL docker卷挂载的目录的全局变量的key值为 基础工作目录 + /PIP_${pipelineId}
* */ * */
public static final String AFL_DOCKER_WORK_DIR_KEY = "AFL_DOCKER_WORK_DIR_KEY"; public static final String AFL_DOCKER_WORK_DIR_KEY = "AFL_DOCKER_WORK_DIR_KEY";
/**
* 目标上传到工作目录下文件夹的前缀TASK_${taskId}
* */
public static final String AFL_DOCKER_SLOT_COMPILE_PATH_PREFIX = "TASK_";
/**
* 执行插装编译节点以后输出目录用于设置到全局变量给后续节点使用
*
* */
public static final String AFL_DOCKER_SLOT_COMPILE_PATH_KEY = "AFL_DOCKER_SLOT_COMPILE_PATH_KEY";
/** /**
* 测试用例生成ai生成或者直接文件上传的文件夹名称SEED * 测试用例生成ai生成或者直接文件上传的文件夹名称SEED
* */ * */
@ -30,4 +24,7 @@ public class PipelineGlobalVariableConstant {
* AFL输出目录 * AFL输出目录
* */ * */
public static final String AFL_DOCKER_OUTPUT = "ai_afl"; public static final String AFL_DOCKER_OUTPUT = "ai_afl";
public static final String COMPILE_RESULT_KEY ="compileResult";
public static final String AFL_RESOURCE_MANAGER_ID_KEY = "managerId";
public static final String IMAGE_NAME = "IMAGE_NAME";
} }

View File

@ -4,13 +4,12 @@ public class TestCaseGenerationConstant {
/** /**
* 脚本内容key * 脚本内容key
*/ */
public static final String COMMAND_SCRIPT ="buildScript";
public static final String CASE_TYPE_KEY ="caseType"; public static final String CASE_TYPE_KEY ="caseType";
public static final String CASE_TYPE_AI ="AI"; public static final String CASE_TYPE_AI ="AI";
public static final String CASE_TYPE_FILE ="FILE"; public static final String CASE_TYPE_FILE ="FILE";
public static final String CASE_TYPE_ITEM = "TEST"; public static final String CASE_TYPE_ITEM = "TEST";
public static final String SEED_SOURCE = "filePath"; public static final String SEED_SOURCE = "filePath";
public static final String SEED_TARGET = "outputDir";
public static final String ITEM_LIST = "itemList"; public static final String ITEM_LIST = "itemList";
public static final String MANAGER_ID = "managerId"; public static final String BINARY_NAME="binaryName";
} }

View File

@ -5,20 +5,15 @@ import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp; import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.AFLSlotCompileConstant; import cd.casic.ci.process.engine.constant.AFLSlotCompileConstant;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker; import cd.casic.ci.process.engine.worker.base.DockerWorker;
import cd.casic.ci.process.process.dataObject.machine.MachineInfo;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine; import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.target.TargetVersion;
import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.service.resource.ResourceManagerService;
import cd.casic.ci.process.process.service.target.TargetVersionService; import cd.casic.ci.process.process.service.target.TargetVersionService;
import cd.casic.ci.process.util.CryptogramUtil; import cd.casic.ci.process.util.CryptogramUtil;
import cd.casic.ci.process.util.SftpUploadUtil; import cd.casic.ci.process.util.SftpUploadUtil;
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo; import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.jcraft.jsch.*; import com.jcraft.jsch.*;
@ -28,10 +23,10 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Map; import static cd.casic.ci.process.engine.constant.PipelineVariableConstant.*;
import java.util.Vector;
@WorkAtom(taskType = "AFL_SLOT_COMPILE") @WorkAtom(taskType = "AFL_SLOT_COMPILE")
@Slf4j @Slf4j
@ -42,10 +37,10 @@ public class AFLSlotCompileWorker extends DockerWorker {
public void execute(TaskRunContext context) { public void execute(TaskRunContext context) {
PipTask task = context.getContextDef() instanceof PipTask ? ((PipTask) context.getContextDef()) : null; PipTask task = context.getContextDef() instanceof PipTask ? ((PipTask) context.getContextDef()) : null;
// 每一个流水线不同目录互不干扰 // 每一个流水线不同目录互不干扰
String workDir = PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_PREFIX + task.getPipelineId(); String workDir = AFL_DOCKER_WORK_DIR_PREFIX + task.getPipelineId();
Map<String, Object> globalVariables = context.getGlobalVariables(); Map<String, Object> globalVariables = context.getGlobalVariables();
if (!globalVariables.containsKey(workDir)) { if (!globalVariables.containsKey(workDir)) {
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); globalVariables.put(AFL_DOCKER_WORK_DIR_KEY,workDir);
} }
Map<String, Object> taskProperties = task.getTaskProperties(); Map<String, Object> taskProperties = task.getTaskProperties();
String managerId = taskProperties.get(AFLSlotCompileConstant.MANAGER_ID) instanceof String ? ((String) taskProperties.get(AFLSlotCompileConstant.MANAGER_ID)) : null; String managerId = taskProperties.get(AFLSlotCompileConstant.MANAGER_ID) instanceof String ? ((String) taskProperties.get(AFLSlotCompileConstant.MANAGER_ID)) : null;
@ -80,7 +75,7 @@ public class AFLSlotCompileWorker extends DockerWorker {
} }
// 工作目录下的相对路径,/test 作为根路径 // 工作目录下的相对路径,/test 作为根路径
// 卷挂载目录+流水线目录 + task编译目录 // 卷挂载目录+流水线目录 + task编译目录
String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir; String realPath = AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir;
// 上传目标文件 到指定资源服务器 // 上传目标文件 到指定资源服务器
try { try {
append(context,"AFL编译上传文件路径:"+realPath); append(context,"AFL编译上传文件路径:"+realPath);
@ -93,9 +88,10 @@ public class AFLSlotCompileWorker extends DockerWorker {
toBadEnding(); toBadEnding();
} }
// 执行预设命令 ,进入目录 // 执行预设命令 ,进入目录
String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" + String allCommand = "docker run -v "+ AFL_VOLUME_WORK_DIR_PREFIX +":"+ AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" +
"cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录 "cd "+ AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录
"mkdir -p "+workDir+"\n"+ "mkdir -p "+workDir+"\n"+
"chmod -R o+rwx "+ workDir +"\n"+
"cd "+workDir+"\n"+handleZipFile(fileName); "cd "+workDir+"\n"+handleZipFile(fileName);
Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT); Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null; String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
@ -105,8 +101,27 @@ public class AFLSlotCompileWorker extends DockerWorker {
dockerRun(allCommand,dockerInfo,context); dockerRun(allCommand,dockerInfo,context);
// 更新全局变量 // 更新全局变量
// /test目录下当前流水线工作目录 // /test目录下当前流水线工作目录
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); globalVariables.put(AFL_DOCKER_WORK_DIR_KEY,workDir);
putInCompileResult(context,realPath+File.separator+getSourceName(fileName),machineInfo); // 获取编译结果并存放
List<String> compileResult = getCompileResult(context, realPath + File.separator + getSourceName(fileName), machineInfo);
if (Objects.isNull(getGlobalVariable(context, COMPILE_RESULT_KEY))) {
// 如果全局变量不存在则放入全局变量中
putGlobalVariable(context,COMPILE_RESULT_KEY,compileResult);;
}
// 放入最近一级作用域
putSecondStageVariable(context,COMPILE_RESULT_KEY,compileResult);
// 放入 所选资源供测试用例生成AFL模糊测试节点使用
if (Objects.isNull(getGlobalVariable(context, AFL_RESOURCE_MANAGER_ID_KEY))) {
// 如果全局变量不存在则放入全局变量中
putGlobalVariable(context, AFL_RESOURCE_MANAGER_ID_KEY,managerId);;
}
putSecondStageVariable(context,AFL_RESOURCE_MANAGER_ID_KEY,managerId);
// 放入镜像名称
if (Objects.isNull(getGlobalVariable(context, IMAGE_NAME))) {
// 如果全局变量不存在则放入全局变量中
putGlobalVariable(context, IMAGE_NAME,imageName);;
}
putSecondStageVariable(context,IMAGE_NAME,imageName);
} }
private String handleZipFile(String fileName){ private String handleZipFile(String fileName){
if (StringUtils.isEmpty(fileName)) { if (StringUtils.isEmpty(fileName)) {
@ -133,10 +148,11 @@ public class AFLSlotCompileWorker extends DockerWorker {
return ""; return "";
} }
private void putInCompileResult(TaskRunContext context,String path, PipResourceMachine machineInfo){ private List<String> getCompileResult(TaskRunContext context, String path, PipResourceMachine machineInfo){
String host = machineInfo.getMachineHost(); String host = machineInfo.getMachineHost();
String user = machineInfo.getPassword(); String user = machineInfo.getUsername();
String password = machineInfo.getPassword(); String password = CryptogramUtil.doDecrypt(machineInfo.getPassword());
append(context,"AFL编译完毕");
List<String> fileNameList = null; List<String> fileNameList = null;
try { try {
JSch jsch = new JSch(); JSch jsch = new JSch();
@ -149,7 +165,7 @@ public class AFLSlotCompileWorker extends DockerWorker {
Vector<ChannelSftp.LsEntry> files = sftpChannel.ls(path); Vector<ChannelSftp.LsEntry> files = sftpChannel.ls(path);
fileNameList = new ArrayList<>(files.size()); fileNameList = new ArrayList<>(files.size());
for (ChannelSftp.LsEntry file : files) { for (ChannelSftp.LsEntry file : files) {
if (!file.getAttrs().isDir()) { if (!file.getAttrs().isDir()&&!file.getFilename().contains(".")) {
fileNameList.add(file.getFilename()); fileNameList.add(file.getFilename());
} }
} }
@ -161,8 +177,11 @@ public class AFLSlotCompileWorker extends DockerWorker {
if (!CollectionUtils.isEmpty(fileNameList)&&context!=null) { if (!CollectionUtils.isEmpty(fileNameList)&&context!=null) {
Map<String, Object> taskProperties = context.getGlobalVariables(); Map<String, Object> taskProperties = context.getGlobalVariables();
// 放入全局上下文中 // 放入全局上下文中
taskProperties.put(AFLSlotCompileConstant.COMPILE_RESULT,fileNameList); append(context,"AFL编译完毕:"+JSON.toJSONString(fileNameList));
return fileNameList;
} }
return Collections.emptyList();
} }
public String getSourceName(String fileName){ public String getSourceName(String fileName){
int dotIndex = fileName.lastIndexOf("."); int dotIndex = fileName.lastIndexOf(".");

View File

@ -2,10 +2,12 @@ package cd.casic.ci.process.engine.worker.afl;
import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.common.WorkAtom;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq; import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.AFLConstant; import cd.casic.ci.process.engine.constant.AFLConstant;
import cd.casic.ci.process.engine.constant.AFLSlotCompileConstant;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant; import cd.casic.ci.process.engine.constant.PipelineVariableConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker; import cd.casic.ci.process.engine.worker.base.DockerWorker;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
@ -20,6 +22,9 @@ import org.springframework.util.CollectionUtils;
import java.io.File; import java.io.File;
import java.util.Map; import java.util.Map;
import static cd.casic.ci.process.engine.constant.AFLConstant.*;
import static cd.casic.ci.process.engine.constant.PipelineVariableConstant.*;
@WorkAtom(taskType = "AFL") @WorkAtom(taskType = "AFL")
@Slf4j @Slf4j
public class AFLWorker extends DockerWorker { public class AFLWorker extends DockerWorker {
@ -29,22 +34,30 @@ public class AFLWorker extends DockerWorker {
public void execute(TaskRunContext context) { public void execute(TaskRunContext context) {
int statusCode = -1; int statusCode = -1;
Map<String, Object> localVariables = context.getLocalVariables(); Map<String, Object> localVariables = context.getLocalVariables();
Map<String, Object> globalVariables = context.getGlobalVariables();
if (context.getContextDef() instanceof PipTask taskDef) { if (context.getContextDef() instanceof PipTask taskDef) {
log.info(taskDef.getTaskName()); log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties(); Map<String, Object> taskProperties = taskDef.getTaskProperties();
String managerId = getVariableNearby(context, AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) getVariableNearby(context, AFL_RESOURCE_MANAGER_ID_KEY)) : null;
// ssh 上传目标文件
ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId);
if (resourceById==null) {
append(context,"缺少资源信息,请添加编译节点");
}
//从taskProperties中获取资源id //从taskProperties中获取资源id
String resourceType = taskProperties.get("resourceType").toString(); String resourceType = "docker";
String resourceId = taskProperties.get("resourceId").toString(); String resourceId = resourceById.getDockerId();
String commandEnd = taskProperties.get("commandEnd").toString(); String commandEnd = taskProperties.get(COMMAND_END) instanceof String ? ((String) taskProperties.get(COMMAND_END)) : null;
String imageName = taskProperties.get("imageName").toString(); String imageName = getVariableNearby(context,IMAGE_NAME) instanceof String ? ((String) getVariableNearby(context,IMAGE_NAME)) : null;
String executableName = taskProperties.get(EXECUTABLE_NAME) instanceof String ? ((String) taskProperties.get(EXECUTABLE_NAME)) : null;
// 代码编译目录 // 代码编译目录
String workDir = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY); String workDir = (String)getGlobalVariable(context,AFL_DOCKER_WORK_DIR_KEY);
// 入参目录 // 入参目录
String seedPath = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY); String seedPath = (String)getGlobalVariable(context,AFL_DOCKER_SEED_PATH_KEY);
if (StringUtils.isEmpty(seedPath) || if (StringUtils.isEmpty(seedPath) ||
StringUtils.isEmpty(workDir) || StringUtils.isEmpty(workDir) ||
StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceId) ||
StringUtils.isEmpty(imageName) ||
StringUtils.isEmpty(executableName) ||
StringUtils.isEmpty(commandEnd) || StringUtils.isEmpty(commandEnd) ||
StringUtils.isEmpty(resourceType)) { StringUtils.isEmpty(resourceType)) {
// 缺少参数 // 缺少参数
@ -58,20 +71,19 @@ public class AFLWorker extends DockerWorker {
append(context,"当前机器不支持docker"); append(context,"当前机器不支持docker");
return; return;
} }
String output = workDir +File.separator+ PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT; String output = workDir +File.separator+ AFL_DOCKER_OUTPUT;
// String volumeWorkDirPath = AFL_VOLUME_WORK_DIR_PREFIX;
String volumeWorkDirPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX;
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef(); PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef();
TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId()); TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId());
String fileName = targetVersion.getFileName(); String fileName = targetVersion.getFileName();
append(context,"AFL 输出路径:"+output); append(context,"AFL 输出路径:"+output);
Object timeObj = taskProperties.get("runningTime"); Object timeObj = taskProperties.get(RUNNING_TIME);
Double timeHours = Double.valueOf(String.valueOf(timeObj)); Double timeHours = Double.valueOf(String.valueOf(timeObj));
String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" + String commandScript = "docker run -v "+volumeWorkDirPath+":"+AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" +
"cd " +PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+ "\n"+ "cd " + AFL_DOCKER_BASE_DIR+File.separator+workDir+ "\n"+
cdSourceName(fileName) + cdSourceName(fileName) +
"mkdir -p "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+"\n" + "mkdir -p "+ AFL_DOCKER_BASE_DIR+File.separator+output+"\n" +
"afl-fuzz -i "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+" "+commandEnd; "afl-fuzz -i "+ AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+ AFL_DOCKER_BASE_DIR+File.separator+output+" ./"+ executableName + " " +commandEnd;
try { try {
//将节点的配置信息反编译成对象 //将节点的配置信息反编译成对象
log.info("AFL模糊测试执行脚本{}",commandScript); log.info("AFL模糊测试执行脚本{}",commandScript);

View File

@ -3,16 +3,12 @@ package cd.casic.ci.process.engine.worker.afl;
import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.common.WorkAtom;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq; import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.req.testCase.TestCaseInfoReq;
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp; import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; import cd.casic.ci.process.engine.constant.PipelineVariableConstant;
import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant;
import cd.casic.ci.process.engine.constant.TestCaseGenerationConstant; import cd.casic.ci.process.engine.constant.TestCaseGenerationConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker; import cd.casic.ci.process.engine.worker.base.DockerWorker;
import cd.casic.ci.process.engine.worker.base.SshWorker;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine; import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.dataObject.testCase.TestCaseInfo; import cd.casic.ci.process.process.dataObject.testCase.TestCaseInfo;
@ -24,13 +20,15 @@ import com.alibaba.fastjson.JSON;
import jakarta.annotation.Resource; import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.io.File; import java.io.File;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import static cd.casic.ci.process.engine.constant.PipelineVariableConstant.*;
import static cd.casic.ci.process.engine.constant.TestCaseGenerationConstant.*;
@WorkAtom(taskType = "TEST_CASE_GENERATION") @WorkAtom(taskType = "TEST_CASE_GENERATION")
@Slf4j @Slf4j
public class TestCaseGenerationWorker extends DockerWorker { public class TestCaseGenerationWorker extends DockerWorker {
@ -40,14 +38,14 @@ public class TestCaseGenerationWorker extends DockerWorker {
public void execute(TaskRunContext context) { public void execute(TaskRunContext context) {
int statusCode = -1; int statusCode = -1;
Map<String, Object> globalVariables = context.getGlobalVariables(); Map<String, Object> globalVariables = context.getGlobalVariables();
String workDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null; String workDir = globalVariables.get(AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(AFL_DOCKER_WORK_DIR_KEY)) : null;
if (context.getContextDef() instanceof PipTask taskDef) { if (context.getContextDef() instanceof PipTask taskDef) {
String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED; String seedPath= workDir+File.separator+ AFL_DOCKER_SEED;
Map<String, Object> taskProperties = taskDef.getTaskProperties(); Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY); Object caseType = taskProperties.get(CASE_TYPE_KEY);
String binaryName = taskProperties.get("binaryName") instanceof String ? ((String) taskProperties.get("binaryName")) : null; String binaryName = taskProperties.get(BINARY_NAME) instanceof String ? ((String) taskProperties.get("binaryName")) : null;
String managerId = taskProperties.get("managerId") instanceof String ? ((String) taskProperties.get("managerId")) : null; String managerId = getVariableNearby(context, AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) getVariableNearby(context, AFL_RESOURCE_MANAGER_ID_KEY)) : null;
Object itemListObject = taskProperties.get(TestCaseGenerationConstant.ITEM_LIST); Object itemListObject = taskProperties.get(ITEM_LIST);
List<String> itemList = JSON.parseArray(JSON.toJSONString(itemListObject),String.class); List<String> itemList = JSON.parseArray(JSON.toJSONString(itemListObject),String.class);
ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId); ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId);
String machineId = resourceById.getMachineId(); String machineId = resourceById.getMachineId();
@ -65,10 +63,10 @@ public class TestCaseGenerationWorker extends DockerWorker {
PipResourceMachine machineInfo = machineQuery.getResourceMachineList().get(0); PipResourceMachine machineInfo = machineQuery.getResourceMachineList().get(0);
DockerEndpointDo dockerInfo = dockerQuery.getDockerEndpointList().get(0); DockerEndpointDo dockerInfo = dockerQuery.getDockerEndpointList().get(0);
//如果machineId为0则说明该节点没有配置机器则使用开始节点的机器 //如果machineId为0则说明该节点没有配置机器则使用开始节点的机器
if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) { if (CASE_TYPE_AI.equals(caseType)) {
// 容器内部test 目录, 获取编译后的文件 // 容器内部test 目录, 获取编译后的文件
String runScript = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" // 目前测试用例ai生成只有这个路径能跑 String runScript = "docker run -v "+ AFL_VOLUME_WORK_DIR_PREFIX +":"+ AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" // 目前测试用例ai生成只有这个路径能跑
+ "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n" + + "cd "+ AFL_DOCKER_BASE_DIR+"\n" +
"mkdir -p "+seedPath +"\n"+ "mkdir -p "+seedPath +"\n"+
"cd "+seedPath +"\n"+ "cd "+seedPath +"\n"+
"PYTHONPATH=CaseGenerator/src python3 CaseGenerator/src/CaseGenerator/main.py --work-dir "+workDir+" --binary "+workDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100"; "PYTHONPATH=CaseGenerator/src python3 CaseGenerator/src/CaseGenerator/main.py --work-dir "+workDir+" --binary "+workDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100";
@ -76,9 +74,9 @@ public class TestCaseGenerationWorker extends DockerWorker {
log.info("测试用例生成-AI生成实际执行命令:{}" , runScript); log.info("测试用例生成-AI生成实际执行命令:{}" , runScript);
//获取机器 //获取机器
dockerRun(runScript,dockerInfo,context); dockerRun(runScript,dockerInfo,context);
} else if (TestCaseGenerationConstant.CASE_TYPE_FILE.equals(caseType)){ } else if (CASE_TYPE_FILE.equals(caseType)){
// 文件上传 // 文件上传
Object filePathObj = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE); Object filePathObj = taskProperties.get(SEED_SOURCE);
log.info("当前测试用例taskProperties:{}",taskProperties); log.info("当前测试用例taskProperties:{}",taskProperties);
if (Objects.isNull(filePathObj)) { if (Objects.isNull(filePathObj)) {
append(context,"缺少文件"); append(context,"缺少文件");
@ -87,7 +85,7 @@ public class TestCaseGenerationWorker extends DockerWorker {
List<String> filePathList = JSON.parseArray(JSON.toJSONString(filePathObj),String.class); List<String> filePathList = JSON.parseArray(JSON.toJSONString(filePathObj),String.class);
for (String filePath : filePathList) { for (String filePath : filePathList) {
File file = new File(filePath); File file = new File(filePath);
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; String seedTarget = AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath;
// 将文件上传到服务器的 目录底下 // 将文件上传到服务器的 目录底下
log.info("测试用例选用上传模式,种子文件路径:{},种子上传路径:{}",filePath,seedTarget); log.info("测试用例选用上传模式,种子文件路径:{},种子上传路径:{}",filePath,seedTarget);
append(context,"测试用例选用上传模式,种子文件路径:"+filePath+",种子上传路径:"+seedPath); append(context,"测试用例选用上传模式,种子文件路径:"+filePath+",种子上传路径:"+seedPath);
@ -101,9 +99,9 @@ public class TestCaseGenerationWorker extends DockerWorker {
} }
append(context,file.getName()+"上传文件成功!"); append(context,file.getName()+"上传文件成功!");
} }
}else if (TestCaseGenerationConstant.CASE_TYPE_ITEM.equals(caseType)){ }else if (CASE_TYPE_ITEM.equals(caseType)){
List<TestCaseInfo> testCaseInfos = testCaseInfoService.listByIds(itemList); List<TestCaseInfo> testCaseInfos = testCaseInfoService.listByIds(itemList);
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; String seedTarget = AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath;
for (TestCaseInfo testCaseInfo : testCaseInfos) { for (TestCaseInfo testCaseInfo : testCaseInfos) {
String testFilePath = testCaseInfo.getTestFilePath(); String testFilePath = testCaseInfo.getTestFilePath();
if (StringUtils.isEmpty(testFilePath)) { if (StringUtils.isEmpty(testFilePath)) {
@ -122,7 +120,7 @@ public class TestCaseGenerationWorker extends DockerWorker {
} }
} }
} }
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY,seedPath); globalVariables.put(AFL_DOCKER_SEED_PATH_KEY,seedPath);
} }
} }
} }

View File

@ -106,7 +106,46 @@ public abstract class BaseWorker implements Runnable{
} }
} }
public abstract void execute(TaskRunContext context); public abstract void execute(TaskRunContext context);
public void toBadEnding(){ protected void toBadEnding(){
throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),""); throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"");
} }
/**
* 往全局变量中保存变量作用域为全局
* */
protected void putGlobalVariable(TaskRunContext context,String key,Object value){
Map<String, Object> globalVariables = context.getGlobalVariables();
globalVariables.put(key,value);
}
/**
* 往当前执行链路中保存变量作用域为当前二级stage
* */
protected void putSecondStageVariable(TaskRunContext context,String key,Object value){
BaseRunContext stageContext = context.getParentContext();
Map<String, Object> localVariables = stageContext.getLocalVariables();
localVariables.put(key,value);
}
protected Object getGlobalVariable(TaskRunContext context,String key){
Map<String, Object> globalVariables = context.getGlobalVariables();
return globalVariables.get(key);
}
protected Object getSecondStageVariable(TaskRunContext context,String key){
BaseRunContext stageContext = context.getParentContext();
Map<String, Object> localVariables = stageContext.getLocalVariables();
return localVariables.get(key);
}
/**
* 就近获取从secondStage 到pipeline 逐级获取获取到了直接返回
* */
protected Object getVariableNearby(TaskRunContext context,String key){
BaseRunContext parentContext = context.getParentContext();
Map<String, Object> localVariables = parentContext.getLocalVariables();
Map<String, Object> globalVariables = context.getGlobalVariables();
if (localVariables.containsKey(key)) {
return localVariables.get(key);
}
if (globalVariables.containsKey(key)) {
return globalVariables.get(key);
}
return null;
}
} }

View File

@ -1,5 +1,8 @@
package cd.casic.ci.process.process.service.variable; package cd.casic.ci.process.process.service.variable;
import java.util.Map;
public interface VariableService { public interface VariableService {
Object getPipelineContextGlobalVariable(String pipelineId,String varName); Object getPipelineContextGlobalVariable(String pipelineId,String varName);
Map<String,Object> getPipelineContextGlobalVariables(String pipelineId);
} }

View File

@ -5,11 +5,14 @@ import cd.casic.ci.process.engine.manager.RunContextManager;
import cd.casic.ci.process.engine.runContext.BaseRunContext; import cd.casic.ci.process.engine.runContext.BaseRunContext;
import cd.casic.ci.process.process.service.variable.VariableService; import cd.casic.ci.process.process.service.variable.VariableService;
import jakarta.annotation.Resource; import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
@Service @Service
@Slf4j
public class VariableServiceImpl implements VariableService { public class VariableServiceImpl implements VariableService {
@Resource @Resource
private RunContextManager contextManager; private RunContextManager contextManager;
@ -20,6 +23,18 @@ public class VariableServiceImpl implements VariableService {
return ""; return "";
} }
Map<String, Object> globalVariables = context.getGlobalVariables(); Map<String, Object> globalVariables = context.getGlobalVariables();
log.info("当前全局变量:{}",globalVariables);
return globalVariables.get(varName); return globalVariables.get(varName);
} }
@Override
public Map<String,Object> getPipelineContextGlobalVariables(String pipelineId) {
BaseRunContext context = contextManager.getContext(pipelineId);
if (context== null) {
return new HashMap<>();
}
Map<String, Object> globalVariables = context.getGlobalVariables();
log.info("当前全局变量:{}",globalVariables);
return globalVariables;
}
} }

View File

@ -36,6 +36,10 @@ public class SftpTest {
System.out.println(CryptogramUtil.doEncrypt("hnidc0327cn!@#xhh")); System.out.println(CryptogramUtil.doEncrypt("hnidc0327cn!@#xhh"));
} }
@Test @Test
public void ssh(){
System.out.println(CryptogramUtil.doDecrypt("cf917c4ff8463a54df75120e2ff802e8"));
}
@Test
public void getFileList(){ public void getFileList(){
String host = "175.6.27.158"; String host = "175.6.27.158";
String user = "ubuntu"; String user = "ubuntu";