mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AFL相关work修改,编译结果查询

Browse Source
This commit is contained in:
even 2025-06-26 15:44:00 +08:00
parent 37718e9fa3
commit 67eade8fec
12 changed files with 186 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
modules/module-ci-process-api/src/main/java/cd/casic/ci/api/VariableController.java
View File

@ -3,15 +3,14 @@ package cd.casic.ci.api;
import cd.casic.ci.process.dto.req.variable.VariableReq;
import cd.casic.ci.process.dto.resp.variable.VariableResp;
import cd.casic.ci.process.process.service.variable.VariableService;
import cd.casic.framework.commons.pojo.CommonResult;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import jakarta.annotation.Resource;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.util.ArrayList;
import java.util.List;
@ -23,76 +22,11 @@ import java.util.List;
@RestController
@RequestMapping("/pipelineVariable")
public class VariableController {
/**
* @pi.name:创建流水线变量
* @pi.path:/pipelineVariable/createVariable
* @pi.methodType:post
* @pi.request-type:json
* @pi.param: model=variable
*/
@RequestMapping(path="/createVariable",method = RequestMethod.POST)
public CommonResult<String> createVariable(@RequestBody @NotNull @Valid VariableReq variable){
String variableId = "";
// String variableId = variableServer.createVariable(variable);
return CommonResult.success(variableId);
}
/**
* @pi.name:删除流水线变量
* @pi.path:/pipelineVariable/deleteVariable
* @pi.methodType:post
* @pi.request-type: formdata
* @pi.param: name=varId;dataType=string;value=varId;
*/
@RequestMapping(path="/deleteVariable",method = RequestMethod.POST)
public CommonResult<String> deleteVariable( @NotNull String varId){
// variableServer.deleteVariable(varId);
return CommonResult.success("");
}
/**
* @pi.name:更新流水线变量
* @pi.path:/pipelineVariable/createVariable
* @pi.methodType:post
* @pi.request-type:json
* @pi.param: model=variable
*/
@RequestMapping(path="/updateVariable",method = RequestMethod.POST)
public CommonResult<String> updateVariable(@RequestBody @NotNull @Valid VariableReq variable){
// variableServer.updateVariable(variable);
return CommonResult.success("");
}
/**
* @pi.name:查询流水线任务变量
* @pi.path:/pipelineVariable/findAllVariable
* @pi.methodType:post
* @pi.request-type: formdata
* @pi.param: name=taskId;dataType=string;value=任务id;
*/
@RequestMapping(path="/findAllVariable",method = RequestMethod.POST)
public CommonResult<List<VariableResp>> findAllVariable(@NotNull String taskId){
// List<VariableResp> allVariable = variableServer.findAllVariable(taskId);
String varListStr = "[\n" +
" {\n" +
" \"varId\": \"04b629ce9234\",\n" +
" \"createTime\": \"2025-05-10 14:25:24\",\n" +
" \"varType\": \"str\",\n" +
" \"type\": 1,\n" +
" \"taskId\": \"a69119cfbbde\",\n" +
" \"pipelineId\": null,\n" +
" \"varKey\": \"testVal\",\n" +
" \"varValue\": \"123\",\n" +
" \"varValues\": null,\n" +
" \"valueList\": null\n" +
" }\n" +
"]";
JSONArray objects = JSON.parseArray(varListStr);
List<VariableResp> allVariable = new ArrayList<>(objects.size());
for (int i = 0; i < objects.size(); i++) {
VariableResp object = objects.getObject(i, VariableResp.class);
allVariable.add(object);
}
return CommonResult.success(allVariable);
@Resource
private VariableService variableService;
@GetMapping("/getPipelineContextGlobalVariable/{pipelineId}/{varName}")
public CommonResult<Object> getPipelineContextGlobalVariable(@PathVariable("pipelineId") String pipelineId,@PathVariable("varName") String varName){
Object variable = variableService.getPipelineContextGlobalVariable(pipelineId, varName);
return CommonResult.success(variable);
}
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/testCase/TestCaseInfoReq.java
View File

@ -18,6 +18,8 @@ public class TestCaseInfoReq {
// 测试文件地址（存储路径或URL）
private String testFileList;
private String testFilePath;
// 描述信息
private String remark;
}

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLSlotCompileConstant.java
View File

@ -4,4 +4,5 @@ public class AFLSlotCompileConstant {
public static final String MANAGER_ID = "managerId";
public static final String IMAGE_NAME = "imageName";
public static final String COMMAND_SCRIPT ="buildScript";
public static final String COMPILE_RESULT ="compileResult";
}

5
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/TestCaseGenerationConstant.java
View File

@ -7,7 +7,10 @@ public class TestCaseGenerationConstant {
public static final String COMMAND_SCRIPT ="buildScript";
public static final String CASE_TYPE_KEY ="caseType";
public static final String CASE_TYPE_AI ="AI";
public static final String CASE_TYPE_TYPE ="FILE";
public static final String CASE_TYPE_FILE ="FILE";
public static final String CASE_TYPE_ITEM = "TEST";
public static final String SEED_SOURCE = "filePath";
public static final String SEED_TARGET = "outputDir";
public static final String ITEM_LIST = "itemList";
public static final String MANAGER_ID = "managerId";
}

48
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java
View File

@ -9,6 +9,7 @@ import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker;
import cd.casic.ci.process.process.dataObject.machine.MachineInfo;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.dataObject.target.TargetVersion;
@ -20,12 +21,17 @@ import cd.casic.ci.process.util.SftpUploadUtil;
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo;
import com.alibaba.fastjson.JSON;
import com.jcraft.jsch.*;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Vector;
@WorkAtom(taskType = "AFL_SLOT_COMPILE")
@Slf4j
@ -100,6 +106,7 @@ public class AFLSlotCompileWorker extends DockerWorker {
// 更新全局变量
// /test目录下当前流水线工作目录
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir);
putInCompileResult(context,realPath+File.separator+getSourceName(fileName),machineInfo);
}
private String handleZipFile(String fileName){
if (StringUtils.isEmpty(fileName)) {
@ -113,14 +120,55 @@ public class AFLSlotCompileWorker extends DockerWorker {
String suffix = fileName.substring(i + 1);
if ("zip".equals(suffix)) {
String cmd ="unzip -o "+fileName+"\n" +
"chmod -R o+rwx "+prefix+"/\n"+
"cd "+prefix+"\n";
return cmd;
}
if ("tar".equals(suffix)) {
String cmd ="tar -zxvf "+fileName+"\n" +
"chmod -R o+rwx "+prefix+"/\n"+
"cd "+prefix+"\n";
return cmd;
}
return "";
}
private void putInCompileResult(TaskRunContext context,String path, PipResourceMachine machineInfo){
String host = machineInfo.getMachineHost();
String user = machineInfo.getPassword();
String password = machineInfo.getPassword();
List<String> fileNameList = null;
try {
JSch jsch = new JSch();
Session session = jsch.getSession(user, host, 22);
session.setPassword(password);
session.setConfig("StrictHostKeyChecking", "no");
session.connect();
ChannelSftp sftpChannel = (ChannelSftp) session.openChannel("sftp");
sftpChannel.connect();
Vector<ChannelSftp.LsEntry> files = sftpChannel.ls(path);
fileNameList = new ArrayList<>(files.size());
for (ChannelSftp.LsEntry file : files) {
if (!file.getAttrs().isDir()) {
fileNameList.add(file.getFilename());
}
}
sftpChannel.disconnect();
session.disconnect();
} catch (JSchException | SftpException e) {
e.printStackTrace();
}
if (!CollectionUtils.isEmpty(fileNameList)&&context!=null) {
Map<String, Object> taskProperties = context.getGlobalVariables();
// 放入全局上下文中
taskProperties.put(AFLSlotCompileConstant.COMPILE_RESULT,fileNameList);
}
}
public String getSourceName(String fileName){
int dotIndex = fileName.lastIndexOf(".");
if (dotIndex!=-1) {
return fileName.substring(0, dotIndex);
}
return "";
}
}

4
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java
View File

@ -69,7 +69,7 @@ public class AFLWorker extends DockerWorker {
Double timeHours = Double.valueOf(String.valueOf(timeObj));
String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" +
"cd " +PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+ "\n"+
getSourceName(fileName) +
cdSourceName(fileName) +
"mkdir -p "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+"\n" +
"afl-fuzz -i "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+output+" "+commandEnd;
try {
@ -89,7 +89,7 @@ public class AFLWorker extends DockerWorker {
localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode);
}
}
public String getSourceName(String fileName){
public String cdSourceName(String fileName){
int dotIndex = fileName.lastIndexOf(".");
if (dotIndex!=-1) {
return "cd "+fileName.substring(0, dotIndex)+"\n";

67
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java
View File

@ -3,6 +3,7 @@ package cd.casic.ci.process.engine.worker.afl;
import cd.casic.ci.process.common.WorkAtom;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.req.testCase.TestCaseInfoReq;
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
@ -14,19 +15,27 @@ import cd.casic.ci.process.engine.worker.base.SshWorker;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.dataObject.testCase.TestCaseInfo;
import cd.casic.ci.process.process.service.testCase.TestCaseInfoService;
import cd.casic.ci.process.util.CryptogramUtil;
import cd.casic.ci.process.util.SftpUploadUtil;
import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo;
import com.alibaba.fastjson.JSON;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@WorkAtom(taskType = "TEST_CASE_GENERATION")
@Slf4j
public class TestCaseGenerationWorker extends DockerWorker {
@Resource
private TestCaseInfoService testCaseInfoService;
@Override
public void execute(TaskRunContext context) {
int statusCode = -1;
@ -38,6 +47,8 @@ public class TestCaseGenerationWorker extends DockerWorker {
Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY);
String binaryName = taskProperties.get("binaryName") instanceof String ? ((String) taskProperties.get("binaryName")) : null;
String managerId = taskProperties.get("managerId") instanceof String ? ((String) taskProperties.get("managerId")) : null;
Object itemListObject = taskProperties.get(TestCaseGenerationConstant.ITEM_LIST);
List<String> itemList = JSON.parseArray(JSON.toJSONString(itemListObject),String.class);
ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId);
String machineId = resourceById.getMachineId();
String dockerId = resourceById.getDockerId();
@ -65,27 +76,51 @@ public class TestCaseGenerationWorker extends DockerWorker {
log.info("测试用例生成-AI生成，实际执行命令:{}" , runScript);
//获取机器
dockerRun(runScript,dockerInfo,context);
} else {
} else if (TestCaseGenerationConstant.CASE_TYPE_FILE.equals(caseType)){
// 文件上传
String filePath = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE)) : null;
if (StringUtils.isEmpty(filePath)) {
Object filePathObj = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE);
log.info("当前测试用例taskProperties:{}",taskProperties);
if (Objects.isNull(filePathObj)) {
append(context,"缺少文件");
toBadEnding();
}
File file = new File(filePath);
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath;
// 将文件上传到服务器的 目录底下
log.info("测试用例选用上传模式,种子文件路径：{},种子上传路径:{}",filePath,seedTarget);
append(context,"测试用例选用上传模式,种子文件路径："+filePath+",种子上传路径:"+seedPath);
try {
SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),CryptogramUtil.doDecrypt(machineInfo.getPassword()),"",filePath,seedTarget,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) {
append(context,"seed文件上传失败");
log.error("seed文件上传失败",e);
List<String> filePathList = JSON.parseArray(JSON.toJSONString(filePathObj),String.class);
for (String filePath : filePathList) {
File file = new File(filePath);
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath;
// 将文件上传到服务器的 目录底下
log.info("测试用例选用上传模式,种子文件路径：{},种子上传路径:{}",filePath,seedTarget);
append(context,"测试用例选用上传模式,种子文件路径："+filePath+",种子上传路径:"+seedPath);
try {
SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),CryptogramUtil.doDecrypt(machineInfo.getPassword()),"",filePath,seedTarget,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) {
append(context,"seed文件上传失败");
log.error("seed文件上传失败",e);
}
append(context,file.getName()+"上传文件成功！");
}
}else if (TestCaseGenerationConstant.CASE_TYPE_ITEM.equals(caseType)){
List<TestCaseInfo> testCaseInfos = testCaseInfoService.listByIds(itemList);
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath;
for (TestCaseInfo testCaseInfo : testCaseInfos) {
String testFilePath = testCaseInfo.getTestFilePath();
if (StringUtils.isEmpty(testFilePath)) {
continue;
}
try {
append(context,"当前上传文件:"+testCaseInfo.getFileName());
SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort())
,machineInfo.getUsername()
,CryptogramUtil.doDecrypt(machineInfo.getPassword())
,"", testFilePath,seedTarget,testCaseInfo.getFileName());
} catch (SftpUploadUtil.SftpUploadException e) {
toBadEnding();
}
}
append(context,"上传文件成功！");
}
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY,seedPath);
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/testCase/TestCaseInfo.java
View File

@ -29,6 +29,8 @@ public class TestCaseInfo extends PipBaseElement {
// 测试文件地址（存储路径或URL）
private String testFileList;
private String testFilePath;
// 描述信息
private String remark;
}

5
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/VariableService.java Normal file
View File

@ -0,0 +1,5 @@
package cd.casic.ci.process.process.service.variable;
public interface VariableService {
Object getPipelineContextGlobalVariable(String pipelineId,String varName);
}

25
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/variable/impl/VariableServiceImpl.java Normal file
View File

@ -0,0 +1,25 @@
package cd.casic.ci.process.process.service.variable.impl;
import cd.casic.ci.process.engine.constant.AFLSlotCompileConstant;
import cd.casic.ci.process.engine.manager.RunContextManager;
import cd.casic.ci.process.engine.runContext.BaseRunContext;
import cd.casic.ci.process.process.service.variable.VariableService;
import jakarta.annotation.Resource;
import org.springframework.stereotype.Service;
import java.util.Map;
@Service
public class VariableServiceImpl implements VariableService {
@Resource
private RunContextManager contextManager;
@Override
public Object getPipelineContextGlobalVariable(String pipelineId, String varName) {
BaseRunContext context = contextManager.getContext(pipelineId);
if (context== null) {
return "";
}
Map<String, Object> globalVariables = context.getGlobalVariables();
return globalVariables.get(varName);
}
}

8
ops-server/src/test/java/cd/casic/server/DockerTest.java
View File

@ -1,4 +1,5 @@
package cd.casic.server;
import cd.casic.ci.process.util.CryptogramUtil;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.command.ExecCreateCmdResponse;
@ -199,7 +200,7 @@ public class DockerTest {
}
@Test
public void imageList() throws URISyntaxException {
URI uri = new URI(format("tcp://%s:%s", "175.6.27.228", "22375"));
URI uri = new URI(format("tcp://%s:%s", "175.6.27.158", "22375"));
ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder().dockerHost(uri).build();
DockerClient build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build();
@ -216,7 +217,10 @@ public class DockerTest {
}
System.out.println(exec);
}
@Test
public void encode(){
System.out.println(CryptogramUtil.doEncrypt("Hunter@2024"));
}
}

31
ops-server/src/test/java/cd/casic/server/SftpTest.java
View File

@ -4,12 +4,14 @@ import cd.casic.ci.process.process.dataObject.machine.MachineInfo;
import cd.casic.ci.process.process.service.machine.MachineInfoService;
import cd.casic.ci.process.util.CryptogramUtil;
import cd.casic.ci.process.util.SftpUploadUtil;
import com.jcraft.jsch.*;
import jakarta.annotation.Resource;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import java.io.File;
import java.util.Vector;
@SpringBootTest(classes = {OpsServerApplication.class})
@ActiveProfiles("local")
@ -33,4 +35,33 @@ public class SftpTest {
public void test03(){
System.out.println(CryptogramUtil.doEncrypt("hnidc0327cn!@#xhh"));
}
@Test
public void getFileList(){
String host = "175.6.27.158";
String user = "ubuntu";
String password = "Hunter@2024";
String remotePath = "/home/casic/706/yunqi/PIP_725054435889385472/pcre2-master";
try {
JSch jsch = new JSch();
Session session = jsch.getSession(user, host, 22);
session.setPassword(password);
session.setConfig("StrictHostKeyChecking", "no");
session.connect();
ChannelSftp sftpChannel = (ChannelSftp) session.openChannel("sftp");
sftpChannel.connect();
Vector<ChannelSftp.LsEntry> files = sftpChannel.ls(remotePath);
for (ChannelSftp.LsEntry file : files) {
if (!file.getAttrs().isDir()) {
System.out.println("文件: " + file.getFilename());
}
}
sftpChannel.disconnect();
session.disconnect();
} catch (JSchException | SftpException e) {
e.printStackTrace();
}
}
}