mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: mianbin:07544335cab67633706f9f25c52ff2caf44d5b4d
Branches Tags
mianbin:master
mianbin:jenkins-engin
mianbin:temp
..
compare: mianbin:6b0e4889b7325ccd9aa5c09bc6b8ce7755e9c252
Branches Tags
mianbin:jenkins-engin
mianbin:master
mianbin:temp

No commits in common. "07544335cab67633706f9f25c52ff2caf44d5b4d" and "6b0e4889b7325ccd9aa5c09bc6b8ce7755e9c252" have entirely different histories.
07544335ca ... 6b0e4889b7

20 changed files with 144 additions and 395 deletions
Show Stats Expand all files Collapse all files

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java
View File

@ -18,5 +18,4 @@ public class SastUrlConstant {
public static final String reportDelete = "/api/report/index";
public static final String getApplicationStatus = "/invoke/application/getStatus/";
public static final String jsonReportDownload = "/api/report/index/downloadReport";
public static final String getReportInfo = "/api/report/index/getReportInfo";
}

69
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastReportStatusResp.java
View File

@ -1,69 +0,0 @@
package cd.casic.ci.process.dto.resp.sast;
import lombok.Data;
import java.util.List;
@Data
public class SastReportStatusResp {
/**
* 报告模式：
* SUMMARY - 概要模式
* DETAILS - 详细模式
* */
private String mode;
/**
* 报告类型：
* PROJECT - 项目报告
* APPLICATION - 应用报告
* */
private String contacts;
/**
* 报告模式：
* SUMMARY - 概要模式
* DETAILS - 详细模式
* */
private String reportType;
/**
* 文件格式：
* DOCX - docx
* XLSX - xlsx
* JSON - json
* PDF - pdf
* */
private String format;
private String projectId;
private List<String> applicationIds;
/**
* 缺陷等级集合
* UN_KNOWN - 未知
* DEADLY - 严重
* SEVERITY - 高危
* POOR_RISK - 中危
* LOW_RISK - 低危
* SAFE - 安全
* */
private List<String> bugLevels;
/**
* WAITING - 待审计
* IGNORE - 忽略
* HIGH - 高风险
* MEDIUM - 中风险
* LOW - 低风险
* NONE - 不受影响/误报
* */
private List<String> audRiskLevels;
/**
* 应用范围：
* ALL - 全部
* CUSTOMIZE - 自定义
* */
private String applicationScope;
/**
* WAITING - 等待中
* GENERATING - 正在生成
* SUCCESS - 生成成功
* FAIL - 生成失败
* */
private String status;
}

4
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java
View File

@ -5,8 +5,4 @@ public class AFLConstant {
* 脚本内容
*/
public static final String COMMAND_SCRIPT ="buildScript";
public static final String WORK_DIR = "workDir";
public static final String BINARY="binary";
public static final String OUTPUT = "output";
public static final String INPUT="input";
}

29
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java
View File

@ -1,29 +0,0 @@
package cd.casic.ci.process.engine.constant;
public class PipelineGlobalVariableConstant {
/**
* AFL线基础工作目录，完整的工作目录为 基础工作目录 + /PIP_${pipelineId}
* */
public static final String AFL_WORK_DIR_PREFIX = "/home/casic/706/yunqi/PIP_";
/**
* 真正用于绑定AFL docker卷挂载的目录的全局变量的key，值为 基础工作目录 + /PIP_${pipelineId}
* */
public static final String AFL_WORK_DIR_KEY = "AFL_PIPELINE_WORK_DIR";
/**
* 目标上传到工作目录下文件夹的前缀（TASK_${taskId}）
* */
public static final String AFL_SLOT_COMPILE_PATH_PREFIX = "TASK_";
/**
* 执行插装编译节点以后输出目录，用于设置到全局变量给后续节点使用
*
* */
public static final String AFL_SLOT_COMPILE_PATH_KEY = "AFL_SLOT_COMPILE_PATH";
/**
* 测试用例生成，ai生成或者直接文件上传的路径，工作目录下 SEED
* */
public static final String AFL_SEED_PATH="SEED";
/**
* AFL输出目录
* */
public static final String AFL_OUTPUT = "ai_afl";
}

5
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/TestCaseGenerationConstant.java
View File

@ -5,9 +5,4 @@ public class TestCaseGenerationConstant {
* 脚本内容key
*/
public static final String COMMAND_SCRIPT ="buildScript";
public static final String CASE_TYPE_KEY ="caseType";
public static final String CASE_TYPE_AI ="AI";
public static final String CASE_TYPE_TYPE ="FILE";
public static final String SEED_SOURCE = "filePath";
public static final String SEED_TARGET = "outputDir";
}

33
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java → modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/AFLWorker.java
View File

@ -1,4 +1,4 @@
package cd.casic.ci.process.engine.worker.afl;
package cd.casic.ci.process.engine.worker;
import cd.casic.ci.process.common.WorkAtom;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
@ -26,24 +26,22 @@ public class AFLWorker extends DockerWorker {
if (context.getContextDef() instanceof PipTask taskDef) {
log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object commandScriptObj = taskProperties.get(AFLConstant.COMMAND_SCRIPT);
// Object machineIdObj = taskProperties.get(DIYImageExecuteCommandConstant.MACHINE_ID);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
// PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef();
// String resourceId = pipeline.getResourceId();
//从taskProperties中获取资源id
String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString();
// 待测试文件路径
String binaryPath = taskProperties.get(AFLConstant.BINARY) instanceof String ? ((String) taskProperties.get(AFLConstant.BINARY)) : null;
String output = taskProperties.get(AFLConstant.OUTPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.OUTPUT)) : null;
String input = taskProperties.get(AFLConstant.INPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.INPUT)) : null;
String workDir = taskProperties.get(AFLConstant.WORK_DIR) instanceof String ? ((String) taskProperties.get(AFLConstant.WORK_DIR)) : null;
if (StringUtils.isEmpty(binaryPath) ||
StringUtils.isEmpty(output) ||
StringUtils.isEmpty(input) ||
StringUtils.isEmpty(workDir) ||
StringUtils.isEmpty(resourceId) ||
StringUtils.isEmpty(resourceType)) {
if (StringUtils.isEmpty(commandScript) || StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceType)) {
// 缺少参数
toBadEnding();
}
// ResourceFindResp resourceById = getResourceManagerService().findResourceById(resourceId);
// DockerEndpointDo dockerEndpoint = resourceById.getDockerEndpoint();
ResourceQueryReq req = new ResourceQueryReq();
req.setId(resourceId);
req.setType(resourceType);
@ -52,13 +50,16 @@ public class AFLWorker extends DockerWorker {
append(context,"当前机器不支持docker");
return;
}
String commandScript = "docker run -v "+workDir+":/test -it aflplusplus/aflplusplus bash\n" +
"cd /test\n" +
"afl-fuzz -i "+input+" -o "+output+" -t 3000 -Q "+binaryPath+" @@";
try {
//将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript);
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
//获取机器
// MachineInfo machineInfoDO = this.getMachineInfoService().getById(machineId);
// 获取docker 暂时先写固定值
// TODO dockerEndpoint替换为查询
dockerRun(commandScript,resourceListByType.getDockerEndpointList().get(0),context);
} catch (Exception e) {
String errorMessage = "该节点配置信息为空，请先配置该节点信息" + "\r\n";

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ApplicationWorker.java → modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/ApplicationWorker.java
View File

@ -1,4 +1,4 @@
package cd.casic.ci.process.engine.worker.sca;
package cd.casic.ci.process.engine.worker;
import cd.casic.ci.process.common.WorkAtom;

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/CodingWorker.java → modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/CodingWorker.java
View File

@ -1,4 +1,4 @@
package cd.casic.ci.process.engine.worker.sca;
package cd.casic.ci.process.engine.worker;
import cd.casic.ci.process.common.WorkAtom;

23
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/CustomCompilerWorker.java
View File

@ -41,7 +41,20 @@ public class CustomCompilerWorker extends SshWorker {
log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object commandScriptObj = taskProperties.get(DIYImageExecuteCommandConstant.COMMAND_SCRIPT);
Object machineIdObj = taskProperties.get(DIYImageExecuteCommandConstant.MACHINE_ID);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
Long machineId = null;
try {
machineId=Long.valueOf(String.valueOf(machineIdObj));
} catch (NumberFormatException e) {
log.error("缺少参数：{}",DIYImageExecuteCommandConstant.MACHINE_ID);
toBadEnding();
}
if (StringUtils.isEmpty(commandScript) ||machineIdObj == null) {
// 缺少参数
toBadEnding();
}
try {
//将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript);
@ -49,11 +62,14 @@ public class CustomCompilerWorker extends SshWorker {
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef();
// String resourceId = pipeline.getResourceId();
// ResourceFindResp resourceById = getResourceManagerService().findResourceById(resourceId);
// PipResourceMachine resourceMachine = resourceById.getResourceMachine();
//获取机器
String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString();
if (StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceType)) {
append(context,"缺少参数");
// 缺少参数
toBadEnding();
}
@ -85,13 +101,12 @@ public class CustomCompilerWorker extends SshWorker {
append(context,"上传路径："+"/home/casic/706/ai_test_527");
// 上传目标 到目标服务器
//通过taskProperties获取制品路径
// String remoteDir = taskProperties.get("remoteDir").toString();
String remoteDir = null;
String remoteDir = taskProperties.get("remoteDir").toString();
if (StringUtils.isEmpty(remoteDir)){
//则默认路径写死
remoteDir = "/home/casic/706/ai_test_527";
}
SftpUploadUtil.uploadFileViaSftp(resourceMachine.getMachineHost(),Integer.valueOf(resourceMachine.getSshPort()),resourceMachine.getUsername(), CryptogramUtil.doDecrypt(resourceMachine.getPassword()),null,file.getAbsolutePath(),remoteDir,file.getName());
SftpUploadUtil.uploadFileViaSftp(resourceMachine.getMachineHost(),Integer.valueOf(resourceMachine.getSshPort()),resourceMachine.getUsername(), CryptogramUtil.doDecrypt(resourceMachine.getPassword()),null,file.getAbsolutePath(),"/home/casic/706/ai_test_527",file.getName());
//TODO 得改一下
statusCode = shell(resourceMachine, null,context,

50
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastWorker.java → modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/SastWorker.java
View File

@ -1,4 +1,4 @@
package cd.casic.ci.process.engine.worker.sast;
package cd.casic.ci.process.engine.worker;
import cd.casic.ci.process.common.WorkAtom;
@ -7,7 +7,6 @@ import cd.casic.ci.process.dto.req.sast.SastReportCreateReq;
import cd.casic.ci.process.dto.resp.sast.SastApplicationCreateResp;
import cd.casic.ci.process.dto.resp.sast.SastApplicationEchoResp;
import cd.casic.ci.process.dto.resp.sast.SastApplicationStatusResp;
import cd.casic.ci.process.dto.resp.sast.SastReportStatusResp;
import cd.casic.ci.process.engine.constant.SastConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.BaseWorker;
@ -16,8 +15,6 @@ import cd.casic.ci.process.process.dataObject.base.PipBaseElement;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.service.sast.SastService;
import cd.casic.ci.process.process.service.task.TaskService;
import cd.casic.framework.commons.exception.ServiceException;
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
import com.alibaba.excel.util.StringUtils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
@ -52,21 +49,20 @@ public class SastWorker extends BaseWorker {
toBadEnding();
}
SastApplicationStatusResp applicationStatus = sastService.getApplicationStatus(applicationId);
SastApplicationEchoResp applicationEcho = sastService.getApplicationEcho(applicationId);
append(context,"获取到sast任务配置"+JSON.toJSONString(applicationEcho));
if (SastConstant.STATUS_PROGRESSING.equals(applicationStatus.getJobStatus())||SastConstant.STATUS_PENDING.equals(applicationStatus.getJobStatus())) {
append(context,"当前任务正在运行中");
// toBadEnding();
} else{
SastApplicationCreateReq sastApplicationCreateReq = converter.converter(applicationEcho);
List<String> list = applicationEcho.getLodeFiles().stream().map(SastApplicationEchoResp.LodeFile::getFileId).toList();
sastApplicationCreateReq.setFileId(list);
append(context,"开始启动SAST任务,检测文件信息：{}"+JSON.toJSONString(applicationEcho.getLodeFiles()));
log.info("SAST启动任务入参:{}",JSON.toJSONString(sastApplicationCreateReq));
SastApplicationCreateResp sastApplicationCreateResp = sastService.applicationCreate(sastApplicationCreateReq);
log.info("SAST启动任务返回值:{}",JSON.toJSONString(sastApplicationCreateResp));
append(context,"启动任务完毕");
toBadEnding();
}
SastApplicationEchoResp applicationEcho = sastService.getApplicationEcho(applicationId);
append(context,"获取到sast任务配置"+JSON.toJSONString(applicationEcho));
SastApplicationCreateReq sastApplicationCreateReq = converter.converter(applicationEcho);
List<String> list = applicationEcho.getLodeFiles().stream().map(SastApplicationEchoResp.LodeFile::getFileId).toList();
sastApplicationCreateReq.setFileId(list);
append(context,"开始启动SAST任务,检测文件信息：{}"+JSON.toJSONString(applicationEcho.getLodeFiles()));
log.info("SAST启动任务入参:{}",JSON.toJSONString(sastApplicationCreateReq));
SastApplicationCreateResp sastApplicationCreateResp = sastService.applicationCreate(sastApplicationCreateReq);
log.info("SAST启动任务返回值:{}",JSON.toJSONString(sastApplicationCreateResp));
append(context,"启动任务完毕");
// 检测状态是否完毕
applicationStatus = sastService.getApplicationStatus(applicationId);
int repeat = 0;
@ -88,20 +84,20 @@ public class SastWorker extends BaseWorker {
continue;
}
append(context,"当前SAST运行中，运行状态："+applicationStatus.getJobStatus());
if (SastConstant.STATUS_CANCEL.equals(applicationStatus.getJobStatus())) {
if (SastConstant.STATUS_PENDING.equals(applicationStatus.getJobStatus())) {
append(context,"任务被取消");
toBadEnding();
}
repeat = 0;
}
JSONObject reportJSON = getReportJSON(applicationId, SastConstant.REPORT_MODE_DETAILS,context);
JSONObject reportJSON = getReportJSON(applicationId, SastConstant.REPORT_MODE_DETAILS);
String reportUrl = getReportUrl(applicationId, SastConstant.REPORT_MODE_DETAILS);
task.getTaskProperties().put(SastConstant.REPORT_JSON,reportJSON);
task.getTaskProperties().put(SastConstant.REPORT_URL,reportUrl);
taskService.updateById(task);
}
}
private JSONObject getReportJSON(String applicationId,String model,TaskRunContext context){
private JSONObject getReportJSON(String applicationId,String model){
String reqJSON = "{\"mode\":\"DETAILS\",\"contacts\":\"admin@clouditera.com\",\"reportType\":\"APPLICATION\",\"format\":\"JSON\",\"projectId\":\"8a863857-bab3-40d9-a79a-2cb6617a9dd3\",\"applicationIds\":[\"9e3e55d6-6d28-4fd1-adc2-15b6cf0702db\"],\"bugLevels\":[\"DEADLY\",\"SEVERITY\",\"POOR_RISK\",\"LOW_RISK\"],\"audRiskLevels\":[\"WAITING\",\"IGNORE\",\"HIGH\",\"MEDIUM\",\"LOW\",\"NONE\"],\"applicationScope\":\"ALL\"}";
SastReportCreateReq sastReportCreateReq = JSON.parseObject(reqJSON, SastReportCreateReq.class);
sastReportCreateReq.setMode(model);
@ -109,21 +105,7 @@ public class SastWorker extends BaseWorker {
sastReportCreateReq.setApplicationIds(Collections.singletonList(applicationId));
log.info("获取JSON报告请求参数{}",JSON.toJSONString(sastReportCreateReq));
String reportId = sastService.reportIndex(sastReportCreateReq);
log.info("获取JSON报告获得reportId{}",reportId);
// todo 获取报告状态
SastReportStatusResp reportStatus = sastService.getReportStatus(reportId);
try {
while (reportStatus.getStatus().equals("WAITING")||reportStatus.getStatus().equals("GENERATING")){
reportStatus = sastService.getReportStatus(reportId);
append(context,"当前生成报告状态为:"+reportStatus.getStatus());
Thread.sleep(1000L);
}
} catch (Exception e) {
throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"获取sast报告状态失败");
}
if (reportStatus.getStatus().equals("FAIL")) {
throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"生成sast报告失败");
}
log.info("获取JSON报告获得url{}",reportId);
JSONObject jsonObject = getJSONString(reportId).getJSONObject("task_summary");
JSONObject severity = jsonObject.getJSONObject("defect_severity_distribution");
JSONObject rule = jsonObject.getJSONObject("detection_rule_distribution");

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java → modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/ScaBinaryWorker.java
View File

@ -1,4 +1,4 @@
package cd.casic.ci.process.engine.worker.sca;
package cd.casic.ci.process.engine.worker;
import cd.casic.ci.process.common.WorkAtom;

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java → modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/ScaMirrorWorker.java
View File

@ -1,4 +1,4 @@
package cd.casic.ci.process.engine.worker.sca;
package cd.casic.ci.process.engine.worker;
import cd.casic.ci.process.common.WorkAtom;

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java → modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/ScaSbomWorker.java
View File

@ -1,4 +1,4 @@
package cd.casic.ci.process.engine.worker.sca;
package cd.casic.ci.process.engine.worker;
import cd.casic.ci.process.common.WorkAtom;

82
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/TestCaseGenerationWorker.java Normal file
View File

@ -0,0 +1,82 @@
package cd.casic.ci.process.engine.worker;
import cd.casic.ci.process.common.WorkAtom;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.TestCaseGenerationConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.SshWorker;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.util.CryptogramUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.Map;
@WorkAtom(taskType = "TEST_CASE_GENERATION")
@Slf4j
public class TestCaseGenerationWorker extends SshWorker {
@Override
public void execute(TaskRunContext context) {
int statusCode = -1;
Map<String, Object> localVariables = context.getLocalVariables();
if (context.getContextDef() instanceof PipTask taskDef) {
log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object commandScriptObj = taskProperties.get(TestCaseGenerationConstant.COMMAND_SCRIPT);
// Object machineIdObj = taskProperties.get(DIYImageExecuteCommandConstant.MACHINE_ID);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef();
if (StringUtils.isEmpty(commandScript)) {
// 缺少参数
toBadEnding();
}
try {
//将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript);
String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString();
if (StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceType)) {
// 缺少参数
toBadEnding();
}
ResourceQueryReq req = new ResourceQueryReq();
req.setId(resourceId);
req.setType(resourceType);
TaskResourceFindResp resourceListByType = getResourceManagerService().findResourceListByType(req);
if (CollectionUtils.isEmpty(resourceListByType.getResourceMachineList())) {
append(context,"当前机器不支持machine");
return;
}
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
PipResourceMachine resourceMachine = resourceListByType.getResourceMachineList().get(0);
// ResourceFindResp resourceById = getResourceManagerService().findResourceById(resourceId);
// PipResourceMachine resourceMachine = resourceById.getResourceMachine();
//获取机器
statusCode = shell(resourceMachine, CryptogramUtil.doDecrypt(resourceMachine.getPassword()), context,
"echo \"自定义镜像执行命令\"",
commandScript
);
} catch (Exception e) {
String errorMessage = "该节点配置信息为空，请先配置该节点信息" + "\r\n";
log.error("执行ssh失败:", e);
append(context, errorMessage);
toBadEnding();
}
if (statusCode == 0) {
log.info("节点执行完成");
} else {
log.error("节点执行失败");
}
localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode);
}
}
}

92
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java
View File

@ -1,92 +0,0 @@
package cd.casic.ci.process.engine.worker.afl;
import cd.casic.ci.process.common.WorkAtom;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.dataObject.target.TargetVersion;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.service.resource.ResourceManagerService;
import cd.casic.ci.process.process.service.target.TargetVersionService;
import cd.casic.ci.process.util.SftpUploadUtil;
import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo;
import com.alibaba.fastjson.JSON;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.util.Map;
@WorkAtom(taskType = "AFL_SLOT_COMPILE")
@Slf4j
public class AFLSlotCompileWorker extends DockerWorker {
@Resource
private TargetVersionService targetVersionService;
@Override
public void execute(TaskRunContext context) {
PipTask task = context.getContextDef() instanceof PipTask ? ((PipTask) context.getContextDef()) : null;
String workDir = PipelineGlobalVariableConstant.AFL_WORK_DIR_PREFIX + task.getPipelineId();
Map<String, Object> globalVariables = context.getGlobalVariables();
if (!globalVariables.containsKey(workDir)) {
globalVariables.put(PipelineGlobalVariableConstant.AFL_WORK_DIR_KEY,workDir);
}
Map<String, Object> taskProperties = task.getTaskProperties();
String managerId = taskProperties.get("managerId") instanceof String ? ((String) taskProperties.get("resourceId")) : null;
// ssh 上传目标文件
ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId);
String machineId = resourceById.getMachineId();
String dockerId = resourceById.getDockerId();
if (StringUtils.isEmpty(machineId)||StringUtils.isEmpty(dockerId)) {
append(context,"该资源不支持docker或者ssh");
}
ResourceQueryReq req = new ResourceQueryReq();
req.setId(machineId);
req.setType("machine");
TaskResourceFindResp machineQuery = resourceManagerService.findResourceListByType(req);
req.setId(dockerId);
req.setType("docker");
TaskResourceFindResp dockerQuery = resourceManagerService.findResourceListByType(req);
PipResourceMachine machineInfo = machineQuery.getResourceMachineList().get(0);
DockerEndpointDo dockerInfo = dockerQuery.getDockerEndpointList().get(0);
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(task.getPipelineId()).getContextDef();
// 获取目标文件
TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId());
String filePath = targetVersion.getFilePath();
File file = new File(filePath);
if (!file.exists() || !file.canRead()) {
log.error("目标文件不存在或不可读");
append(context,"目标文件不存在或不可读");
toBadEnding();
}
// 工作目录下的相对路径
String compilePath = PipelineGlobalVariableConstant.AFL_SLOT_COMPILE_PATH_PREFIX+task.getId();
String realWorkDir = workDir + compilePath;
// 上传目标文件 到指定资源服务器
try {
SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",compilePath,filePath,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) {
append(context,"上传文件失败，请确认资源信息是否有误:"+JSON.toJSONString(machineInfo));
toBadEnding();
}
// 执行预设命令 ,进入目录
String allCommand = "docker run -v "+workDir+":/test -it aflplusplus/aflplusplus bash\n" +
"cd /test\n"+
"cd "+compilePath+"\n";
Object commandScriptObj = taskProperties.get(DIYImageExecuteCommandConstant.COMMAND_SCRIPT);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
allCommand += commandScript;
dockerRun(allCommand,dockerInfo,context);
// 更新全局变量
globalVariables.put(PipelineGlobalVariableConstant.AFL_SLOT_COMPILE_PATH_KEY,compilePath);
globalVariables.put(PipelineGlobalVariableConstant.AFL_WORK_DIR_KEY,workDir);
}
}

110
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java
View File

@ -1,110 +0,0 @@
package cd.casic.ci.process.engine.worker.afl;
import cd.casic.ci.process.common.WorkAtom;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.TestCaseGenerationConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.SshWorker;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.util.CryptogramUtil;
import cd.casic.ci.process.util.SftpUploadUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.util.Map;
@WorkAtom(taskType = "TEST_CASE_GENERATION")
@Slf4j
public class TestCaseGenerationWorker extends SshWorker {
@Override
public void execute(TaskRunContext context) {
int statusCode = -1;
Map<String, Object> localVariables = context.getLocalVariables();
if (context.getContextDef() instanceof PipTask taskDef) {
log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY);
String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString();
if (StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceType)) {
// 缺少参数
toBadEnding();
}
ResourceQueryReq req = new ResourceQueryReq();
req.setId(resourceId);
req.setType(resourceType);
TaskResourceFindResp resourceListByType = getResourceManagerService().findResourceListByType(req);
if (CollectionUtils.isEmpty(resourceListByType.getResourceMachineList())) {
append(context,"当前机器不支持machine");
return;
}
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
PipResourceMachine resourceMachine = resourceListByType.getResourceMachineList().get(0);
if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) {
Object commandScriptObj = taskProperties.get(TestCaseGenerationConstant.COMMAND_SCRIPT);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
String seedTarget = taskProperties.get(TestCaseGenerationConstant.SEED_TARGET) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_TARGET)) : null;
if (StringUtils.isEmpty(commandScript)) {
// 缺少参数
toBadEnding();
}
if (StringUtils.isNotEmpty(seedTarget)) {
commandScript += " --output-dir " + seedTarget + " --count 100";
} else {
commandScript += " --output-dir case --count 100";
}
try {
//将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript);
//获取机器
statusCode = shell(resourceMachine, null, context,
"echo \"测试用例生成\"",
"cd /home/casic/706/yunqi",// 基础目录
commandScript
);
} catch (Exception e) {
String errorMessage = "该节点配置信息为空，请先配置该节点信息" + "\r\n";
log.error("执行ssh失败:", e);
append(context, errorMessage);
toBadEnding();
}
if (statusCode == 0) {
log.info("节点执行完成");
} else {
log.error("节点执行失败");
}
localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode);
} else {
// 文件上传
String filePath = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE)) : null;
if (StringUtils.isEmpty(filePath)) {
append(context,"缺少文件");
toBadEnding();
}
File file = new File(filePath);
String seedTarget = taskProperties.get(TestCaseGenerationConstant.SEED_TARGET) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_TARGET)) : null;
String basePath = "/home/casic/706/yunqi/";
if (seedTarget.startsWith("/")) {
seedTarget=seedTarget.substring(1);
}
seedTarget = basePath+seedTarget;
// 将文件上传到服务器的 目录底下
try {
SftpUploadUtil.uploadFileViaSftp(
resourceMachine.getMachineHost()
,Integer.valueOf(resourceMachine.getSshPort()),resourceMachine.getUsername(),resourceMachine.getPassword(),"",filePath,seedTarget,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) {
append(context,"seed文件上传失败");
log.error("seed文件上传失败",e);
}
}
}
}
}

10
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java
View File

@ -41,14 +41,14 @@ import java.util.Map;
public abstract class BaseWorker implements Runnable{
// 一些属性
@Resource
protected RunContextManager contextManager;
protected String contextKey;
private RunContextManager contextManager;
private String contextKey;
@Resource
protected MachineInfoService machineInfoService;
private MachineInfoService machineInfoService;
@Resource
protected LoggerManager loggerManager;
private LoggerManager loggerManager;
@Resource
protected ResourceManagerService resourceManagerService;
private ResourceManagerService resourceManagerService;
@Override

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java
View File

@ -33,5 +33,4 @@ public interface SastService {
public SastApplicationStatusResp getApplicationStatus(String applicationId);
public JSONObject jsonReportDownload(String reportId);
public SastReportStatusResp getReportStatus(String reportId);
}

15
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java
View File

@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletResponse;
import kotlin.text.Charsets;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.luaj.vm2.ast.Str;
import org.springframework.core.io.FileSystemResource;
@ -40,7 +39,6 @@ import java.util.Map;
import static cd.casic.ci.process.constant.SastUrlConstant.*;
@Service
@Slf4j
public class SastServiceImpl implements SastService {
@Resource
private RedisTemplateUtils redisTemplateUtils;
@ -250,23 +248,10 @@ public class SastServiceImpl implements SastService {
HttpEntity<Map<String, String>> entity = new HttpEntity<>(entityMap,httpHeaders);
String uriString=UriComponentsBuilder.fromUriString(sastProperties.getBaseUrl()+jsonReportDownload)
.queryParam("reportId",reportId).toUriString();
log.info("sast 下载报告url:{}",uriString);
ResponseEntity<String> exchange = restTemplate.exchange(uriString, HttpMethod.GET,entity, String.class,new HashMap<>());
return JSON.parseObject(exchange.getBody());
}
@Override
public SastReportStatusResp getReportStatus(String reportId) {
HttpHeaders httpHeaders = getHeaders();
Map<String, String> entityMap = new HashMap<>();
HttpEntity<Map<String, String>> entity = new HttpEntity<>(entityMap,httpHeaders);
String uriString=UriComponentsBuilder.fromUriString(sastProperties.getBaseUrl()+getReportInfo)
.queryParam("id",reportId).toUriString();
log.info("sast 下载报告url:{}",uriString);
ResponseEntity<SastReportStatusResp> exchange = restTemplate.exchange(uriString, HttpMethod.GET,entity, SastReportStatusResp.class,new HashMap<>());
return exchange.getBody();
}
private HttpHeaders getHeaders(){
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.set(TOKEN_HEADER_KEY,TOKEN_PREFIX+getToken());

5
ops-server/src/test/java/cd/casic/server/SastTest.java
View File

@ -201,9 +201,4 @@ public class SastTest {
JSONObject jsonObject = sastService.jsonReportDownload("3300b586-52d7-4a46-b656-e653a0b2a024");
System.out.println(jsonObject);
}
@Test
public void getReportStatus(){
SastReportStatusResp reportStatus = sastService.getReportStatus("34cd37e6-70b8-4b93-8387-5f666de87264");
System.out.println(reportStatus);
}
}