mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

测试用例生成

Browse Source
This commit is contained in:
even 2025-06-20 14:25:08 +08:00
parent ee585fe263
commit 3ebdb6cb7a
10 changed files with 240 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java
View File

@ -18,4 +18,5 @@ public class SastUrlConstant {
public static final String reportDelete = "/api/report/index";
public static final String getApplicationStatus = "/invoke/application/getStatus/";
public static final String jsonReportDownload = "/api/report/index/downloadReport";
public static final String getReportInfo = "/api/report/index/getReportInfo";
}

69
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastReportStatusResp.java Normal file
View File

@ -0,0 +1,69 @@
package cd.casic.ci.process.dto.resp.sast;
import lombok.Data;
import java.util.List;
@Data
public class SastReportStatusResp {
/**
* 报告模式：
* SUMMARY - 概要模式
* DETAILS - 详细模式
* */
private String mode;
/**
* 报告类型：
* PROJECT - 项目报告
* APPLICATION - 应用报告
* */
private String contacts;
/**
* 报告模式：
* SUMMARY - 概要模式
* DETAILS - 详细模式
* */
private String reportType;
/**
* 文件格式：
* DOCX - docx
* XLSX - xlsx
* JSON - json
* PDF - pdf
* */
private String format;
private String projectId;
private List<String> applicationIds;
/**
* 缺陷等级集合
* UN_KNOWN - 未知
* DEADLY - 严重
* SEVERITY - 高危
* POOR_RISK - 中危
* LOW_RISK - 低危
* SAFE - 安全
* */
private List<String> bugLevels;
/**
* WAITING - 待审计
* IGNORE - 忽略
* HIGH - 高风险
* MEDIUM - 中风险
* LOW - 低风险
* NONE - 不受影响/误报
* */
private List<String> audRiskLevels;
/**
* 应用范围：
* ALL - 全部
* CUSTOMIZE - 自定义
* */
private String applicationScope;
/**
* WAITING - 等待中
* GENERATING - 正在生成
* SUCCESS - 生成成功
* FAIL - 生成失败
* */
private String status;
}

4
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java
View File

@ -5,4 +5,8 @@ public class AFLConstant {
* 脚本内容
*/
public static final String COMMAND_SCRIPT ="buildScript";
public static final String WORK_DIR = "workDir";
public static final String BINARY="binary";
public static final String OUTPUT = "output";
public static final String INPUT="input";
}

5
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/TestCaseGenerationConstant.java
View File

@ -5,4 +5,9 @@ public class TestCaseGenerationConstant {
* 脚本内容key
*/
public static final String COMMAND_SCRIPT ="buildScript";
public static final String CASE_TYPE_KEY ="caseType";
public static final String CASE_TYPE_AI ="AI";
public static final String CASE_TYPE_TYPE ="FILE";
public static final String SEED_SOURCE = "filePath";
public static final String SEED_TARGET = "outputDir";
}

31
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/AFLWorker.java
View File

@ -26,22 +26,24 @@ public class AFLWorker extends DockerWorker {
if (context.getContextDef() instanceof PipTask taskDef) {
log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object commandScriptObj = taskProperties.get(AFLConstant.COMMAND_SCRIPT);
// Object machineIdObj = taskProperties.get(DIYImageExecuteCommandConstant.MACHINE_ID);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
// PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef();
// String resourceId = pipeline.getResourceId();
//从taskProperties中获取资源id
String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString();
if (StringUtils.isEmpty(commandScript) || StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceType)) {
// 待测试文件路径
String binaryPath = taskProperties.get(AFLConstant.BINARY) instanceof String ? ((String) taskProperties.get(AFLConstant.BINARY)) : null;
String output = taskProperties.get(AFLConstant.OUTPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.OUTPUT)) : null;
String input = taskProperties.get(AFLConstant.INPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.INPUT)) : null;
String workDir = taskProperties.get(AFLConstant.WORK_DIR) instanceof String ? ((String) taskProperties.get(AFLConstant.WORK_DIR)) : null;
if (StringUtils.isEmpty(binaryPath) ||
StringUtils.isEmpty(output) ||
StringUtils.isEmpty(input) ||
StringUtils.isEmpty(workDir) ||
StringUtils.isEmpty(resourceId) ||
StringUtils.isEmpty(resourceType)) {
// 缺少参数
toBadEnding();
}
// ResourceFindResp resourceById = getResourceManagerService().findResourceById(resourceId);
// DockerEndpointDo dockerEndpoint = resourceById.getDockerEndpoint();
ResourceQueryReq req = new ResourceQueryReq();
req.setId(resourceId);
req.setType(resourceType);
@ -50,16 +52,13 @@ public class AFLWorker extends DockerWorker {
append(context,"当前机器不支持docker");
return;
}
String commandScript = "docker run -v "+workDir+":/test -it aflplusplus/aflplusplus bash\n" +
"cd /test\n" +
"afl-fuzz -i "+input+" -o "+output+" -t 3000 -Q "+binaryPath+" @@";
try {
//将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript);
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
//获取机器
// MachineInfo machineInfoDO = this.getMachineInfoService().getById(machineId);
// 获取docker 暂时先写固定值
// TODO dockerEndpoint替换为查询
dockerRun(commandScript,resourceListByType.getDockerEndpointList().get(0),context);
} catch (Exception e) {
String errorMessage = "该节点配置信息为空，请先配置该节点信息" + "\r\n";

100
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/SastWorker.java
View File

@ -7,6 +7,7 @@ import cd.casic.ci.process.dto.req.sast.SastReportCreateReq;
import cd.casic.ci.process.dto.resp.sast.SastApplicationCreateResp;
import cd.casic.ci.process.dto.resp.sast.SastApplicationEchoResp;
import cd.casic.ci.process.dto.resp.sast.SastApplicationStatusResp;
import cd.casic.ci.process.dto.resp.sast.SastReportStatusResp;
import cd.casic.ci.process.engine.constant.SastConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.BaseWorker;
@ -15,6 +16,8 @@ import cd.casic.ci.process.process.dataObject.base.PipBaseElement;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.service.sast.SastService;
import cd.casic.ci.process.process.service.task.TaskService;
import cd.casic.framework.commons.exception.ServiceException;
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
import com.alibaba.excel.util.StringUtils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
@ -49,55 +52,56 @@ public class SastWorker extends BaseWorker {
toBadEnding();
}
SastApplicationStatusResp applicationStatus = sastService.getApplicationStatus(applicationId);
if (SastConstant.STATUS_PROGRESSING.equals(applicationStatus.getJobStatus())||SastConstant.STATUS_PENDING.equals(applicationStatus.getJobStatus())) {
append(context,"当前任务正在运行中");
toBadEnding();
}
SastApplicationEchoResp applicationEcho = sastService.getApplicationEcho(applicationId);
append(context,"获取到sast任务配置"+JSON.toJSONString(applicationEcho));
SastApplicationCreateReq sastApplicationCreateReq = converter.converter(applicationEcho);
List<String> list = applicationEcho.getLodeFiles().stream().map(SastApplicationEchoResp.LodeFile::getFileId).toList();
sastApplicationCreateReq.setFileId(list);
append(context,"开始启动SAST任务,检测文件信息：{}"+JSON.toJSONString(applicationEcho.getLodeFiles()));
log.info("SAST启动任务入参:{}",JSON.toJSONString(sastApplicationCreateReq));
// SastApplicationCreateResp sastApplicationCreateResp = sastService.applicationCreate(sastApplicationCreateReq);
// log.info("SAST启动任务返回值:{}",JSON.toJSONString(sastApplicationCreateResp));
// append(context,"启动任务完毕");
//// 检测状态是否完毕
// applicationStatus = sastService.getApplicationStatus(applicationId);
// int repeat = 0;
// while (!SastConstant.STATUS_JOB_DONE.equals(applicationStatus.getJobStatus())) {
// try {
// Thread.sleep(20000L);
// } catch (InterruptedException e) {
// log.error("SAST WORKER线程中断");
// }
// try {
// applicationStatus=sastService.getApplicationStatus(applicationId);
// } catch (Exception e) {
// log.error("sastWorker执行失败",e);
// append(context,"获取状态失败");
// if (repeat++>3) {
// append(context,"获取状态失败,尝试超过三次执行失败");
// toBadEnding();
// }
// continue;
// }
// append(context,"当前SAST运行中，运行状态："+applicationStatus.getJobStatus());
// if (SastConstant.STATUS_PENDING.equals(applicationStatus.getJobStatus())) {
// append(context,"任务被取消");
// toBadEnding();
// }
// repeat = 0;
// }
JSONObject reportJSON = getReportJSON(applicationId, SastConstant.REPORT_MODE_DETAILS);
if (SastConstant.STATUS_PROGRESSING.equals(applicationStatus.getJobStatus())||SastConstant.STATUS_PENDING.equals(applicationStatus.getJobStatus())) {
append(context,"当前任务正在运行中");
// toBadEnding();
} else{
SastApplicationCreateReq sastApplicationCreateReq = converter.converter(applicationEcho);
List<String> list = applicationEcho.getLodeFiles().stream().map(SastApplicationEchoResp.LodeFile::getFileId).toList();
sastApplicationCreateReq.setFileId(list);
append(context,"开始启动SAST任务,检测文件信息：{}"+JSON.toJSONString(applicationEcho.getLodeFiles()));
log.info("SAST启动任务入参:{}",JSON.toJSONString(sastApplicationCreateReq));
SastApplicationCreateResp sastApplicationCreateResp = sastService.applicationCreate(sastApplicationCreateReq);
log.info("SAST启动任务返回值:{}",JSON.toJSONString(sastApplicationCreateResp));
append(context,"启动任务完毕");
}
// 检测状态是否完毕
applicationStatus = sastService.getApplicationStatus(applicationId);
int repeat = 0;
while (!SastConstant.STATUS_JOB_DONE.equals(applicationStatus.getJobStatus())) {
try {
Thread.sleep(20000L);
} catch (InterruptedException e) {
log.error("SAST WORKER线程中断");
}
try {
applicationStatus=sastService.getApplicationStatus(applicationId);
} catch (Exception e) {
log.error("sastWorker执行失败",e);
append(context,"获取状态失败");
if (repeat++>3) {
append(context,"获取状态失败,尝试超过三次执行失败");
toBadEnding();
}
continue;
}
append(context,"当前SAST运行中，运行状态："+applicationStatus.getJobStatus());
if (SastConstant.STATUS_CANCEL.equals(applicationStatus.getJobStatus())) {
append(context,"任务被取消");
toBadEnding();
}
repeat = 0;
}
JSONObject reportJSON = getReportJSON(applicationId, SastConstant.REPORT_MODE_DETAILS,context);
String reportUrl = getReportUrl(applicationId, SastConstant.REPORT_MODE_DETAILS);
task.getTaskProperties().put(SastConstant.REPORT_JSON,reportJSON);
task.getTaskProperties().put(SastConstant.REPORT_URL,reportUrl);
taskService.updateById(task);
}
}
private JSONObject getReportJSON(String applicationId,String model){
private JSONObject getReportJSON(String applicationId,String model,TaskRunContext context){
String reqJSON = "{\"mode\":\"DETAILS\",\"contacts\":\"admin@clouditera.com\",\"reportType\":\"APPLICATION\",\"format\":\"JSON\",\"projectId\":\"8a863857-bab3-40d9-a79a-2cb6617a9dd3\",\"applicationIds\":[\"9e3e55d6-6d28-4fd1-adc2-15b6cf0702db\"],\"bugLevels\":[\"DEADLY\",\"SEVERITY\",\"POOR_RISK\",\"LOW_RISK\"],\"audRiskLevels\":[\"WAITING\",\"IGNORE\",\"HIGH\",\"MEDIUM\",\"LOW\",\"NONE\"],\"applicationScope\":\"ALL\"}";
SastReportCreateReq sastReportCreateReq = JSON.parseObject(reqJSON, SastReportCreateReq.class);
sastReportCreateReq.setMode(model);
@ -106,6 +110,20 @@ public class SastWorker extends BaseWorker {
log.info("获取JSON报告请求参数{}",JSON.toJSONString(sastReportCreateReq));
String reportId = sastService.reportIndex(sastReportCreateReq);
log.info("获取JSON报告获得reportId{}",reportId);
// todo 获取报告状态
SastReportStatusResp reportStatus = sastService.getReportStatus(reportId);
try {
while (reportStatus.getStatus().equals("WAITING")||reportStatus.getStatus().equals("GENERATING")){
reportStatus = sastService.getReportStatus(reportId);
append(context,"当前生成报告状态为:"+reportStatus.getStatus());
Thread.sleep(1000L);
}
} catch (Exception e) {
throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"获取sast报告状态失败");
}
if (reportStatus.getStatus().equals("FAIL")) {
throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"生成sast报告失败");
}
JSONObject jsonObject = getJSONString(reportId).getJSONObject("task_summary");
JSONObject severity = jsonObject.getJSONObject("defect_severity_distribution");
JSONObject rule = jsonObject.getJSONObject("detection_rule_distribution");

110
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/TestCaseGenerationWorker.java
View File

@ -12,10 +12,12 @@ import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.util.CryptogramUtil;
import cd.casic.ci.process.util.SftpUploadUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.util.Map;
@WorkAtom(taskType = "TEST_CASE_GENERATION")
@ -28,55 +30,81 @@ public class TestCaseGenerationWorker extends SshWorker {
if (context.getContextDef() instanceof PipTask taskDef) {
log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object commandScriptObj = taskProperties.get(TestCaseGenerationConstant.COMMAND_SCRIPT);
// Object machineIdObj = taskProperties.get(DIYImageExecuteCommandConstant.MACHINE_ID);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef();
if (StringUtils.isEmpty(commandScript)) {
// 缺少参数
Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY);
String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString();
if (StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceType)) {
// 缺少参数
toBadEnding();
}
try {
//将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript);
String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString();
if (StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceType)) {
ResourceQueryReq req = new ResourceQueryReq();
req.setId(resourceId);
req.setType(resourceType);
TaskResourceFindResp resourceListByType = getResourceManagerService().findResourceListByType(req);
if (CollectionUtils.isEmpty(resourceListByType.getResourceMachineList())) {
append(context,"当前机器不支持machine");
return;
}
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
PipResourceMachine resourceMachine = resourceListByType.getResourceMachineList().get(0);
if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) {
Object commandScriptObj = taskProperties.get(TestCaseGenerationConstant.COMMAND_SCRIPT);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
String seedTarget = taskProperties.get(TestCaseGenerationConstant.SEED_TARGET) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_TARGET)) : null;
if (StringUtils.isEmpty(commandScript)) {
// 缺少参数
toBadEnding();
}
ResourceQueryReq req = new ResourceQueryReq();
req.setId(resourceId);
req.setType(resourceType);
TaskResourceFindResp resourceListByType = getResourceManagerService().findResourceListByType(req);
if (CollectionUtils.isEmpty(resourceListByType.getResourceMachineList())) {
append(context,"当前机器不支持machine");
return;
if (StringUtils.isNotEmpty(seedTarget)) {
commandScript += "--output-dir " + seedTarget + " --count 100";
} else {
commandScript += "--output-dir case --count 100";
}
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
PipResourceMachine resourceMachine = resourceListByType.getResourceMachineList().get(0);
// ResourceFindResp resourceById = getResourceManagerService().findResourceById(resourceId);
// PipResourceMachine resourceMachine = resourceById.getResourceMachine();
//获取机器
statusCode = shell(resourceMachine, CryptogramUtil.doDecrypt(resourceMachine.getPassword()), context,
"echo \"自定义镜像执行命令\"",
commandScript
);
} catch (Exception e) {
String errorMessage = "该节点配置信息为空，请先配置该节点信息" + "\r\n";
log.error("执行ssh失败:", e);
append(context, errorMessage);
toBadEnding();
}
if (statusCode == 0) {
log.info("节点执行完成");
try {
//将节点的配置信息反编译成对象
log.info("构建脚本" + commandScript);
//获取机器
statusCode = shell(resourceMachine, CryptogramUtil.doDecrypt(resourceMachine.getPassword()), context,
"echo \"测试用例生成\"",
"cd /home/casic/706/yunqi",// 基础目录
commandScript
);
} catch (Exception e) {
String errorMessage = "该节点配置信息为空，请先配置该节点信息" + "\r\n";
log.error("执行ssh失败:", e);
append(context, errorMessage);
toBadEnding();
}
if (statusCode == 0) {
log.info("节点执行完成");
} else {
log.error("节点执行失败");
}
localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode);
} else {
log.error("节点执行失败");
// 文件上传
String filePath = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE)) : null;
if (StringUtils.isEmpty(filePath)) {
append(context,"缺少文件");
toBadEnding();
}
File file = new File(filePath);
String seedTarget = taskProperties.get(TestCaseGenerationConstant.SEED_TARGET) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_TARGET)) : null;
String basePath = "/home/casic/706/yunqi/";
if (seedTarget.startsWith("/")) {
seedTarget=seedTarget.substring(1);
}
seedTarget = basePath+seedTarget;
// 将文件上传到服务器的 目录底下
try {
SftpUploadUtil.uploadFileViaSftp(
resourceMachine.getMachineHost()
,Integer.valueOf(resourceMachine.getSshPort()),resourceMachine.getUsername(),resourceMachine.getPassword(),"",filePath,seedTarget,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) {
append(context,"seed文件上传失败");
log.error("seed文件上传失败",e);
}
}
localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode);
}
}
}

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java
View File

@ -33,4 +33,5 @@ public interface SastService {
public SastApplicationStatusResp getApplicationStatus(String applicationId);
public JSONObject jsonReportDownload(String reportId);
public SastReportStatusResp getReportStatus(String reportId);
}

12
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java
View File

@ -255,6 +255,18 @@ public class SastServiceImpl implements SastService {
return JSON.parseObject(exchange.getBody());
}
@Override
public SastReportStatusResp getReportStatus(String reportId) {
HttpHeaders httpHeaders = getHeaders();
Map<String, String> entityMap = new HashMap<>();
HttpEntity<Map<String, String>> entity = new HttpEntity<>(entityMap,httpHeaders);
String uriString=UriComponentsBuilder.fromUriString(sastProperties.getBaseUrl()+getReportInfo)
.queryParam("id",reportId).toUriString();
log.info("sast 下载报告url:{}",uriString);
ResponseEntity<SastReportStatusResp> exchange = restTemplate.exchange(uriString, HttpMethod.GET,entity, SastReportStatusResp.class,new HashMap<>());
return exchange.getBody();
}
private HttpHeaders getHeaders(){
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.set(TOKEN_HEADER_KEY,TOKEN_PREFIX+getToken());

5
ops-server/src/test/java/cd/casic/server/SastTest.java
View File

@ -201,4 +201,9 @@ public class SastTest {
JSONObject jsonObject = sastService.jsonReportDownload("3300b586-52d7-4a46-b656-e653a0b2a024");
System.out.println(jsonObject);
}
@Test
public void getReportStatus(){
SastReportStatusResp reportStatus = sastService.getReportStatus("34cd37e6-70b8-4b93-8387-5f666de87264");
System.out.println(reportStatus);
}
}