diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java index 56bd7be0..48d40ffb 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java @@ -11,6 +11,7 @@ public class SastUrlConstant { public static final String detectionConfig = "/invoke/project/setting/getDetectionConfig"; public static final String applicationCreate = "/invoke/application/create"; public static final String applicationStash = "/invoke/application/stash"; + public static final String applicationBinaryStash = "/invoke/application/binary/stash"; public static final String getApplicationEcho = "/invoke/application/getApplicationEcho"; public static final String reportIndex = "/api/report/index"; public static final String reportDownload = "/api/report/index/download"; @@ -21,5 +22,6 @@ public class SastUrlConstant { public static final String jsonReportDownload = "/api/report/index/downloadReport"; public static final String getReportInfo = "/api/report/index/getReportInfo"; public static final String stashScan = "/invoke/application/stash/scan/"; + public static final String binaryStashScan = "/invoke/application/binary/stash/scan/"; public static final String engineLog = "/invoke/engine/%s/log"; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationBinaryStashReq.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationBinaryStashReq.java new file mode 100644 index 00000000..9c360b81 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationBinaryStashReq.java @@ -0,0 +1,58 @@ +package cd.casic.ci.process.dto.req.sast; + +import lombok.Data; + +import java.util.List; + +@Data +public class SastApplicationBinaryStashReq { + private String applicationId;//不传 + /** + * 当前项目id 写死"893ed995-5b81-474a-96a9-2800281421cd" + * */ + private String projectId;//不传 + /** + * 应用名称，不可重复 + * */ + private String applicationName;//可以不传 + /** + * 写死 BINARY + * */ + private String codeSourceFrom;//不传 + /** + * 目标指令集 true-自动识别 false-手动识别 + * */ + private Boolean archAutoIdentify;// TODO 询问是否需要写死自动识别 + /** + * + * */ + private String targetArchitectures;// TODO 询问作用 + /** + * 上传文件返回的id + * */ + private List fileId;// 上传目标文件，省去页面上传 + /** + * 模板id + * */ + private String templateId;//需要用户选择 + /** + * 分析超时时长 + * */ + private Integer analysisTimeoutMs;//需要用户填写 + /** + * k集合大小 + * */ + private Integer kSetSize;//需要用户填写 + /** + * CallString长度 + * */ + private Integer callStringLength;//需要用户填写 + /** + * 入口地址entryAddress + * */ + private String entryAddress;//需要用户填写 + /** + * z3 是否能够超时 + * */ + private Boolean z3TimeoutMsStatus;//需要用户填写 +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationBinaryStashResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationBinaryStashResp.java new file mode 100644 index 00000000..18ad4517 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationBinaryStashResp.java @@ -0,0 +1,19 @@ +package cd.casic.ci.process.dto.resp.sast; + +import lombok.Data; + +@Data +public class SastApplicationBinaryStashResp { + private String applicationId; + private String applicationName; + private String taskId; + private String fileId; + private String jobAddBy; + private String jobStatus; + private String checkStartTime; + private String checkEndTime; + private String codeQualityMetric; // 这里是JSON字符串 + private String engineType; + private String progress; // 允许为null + private String type; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java index 4f093392..49dc42c6 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java @@ -6,6 +6,7 @@ import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; import cd.casic.framework.commons.exception.ServiceException; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Component; @@ -16,6 +17,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @Component +@Slf4j public class MemoryPostHandlerManager implements PostHandlerManager { private final ConcurrentHashMap> handlerMap = new ConcurrentHashMap<>(); @Resource @@ -38,7 +40,13 @@ public class MemoryPostHandlerManager implements PostHandlerManager { return; } for (ExecuteTaskPostHandler postHandler : pipExecutePostHandlers) { - postHandlerExecutor.submit(()->postHandler.executeAfterDone(hisInstance)); + postHandlerExecutor.submit(()->{ + try { + postHandler.executeAfterDone(hisInstance); + } catch (Exception e) { + log.error("后置执行失败",e); + } + }); } } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastBinaryWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastBinaryWorker.java new file mode 100644 index 00000000..7cd0d328 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastBinaryWorker.java @@ -0,0 +1,105 @@ +package cd.casic.ci.process.engine.worker.sast; + +import cd.casic.ci.process.common.WorkAtom; +import cd.casic.ci.process.dto.resp.sast.SastApplicationEchoResp; +import cd.casic.ci.process.dto.resp.sast.SastApplicationStashResp; +import cd.casic.ci.process.dto.resp.sast.SastApplicationStatusResp; +import cd.casic.ci.process.dto.resp.sast.SastEngineLogResp; +import cd.casic.ci.process.engine.constant.SastConstant; +import cd.casic.ci.process.engine.runContext.TaskRunContext; +import cd.casic.ci.process.engine.worker.base.BaseWorker; +import cd.casic.ci.process.process.converter.SastConverter; +import cd.casic.ci.process.process.dataObject.base.PipBaseElement; +import cd.casic.ci.process.process.dataObject.task.PipTask; +import cd.casic.ci.process.process.service.sast.SastService; +import cd.casic.ci.process.process.service.task.TaskService; +import com.alibaba.excel.util.StringUtils; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +@WorkAtom(taskType = "SAST_BINARY") +public class SastBinaryWorker extends BaseWorker { + @Resource + private SastService sastService; + @Resource + private SastConverter converter; + @Resource + private TaskService taskService; + @Override + public void execute(TaskRunContext context) { + PipBaseElement contextDef = context.getContextDef(); + String id = contextDef.getId(); + log.info("==============触发worker执行========"); + log.info("==========运行context：{}===========", JSON.toJSONString(context)); + if (contextDef instanceof PipTask task) { + String applicationId = (String)(task.getTaskProperties().get(SastConstant.APPLICATION_ID)); + if (StringUtils.isEmpty(applicationId)) { + append(context,"未获取到sast信息"); + toBadEnding(); + } + SastApplicationStatusResp applicationStatus = sastService.getApplicationStatus(applicationId); + SastApplicationEchoResp applicationEcho = sastService.getApplicationEcho(applicationId); + append(context,"获取到sast任务配置"+JSON.toJSONString(applicationEcho)); + if (SastConstant.STATUS_PROGRESSING.equals(applicationStatus.getJobStatus())||SastConstant.STATUS_PENDING.equals(applicationStatus.getJobStatus())) { + append(context,"当前任务正在运行中"); +// toBadEnding(); + } else{ +// SastApplicationCreateReq sastApplicationCreateReq = converter.converter(applicationEcho); +// List list = applicationEcho.getLodeFiles().stream().map(SastApplicationEchoResp.LodeFile::getFileId).toList(); +// sastApplicationCreateReq.setFileId(list); +// append(context,"开始启动SAST任务,检测文件信息：{}"+JSON.toJSONString(applicationEcho.getLodeFiles())); +// log.info("SAST启动任务入参:{}",JSON.toJSONString(sastApplicationCreateReq)); + SastApplicationStashResp sastApplicationCreateResp = sastService.binaryStashScan(applicationId); + log.info("SAST启动二进制任务返回值:{}",JSON.toJSONString(sastApplicationCreateResp)); + append(context,"启动任务完毕"); + } +// 检测状态是否完毕 + applicationStatus = sastService.getApplicationStatus(applicationId); + List oldLogList = new ArrayList<>(); + int repeat = 0; + while (!SastConstant.STATUS_JOB_DONE.equals(applicationStatus.getJobStatus())) { + try { + Thread.sleep(20000L); + } catch (InterruptedException e) { + log.error("二进制任务执行失败，R线程中断"); + } + try { + applicationStatus=sastService.getApplicationStatus(applicationId); + } catch (Exception e) { + log.error("二进制任务执行失败",e); + append(context,"二进制任务获取状态失败"); + if (repeat++>3) { + append(context,"获取状态失败,尝试超过三次执行失败"); + toBadEnding(); + } + continue; + } + append(context,"当前SAST二进制任务运行中，运行状态："+applicationStatus.getJobStatus()); + if (SastConstant.STATUS_CANCEL.equals(applicationStatus.getJobStatus())) { + append(context,"任务被取消"); + toBadEnding(); + } + List logList = sastService.engineLog(applicationId); + if (logList.size()>oldLogList.size()) { + for (int i = oldLogList.size(); i < logList.size(); i++) { + SastEngineLogResp logResp = logList.get(i); + append(context,logResp.getPrettyPrint()); + } + oldLogList = logList; + } + repeat = 0; + } +// JSONObject reportJSON = getReportJSON(applicationId, SastConstant.REPORT_MODE_DETAILS,context); +// String reportUrl = getReportUrl(applicationId, SastConstant.REPORT_MODE_DETAILS); +// task.getTaskProperties().put(SastConstant.REPORT_JSON,reportJSON); +// task.getTaskProperties().put(SastConstant.REPORT_URL,reportUrl); + taskService.updateById(task); + } + } +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java index 72729993..74fcf623 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java @@ -126,7 +126,7 @@ public class ScaBinaryWorker extends HttpWorker { postHandlerManager.registerPostHandler(new ExecuteTaskPostHandler(pipTask.getId(),pipelineId) { @Override public void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance) { - vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"APPLICATION"); + vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"BINARY"); } }); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java index 2a5e24b3..601a770c 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java @@ -124,7 +124,7 @@ public class ScaMirrorWorker extends HttpWorker { postHandlerManager.registerPostHandler(new ExecuteTaskPostHandler(pipTask.getId(),pipelineId) { @Override public void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance) { - vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"APPLICATION"); + vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"MIRROR"); } }); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java index 069a0da0..7f1d412a 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java @@ -122,7 +122,7 @@ public class ScaSbomWorker extends HttpWorker { postHandlerManager.registerPostHandler(new ExecuteTaskPostHandler(pipTask.getId(),pipelineId) { @Override public void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance) { - vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"APPLICATION"); + vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"SCA_S_BOM"); } }); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java index 8cbce99c..8886444c 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java @@ -1,9 +1,6 @@ package cd.casic.ci.process.process.service.sast; -import cd.casic.ci.process.dto.req.sast.SastApplicationCreateReq; -import cd.casic.ci.process.dto.req.sast.SastApplicationStashReq; -import cd.casic.ci.process.dto.req.sast.SastIdentifiedLanguageTaskReq; -import cd.casic.ci.process.dto.req.sast.SastReportCreateReq; +import cd.casic.ci.process.dto.req.sast.*; import cd.casic.ci.process.dto.resp.sast.*; import cd.casic.framework.commons.pojo.PageParam; import cd.casic.framework.commons.pojo.PageResult; @@ -38,4 +35,6 @@ public interface SastService { public SastReportStatusResp getReportStatus(String reportId); SastApplicationStashResp stashScan(String applicationId); List engineLog(String applicationId); + SastApplicationBinaryStashResp applicationBinaryStash(SastApplicationBinaryStashReq req) ; + SastApplicationStashResp binaryStashScan(String applicationId); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java index f5423223..0163ff75 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java @@ -1,9 +1,6 @@ package cd.casic.ci.process.process.service.sast.impl; -import cd.casic.ci.process.dto.req.sast.SastApplicationCreateReq; -import cd.casic.ci.process.dto.req.sast.SastApplicationStashReq; -import cd.casic.ci.process.dto.req.sast.SastIdentifiedLanguageTaskReq; -import cd.casic.ci.process.dto.req.sast.SastReportCreateReq; +import cd.casic.ci.process.dto.req.sast.*; import cd.casic.ci.process.dto.resp.report.ReportResp; import cd.casic.ci.process.dto.resp.sast.*; import cd.casic.ci.process.process.service.sast.SastService; @@ -299,6 +296,25 @@ public class SastServiceImpl implements SastService { return JSONArray.parseArray(exchange.getBody(), SastEngineLogResp.class); } + @Override + public SastApplicationBinaryStashResp applicationBinaryStash(SastApplicationBinaryStashReq req) { + HttpHeaders httpHeaders = getHeaders(); + HttpEntity entity = new HttpEntity<>(req,httpHeaders); + httpHeaders.setContentType(MediaType.APPLICATION_JSON); + ResponseEntity exchange = restTemplate.exchange(sastProperties.getBaseUrl() +applicationBinaryStash, HttpMethod.POST,entity,SastApplicationBinaryStashResp.class,new HashMap<>()); + return exchange.getBody(); + } + @Override + public SastApplicationStashResp binaryStashScan(String applicationId) { + HttpHeaders httpHeaders = getHeaders(); + Map entityMap = new HashMap<>(); + HttpEntity> entity = new HttpEntity<>(entityMap,httpHeaders); + String uriString=sastProperties.getBaseUrl()+binaryStashScan+applicationId; + log.info("sast 开始任务url:{}",uriString); + ResponseEntity exchange = restTemplate.exchange(uriString, HttpMethod.POST,entity, SastApplicationStashResp.class,new HashMap<>()); + return exchange.getBody(); + } + private HttpHeaders getHeaders(){ HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.set(TOKEN_HEADER_KEY,TOKEN_PREFIX+getToken());