diff --git a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/ShuiMuController.java b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/ShuiMuController.java index b1660c3e..20b7b396 100644 --- a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/ShuiMuController.java +++ b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/ShuiMuController.java @@ -5,6 +5,7 @@ import cd.casic.ci.process.dto.req.shuimu.ShuiMuProjectCreateReq; import cd.casic.ci.process.dto.req.shuimu.ShuiMuTestProblemReq; import cd.casic.ci.process.dto.req.shuimu.ShuiMuUpdateWFuzzJsonReq; import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectInfoResp; +import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectReportResp; import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectVersionResp; import cd.casic.ci.process.dto.resp.shuimu.ShuiMuTestProblemResp; import cd.casic.ci.process.process.service.shuimu.ShuiMuService; @@ -69,4 +70,8 @@ public class ShuiMuController { public CommonResult> problemsOfTest(@RequestBody ShuiMuTestProblemReq req){ return CommonResult.success(shuiMuService.problemsOfTest(req)); } + @GetMapping("/getProjectReport") + public CommonResult getProjectReport(@RequestParam String projectId){ + return CommonResult.success(shuiMuService.getProjectReport(projectId)); + } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/ShuiMuUrlConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/ShuiMuUrlConstant.java index 64f371e5..793dfbeb 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/ShuiMuUrlConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/ShuiMuUrlConstant.java @@ -18,4 +18,5 @@ public class ShuiMuUrlConstant { public static final String executeAllEntrypointsTest ="/api/driverGeneration/executeAllEntrypointsTest"; public static final String problemsOfTest = "/api/test/problemsOfTest"; + public static final String getProjectReport = "/api/project"; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/shuimu/ShuiMuProjectReportResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/shuimu/ShuiMuProjectReportResp.java new file mode 100644 index 00000000..c4b60b3a --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/shuimu/ShuiMuProjectReportResp.java @@ -0,0 +1,22 @@ +package cd.casic.ci.process.dto.resp.shuimu; + +import lombok.Data; + +@Data +public class ShuiMuProjectReportResp { + private String id; + private String name; + private String language; + private String description; + private String product; + private String ownerId; + private String createBy; + private String createAt; + private String updateAt; + private String deleteAt; + private String projectRunning; + private String problemsCount; + private String fixedProblemsCount; + private String noFixedProblemsCount; + private String projectFlag; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java index 91894e83..9ba939d2 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLConstant.java @@ -5,4 +5,6 @@ public class AFLConstant { public static final String COMMAND_END="commandEnd"; public static final String EXECUTABLE_NAME ="executableName"; public static final String RUNNING_TIME ="runningTime"; + + public static final String GROUP_ID= "groupId"; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/PostHandlerManager.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/PostHandlerManager.java new file mode 100644 index 00000000..cf987075 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/PostHandlerManager.java @@ -0,0 +1,9 @@ +package cd.casic.ci.process.engine.manager; + +import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; +import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; + +public interface PostHandlerManager { + public void registerPostHandler(ExecuteTaskPostHandler handler); + public void executePostHandler(PipPipelineHisInstance hisInstance); +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/DefaultRunContextManager.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/DefaultRunContextManager.java index 30b3080c..867e3e08 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/DefaultRunContextManager.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/DefaultRunContextManager.java @@ -5,6 +5,7 @@ import cd.casic.ci.process.dto.resp.context.TreeRunContextResp; import cd.casic.ci.process.engine.constant.EngineRuntimeConstant; import cd.casic.ci.process.engine.enums.ContextStateEnum; import cd.casic.ci.process.engine.manager.LoggerManager; +import cd.casic.ci.process.engine.manager.PostHandlerManager; import cd.casic.ci.process.engine.manager.RunContextManager; import cd.casic.ci.process.engine.runContext.BaseRunContext; import cd.casic.ci.process.engine.runContext.PipelineRunContext; @@ -46,11 +47,12 @@ public class DefaultRunContextManager implements RunContextManager { @Resource private LoggerManager loggerManager; @Resource - private PipTaskDao taskDao; - @Resource private PipPipelineHisInstanceDao hisInstanceDao; @Resource private PipelineService pipelineService; + @Resource + private PostHandlerManager postHandlerManager; + @Override public Boolean stopPipeline(String pipelineId) { return null; @@ -148,7 +150,7 @@ public class DefaultRunContextManager implements RunContextManager { } pipelineRunContext.changeContextStateAndChild(stateEnum); } - // 记录日志需要新事物，成功或失败不影响流水线执行 + // 记录日志需要新事务，成功或失败不影响流水线执行 @Transactional(rollbackFor = Exception.class,propagation = Propagation.REQUIRES_NEW) public void toHistory(String pipelineId){ log.info("========================开始入库"); @@ -193,6 +195,8 @@ public class DefaultRunContextManager implements RunContextManager { pipPipelineHisInstance.setTargetVersionName(""); pipPipelineHisInstance.setTargetVersionId(pipeline.getTargetVersionId()); hisInstanceDao.insert(pipPipelineHisInstance); + // 执行后置处理逻辑 + postHandlerManager.executePostHandler(pipPipelineHisInstance); } } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java new file mode 100644 index 00000000..bc63083a --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java @@ -0,0 +1,40 @@ +package cd.casic.ci.process.engine.manager.impl; + +import cd.casic.ci.process.engine.manager.PostHandlerManager; +import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; +import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; +import cd.casic.framework.commons.exception.ServiceException; +import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; + +@Component +public class MemoryPostHandlerManager implements PostHandlerManager { + private final ConcurrentHashMap> handlerMap = new ConcurrentHashMap<>(); + @Override + public void registerPostHandler(ExecuteTaskPostHandler handler) { + if (handler==null|| StringUtils.isNotEmpty(handler.getPipelineId())) { + throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"注册后置处理器失败"); + } + List orDefault = handlerMap.getOrDefault(handler.getPipelineId(), new CopyOnWriteArrayList<>()); + orDefault.add(handler); + handlerMap.put(handler.getPipelineId(),orDefault); + } + + @Override + public void executePostHandler(PipPipelineHisInstance hisInstance) { + String pipelineId = hisInstance.getPipelineId(); + List pipExecutePostHandlers = handlerMap.get(pipelineId); + if (CollectionUtils.isEmpty(pipExecutePostHandlers)) { + return; + } + for (ExecuteTaskPostHandler postHandler : pipExecutePostHandlers) { + postHandler.executeAfterDone(hisInstance); + } + } +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/postHandler/ExecuteTaskPostHandler.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/postHandler/ExecuteTaskPostHandler.java new file mode 100644 index 00000000..007cbd32 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/postHandler/ExecuteTaskPostHandler.java @@ -0,0 +1,17 @@ +package cd.casic.ci.process.engine.postHandler; + +import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public abstract class ExecuteTaskPostHandler { + private String taskId; + private String pipelineId; + + /** + * 流水线执行结束需要执行的操作 + * */ + public abstract void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance); +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java index 86b163d9..b847f502 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java @@ -1,26 +1,35 @@ package cd.casic.ci.process.engine.worker.afl; import cd.casic.ci.process.common.WorkAtom; +import cd.casic.ci.process.dto.req.aflManager.AflManagerReq; import cd.casic.ci.process.dto.req.resource.ResourceQueryReq; import cd.casic.ci.process.dto.resp.resource.ResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.engine.constant.AFLConstant; -import cd.casic.ci.process.engine.constant.AFLSlotCompileConstant; import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; -import cd.casic.ci.process.engine.constant.PipelineVariableConstant; +import cd.casic.ci.process.engine.manager.PostHandlerManager; +import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.DockerWorker; +import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.task.PipTask; +import cd.casic.ci.process.process.service.aflManager.AflInfoService; +import cd.casic.ci.process.process.service.aflManager.AflPlotInfoService; +import cd.casic.ci.process.process.service.aflManager.AflSeedInfoService; import cd.casic.ci.process.process.service.target.TargetVersionService; +import cd.casic.ci.process.util.SftpUploadUtil; +import com.alibaba.fastjson.JSON; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.scheduling.annotation.Async; import org.springframework.util.CollectionUtils; import java.io.File; import java.util.Map; +import java.util.UUID; import static cd.casic.ci.process.engine.constant.AFLConstant.*; import static cd.casic.ci.process.engine.constant.PipelineVariableConstant.*; @@ -30,6 +39,14 @@ import static cd.casic.ci.process.engine.constant.PipelineVariableConstant.*; public class AFLWorker extends DockerWorker { @Resource private TargetVersionService targetVersionService; + @Resource + private AflInfoService aflInfoService; + @Resource + private AflSeedInfoService aflSeedInfoService; + @Resource + private AflPlotInfoService aflPlotInfoService; + @Resource + private PostHandlerManager postHandlerManager; @Override public void execute(TaskRunContext context) { int statusCode = -1; @@ -49,6 +66,7 @@ public class AFLWorker extends DockerWorker { String commandEnd = taskProperties.get(COMMAND_END) instanceof String ? ((String) taskProperties.get(COMMAND_END)) : null; String imageName = getVariableNearby(context,IMAGE_NAME) instanceof String ? ((String) getVariableNearby(context,IMAGE_NAME)) : null; String executableName = taskProperties.get(EXECUTABLE_NAME) instanceof String ? ((String) taskProperties.get(EXECUTABLE_NAME)) : null; + putGlobalVariable(context, EXECUTABLE_NAME_KEY,imageName); // 代码编译目录 String workDir = (String)getGlobalVariable(context,AFL_DOCKER_WORK_DIR_KEY); // 入参目录 @@ -102,20 +120,67 @@ public class AFLWorker extends DockerWorker { toBadEnding(); } localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode); - putGlobalVariable(context, EXECUTABLE_NAME_KEY,imageName); + afterTaskExecute(context); + afterPipelineExecute(context); } } - public String cdSourceName(String fileName){ + private String cdSourceName(String fileName){ int dotIndex = fileName.lastIndexOf("."); if (dotIndex!=-1) { return "cd "+fileName.substring(0, dotIndex)+"\n"; } return ""; } - public Long getRunningTime(Double runningTimeHours){ + private Long getRunningTime(Double runningTimeHours){ if (runningTimeHours!=null && runningTimeHours >0) { return (long)(runningTimeHours*60*60*1000); } return null; } + @Async + public void afterTaskExecute(TaskRunContext context){ + append(context,"开始拉取afl输出信息"); + PipTask contextDef = (PipTask)context.getContextDef(); + Map localVariables = context.getLocalVariables(); + String pipelineId =contextDef.getPipelineId(); + String taskId = contextDef.getId(); + String groupId = UUID.randomUUID().toString().replace("-",""); + localVariables.put(GROUP_ID,groupId); + AflManagerReq req = new AflManagerReq(); + req.setGroupIdentifier(groupId); + req.setTaskId(taskId); + req.setPipelineId(pipelineId); + try { + aflInfoService.saveAflInfo(req); + } catch (SftpUploadUtil.SftpUploadException e) { + log.error("保存afl信息失败,入参{}", JSON.toJSONString(req),e); + } + aflSeedInfoService.saveAflSeedInfo(req); + try { + aflPlotInfoService.saveAflPlotInfo(req); + } catch (SftpUploadUtil.SftpUploadException e) { + log.error("保存afl信息失败,入参{}", JSON.toJSONString(req),e); + } + } + private void afterPipelineExecute(TaskRunContext context){ + append(context,"开始拉取afl输出信息"); + PipTask contextDef = (PipTask)context.getContextDef(); + String pipelineId =contextDef.getPipelineId(); + String taskId = contextDef.getId(); + AflManagerReq req = new AflManagerReq(); + req.setTaskId(taskId); + req.setPipelineId(pipelineId); + postHandlerManager.registerPostHandler(new ExecuteTaskPostHandler(taskId,pipelineId) { + @Override + public void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance) { + Map localVariables = context.getLocalVariables(); + String groupId = localVariables.get(GROUP_ID) instanceof String ? ((String) localVariables.get(GROUP_ID)) : null; + req.setGroupIdentifier(groupId); + req.setPipelineHistoryId(pipPipelineHisInstance.getId()); + aflInfoService.updateHistoryPipelineIdByAflInfo(req); + aflSeedInfoService.updateHistoryPipelineIdByAflSeedInfo(req); + aflPlotInfoService.updateHistoryPipelineIdByAflPlotInfo(req); + } + }); + } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java index 7c7feea4..18486107 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java @@ -7,6 +7,7 @@ import cd.casic.ci.process.engine.constant.PipelineBehaviorConstant; import cd.casic.ci.process.engine.context.ConstantContext; import cd.casic.ci.process.engine.enums.ContextStateEnum; import cd.casic.ci.process.engine.manager.LoggerManager; +import cd.casic.ci.process.engine.manager.PostHandlerManager; import cd.casic.ci.process.engine.manager.RunContextManager; import cd.casic.ci.process.engine.runContext.BaseRunContext; @@ -50,7 +51,8 @@ public abstract class BaseWorker implements Runnable{ protected LoggerManager loggerManager; @Resource protected ResourceManagerService resourceManagerService; - + @Resource + protected PostHandlerManager postHandlerManager; @Override public void run() { diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/ShuiMuService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/ShuiMuService.java index 2685b6bb..469ce6fc 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/ShuiMuService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/ShuiMuService.java @@ -4,10 +4,7 @@ import cd.casic.ci.process.dto.req.shuimu.ShuiMuBuildAndSaveReq; import cd.casic.ci.process.dto.req.shuimu.ShuiMuProjectCreateReq; import cd.casic.ci.process.dto.req.shuimu.ShuiMuTestProblemReq; import cd.casic.ci.process.dto.req.shuimu.ShuiMuUpdateWFuzzJsonReq; -import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectInfoResp; -import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectResourceResp; -import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectVersionResp; -import cd.casic.ci.process.dto.resp.shuimu.ShuiMuTestProblemResp; +import cd.casic.ci.process.dto.resp.shuimu.*; import cd.casic.framework.commons.pojo.CommonResult; import org.springframework.web.bind.annotation.RequestBody; @@ -38,4 +35,7 @@ public interface ShuiMuService { ShuiMuProjectResourceResp getProjectResource(String projectId); List problemsOfTest(ShuiMuTestProblemReq req); + + ShuiMuProjectReportResp getProjectReport(String projectId); + } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/impl/ShuiMuServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/impl/ShuiMuServiceImpl.java index 93a9be22..75456b7f 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/impl/ShuiMuServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/impl/ShuiMuServiceImpl.java @@ -24,6 +24,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.core.io.FileSystemResource; import org.springframework.http.*; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestTemplate; @@ -296,6 +297,28 @@ public class ShuiMuServiceImpl implements ShuiMuService { String data = jsonObject.getString("data"); return JSONArray.parseArray(data, ShuiMuTestProblemResp.class); } + @Override + public ShuiMuProjectReportResp getProjectReport(String projectId) { + HttpEntity entity = new HttpEntity<>(null,getHeaders()); + String uriString = UriComponentsBuilder.fromUriString(shuiMuProperties.getBaseUrl() + getProjectReport ) + .queryParam("skip",0) + .queryParam("limit",1) + .queryParam("product","code") + .queryParam("projectId",projectId) + .queryParam("showMine",0) + .toUriString(); + ResponseEntity exchange = restTemplate.exchange(uriString, HttpMethod.GET, entity, String.class, new HashMap<>()); + if (exchange.getStatusCode().isError()) { + throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"保存并且构建失败"); + } + String data = JSON.parseObject(exchange.getBody()).getString("data"); + + List list = JSON.parseArray(data, ShuiMuProjectReportResp.class); + if (CollectionUtils.isEmpty(list)) { + return null; + } + return list.get(0); + } private HttpHeaders getHeaders(){ HttpHeaders httpHeaders = new HttpHeaders();