diff --git a/modules/module-ci-execute/src/main/java/cd/casic/module/execute/docker/service/impl/ImageService.java b/modules/module-ci-execute/src/main/java/cd/casic/module/execute/docker/service/impl/ImageService.java index 7bebb873..a014ae2d 100644 --- a/modules/module-ci-execute/src/main/java/cd/casic/module/execute/docker/service/impl/ImageService.java +++ b/modules/module-ci-execute/src/main/java/cd/casic/module/execute/docker/service/impl/ImageService.java @@ -2,6 +2,7 @@ package cd.casic.module.execute.docker.service.impl; import cd.casic.ci.commons.properties.TargetFileUploadProperties; import cd.casic.framework.commons.exception.ServerException; +import cd.casic.framework.commons.exception.util.ServiceExceptionUtil; import cd.casic.framework.commons.pojo.PageResult; import cd.casic.framework.mybatis.core.query.LambdaQueryWrapperX; import cd.casic.module.execute.docker.DockerClientFactory; @@ -179,7 +180,7 @@ public class ImageService implements IImageService { } /** - * 先下载后上传到目标主机 + * 先下载后上传到目标主机 先确定目标主机是否可连接在继续 * @param imageId * @param machineId * @return @@ -189,15 +190,7 @@ public class ImageService implements IImageService { if (Objects.isNull(imageId) || Objects.isNull(machineId)) { throw new ServerException(MachineErrorCodeConstants.MACHINE_INFO_TAG_NULL); } - DockerImageDo imageDo = dockerImageDao.selectById(imageId); - //1 远程下载 - // 1.1 建立连接 下载 - Sftp sftp = getSftp(FtpConfig.create().setHost(fileUploadProperties.getRemoteHost()).setPort(fileUploadProperties.getRemotePort()).setUser(fileUploadProperties.getUsername()).setPassword(fileUploadProperties.getPassword())); - String srcPath = imageDo.getPath(); - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - sftp.download(srcPath, byteOut); //远程下载到中间缓存区 - sftp.close(); - // 2 建立连接 区分密码还是秘钥 + // 1 建立连接 区分密码还是秘钥 MachineInfoDO machineInfoDO = machineInfoMapper.selectById(machineId); FtpConfig targetFtp = FtpConfig.create().setHost(machineInfoDO.getHostIp()).setPort(machineInfoDO.getSshPort()).setUser(machineInfoDO.getUsername()).setPassword(CryptogramUtil.doDecrypt(machineInfoDO.getPassword())); if (AuthenticationType.of(machineInfoDO.getAuthenticationType()).equals(AuthenticationType.SECRET_KEY)) { @@ -205,8 +198,17 @@ public class ImageService implements IImageService { targetFtp.setSystemKey(CryptogramUtil.doDecrypt(secretKeyDO.getPrivateKey())); targetFtp.setPassword(CryptogramUtil.doDecrypt(secretKeyDO.getPassword())); } - // 2.1 上传 + // 2 拿到目标主机连接 Sftp uploadSftp = getSftp(targetFtp); + + //远程下载 + DockerImageDo imageDo = dockerImageDao.selectById(imageId); + Sftp sftp = getSftp(FtpConfig.create().setHost(fileUploadProperties.getRemoteHost()).setPort(fileUploadProperties.getRemotePort()).setUser(fileUploadProperties.getUsername()).setPassword(fileUploadProperties.getPassword())); + String srcPath = imageDo.getPath(); + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + sftp.download(srcPath, byteOut); //远程下载到中间缓存区 + sftp.close(); + //远程上传 String suffix = FileUtil.getSuffix(srcPath); String fileName = imageDo.getName() + "." + suffix; ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(byteOut.toByteArray()); //转换 @@ -225,14 +227,16 @@ public class ImageService implements IImageService { */ private Sftp getSftp(FtpConfig config) { Session session; - if (Objects.nonNull(config.getSystemKey())) { - session = JschUtil.getSession(config.getHost(), config.getPort(), config.getUser(), config.getSystemKey().getBytes(StandardCharsets.UTF_8), config.getPassword().getBytes()); - } else { - session = JschUtil.getSession(config.getHost(), config.getPort(), config.getUser(), config.getPassword()); - } - if (!session.isConnected()) { + Integer timeOut = 5000; + try { + if (Objects.nonNull(config.getSystemKey())) { + session = JschUtil.openSession(config.getHost(), config.getPort(), config.getUser(), config.getSystemKey().getBytes(StandardCharsets.UTF_8), config.getPassword().getBytes(), timeOut); + } else { + session = JschUtil.openSession(config.getHost(), config.getPort(), config.getUser(), config.getPassword(), timeOut); + } + } catch (Exception e) { log.error("与主机IP:{} 建立SSH连接失败", config.getHost()); - throw new ServerException(500, "部署失败，请检测主机是否可用"); + throw ServiceExceptionUtil.invalidParamException("部署失败，请检测主机: {} 是否可用",config.getHost()); } return new Sftp(session); } diff --git a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/DataAnalysisController.java b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/DataAnalysisController.java new file mode 100644 index 00000000..98654a70 --- /dev/null +++ b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/DataAnalysisController.java @@ -0,0 +1,53 @@ +package cd.casic.ci.api; + +import cd.casic.ci.process.process.service.dataAnalysis.DataAnalysisService; +import cd.casic.framework.commons.pojo.CommonResult; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.Resource; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +/** + * 数据分析 NUDT数据安全态势 + * @author: Paul + * @create: 2025-07-29 13:52 + */ +@Tag(name = "NUDT数据安全态势") +@RestController +@RequestMapping("/dataAnalysis") +public class DataAnalysisController { + + @Resource + private DataAnalysisService dataAnalysisService; + + @Operation(summary = "人数，任务次数，漏洞总数，高危漏洞 统计") + @GetMapping("/userAndTaskCount") + public CommonResult userAndTaskDataCount(@RequestParam(required = false,value = "country") String country){ + return CommonResult.success(dataAnalysisService.getUserAndTaskDataCount()); + } + + @Operation(summary = "漏洞监测分布概览") + @GetMapping("/HoopVulCount") + public CommonResult HoopVulDataCount(@RequestParam(required = false,value = "country") String country){ + return CommonResult.success(dataAnalysisService.getHoopVulCountList()); + } + + @Operation(summary = "资源分配数据概览") + @GetMapping("/ResourceDistList") + public CommonResult ResourceDistList(@RequestParam(required = false,value = "country") String country){ + return CommonResult.success(dataAnalysisService.getResourceDistCountList()); + } + + @Operation(summary = "漏洞监测数据概览") + @GetMapping("/VulMonitorDataList") + public CommonResult VulMonitorDataList(@RequestParam(required = false,value = "country") String country){ + + return CommonResult.success(); + } + + + +} diff --git a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/SastController.java b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/SastController.java index 4990237a..804cd5fe 100644 --- a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/SastController.java +++ b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/SastController.java @@ -1,9 +1,6 @@ package cd.casic.ci.api; -import cd.casic.ci.process.dto.req.sast.SastApplicationCreateReq; -import cd.casic.ci.process.dto.req.sast.SastEngineConfigReq; -import cd.casic.ci.process.dto.req.sast.SastIdentifiedLanguageTaskReq; -import cd.casic.ci.process.dto.req.sast.SastReportCreateReq; +import cd.casic.ci.process.dto.req.sast.*; import cd.casic.ci.process.dto.resp.sast.*; import cd.casic.ci.process.process.service.sast.SastService; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; @@ -97,8 +94,8 @@ public class SastController { * sast上传 应用创建或者更新 * */ @PostMapping("/applicationCreateOrUpdate") - public CommonResult applicationCreateOrUpdate(@RequestBody SastApplicationCreateReq req){ - SastApplicationCreateResp sastApplicationCreateResp = sastService.applicationCreate(req); + public CommonResult applicationCreateOrUpdate(@RequestBody SastApplicationStashReq req){ + SastApplicationCreateResp sastApplicationCreateResp = sastService.applicationStash(req); return CommonResult.success(sastApplicationCreateResp); } /** diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java index 3f14a5be..48d40ffb 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/SastUrlConstant.java @@ -10,6 +10,8 @@ public class SastUrlConstant { public static final String engineConfig = "/invoke/engine/config"; public static final String detectionConfig = "/invoke/project/setting/getDetectionConfig"; public static final String applicationCreate = "/invoke/application/create"; + public static final String applicationStash = "/invoke/application/stash"; + public static final String applicationBinaryStash = "/invoke/application/binary/stash"; public static final String getApplicationEcho = "/invoke/application/getApplicationEcho"; public static final String reportIndex = "/api/report/index"; public static final String reportDownload = "/api/report/index/download"; @@ -19,4 +21,7 @@ public class SastUrlConstant { public static final String getApplicationStatus = "/invoke/application/getStatus/"; public static final String jsonReportDownload = "/api/report/index/downloadReport"; public static final String getReportInfo = "/api/report/index/getReportInfo"; + public static final String stashScan = "/invoke/application/stash/scan/"; + public static final String binaryStashScan = "/invoke/application/binary/stash/scan/"; + public static final String engineLog = "/invoke/engine/%s/log"; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationBinaryStashReq.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationBinaryStashReq.java new file mode 100644 index 00000000..9c360b81 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationBinaryStashReq.java @@ -0,0 +1,58 @@ +package cd.casic.ci.process.dto.req.sast; + +import lombok.Data; + +import java.util.List; + +@Data +public class SastApplicationBinaryStashReq { + private String applicationId;//不传 + /** + * 当前项目id 写死"893ed995-5b81-474a-96a9-2800281421cd" + * */ + private String projectId;//不传 + /** + * 应用名称，不可重复 + * */ + private String applicationName;//可以不传 + /** + * 写死 BINARY + * */ + private String codeSourceFrom;//不传 + /** + * 目标指令集 true-自动识别 false-手动识别 + * */ + private Boolean archAutoIdentify;// TODO 询问是否需要写死自动识别 + /** + * + * */ + private String targetArchitectures;// TODO 询问作用 + /** + * 上传文件返回的id + * */ + private List fileId;// 上传目标文件，省去页面上传 + /** + * 模板id + * */ + private String templateId;//需要用户选择 + /** + * 分析超时时长 + * */ + private Integer analysisTimeoutMs;//需要用户填写 + /** + * k集合大小 + * */ + private Integer kSetSize;//需要用户填写 + /** + * CallString长度 + * */ + private Integer callStringLength;//需要用户填写 + /** + * 入口地址entryAddress + * */ + private String entryAddress;//需要用户填写 + /** + * z3 是否能够超时 + * */ + private Boolean z3TimeoutMsStatus;//需要用户填写 +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationStashReq.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationStashReq.java new file mode 100644 index 00000000..b99dec5d --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationStashReq.java @@ -0,0 +1,64 @@ +package cd.casic.ci.process.dto.req.sast; + +import lombok.Data; + +import java.util.List; + +@Data +public class SastApplicationStashReq { + /** + * 项目Id + * */ + private String projectId; + /** + * 应用名称 + * */ + private String applicationName; + /** + * 源代码ONLINE 或者字节码COMPRESS_FILE + * */ + private String codeSourceFrom; + /** + * GITLAB，GITEE，GITHUB，LOCAL (本地) + * */ + private String codeRepoType; + /** + * git连接状态 目前不用git 固定传false + * */ + private Boolean connectionStatus; + /** + * 语言和模板配置，是否开启编译模式（目前不开启传false） + * */ + private List config; + /** + * 传接口获取到的引擎id + * */ + private List openEngine; + /** + * 传上传文件接口返回的文件id + * */ + private List fileId; + /** + * 编辑时使用 applicationId + * */ + private String applicationId; + /** + * 是否开启缺陷自动研判 + * */ + private Boolean autoAudit; + /** + * 函数过滤内容 + * */ + private String functionFilter; + + @Data + public static class SastApplicationConfig{ + private String languageId; + private String templateId; + private String version; + private String compileTool; + private String compileCommand; + private boolean compile; + private boolean isCompile; + } +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/ScaVulInfoResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/ScaVulInfoResp.java new file mode 100644 index 00000000..d068aa54 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/ScaVulInfoResp.java @@ -0,0 +1,148 @@ +package cd.casic.ci.process.dto.resp; + +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +@Data +public class ScaVulInfoResp { + /** + * 漏洞UID + */ + private String vulUid; + + /** + * CVE编号 + */ + private String cve; + + /** + * CNNVD编号 + */ + private String cnnvd; + + /** + * CNVD编号 + */ + private String cnvd; + + /** + * CWE编号 + */ + private String cwe; + + /** + * 漏洞名称 + */ + private String vulName; + + /** + * 漏洞类型 + */ + private String vulType; + + /** + * 攻击类型 + */ + private String attackType; + + /** + * 是否可利用 + */ + private String isExploit; + + /** + * 安全等级 + */ + private String securityLevel; + + /** + * CVSS4评分 + */ + private Double cvss4; + + /** + * 发布时间 + */ + @JsonFormat(pattern = "yyyy-MM-dd") + private LocalDate releaseDate; + + /** + * 更新时间 + */ + @JsonFormat(pattern = "yyyy-MM-dd") + private LocalDate updateDate; + + /** + * 漏洞描述 + */ + private String vulDescription; + + /** + * 修复建议 + */ + private String suggestion; + + /** + * 检测时间 + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private LocalDateTime detectionTime; + + /** + * 影响组件数量 + */ + private Integer influenceComponentNum; + + /** + * 影响包数量 + */ + private Integer influencePackageNum; + + /** + * 影响组件 + */ + private String influenceComponent; + + /** + * 影响组件镜像 + */ + private String influenceComponentImage; + + /** + * 漏洞状态 + */ + private String vulStatus; + + /** + * 审计状态 + */ + private String auditStatus; + + /** + * 可达类型字符串 + */ + private String reachTypeStr; + + /** + * 是否可修复 + */ + private String fixAvailable; + + /** + * 是否隐藏 + */ + private Integer isHide; + + /** + * CVSS2评分 + */ + private String cvss2; + + /** + * CVSS3评分 + */ + private String cvss3; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/dataAnalysis/DataAnalysisResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/dataAnalysis/DataAnalysisResp.java new file mode 100644 index 00000000..edbc0831 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/dataAnalysis/DataAnalysisResp.java @@ -0,0 +1,115 @@ +package cd.casic.ci.process.dto.resp.dataAnalysis; + +import cd.casic.ci.process.enums.MachineSystemEnum; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Builder; +import lombok.Data; +import java.math.BigDecimal; +import java.util.List; +import java.util.Random; + +/** + * NUDT数据安全态势 + * @author: Paul + * @create: 2025-07-29 15:16 + */ +@Data +@Builder +public class DataAnalysisResp { + + + /** + * 漏洞扫描 + */ + private String scanVulSUM; + /** + * 使用人数 + */ + private String useUserSUM; + /** + * 执行任务 + */ + private String taskSUM; + /** + * 漏洞总数 + */ + private String vulSUM; + /** + * 高危漏洞 + */ + private String perilousVulSUM; + + + private List hoopVulCountList; + + + private List resourceDistCountList; + + + /** + * 漏洞监测分布概览 + */ + @Data + public static class HoopVulCount{ + /** + * 城市 + */ + private String country; + /** + * 百分比 两位小数 + */ + private BigDecimal value; + } + + /** + * 资源分配数据概览 + */ + @Data + public static class ResourceDistCount{ + private static final List CITIES = List.of( + "北京", "上海", "广州", "深圳", "杭州", + "成都", "武汉", "西安", "长沙", "厦门" + ); + /** + * 机器id + */ + private Long id; + + private String name; + + /** + * 机器唯一标识 + */ + private String tag; + + private String hostIp; + + private String description; + + //1：Linux 2：Windows + private Integer machineInfoType; + + private String type; + + private String country; + + + private Integer sshPort; + + public void setMachineInfoType(Integer machineInfoType) { + this.country = CITIES.get(new Random().nextInt(CITIES.size())); + this.machineInfoType = machineInfoType; + this.type = machineInfoType.equals(MachineSystemEnum.WINDOWS.getSystem())?MachineSystemEnum.WINDOWS.name():MachineSystemEnum.LINUX.name(); + } + } + + /** + * + */ + /*@Data + static class VulMonitorCount{ + + }*/ + +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationBinaryStashResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationBinaryStashResp.java new file mode 100644 index 00000000..18ad4517 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationBinaryStashResp.java @@ -0,0 +1,19 @@ +package cd.casic.ci.process.dto.resp.sast; + +import lombok.Data; + +@Data +public class SastApplicationBinaryStashResp { + private String applicationId; + private String applicationName; + private String taskId; + private String fileId; + private String jobAddBy; + private String jobStatus; + private String checkStartTime; + private String checkEndTime; + private String codeQualityMetric; // 这里是JSON字符串 + private String engineType; + private String progress; // 允许为null + private String type; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationStashResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationStashResp.java new file mode 100644 index 00000000..f3e69be7 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastApplicationStashResp.java @@ -0,0 +1,19 @@ +package cd.casic.ci.process.dto.resp.sast; + +import lombok.Data; + +@Data +public class SastApplicationStashResp { + private String applicationId; + private String applicationName; + private String taskId; + private String fileId; + private String jobAddBy; + private String jobStatus; + private String checkStartTime; + private String checkEndTime; + private String codeQualityMetric; + private String engineType; + private String progress; + private String type; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastEngineLogResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastEngineLogResp.java new file mode 100644 index 00000000..fcc8e4af --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/sast/SastEngineLogResp.java @@ -0,0 +1,18 @@ +package cd.casic.ci.process.dto.resp.sast; + +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +public class SastEngineLogResp { + private String taskId; + private LocalDateTime createTime; + private String serverFrom; + private String ipFrom; + private String containerId; + private String stageInfo; + private String message; + private String infoLevel; + private String prettyPrint; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/config/ExecutorConfig.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/config/ExecutorConfig.java index 05adf685..bdd8f573 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/config/ExecutorConfig.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/config/ExecutorConfig.java @@ -46,4 +46,16 @@ public class ExecutorConfig { executor.initialize(); // 必须手动触发初始化 return executor; } + @Bean("postHandlerExecutor") + public ThreadPoolTaskExecutor postHandlerExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(10); + executor.setMaxPoolSize(10); + executor.setQueueCapacity(100); + executor.setThreadNamePrefix("postHandler-"); + ThreadPoolExecutor.CallerRunsPolicy callerRunsPolicy = new ThreadPoolExecutor.CallerRunsPolicy(); + executor.setRejectedExecutionHandler(callerRunsPolicy); + executor.initialize(); // 必须手动触发初始化 + return executor; + } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java index 0a8f34ca..49dc42c6 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryPostHandlerManager.java @@ -5,7 +5,10 @@ import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; import cd.casic.framework.commons.exception.ServiceException; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -14,8 +17,11 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @Component +@Slf4j public class MemoryPostHandlerManager implements PostHandlerManager { private final ConcurrentHashMap> handlerMap = new ConcurrentHashMap<>(); + @Resource + private ThreadPoolTaskExecutor postHandlerExecutor; @Override public void registerPostHandler(ExecuteTaskPostHandler handler) { if (handler==null|| StringUtils.isEmpty(handler.getPipelineId())) { @@ -34,7 +40,13 @@ public class MemoryPostHandlerManager implements PostHandlerManager { return; } for (ExecuteTaskPostHandler postHandler : pipExecutePostHandlers) { - postHandler.executeAfterDone(hisInstance); + postHandlerExecutor.submit(()->{ + try { + postHandler.executeAfterDone(hisInstance); + } catch (Exception e) { + log.error("后置执行失败",e); + } + }); } } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/postHandler/ExecuteTaskPostHandler.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/postHandler/ExecuteTaskPostHandler.java index 007cbd32..6932dc50 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/postHandler/ExecuteTaskPostHandler.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/postHandler/ExecuteTaskPostHandler.java @@ -7,8 +7,8 @@ import lombok.Data; @Data @AllArgsConstructor public abstract class ExecuteTaskPostHandler { - private String taskId; - private String pipelineId; + protected String taskId; + protected String pipelineId; /** * 流水线执行结束需要执行的操作 diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java index 648e9f08..ca9c55cc 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java @@ -153,6 +153,8 @@ public class TestCaseGenerationWorker extends DockerWorker { Properties config = new Properties(); config.put("StrictHostKeyChecking", "no"); session.setConfig(config); + + session.connect(); sftp = (ChannelSftp) session.openChannel("sftp"); sftp.connect(); @@ -167,6 +169,7 @@ public class TestCaseGenerationWorker extends DockerWorker { String binaryId = JSON.parseObject(targetUploadRes).getString("id"); append(context,"开始创建测试用例生成task"); sftp.disconnect(); + // 创建task TestCaseAITaskCreateReq req = new TestCaseAITaskCreateReq(); req.setBinaryId(binaryId); @@ -186,7 +189,9 @@ public class TestCaseGenerationWorker extends DockerWorker { } else { String logRes = aiGeneratorService.getTaskLog(taskId); String logStr = JSON.parseObject(logRes).getString("log"); - append(context,logStr); + if (StringUtils.isNotEmpty(logStr)) { + append(context,logStr); + } } // append(context,"task当前执行状态"+taskStatusRes); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastBinaryWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastBinaryWorker.java new file mode 100644 index 00000000..7cd0d328 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastBinaryWorker.java @@ -0,0 +1,105 @@ +package cd.casic.ci.process.engine.worker.sast; + +import cd.casic.ci.process.common.WorkAtom; +import cd.casic.ci.process.dto.resp.sast.SastApplicationEchoResp; +import cd.casic.ci.process.dto.resp.sast.SastApplicationStashResp; +import cd.casic.ci.process.dto.resp.sast.SastApplicationStatusResp; +import cd.casic.ci.process.dto.resp.sast.SastEngineLogResp; +import cd.casic.ci.process.engine.constant.SastConstant; +import cd.casic.ci.process.engine.runContext.TaskRunContext; +import cd.casic.ci.process.engine.worker.base.BaseWorker; +import cd.casic.ci.process.process.converter.SastConverter; +import cd.casic.ci.process.process.dataObject.base.PipBaseElement; +import cd.casic.ci.process.process.dataObject.task.PipTask; +import cd.casic.ci.process.process.service.sast.SastService; +import cd.casic.ci.process.process.service.task.TaskService; +import com.alibaba.excel.util.StringUtils; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +@WorkAtom(taskType = "SAST_BINARY") +public class SastBinaryWorker extends BaseWorker { + @Resource + private SastService sastService; + @Resource + private SastConverter converter; + @Resource + private TaskService taskService; + @Override + public void execute(TaskRunContext context) { + PipBaseElement contextDef = context.getContextDef(); + String id = contextDef.getId(); + log.info("==============触发worker执行========"); + log.info("==========运行context：{}===========", JSON.toJSONString(context)); + if (contextDef instanceof PipTask task) { + String applicationId = (String)(task.getTaskProperties().get(SastConstant.APPLICATION_ID)); + if (StringUtils.isEmpty(applicationId)) { + append(context,"未获取到sast信息"); + toBadEnding(); + } + SastApplicationStatusResp applicationStatus = sastService.getApplicationStatus(applicationId); + SastApplicationEchoResp applicationEcho = sastService.getApplicationEcho(applicationId); + append(context,"获取到sast任务配置"+JSON.toJSONString(applicationEcho)); + if (SastConstant.STATUS_PROGRESSING.equals(applicationStatus.getJobStatus())||SastConstant.STATUS_PENDING.equals(applicationStatus.getJobStatus())) { + append(context,"当前任务正在运行中"); +// toBadEnding(); + } else{ +// SastApplicationCreateReq sastApplicationCreateReq = converter.converter(applicationEcho); +// List list = applicationEcho.getLodeFiles().stream().map(SastApplicationEchoResp.LodeFile::getFileId).toList(); +// sastApplicationCreateReq.setFileId(list); +// append(context,"开始启动SAST任务,检测文件信息：{}"+JSON.toJSONString(applicationEcho.getLodeFiles())); +// log.info("SAST启动任务入参:{}",JSON.toJSONString(sastApplicationCreateReq)); + SastApplicationStashResp sastApplicationCreateResp = sastService.binaryStashScan(applicationId); + log.info("SAST启动二进制任务返回值:{}",JSON.toJSONString(sastApplicationCreateResp)); + append(context,"启动任务完毕"); + } +// 检测状态是否完毕 + applicationStatus = sastService.getApplicationStatus(applicationId); + List oldLogList = new ArrayList<>(); + int repeat = 0; + while (!SastConstant.STATUS_JOB_DONE.equals(applicationStatus.getJobStatus())) { + try { + Thread.sleep(20000L); + } catch (InterruptedException e) { + log.error("二进制任务执行失败，R线程中断"); + } + try { + applicationStatus=sastService.getApplicationStatus(applicationId); + } catch (Exception e) { + log.error("二进制任务执行失败",e); + append(context,"二进制任务获取状态失败"); + if (repeat++>3) { + append(context,"获取状态失败,尝试超过三次执行失败"); + toBadEnding(); + } + continue; + } + append(context,"当前SAST二进制任务运行中，运行状态："+applicationStatus.getJobStatus()); + if (SastConstant.STATUS_CANCEL.equals(applicationStatus.getJobStatus())) { + append(context,"任务被取消"); + toBadEnding(); + } + List logList = sastService.engineLog(applicationId); + if (logList.size()>oldLogList.size()) { + for (int i = oldLogList.size(); i < logList.size(); i++) { + SastEngineLogResp logResp = logList.get(i); + append(context,logResp.getPrettyPrint()); + } + oldLogList = logList; + } + repeat = 0; + } +// JSONObject reportJSON = getReportJSON(applicationId, SastConstant.REPORT_MODE_DETAILS,context); +// String reportUrl = getReportUrl(applicationId, SastConstant.REPORT_MODE_DETAILS); +// task.getTaskProperties().put(SastConstant.REPORT_JSON,reportJSON); +// task.getTaskProperties().put(SastConstant.REPORT_URL,reportUrl); + taskService.updateById(task); + } + } +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastWorker.java index 0d6bbf83..840ccfc4 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastWorker.java @@ -4,10 +4,7 @@ package cd.casic.ci.process.engine.worker.sast; import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.dto.req.sast.SastApplicationCreateReq; import cd.casic.ci.process.dto.req.sast.SastReportCreateReq; -import cd.casic.ci.process.dto.resp.sast.SastApplicationCreateResp; -import cd.casic.ci.process.dto.resp.sast.SastApplicationEchoResp; -import cd.casic.ci.process.dto.resp.sast.SastApplicationStatusResp; -import cd.casic.ci.process.dto.resp.sast.SastReportStatusResp; +import cd.casic.ci.process.dto.resp.sast.*; import cd.casic.ci.process.engine.constant.SastConstant; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.BaseWorker; @@ -24,6 +21,7 @@ import com.alibaba.fastjson.JSONObject; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -58,17 +56,18 @@ public class SastWorker extends BaseWorker { append(context,"当前任务正在运行中"); // toBadEnding(); } else{ - SastApplicationCreateReq sastApplicationCreateReq = converter.converter(applicationEcho); - List list = applicationEcho.getLodeFiles().stream().map(SastApplicationEchoResp.LodeFile::getFileId).toList(); - sastApplicationCreateReq.setFileId(list); - append(context,"开始启动SAST任务,检测文件信息：{}"+JSON.toJSONString(applicationEcho.getLodeFiles())); - log.info("SAST启动任务入参:{}",JSON.toJSONString(sastApplicationCreateReq)); - SastApplicationCreateResp sastApplicationCreateResp = sastService.applicationCreate(sastApplicationCreateReq); +// SastApplicationCreateReq sastApplicationCreateReq = converter.converter(applicationEcho); +// List list = applicationEcho.getLodeFiles().stream().map(SastApplicationEchoResp.LodeFile::getFileId).toList(); +// sastApplicationCreateReq.setFileId(list); +// append(context,"开始启动SAST任务,检测文件信息：{}"+JSON.toJSONString(applicationEcho.getLodeFiles())); +// log.info("SAST启动任务入参:{}",JSON.toJSONString(sastApplicationCreateReq)); + SastApplicationStashResp sastApplicationCreateResp = sastService.stashScan(applicationId); log.info("SAST启动任务返回值:{}",JSON.toJSONString(sastApplicationCreateResp)); append(context,"启动任务完毕"); } // 检测状态是否完毕 applicationStatus = sastService.getApplicationStatus(applicationId); + List oldLogList = new ArrayList<>(); int repeat = 0; while (!SastConstant.STATUS_JOB_DONE.equals(applicationStatus.getJobStatus())) { try { @@ -92,6 +91,14 @@ public class SastWorker extends BaseWorker { append(context,"任务被取消"); toBadEnding(); } + List logList = sastService.engineLog(applicationId); + if (logList.size()>oldLogList.size()) { + for (int i = oldLogList.size(); i < logList.size(); i++) { + SastEngineLogResp logResp = logList.get(i); + append(context,logResp.getPrettyPrint()); + } + oldLogList = logList; + } repeat = 0; } JSONObject reportJSON = getReportJSON(applicationId, SastConstant.REPORT_MODE_DETAILS,context); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ApplicationWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ApplicationWorker.java index 81e25cdc..6988f199 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ApplicationWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ApplicationWorker.java @@ -5,18 +5,21 @@ import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.dto.req.report.ReportAssetTaskReq; import cd.casic.ci.process.dto.resp.report.ReportResp; import cd.casic.ci.process.engine.context.ConstantContextHolder; +import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; import cd.casic.ci.process.engine.runContext.BaseRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.HttpWorker; import cd.casic.ci.process.process.dao.pipeline.PipTaskDao; import cd.casic.ci.process.process.dataObject.base.BaseIdReq; import cd.casic.ci.process.process.dataObject.base.PipBaseElement; +import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.service.pipeline.PipelineService; import cd.casic.ci.process.process.service.report.impl.ReportServiceImpl; import cd.casic.ci.process.process.service.target.impl.TargetVersionServiceImpl; +import cd.casic.ci.process.process.service.vulInfo.VulInfoService; import cd.casic.framework.commons.exception.ServiceException; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; import cn.hutool.core.util.ObjectUtil; @@ -38,6 +41,8 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.util.*; +import static cd.casic.ci.process.engine.constant.AFLConstant.GROUP_ID; + /** * @author HopeLi * @version v1.0 @@ -62,6 +67,8 @@ public class ApplicationWorker extends HttpWorker { @Resource private ReportServiceImpl reportService; + @Resource + private VulInfoService vulInfoService; @Override @@ -112,6 +119,14 @@ public class ApplicationWorker extends HttpWorker { log.error("==================SCA-应用包审查分析节点执行失败=================",e); throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"SCA-应用包审查分析节点执行失败"); } + Integer scaTaskId = pipTask.getTaskProperties().get("scaTaskId") instanceof Integer ? ((Integer) pipTask.getTaskProperties().get("scaTaskId")) : null; + String fileName = targetVersion.getFileName(); + postHandlerManager.registerPostHandler(new ExecuteTaskPostHandler(pipTask.getId(),pipelineId) { + @Override + public void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance) { + vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"APPLICATION"); + } + }); } localVariables.put("statusCode", statusCode + ""); } @@ -291,5 +306,4 @@ public class ApplicationWorker extends HttpWorker { } return 0; } - } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java index b3c79a12..74fcf623 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaBinaryWorker.java @@ -5,18 +5,21 @@ import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.dto.req.report.ReportAssetTaskReq; import cd.casic.ci.process.dto.resp.report.ReportResp; import cd.casic.ci.process.engine.context.ConstantContextHolder; +import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; import cd.casic.ci.process.engine.runContext.BaseRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.HttpWorker; import cd.casic.ci.process.process.dao.pipeline.PipTaskDao; import cd.casic.ci.process.process.dataObject.base.BaseIdReq; import cd.casic.ci.process.process.dataObject.base.PipBaseElement; +import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.service.pipeline.PipelineService; import cd.casic.ci.process.process.service.report.impl.ReportServiceImpl; import cd.casic.ci.process.process.service.target.impl.TargetVersionServiceImpl; +import cd.casic.ci.process.process.service.vulInfo.VulInfoService; import cd.casic.framework.commons.exception.ServiceException; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; import cn.hutool.core.util.ObjectUtil; @@ -63,7 +66,8 @@ public class ScaBinaryWorker extends HttpWorker { @Resource private ReportServiceImpl reportService; - + @Resource + private VulInfoService vulInfoService; @Override public void execute(TaskRunContext context) { int statusCode = 0; @@ -117,6 +121,14 @@ public class ScaBinaryWorker extends HttpWorker { log.error("==================SCA-BINARY节点执行失败=================",e); throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"SCA-SBOM节点执行失败"); } + Integer scaTaskId = pipTask.getTaskProperties().get("scaTaskId") instanceof Integer ? ((Integer) pipTask.getTaskProperties().get("scaTaskId")) : null; + String fileName = targetVersion.getFileName(); + postHandlerManager.registerPostHandler(new ExecuteTaskPostHandler(pipTask.getId(),pipelineId) { + @Override + public void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance) { + vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"BINARY"); + } + }); } localVariables.put("statusCode", statusCode + ""); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java index 271825f8..601a770c 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaMirrorWorker.java @@ -5,18 +5,21 @@ import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.dto.req.report.ReportAssetTaskReq; import cd.casic.ci.process.dto.resp.report.ReportResp; import cd.casic.ci.process.engine.context.ConstantContextHolder; +import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; import cd.casic.ci.process.engine.runContext.BaseRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.HttpWorker; import cd.casic.ci.process.process.dao.pipeline.PipTaskDao; import cd.casic.ci.process.process.dataObject.base.BaseIdReq; import cd.casic.ci.process.process.dataObject.base.PipBaseElement; +import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.service.pipeline.PipelineService; import cd.casic.ci.process.process.service.report.impl.ReportServiceImpl; import cd.casic.ci.process.process.service.target.impl.TargetVersionServiceImpl; +import cd.casic.ci.process.process.service.vulInfo.VulInfoService; import cd.casic.framework.commons.exception.ServiceException; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; import cn.hutool.core.util.ObjectUtil; @@ -60,7 +63,8 @@ public class ScaMirrorWorker extends HttpWorker { @Resource private PipTaskDao pipTaskDao; - + @Resource + private VulInfoService vulInfoService; @Resource private ReportServiceImpl reportService; @@ -115,6 +119,14 @@ public class ScaMirrorWorker extends HttpWorker { log.error("==================SCA-MIRROR节点执行失败=================",e); throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"SCA-MIRROR节点执行失败"); } + Integer scaTaskId = pipTask.getTaskProperties().get("scaTaskId") instanceof Integer ? ((Integer) pipTask.getTaskProperties().get("scaTaskId")) : null; + String fileName = targetVersion.getFileName(); + postHandlerManager.registerPostHandler(new ExecuteTaskPostHandler(pipTask.getId(),pipelineId) { + @Override + public void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance) { + vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"MIRROR"); + } + }); } localVariables.put("statusCode", statusCode + ""); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java index 199dd842..7f1d412a 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sca/ScaSbomWorker.java @@ -5,18 +5,21 @@ import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.dto.req.report.ReportAssetTaskReq; import cd.casic.ci.process.dto.resp.report.ReportResp; import cd.casic.ci.process.engine.context.ConstantContextHolder; +import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; import cd.casic.ci.process.engine.runContext.BaseRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.HttpWorker; import cd.casic.ci.process.process.dao.pipeline.PipTaskDao; import cd.casic.ci.process.process.dataObject.base.BaseIdReq; import cd.casic.ci.process.process.dataObject.base.PipBaseElement; +import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance; import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.service.pipeline.PipelineService; import cd.casic.ci.process.process.service.report.impl.ReportServiceImpl; import cd.casic.ci.process.process.service.target.impl.TargetVersionServiceImpl; +import cd.casic.ci.process.process.service.vulInfo.VulInfoService; import cd.casic.framework.commons.exception.ServiceException; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; import cn.hutool.core.util.ObjectUtil; @@ -64,7 +67,8 @@ public class ScaSbomWorker extends HttpWorker { @Resource private ReportServiceImpl reportService; - + @Resource + private VulInfoService vulInfoService; @Override public void execute(TaskRunContext context) { int statusCode = 0; @@ -113,6 +117,14 @@ public class ScaSbomWorker extends HttpWorker { log.error("==================SCA-SBOM节点执行失败=================",e); throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"SCA-SBOM节点执行失败"); } + Integer scaTaskId = pipTask.getTaskProperties().get("scaTaskId") instanceof Integer ? ((Integer) pipTask.getTaskProperties().get("scaTaskId")) : null; + String fileName = targetVersion.getFileName(); + postHandlerManager.registerPostHandler(new ExecuteTaskPostHandler(pipTask.getId(),pipelineId) { + @Override + public void executeAfterDone(PipPipelineHisInstance pipPipelineHisInstance) { + vulInfoService.scaToVulInfoSave(scaTaskId,pipeline.getTargetType(),fileName,"长沙",pipPipelineHisInstance.getId(),taskId,"SCA_S_BOM"); + } + }); } localVariables.put("statusCode", statusCode + ""); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/converter/VulInfoConverter.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/converter/VulInfoConverter.java new file mode 100644 index 00000000..471b24ea --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/converter/VulInfoConverter.java @@ -0,0 +1,200 @@ +package cd.casic.ci.process.process.converter; + +import cd.casic.ci.process.dto.resp.ScaVulInfoResp; +import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.Mappings; +import org.mapstruct.Named; +import org.mapstruct.factory.Mappers; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +@Mapper(componentModel = "spring") +public interface VulInfoConverter { + + VulInfoConverter INSTANCE = Mappers.getMapper(VulInfoConverter.class); + + /** + * JSON数据转换为数据库实体 + */ + @Mappings({ + @Mapping(target = "id", ignore = true), + @Mapping(target = "createTime", source = "releaseDate", qualifiedByName = "dateToDateTime"), + @Mapping(target = "modifyTime", source = "updateDate", qualifiedByName = "dateToDateTime"), + @Mapping(target = "verifyStatus", source = "auditStatus"), + @Mapping(target = "vulId", source = "vulUid"), + @Mapping(target = "isOpenSource", ignore = true), + @Mapping(target = "vulTitle", source = "vulName"), + @Mapping(target = "vulDescription", source = "vulDescription"), + @Mapping(target = "severity", source = "securityLevel", qualifiedByName = "securityLevelToSeverity"), + @Mapping(target = "solution", source = "suggestion"), + @Mapping(target = "relieve", ignore = true), + @Mapping(target = "targetName", source = "influenceComponent"), + @Mapping(target = "cwes", source = "cwe", qualifiedByName = "cweToCwes"), + @Mapping(target = "level", source = "isExploit", qualifiedByName = "exploitToLevel"), + @Mapping(target = "targetType", ignore = true), + @Mapping(target = "targetUrl", ignore = true), + @Mapping(target = "affectProjectId", ignore = true), + @Mapping(target = "unaffectedVersions", ignore = true), + @Mapping(target = "affectProductName", source = "influenceComponent") + }) + VulInfo scaToVulInfo(ScaVulInfoResp vulJsonData); + + List scaToVulInfo(List vulJsonData); + /** + * 数据库实体转换为JSON数据 + */ + @Mappings({ + @Mapping(target = "vulUid", source = "vulId"), + @Mapping(target = "cve", ignore = true), + @Mapping(target = "cnnvd", ignore = true), + @Mapping(target = "cnvd", ignore = true), + @Mapping(target = "cwe", source = "cwes", qualifiedByName = "cwesToCwe"), + @Mapping(target = "vulName", source = "vulTitle"), + @Mapping(target = "vulType", ignore = true), + @Mapping(target = "attackType", ignore = true), + @Mapping(target = "isExploit", source = "level", qualifiedByName = "levelToExploit"), + @Mapping(target = "securityLevel", source = "severity", qualifiedByName = "severityToSecurityLevel"), + @Mapping(target = "cvss4", ignore = true), + @Mapping(target = "releaseDate", source = "createTime", qualifiedByName = "dateTimeToDate"), + @Mapping(target = "updateDate", source = "modifyTime", qualifiedByName = "dateTimeToDate"), + @Mapping(target = "vulDescription", source = "vulDescription"), + @Mapping(target = "suggestion", source = "solution"), + @Mapping(target = "detectionTime", ignore = true), + @Mapping(target = "influenceComponentNum", ignore = true), + @Mapping(target = "influencePackageNum", ignore = true), + @Mapping(target = "influenceComponent", source = "targetName"), + @Mapping(target = "influenceComponentImage", source = "targetName"), + @Mapping(target = "vulStatus", ignore = true), + @Mapping(target = "auditStatus", source = "verifyStatus"), + @Mapping(target = "reachTypeStr", ignore = true), + @Mapping(target = "fixAvailable", ignore = true), + @Mapping(target = "isHide", ignore = true), + @Mapping(target = "cvss2", ignore = true), + @Mapping(target = "cvss3", ignore = true) + }) + ScaVulInfoResp vulInfoToSca(VulInfo vulInfo); + + /** + * LocalDate转换为LocalDateTime + */ + @Named("dateToDateTime") + default LocalDateTime dateToDateTime(LocalDate date) { + if (date == null) { + return LocalDateTime.now(); + } + return date.atStartOfDay(); + } + + /** + * LocalDateTime转换为LocalDate + */ + @Named("dateTimeToDate") + default LocalDate dateTimeToDate(LocalDateTime dateTime) { + if (dateTime == null) { + return LocalDate.now(); + } + return dateTime.toLocalDate(); + } + + /** + * CWE转换为CWE列表 + */ + @Named("cweToCwes") + default List cweToCwes(String cwe) { + if (cwe == null || cwe.trim().isEmpty()) { + return Arrays.asList(); + } + return Arrays.asList(cwe.split(",")); + } + + /** + * CWE列表转换为CWE字符串 + */ + @Named("cwesToCwe") + default String cwesToCwe(List cwes) { + if (cwes == null || cwes.isEmpty()) { + return ""; + } + return String.join(",", cwes); + } + + /** + * 利用难度转换为利用级别 + */ + @Named("exploitToLevel") + default Boolean exploitToLevel(String isExploit) { + if (isExploit == null) { + return false; + } + // 映射规则：容易 -> true, 困难 -> false, 一般 -> true + switch (isExploit) { + case "容易": + case "一般": + return true; + case "困难": + default: + return false; + } + } + + /** + * 利用级别转换为利用难度 + */ + @Named("levelToExploit") + default String levelToExploit(Boolean level) { + if (level == null) { + return "困难"; + } + return level ? "容易" : "困难"; + } + + /** + * 安全等级转换为危险等级 + */ + @Named("securityLevelToSeverity") + default Integer securityLevelToSeverity(String securityLevel) { + if (securityLevel == null) { + return 0; + } + // 映射规则：严重 -> 1, 高危 -> 2, 中危 -> 3, 低危 -> 4 + switch (securityLevel) { + case "严重": + return 1; + case "高危": + return 2; + case "中危": + return 3; + case "低危": + return 4; + default: + return 0; + } + } + + /** + * 危险等级转换为安全等级 + */ + @Named("severityToSecurityLevel") + default String severityToSecurityLevel(Integer severity) { + if (severity == null) { + return "未知"; + } + // 映射规则：1 -> 严重, 2 -> 高危, 3 -> 中危, 4 -> 低危 + switch (severity) { + case 1: + return "严重"; + case 2: + return "高危"; + case 3: + return "中危"; + case 4: + return "低危"; + default: + return "未知"; + } + } +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dao/vulInfo/VulInfoDao.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dao/vulInfo/VulInfoDao.java new file mode 100644 index 00000000..379a803d --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dao/vulInfo/VulInfoDao.java @@ -0,0 +1,7 @@ +package cd.casic.ci.process.process.dao.vulInfo; + +import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo; +import cd.casic.framework.mybatis.core.mapper.BaseMapperX; + +public interface VulInfoDao extends BaseMapperX { +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/volumnInfo/VulInfo.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/volumnInfo/VulInfo.java new file mode 100644 index 00000000..0b11df44 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/volumnInfo/VulInfo.java @@ -0,0 +1,134 @@ +package cd.casic.ci.process.process.dataObject.volumnInfo; + +import cd.casic.framework.mybatis.core.type.EncryptTypeHandler; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.apache.ibatis.type.JdbcType; + +import java.time.LocalDateTime; +import java.util.List; + +@Data +@TableName(value = "vul_info",autoResultMap = true) +public class VulInfo { + /** + * 主键ID，自动生成 + */ + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + /** + * 创建时间，自动生成 + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private LocalDateTime createTime; + + /** + * 修改时间，自动生成，修改时间就是审核时间 + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private LocalDateTime modifyTime; + + /** + * 漏洞状态，枚举，未审核、审核中，审核完 + */ + private String verifyStatus; + + /** + * 内部漏洞id，不需要输入，根据类型+日期随机生成 + */ + private String vulId; + + /** + * 是否开源，选项框，对勾，源码时候才会有这个概念 + */ + private Boolean isOpenSource; + + /** + * 中文标题 + */ + private String vulTitle; + + /** + * 中文描述 + */ + private String vulDescription; + + /** + * 危险等级,高中低三种 + */ + private Integer severity; + + /** + * 中文修复方案 + */ + private String solution; + + /** + * 暂时修复的缓解措施 + */ + private String relieve; + + /** + * 目标的名称 + */ + private String targetName; + + /** + * 漏洞类型 + */ + @TableField(typeHandler = JacksonTypeHandler.class,jdbcType = JdbcType.VARCHAR) + private List cwes; + + /** + * 利用级别，可利用和不可利用，默认不要利用 + */ + private Boolean level; + + /** + * 发现的漏洞目标类型，源码，二进制，固件三种 + */ + private String targetType; + + /** + * 目标的链接，来自于人机协同系统，就是一个下载地址的url + */ + private String targetUrl; + + /** + * 受影响的目标，用户自己输入 + */ + private String affectProjectId; + + /** + * 类型是源码的情况才有这个概念，安全分支列表 + */ + private String unaffectedVersions; + + /** + * 受影响产品名 + */ + private String affectProductName; + /** + * 城市 + * */ + private String city; + /** + * 关联的流水线执行记录id + * */ + private String instanceId; + /** + * 流水线节点id + * */ + private String taskId; + /** + * 节点类型 + * */ + private String taskType; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/DataAnalysisService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/DataAnalysisService.java new file mode 100644 index 00000000..8d56bb13 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/DataAnalysisService.java @@ -0,0 +1,26 @@ +package cd.casic.ci.process.process.service.dataAnalysis; + +import cd.casic.ci.process.dto.resp.dataAnalysis.DataAnalysisResp; + +import java.util.List; + +public interface DataAnalysisService { + + String getUseUserSUM(); + + String getTaskSUM(); + + String getVulSUM(); + + DataAnalysisResp getUserAndTaskDataCount(); + + List getHoopVulCountList(); + + List getResourceDistCountList(); + + + + + + +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/Impl/DataAnalysisServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/Impl/DataAnalysisServiceImpl.java new file mode 100644 index 00000000..2f395108 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/Impl/DataAnalysisServiceImpl.java @@ -0,0 +1,78 @@ +package cd.casic.ci.process.process.service.dataAnalysis.Impl; + +import cd.casic.ci.process.dto.resp.dataAnalysis.DataAnalysisResp; +import cd.casic.ci.process.process.dao.history.PipPipelineHisInstanceDao; +import cd.casic.ci.process.process.dao.vulInfo.VulInfoDao; +import cd.casic.ci.process.process.service.dataAnalysis.DataAnalysisService; +import cd.casic.framework.datapermission.core.dal.AdminUserMapper; +import cd.casic.module.machine.dal.dataobject.MachineInfoDO; +import cd.casic.module.machine.dal.mysql.MachineInfoMapper; +import cn.hutool.core.bean.BeanUtil; +import jakarta.annotation.Resource; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author: Paul + * @create: 2025-07-29 16:17 + */ +@Service +public class DataAnalysisServiceImpl implements DataAnalysisService { + + @Resource + private AdminUserMapper userMapper; + + @Resource + private VulInfoDao vulInfoDao; + + @Resource + private PipPipelineHisInstanceDao pipelineHisInstanceDao; + + @Resource + private MachineInfoMapper machineInfoMapper; + + @Override + public String getUseUserSUM() { + return userMapper.selectCount().toString(); + } + + @Override + public String getTaskSUM() { + return pipelineHisInstanceDao.selectCount().toString(); + } + + @Override + public String getVulSUM() { + return vulInfoDao.selectCount().toString(); + } + + public String getPerilousVulSUM(){ + return vulInfoDao.selectCount("severity", 2).toString(); + } + + @Override + public DataAnalysisResp getUserAndTaskDataCount() { + return DataAnalysisResp.builder().scanVulSUM("101125698234").useUserSUM(getUseUserSUM()) + .taskSUM(getTaskSUM()).vulSUM(getVulSUM()).perilousVulSUM(getPerilousVulSUM()).build(); + } + + @Override + public List getHoopVulCountList() { + + return null; + } + + @Override + public List getResourceDistCountList() { + + List machineInfoDOS = machineInfoMapper.selectList(); + List distCountList = machineInfoDOS.stream().map(obj -> { + DataAnalysisResp.ResourceDistCount distCount = new DataAnalysisResp.ResourceDistCount(); + BeanUtil.copyProperties(obj, distCount); + return distCount; + }).collect(Collectors.toList()); + return distCountList; + } +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java index ec47293f..8886444c 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java @@ -1,8 +1,6 @@ package cd.casic.ci.process.process.service.sast; -import cd.casic.ci.process.dto.req.sast.SastApplicationCreateReq; -import cd.casic.ci.process.dto.req.sast.SastIdentifiedLanguageTaskReq; -import cd.casic.ci.process.dto.req.sast.SastReportCreateReq; +import cd.casic.ci.process.dto.req.sast.*; import cd.casic.ci.process.dto.resp.sast.*; import cd.casic.framework.commons.pojo.PageParam; import cd.casic.framework.commons.pojo.PageResult; @@ -22,6 +20,7 @@ public interface SastService { List engineConfig(List languageIdList); SastDetectionConfigResp detectionConfig(String languageId); SastApplicationCreateResp applicationCreate(SastApplicationCreateReq req); + SastApplicationCreateResp applicationStash(SastApplicationStashReq req); SastApplicationEchoResp getApplicationEcho(String applicationId); String reportIndex(SastReportCreateReq req); public String reportDownload(String reportId); @@ -34,4 +33,8 @@ public interface SastService { public JSONObject jsonReportDownload(String reportId); public SastReportStatusResp getReportStatus(String reportId); + SastApplicationStashResp stashScan(String applicationId); + List engineLog(String applicationId); + SastApplicationBinaryStashResp applicationBinaryStash(SastApplicationBinaryStashReq req) ; + SastApplicationStashResp binaryStashScan(String applicationId); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java index 2ffe84f8..0163ff75 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java @@ -1,8 +1,6 @@ package cd.casic.ci.process.process.service.sast.impl; -import cd.casic.ci.process.dto.req.sast.SastApplicationCreateReq; -import cd.casic.ci.process.dto.req.sast.SastIdentifiedLanguageTaskReq; -import cd.casic.ci.process.dto.req.sast.SastReportCreateReq; +import cd.casic.ci.process.dto.req.sast.*; import cd.casic.ci.process.dto.resp.report.ReportResp; import cd.casic.ci.process.dto.resp.sast.*; import cd.casic.ci.process.process.service.sast.SastService; @@ -165,6 +163,15 @@ public class SastServiceImpl implements SastService { return body; } @Override + public SastApplicationCreateResp applicationStash(SastApplicationStashReq req) { + HttpHeaders httpHeaders = getHeaders(); + HttpEntity entity = new HttpEntity<>(req,httpHeaders); + httpHeaders.setContentType(MediaType.APPLICATION_JSON); + ResponseEntity exchange = restTemplate.exchange(sastProperties.getBaseUrl() +applicationStash, HttpMethod.POST,entity,SastApplicationCreateResp.class,new HashMap<>()); + SastApplicationCreateResp body = exchange.getBody(); + return body; + } + @Override public SastApplicationEchoResp getApplicationEcho(String applicationId){ HttpHeaders httpHeaders = getHeaders(); Map entityMap = new HashMap<>(); @@ -267,6 +274,47 @@ public class SastServiceImpl implements SastService { return exchange.getBody(); } + @Override + public SastApplicationStashResp stashScan(String applicationId) { + HttpHeaders httpHeaders = getHeaders(); + Map entityMap = new HashMap<>(); + HttpEntity> entity = new HttpEntity<>(entityMap,httpHeaders); + String uriString=sastProperties.getBaseUrl()+stashScan+applicationId; + log.info("sast 开始任务url:{}",uriString); + ResponseEntity exchange = restTemplate.exchange(uriString, HttpMethod.POST,entity, SastApplicationStashResp.class,new HashMap<>()); + return exchange.getBody(); + } + @Override + public List engineLog(String applicationId) { + HttpHeaders httpHeaders = getHeaders(); + Map entityMap = new HashMap<>(); + HttpEntity> entity = new HttpEntity<>(entityMap,httpHeaders); + String uriString=sastProperties.getBaseUrl()+engineLog; + uriString=String.format(uriString,applicationId); + log.info("sast 开始任务url:{}",uriString); + ResponseEntity exchange = restTemplate.exchange(uriString, HttpMethod.GET,entity, String.class,new HashMap<>()); + return JSONArray.parseArray(exchange.getBody(), SastEngineLogResp.class); + } + + @Override + public SastApplicationBinaryStashResp applicationBinaryStash(SastApplicationBinaryStashReq req) { + HttpHeaders httpHeaders = getHeaders(); + HttpEntity entity = new HttpEntity<>(req,httpHeaders); + httpHeaders.setContentType(MediaType.APPLICATION_JSON); + ResponseEntity exchange = restTemplate.exchange(sastProperties.getBaseUrl() +applicationBinaryStash, HttpMethod.POST,entity,SastApplicationBinaryStashResp.class,new HashMap<>()); + return exchange.getBody(); + } + @Override + public SastApplicationStashResp binaryStashScan(String applicationId) { + HttpHeaders httpHeaders = getHeaders(); + Map entityMap = new HashMap<>(); + HttpEntity> entity = new HttpEntity<>(entityMap,httpHeaders); + String uriString=sastProperties.getBaseUrl()+binaryStashScan+applicationId; + log.info("sast 开始任务url:{}",uriString); + ResponseEntity exchange = restTemplate.exchange(uriString, HttpMethod.POST,entity, SastApplicationStashResp.class,new HashMap<>()); + return exchange.getBody(); + } + private HttpHeaders getHeaders(){ HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.set(TOKEN_HEADER_KEY,TOKEN_PREFIX+getToken()); diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/vulInfo/VulInfoService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/vulInfo/VulInfoService.java new file mode 100644 index 00000000..5699ca57 --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/vulInfo/VulInfoService.java @@ -0,0 +1,19 @@ +package cd.casic.ci.process.process.service.vulInfo; + +import cd.casic.ci.process.dto.resp.ScaVulInfoResp; +import cd.casic.ci.process.process.dao.vulInfo.VulInfoDao; +import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo; +import com.baomidou.mybatisplus.extension.service.IService; +import jakarta.annotation.Resource; +import org.springframework.stereotype.Service; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +public interface VulInfoService extends IService { + void scaToVulInfoSave(Integer scaTaskId,String targetType,String targetName,String city,String instanceId,String taskId,String taskType); + Integer scaVulCountGet(Integer scaTaskId) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException; + List scaVulListGet(Integer scaTaskId) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/vulInfo/impl/VulInfoServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/vulInfo/impl/VulInfoServiceImpl.java new file mode 100644 index 00000000..c1e0e6ca --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/vulInfo/impl/VulInfoServiceImpl.java @@ -0,0 +1,147 @@ +package cd.casic.ci.process.process.service.vulInfo.impl; + +import cd.casic.ci.process.dto.req.report.ReportAssetTaskReq; +import cd.casic.ci.process.dto.resp.ScaVulInfoResp; +import cd.casic.ci.process.engine.context.ConstantContextHolder; +import cd.casic.ci.process.process.converter.VulInfoConverter; +import cd.casic.ci.process.process.dao.vulInfo.VulInfoDao; +import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo; +import cd.casic.ci.process.process.service.vulInfo.VulInfoService; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.*; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import static cd.casic.ci.process.engine.worker.base.HttpWorker.getRestTemplateWithoutSANCheck; + +@Service +@Slf4j +public class VulInfoServiceImpl extends ServiceImpl implements VulInfoService { + @Resource + private VulInfoConverter converter; + @Resource + private VulInfoDao vulInfoDao; + + @Override + public void scaToVulInfoSave(Integer scaTaskId,String targetType,String targetName,String city,String instanceId,String taskId,String taskType) { + List scaVulInfoResps = null; + try { + scaVulInfoResps = scaVulListGet(scaTaskId); + } catch (Exception e) { + return; + } + System.out.println(scaVulInfoResps.size()); + System.out.println(scaVulInfoResps); + List vulInfos = converter.scaToVulInfo(scaVulInfoResps); + vulInfos.forEach(item->{ + item.setTargetType(targetType); + item.setTargetName(targetName); + item.setCity(city); + item.setInstanceId(instanceId); + item.setTaskId(taskId); + item.setTaskType(taskType); + }); + saveBatch(vulInfos); + } + + @Override + public Integer scaVulCountGet(Integer scaTaskId) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + RestTemplate restTemplate= getRestTemplateWithoutSANCheck(); + String url = ConstantContextHolder.getScaIp() + "/openapi/v1/task/vul/list"; + JSONObject json = new JSONObject(); + json.put("taskId", scaTaskId); + json.put("pageNo", 1); + json.put("pageSize", 1); + String body = json.toJSONString(); + HttpEntity entity = new HttpEntity<>(body,createHeaders()); + ResponseEntity exchange = restTemplate.exchange(url, HttpMethod.POST, entity, String.class, new HashMap<>()); + String res = exchange.getBody(); + return JSONObject.parseObject(res).getInteger("total"); + } + + @Override + public List scaVulListGet(Integer scaTaskId) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + HttpHeaders headers = createHeaders(); + RestTemplate restTemplate = getRestTemplateWithoutSANCheck(); + JSONObject json = new JSONObject(); + ReportAssetTaskReq req = new ReportAssetTaskReq(); + //报告维度 5-组件，默认为5-组件 + json.put("dimension", req.getDimension()); + json.put("reportTemplateId", req.getReportTemplateId()); + json.put("type", 4); + json.put("reportName", req.getReportName()); + json.put("scaTaskId", scaTaskId); + String generateUrl = ConstantContextHolder.getScaIp()+"/openapi/v1/asset/report/task"; + HttpEntity entity = new HttpEntity<>(json.toJSONString(),headers); + // 生成报告 + ResponseEntity exchange = restTemplate.exchange(generateUrl, HttpMethod.POST, entity, String.class, new HashMap<>()); + String generateRes = exchange.getBody(); + JSONObject generateResObj = JSON.parseObject(generateRes); + if ("success".equals(generateResObj.getString("message"))) { + Integer id = generateResObj.getInteger("data"); + // 检查报告状态 + if (!checkScaReportStatus(id,restTemplate,headers)) { + return List.of(); + } + // 下载报告 + String downloadUrl = ConstantContextHolder.getScaIp()+"/openapi/v1/asset/report/downLoadReport/batch"; + JSONObject downloadBody = new JSONObject(); + downloadBody.put("reportIds", Collections.singletonList(id)); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity downloadEntity = new HttpEntity<>(downloadBody.toJSONString(),headers); + ResponseEntity downloadExchange = restTemplate.exchange(downloadUrl, HttpMethod.POST, downloadEntity, byte[].class, new HashMap<>()); + byte[] downloadRes = downloadExchange.getBody(); + String reportString = new String(downloadRes, StandardCharsets.UTF_8); + JSONObject reportObj = JSONObject.parseObject(reportString); + // 解析报告 + String vulInfo = reportObj.getString("vulInfo"); + return JSONArray.parseArray(vulInfo, ScaVulInfoResp.class); + } + return List.of(); + } + + private HttpHeaders createHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.add("OpenApiUserToken", ConstantContextHolder.getScaToken()); + return headers; + } + private Boolean checkScaReportStatus(Integer reporterId,RestTemplate restTemplate,HttpHeaders headers){ + HttpEntity checkEntity = new HttpEntity<>("",headers); + String checkUrl = ConstantContextHolder.getScaIp()+"/openapi/v1/asset/report/status/"+reporterId; + while (true){ + ResponseEntity exchange = restTemplate.exchange(checkUrl, HttpMethod.POST, checkEntity, String.class, new HashMap<>()); + JSONObject res = JSONObject.parseObject(exchange.getBody()); + JSONObject data = res.getJSONObject("data"); + Integer status = data.getInteger("status"); + log.info("报告生成状态 -1-排队中 0-生成中 1-已生成 2-生成失败：{}",status); + //报告状态 -1-排队中 0-生成中 1-已生成 2-生成失败 + if (status==null) { + break; + } + if (status.equals(1)) { + return true; + }else if (status.equals(2)) { + return false; + } + try { + Thread.sleep(1000L); + } catch (InterruptedException e) { + } + } + return false; + } +} diff --git a/ops-server/src/test/java/cd/casic/server/VulInfoTest.java b/ops-server/src/test/java/cd/casic/server/VulInfoTest.java new file mode 100644 index 00000000..2ae14a04 --- /dev/null +++ b/ops-server/src/test/java/cd/casic/server/VulInfoTest.java @@ -0,0 +1,65 @@ +package cd.casic.server; + +import cd.casic.ci.process.constant.PipelineTargetTypeConstant; +import cd.casic.ci.process.dto.req.testCase.TestCaseAITaskCreateReq; +import cd.casic.ci.process.dto.resp.ScaVulInfoResp; +import cd.casic.ci.process.process.converter.VulInfoConverter; +import cd.casic.ci.process.process.dataObject.target.TargetVersion; +import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo; +import cd.casic.ci.process.process.service.testCase.TestCaseAIGeneratorService; +import cd.casic.ci.process.process.service.vulInfo.VulInfoService; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.io.ClassPathResource; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.web.WebAppConfiguration; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.List; + + +@SpringBootTest(classes = {OpsServerApplication.class}) +@ActiveProfiles("local") +@Slf4j +@WebAppConfiguration +public class VulInfoTest { + @Resource + private VulInfoService vulInfoService; + @Resource + private VulInfoConverter converter; + @Test + public void test() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + System.out.println(vulInfoService.scaVulCountGet(681)); + } + @Test + public void scaVulListGet() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + String targetType = PipelineTargetTypeConstant.SOURCE_CODE; + Integer scaTaskId = 681; + String targetName="测试目标名称"; + String city = "长沙"; + String instanceId = "instanceId"; + String taskId = "taskId"; + String taskType ="taskType"; + List scaVulInfoResps = vulInfoService.scaVulListGet(scaTaskId); + System.out.println(scaVulInfoResps.size()); + System.out.println(scaVulInfoResps); + List vulInfos = converter.scaToVulInfo(scaVulInfoResps); + vulInfos.forEach(item->{ + item.setTargetType(targetType); + item.setTargetName(targetName); + item.setCity(city); + item.setInstanceId(instanceId); + item.setTaskId(taskId); + item.setTaskType(taskType); + }); + vulInfoService.saveBatch(vulInfos); + + } +}