Merge remote-tracking branch 'origin/master'

modules/module-ci-process-api/src/main/java/cd/casic/ci/api/DataAnalysisController.java

@@ -25,27 +25,26 @@ public class DataAnalysisController {
 
     @Operation(summary = "人数，任务次数，漏洞总数，高危漏洞 统计")
     @GetMapping("/userAndTaskCount")
-    public CommonResult userAndTaskDataCount(@RequestParam(required = false,value = "country") String country){
+    public CommonResult userAndTaskDataCount(@RequestParam(required = false,value = "city") String city){
-        return CommonResult.success(dataAnalysisService.getUserAndTaskDataCount());
+        return CommonResult.success(dataAnalysisService.getUserAndTaskDataCount(city));
     }
 
     @Operation(summary = "漏洞监测分布概览")
     @GetMapping("/HoopVulCount")
-    public CommonResult HoopVulDataCount(@RequestParam(required = false,value = "country") String country){
+    public CommonResult HoopVulDataCount(@RequestParam(required = false,value = "city") String city){
         return CommonResult.success(dataAnalysisService.getHoopVulCountList());
     }
 
     @Operation(summary = "资源分配数据概览")
     @GetMapping("/ResourceDistList")
-    public CommonResult ResourceDistList(@RequestParam(required = false,value = "country") String country){
+    public CommonResult ResourceDistList(@RequestParam(required = false,value = "city") String city){
-        return CommonResult.success(dataAnalysisService.getResourceDistCountList());
+        return CommonResult.success(dataAnalysisService.getResourceDistCountList(city));
     }
 
     @Operation(summary = "漏洞监测数据概览")
     @GetMapping("/VulMonitorDataList")
-    public CommonResult VulMonitorDataList(@RequestParam(required = false,value = "country") String country){
+    public CommonResult VulMonitorDataList(@RequestParam(required = false,value = "city") String city){
-
+        return CommonResult.success(dataAnalysisService.getVulMonitorCountList(city));
-        return CommonResult.success();
     }
 
 

modules/module-ci-process-api/src/main/java/cd/casic/ci/api/SastController.java

@@ -145,4 +145,8 @@ public class SastController {
         SastApplicationStatusResp applicationStatus = sastService.getApplicationStatus(applicationId);
         return CommonResult.success(applicationStatus);
     }
+    @PostMapping("/applicationBinaryStash")
+    CommonResult<SastApplicationBinaryStashResp> applicationBinaryStash(@RequestBody SastApplicationBinaryStashReq req) {
+        return CommonResult.success(sastService.applicationBinaryStash(req));
+    }
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationBinaryStashReq.java

@@ -1,5 +1,6 @@
 package cd.casic.ci.process.dto.req.sast;
 
+import jakarta.validation.constraints.NotBlank;
 import lombok.Data;
 
 import java.util.List;
@@ -55,4 +56,13 @@ public class SastApplicationBinaryStashReq {
      * z3 是否能够超时
      * */
     private Boolean z3TimeoutMsStatus;//需要用户填写
+    /**
+     *  z3超时时间
+     * */
+    private Integer z3TimeoutMs;
+    /**
+     *  流水线id
+     * */
+    @NotBlank
+    private String pipelineId;
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastVulInfoReq.java

@@ -0,0 +1,56 @@
+package cd.casic.ci.process.dto.req.sast;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class SastVulInfoReq {
+    private String id;
+    private String name;
+    /**
+     * UN_KNOWN("未知"),
+     *     DEADLY("严重"),
+     *     SEVERITY("高危"),
+     *     POOR_RISK("中危"),
+     *     LOW_RISK("低危"),
+     *     SAFE("安全");
+     * */
+    private String severity;
+    private int count;
+    private List<DefectDetail> defectDetails = new ArrayList<>();
+
+    // 便捷方法
+    public void addDefectDetail(DefectDetail detail) {
+        if (defectDetails == null) {
+            defectDetails = new ArrayList<>();
+        }
+        defectDetails.add(detail);
+    }
+    @Data
+    @NoArgsConstructor
+    @AllArgsConstructor
+    public static class DefectDetail {
+        private String file;
+        private int line;
+        private String path;
+        private List<String> propagationPath = new ArrayList<>();
+        private String description;
+        private String potentialRisk;
+
+        // 便捷方法
+        public void addPropagationPath(String path) {
+            if (propagationPath == null) {
+                propagationPath = new ArrayList<>();
+            }
+            propagationPath.add(path);
+        }
+}
+
+
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/dataAnalysis/DataAnalysisResp.java

@@ -1,6 +1,7 @@
 package cd.casic.ci.process.dto.resp.dataAnalysis;
 
 import cd.casic.ci.process.enums.MachineSystemEnum;
+import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
 import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableId;
 import lombok.Builder;
@@ -40,26 +41,38 @@ public class DataAnalysisResp {
      */
     private String perilousVulSUM;
 
-
+    /**
+     * 漏洞监测分布概览
+     */
     private List<HoopVulCount> hoopVulCountList;
 
-
+    /**
+     * 资源分配数据概览
+     */
     private List<ResourceDistCount> resourceDistCountList;
 
+    /**
+     * 漏洞监测数据概览
+     */
+    private List<VulInfo> VulMonitorCountList;
+
 
     /**
      * 漏洞监测分布概览
      */
     @Data
+    @Builder
     public static class HoopVulCount{
         /**
          * 城市
          */
-        private String country;
+        private String city;
         /**
          * 百分比 两位小数
          */
         private BigDecimal value;
+
+        private String count;
     }
 
    /**
@@ -68,8 +81,8 @@ public class DataAnalysisResp {
     @Data
     public static class ResourceDistCount{
        private static final List<String> CITIES = List.of(
-               "北京", "上海", "广州", "深圳", "杭州",
+               "北京", "上海", "天津", "重庆",
-               "成都", "武汉", "西安", "长沙", "厦门"
+               "成都", "长沙"
        );
        /**
         * 机器id
@@ -104,12 +117,4 @@ public class DataAnalysisResp {
        }
    }
 
-    /**
-     *
-     */
-    /*@Data
-    static class VulMonitorCount{
-
-    }*/
-
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastWorker.java

@@ -132,7 +132,8 @@ public class SastWorker extends BaseWorker {
         if (reportStatus.getStatus().equals("FAIL")) {
             throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"生成sast报告失败");
         }
-        JSONObject jsonObject = getJSONString(reportId).getJSONObject("task_summary");
+        JSONObject reportJson = getJSONString(reportId);
+        JSONObject jsonObject = reportJson.getJSONObject("task_summary");
         JSONObject severity = jsonObject.getJSONObject("defect_severity_distribution");
         JSONObject rule = jsonObject.getJSONObject("detection_rule_distribution");
         JSONObject res = new JSONObject();

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/pipeline/PipPipeline.java

@@ -138,4 +138,5 @@ public class PipPipeline extends PipBaseElement {
      *  机器id
      * */
     private String machineId;
+    private Long tenantId;
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/DataAnalysisService.java

@@ -1,22 +1,25 @@
 package cd.casic.ci.process.process.service.dataAnalysis;
 
 import cd.casic.ci.process.dto.resp.dataAnalysis.DataAnalysisResp;
+import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
 
 import java.util.List;
 
 public interface DataAnalysisService {
 
-    String getUseUserSUM();
+    String getUseUserSUM(String city);
 
-    String getTaskSUM();
+    String getTaskSUM(String city);
 
-    String getVulSUM();
+    String getVulSUM(String city);
 
-    DataAnalysisResp getUserAndTaskDataCount();
+    DataAnalysisResp getUserAndTaskDataCount(String city);
 
     List<DataAnalysisResp.HoopVulCount> getHoopVulCountList();
 
-    List<DataAnalysisResp.ResourceDistCount> getResourceDistCountList();
+    List<DataAnalysisResp.ResourceDistCount> getResourceDistCountList(String city);
+
+    List<VulInfo> getVulMonitorCountList(String city);
 
 
 

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/Impl/DataAnalysisServiceImpl.java

@@ -3,15 +3,20 @@ package cd.casic.ci.process.process.service.dataAnalysis.Impl;
 import cd.casic.ci.process.dto.resp.dataAnalysis.DataAnalysisResp;
 import cd.casic.ci.process.process.dao.history.PipPipelineHisInstanceDao;
 import cd.casic.ci.process.process.dao.vulInfo.VulInfoDao;
+import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
 import cd.casic.ci.process.process.service.dataAnalysis.DataAnalysisService;
 import cd.casic.framework.datapermission.core.dal.AdminUserMapper;
 import cd.casic.module.machine.dal.dataobject.MachineInfoDO;
 import cd.casic.module.machine.dal.mysql.MachineInfoMapper;
 import cn.hutool.core.bean.BeanUtil;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import jakarta.annotation.Resource;
 import org.springframework.stereotype.Service;
-
+import java.math.BigDecimal;
-import java.util.List;
+import java.math.RoundingMode;
+import java.text.DecimalFormat;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -34,45 +39,66 @@ public class DataAnalysisServiceImpl implements DataAnalysisService {
     private MachineInfoMapper machineInfoMapper;
 
     @Override
-    public String getUseUserSUM() {
+    public String getUseUserSUM(String city) {
         return userMapper.selectCount().toString();
     }
 
     @Override
-    public String getTaskSUM() {
+    public String getTaskSUM(String city) {
         return pipelineHisInstanceDao.selectCount().toString();
     }
 
     @Override
-    public String getVulSUM() {
+    public String getVulSUM(String city) {
-        return vulInfoDao.selectCount().toString();
+        return vulInfoDao.selectCount(new QueryWrapper<VulInfo>().lambda().eq(Objects.nonNull(city),VulInfo::getCity,city)).toString();
     }
 
-    public String getPerilousVulSUM(){
+    public String getPerilousVulSUM(String city){
-        return vulInfoDao.selectCount("severity", 2).toString();
+        return vulInfoDao.selectCount(new LambdaQueryWrapper<VulInfo>().eq(Objects.nonNull(city),VulInfo::getCity,city).eq(VulInfo::getSeverity,2)).toString();
     }
 
     @Override
-    public DataAnalysisResp getUserAndTaskDataCount() {
+    public DataAnalysisResp getUserAndTaskDataCount(String city) {
-        return DataAnalysisResp.builder().scanVulSUM("101125698234").useUserSUM(getUseUserSUM())
+        return DataAnalysisResp.builder().scanVulSUM("101125698234").useUserSUM(getUseUserSUM(city))
-                .taskSUM(getTaskSUM()).vulSUM(getVulSUM()).perilousVulSUM(getPerilousVulSUM()).build();
+                .taskSUM(getTaskSUM(city)).vulSUM(getVulSUM(city)).perilousVulSUM(getPerilousVulSUM(city)).build();
     }
 
     @Override
     public List<DataAnalysisResp.HoopVulCount> getHoopVulCountList() {
-
+        Integer integer = Integer.decode(getVulSUM(null));
-        return null;
+        if (integer == 0) {
+            return null;
+        }
+        QueryWrapper<VulInfo> wrapper = new QueryWrapper<>();
+        wrapper.select("city", "COUNT(*) as value")
+                .groupBy("city");
+        List<Map<String, Object>> listMaps = vulInfoDao.selectMaps(wrapper);
+        List<DataAnalysisResp.HoopVulCount> rest = new ArrayList<>();
+        listMaps.forEach(map -> {
+            DataAnalysisResp.HoopVulCount hoopVulCount = BeanUtil.mapToBean(map, DataAnalysisResp.HoopVulCount.class, false);
+            String format = new DecimalFormat("0.00%").format(hoopVulCount.getValue().divide(new BigDecimal(integer), 4, RoundingMode.HALF_UP));
+            hoopVulCount.setCount(format);
+            rest.add(hoopVulCount);
+        });
+        return rest;
     }
 
     @Override
-    public List<DataAnalysisResp.ResourceDistCount> getResourceDistCountList() {
+    public List<DataAnalysisResp.ResourceDistCount> getResourceDistCountList(String city) {
-
         List<MachineInfoDO> machineInfoDOS = machineInfoMapper.selectList();
         List<DataAnalysisResp.ResourceDistCount> distCountList = machineInfoDOS.stream().map(obj -> {
             DataAnalysisResp.ResourceDistCount distCount = new DataAnalysisResp.ResourceDistCount();
             BeanUtil.copyProperties(obj, distCount);
             return distCount;
         }).collect(Collectors.toList());
+        if (Objects.nonNull(city)) {
+            return distCountList.stream().filter(a -> city.equals(a.getCountry())).toList();
+        }
         return distCountList;
     }
+
+    @Override
+    public List<VulInfo> getVulMonitorCountList(String city) {
+        return vulInfoDao.selectList(new QueryWrapper<VulInfo>().lambda().eq(Objects.nonNull(city),VulInfo::getCity,city).orderByDesc(VulInfo::getCreateTime));
+    }
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/pipeline/impl/PipelineServiceImpl.java

@@ -109,7 +109,8 @@ public class PipelineServiceImpl extends ServiceImpl<PipelineDao, PipPipeline> i
         List<PipTask> pipTaskList = new ArrayList<>(0);
 
         PipPipeline pipeline = pipelineConverter.reqToDO(pipelineReq);
-
+        Long tenantId = WebFrameworkUtils.getLoginUser().getTenantId();
+        pipeline.setTenantId(tenantId);
         // 随机颜色
         int randomNumber = (int)(Math.random() * 5) + 1;
         pipeline.setColor(randomNumber);

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java

@@ -37,4 +37,5 @@ public interface SastService {
     List<SastEngineLogResp> engineLog(String applicationId);
     SastApplicationBinaryStashResp applicationBinaryStash(SastApplicationBinaryStashReq req) ;
     SastApplicationStashResp binaryStashScan(String applicationId);
+    void saveReportVulInfo(List<SastVulInfoReq> list,String targetType,String targetName,String city,String instanceId,String taskId,String taskType);
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java

@@ -3,8 +3,16 @@ package cd.casic.ci.process.process.service.sast.impl;
 import cd.casic.ci.process.dto.req.sast.*;
 import cd.casic.ci.process.dto.resp.report.ReportResp;
 import cd.casic.ci.process.dto.resp.sast.*;
+import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
+import cd.casic.ci.process.process.dataObject.target.TargetVersion;
+import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
+import cd.casic.ci.process.process.service.pipeline.PipelineService;
 import cd.casic.ci.process.process.service.sast.SastService;
+import cd.casic.ci.process.process.service.target.TargetVersionService;
+import cd.casic.ci.process.process.service.vulInfo.VulInfoService;
 import cd.casic.ci.process.properties.SastProperties;
+import cd.casic.framework.commons.exception.ServiceException;
+import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
 import cd.casic.framework.commons.pojo.PageParam;
 import cd.casic.framework.commons.pojo.PageResult;
 import cd.casic.framework.redis.core.RedisTemplateUtils;
@@ -20,6 +28,7 @@ import org.luaj.vm2.ast.Str;
 import org.springframework.core.io.FileSystemResource;
 import org.springframework.http.*;
 import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
 import org.springframework.util.LinkedMultiValueMap;
 import org.springframework.util.MultiValueMap;
 import org.springframework.web.client.RestTemplate;
@@ -31,9 +40,7 @@ import java.io.InputStream;
 import java.net.URLDecoder;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
+import java.util.*;
-import java.util.List;
-import java.util.Map;
 
 import static cd.casic.ci.process.constant.SastUrlConstant.*;
 
@@ -46,10 +53,17 @@ public class SastServiceImpl implements SastService {
     private SastProperties sastProperties;
     @Resource
     private RestTemplate restTemplate;
+    @Resource
+    private PipelineService pipelineService;
+    @Resource
+    private TargetVersionService targetVersionService;
+    @Resource
+    private VulInfoService vulInfoService;
     public static final String TOKEN_PREFIX = "Bearer ";
     public static final String TOKEN_HEADER_KEY = "authorization";
     public static final String REDIS_SAST_TOKEN_KEY = "REDIS_SAST_TOKEN_KEY";
 
+
     private SastTokenResp getTokenRemote(){
         HttpHeaders httpHeaders = new HttpHeaders();
         HttpEntity<SastProperties> httpEntity = new HttpEntity<SastProperties>(sastProperties,httpHeaders);
@@ -299,6 +313,19 @@ public class SastServiceImpl implements SastService {
     @Override
     public SastApplicationBinaryStashResp applicationBinaryStash(SastApplicationBinaryStashReq req) {
         HttpHeaders httpHeaders = getHeaders();
+        req.setProjectId("893ed995-5b81-474a-96a9-2800281421cd");
+        req.setApplicationName("二进制任务"+ UUID.randomUUID().toString());
+        req.setCodeSourceFrom("BINARY");
+        req.setArchAutoIdentify(true);
+        String pipelineId = req.getPipelineId();
+        PipPipeline pipeline = pipelineService.getById(pipelineId);
+        String targetVersionId = pipeline.getTargetVersionId();
+        TargetVersion targetVersion = targetVersionService.getById(targetVersionId);
+        File targetFile=new File(targetVersion.getFilePath());
+        SastFileUploadResp uploadResp = uploadFile(targetFile);
+        String id = uploadResp.getId();
+        req.setFileId(Collections.singletonList(id));
+        log.info("sast二进制上传文件成功，开始调用接口{}",req);
         HttpEntity<SastApplicationBinaryStashReq> entity = new HttpEntity<>(req,httpHeaders);
         httpHeaders.setContentType(MediaType.APPLICATION_JSON);
         ResponseEntity<SastApplicationBinaryStashResp> exchange = restTemplate.exchange(sastProperties.getBaseUrl() +applicationBinaryStash, HttpMethod.POST,entity,SastApplicationBinaryStashResp.class,new HashMap<>());
@@ -315,9 +342,56 @@ public class SastServiceImpl implements SastService {
         return exchange.getBody();
     }
 
+    @Override
+    public void saveReportVulInfo(List<SastVulInfoReq> list,String targetType,String targetName,String city,String instanceId,String taskId,String taskType) {
+        if (CollectionUtils.isEmpty(list)) {
+            return;
+        }
+        List<VulInfo> vulInfos = new ArrayList<>(list.size());
+        for (SastVulInfoReq req : list) {
+            VulInfo vulInfo = new VulInfo();
+            // 设置安全等级、名称
+            vulInfo.setVulTitle(req.getName());
+            String severity = req.getSeverity();
+            List<SastVulInfoReq.DefectDetail> defectDetails = req.getDefectDetails();
+            // 设置描述修复措施
+            if (CollectionUtils.isEmpty(defectDetails)) {
+                continue;
+            }
+            //设置流水线相关属性,因为漏洞描述还有漏洞修复建议列表里面每一项都是一样的，所以直接取第一个
+            SastVulInfoReq.DefectDetail first = defectDetails.getFirst();
+            String description = first.getDescription();
+            String potentialRisk = first.getPotentialRisk();
+            vulInfo.setSolution(potentialRisk);
+            vulInfo.setVulDescription(description);
+            vulInfo.setSeverity(securityLevelToSeverity(severity));
+            vulInfo.setTargetType(targetType);
+            vulInfo.setTargetName(targetName);
+            vulInfo.setCity(city);
+            vulInfo.setInstanceId(instanceId);
+            vulInfo.setTaskId(taskId);
+            vulInfo.setTaskType(taskType);
+            vulInfos.add(vulInfo);
+        }
+        vulInfoService.saveBatch(vulInfos);
+    }
+
     private HttpHeaders getHeaders(){
         HttpHeaders httpHeaders = new HttpHeaders();
         httpHeaders.set(TOKEN_HEADER_KEY,TOKEN_PREFIX+getToken());
         return httpHeaders;
     }
+    private Integer securityLevelToSeverity(String securityLevel) {
+        if (securityLevel == null) {
+            return 0;
+        }
+        // 映射规则：严重 -> 1, 高危 -> 2, 中危 -> 3, 低危 -> 4
+        return switch (securityLevel) {
+            case "严重" -> 1;
+            case "高危" -> 2;
+            case "中危" -> 3;
+            case "安全", "低危" -> 4;
+            default -> 0;
+        };
+    }
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/util/WebFrameworkUtils.java

@@ -40,6 +40,14 @@ public class WebFrameworkUtils {
             return null;
         }
     }
+    public static LoginUser getLoginUser() {
+        try {
+            Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+            return (LoginUser) authentication.getPrincipal();
+        } catch (Exception e){
+            return null;
+        }
+    }
     public static String getLoginUserIdStr(){
         Long loginUserId = getLoginUserId();
         if (loginUserId!=null) {

ops-server/src/test/java/cd/casic/server/VulInfoTest.java

@@ -1,13 +1,17 @@
 package cd.casic.server;
 
 import cd.casic.ci.process.constant.PipelineTargetTypeConstant;
+import cd.casic.ci.process.dto.req.sast.SastVulInfoReq;
 import cd.casic.ci.process.dto.req.testCase.TestCaseAITaskCreateReq;
 import cd.casic.ci.process.dto.resp.ScaVulInfoResp;
 import cd.casic.ci.process.process.converter.VulInfoConverter;
 import cd.casic.ci.process.process.dataObject.target.TargetVersion;
 import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
+import cd.casic.ci.process.process.service.sast.SastService;
 import cd.casic.ci.process.process.service.testCase.TestCaseAIGeneratorService;
 import cd.casic.ci.process.process.service.vulInfo.VulInfoService;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import jakarta.annotation.Resource;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
@@ -34,6 +38,8 @@ public class VulInfoTest {
     private VulInfoService vulInfoService;
     @Resource
     private VulInfoConverter converter;
+    @Resource
+    private SastService sastService;
     @Test
     public void test() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
         System.out.println(vulInfoService.scaVulCountGet(681));
@@ -61,5 +67,9 @@ public class VulInfoTest {
         });
         vulInfoService.saveBatch(vulInfos);
 
+    }
+    @Test
+    public void sastTest(){
+
     }
 }