mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
HopeLi 2025-07-30 17:40:36 +08:00
commit 377f6924b1
14 changed files with 245 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
modules/module-ci-process-api/src/main/java/cd/casic/ci/api/DataAnalysisController.java
View File

@ -25,27 +25,26 @@ public class DataAnalysisController {
@Operation(summary = "人数，任务次数，漏洞总数，高危漏洞 统计")
@GetMapping("/userAndTaskCount")
public CommonResult userAndTaskDataCount(@RequestParam(required = false,value = "country") String country){
return CommonResult.success(dataAnalysisService.getUserAndTaskDataCount());
public CommonResult userAndTaskDataCount(@RequestParam(required = false,value = "city") String city){
return CommonResult.success(dataAnalysisService.getUserAndTaskDataCount(city));
}
@Operation(summary = "漏洞监测分布概览")
@GetMapping("/HoopVulCount")
public CommonResult HoopVulDataCount(@RequestParam(required = false,value = "country") String country){
public CommonResult HoopVulDataCount(@RequestParam(required = false,value = "city") String city){
return CommonResult.success(dataAnalysisService.getHoopVulCountList());
}
@Operation(summary = "资源分配数据概览")
@GetMapping("/ResourceDistList")
public CommonResult ResourceDistList(@RequestParam(required = false,value = "country") String country){
return CommonResult.success(dataAnalysisService.getResourceDistCountList());
public CommonResult ResourceDistList(@RequestParam(required = false,value = "city") String city){
return CommonResult.success(dataAnalysisService.getResourceDistCountList(city));
}
@Operation(summary = "漏洞监测数据概览")
@GetMapping("/VulMonitorDataList")
public CommonResult VulMonitorDataList(@RequestParam(required = false,value = "country") String country){
return CommonResult.success();
public CommonResult VulMonitorDataList(@RequestParam(required = false,value = "city") String city){
return CommonResult.success(dataAnalysisService.getVulMonitorCountList(city));
}

4
modules/module-ci-process-api/src/main/java/cd/casic/ci/api/SastController.java
View File

@ -145,4 +145,8 @@ public class SastController {
SastApplicationStatusResp applicationStatus = sastService.getApplicationStatus(applicationId);
return CommonResult.success(applicationStatus);
}
@PostMapping("/applicationBinaryStash")
CommonResult<SastApplicationBinaryStashResp> applicationBinaryStash(@RequestBody SastApplicationBinaryStashReq req) {
return CommonResult.success(sastService.applicationBinaryStash(req));
}
}

10
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastApplicationBinaryStashReq.java
View File

@ -1,5 +1,6 @@
package cd.casic.ci.process.dto.req.sast;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
import java.util.List;
@ -55,4 +56,13 @@ public class SastApplicationBinaryStashReq {
* z3 是否能够超时
* */
private Boolean z3TimeoutMsStatus;//需要用户填写
/**
* z3超时时间
* */
private Integer z3TimeoutMs;
/**
* 流水线id
* */
@NotBlank
private String pipelineId;
}

56
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/sast/SastVulInfoReq.java Normal file
View File

@ -0,0 +1,56 @@
package cd.casic.ci.process.dto.req.sast;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class SastVulInfoReq {
private String id;
private String name;
/**
* UN_KNOWN("未知"),
* DEADLY("严重"),
* SEVERITY("高危"),
* POOR_RISK("中危"),
* LOW_RISK("低危"),
* SAFE("安全");
* */
private String severity;
private int count;
private List<DefectDetail> defectDetails = new ArrayList<>();
// 便捷方法
public void addDefectDetail(DefectDetail detail) {
if (defectDetails == null) {
defectDetails = new ArrayList<>();
}
defectDetails.add(detail);
}
@Data
@NoArgsConstructor
@AllArgsConstructor
public static class DefectDetail {
private String file;
private int line;
private String path;
private List<String> propagationPath = new ArrayList<>();
private String description;
private String potentialRisk;
// 便捷方法
public void addPropagationPath(String path) {
if (propagationPath == null) {
propagationPath = new ArrayList<>();
}
propagationPath.add(path);
}
}
}

31
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/dataAnalysis/DataAnalysisResp.java
View File

@ -1,6 +1,7 @@
package cd.casic.ci.process.dto.resp.dataAnalysis;
import cd.casic.ci.process.enums.MachineSystemEnum;
import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import lombok.Builder;
@ -40,26 +41,38 @@ public class DataAnalysisResp {
*/
private String perilousVulSUM;
/**
* 漏洞监测分布概览
*/
private List<HoopVulCount> hoopVulCountList;
/**
* 资源分配数据概览
*/
private List<ResourceDistCount> resourceDistCountList;
/**
* 漏洞监测数据概览
*/
private List<VulInfo> VulMonitorCountList;
/**
* 漏洞监测分布概览
*/
@Data
@Builder
public static class HoopVulCount{
/**
* 城市
*/
private String country;
private String city;
/**
* 百分比 两位小数
*/
private BigDecimal value;
private String count;
}
/**
@ -68,8 +81,8 @@ public class DataAnalysisResp {
@Data
public static class ResourceDistCount{
private static final List<String> CITIES = List.of(
"北京", "上海", "广州", "深圳", "杭州",
"成都", "武汉", "西安", "长沙", "厦门"
"北京", "上海", "天津", "重庆",
"成都", "长沙"
);
/**
* 机器id
@ -104,12 +117,4 @@ public class DataAnalysisResp {
}
}
/**
*
*/
/*@Data
static class VulMonitorCount{
}*/
}

3
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/sast/SastWorker.java
View File

@ -132,7 +132,8 @@ public class SastWorker extends BaseWorker {
if (reportStatus.getStatus().equals("FAIL")) {
throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"生成sast报告失败");
}
JSONObject jsonObject = getJSONString(reportId).getJSONObject("task_summary");
JSONObject reportJson = getJSONString(reportId);
JSONObject jsonObject = reportJson.getJSONObject("task_summary");
JSONObject severity = jsonObject.getJSONObject("defect_severity_distribution");
JSONObject rule = jsonObject.getJSONObject("detection_rule_distribution");
JSONObject res = new JSONObject();

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/pipeline/PipPipeline.java
View File

@ -138,4 +138,5 @@ public class PipPipeline extends PipBaseElement {
* 机器id
* */
private String machineId;
private Long tenantId;
}

13
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/DataAnalysisService.java
View File

@ -1,22 +1,25 @@
package cd.casic.ci.process.process.service.dataAnalysis;
import cd.casic.ci.process.dto.resp.dataAnalysis.DataAnalysisResp;
import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
import java.util.List;
public interface DataAnalysisService {
String getUseUserSUM();
String getUseUserSUM(String city);
String getTaskSUM();
String getTaskSUM(String city);
String getVulSUM();
String getVulSUM(String city);
DataAnalysisResp getUserAndTaskDataCount();
DataAnalysisResp getUserAndTaskDataCount(String city);
List<DataAnalysisResp.HoopVulCount> getHoopVulCountList();
List<DataAnalysisResp.ResourceDistCount> getResourceDistCountList();
List<DataAnalysisResp.ResourceDistCount> getResourceDistCountList(String city);
List<VulInfo> getVulMonitorCountList(String city);

56
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/dataAnalysis/Impl/DataAnalysisServiceImpl.java
View File

@ -3,15 +3,20 @@ package cd.casic.ci.process.process.service.dataAnalysis.Impl;
import cd.casic.ci.process.dto.resp.dataAnalysis.DataAnalysisResp;
import cd.casic.ci.process.process.dao.history.PipPipelineHisInstanceDao;
import cd.casic.ci.process.process.dao.vulInfo.VulInfoDao;
import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
import cd.casic.ci.process.process.service.dataAnalysis.DataAnalysisService;
import cd.casic.framework.datapermission.core.dal.AdminUserMapper;
import cd.casic.module.machine.dal.dataobject.MachineInfoDO;
import cd.casic.module.machine.dal.mysql.MachineInfoMapper;
import cn.hutool.core.bean.BeanUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import jakarta.annotation.Resource;
import org.springframework.stereotype.Service;
import java.util.List;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.text.DecimalFormat;
import java.util.*;
import java.util.stream.Collectors;
/**
@ -34,45 +39,66 @@ public class DataAnalysisServiceImpl implements DataAnalysisService {
private MachineInfoMapper machineInfoMapper;
@Override
public String getUseUserSUM() {
public String getUseUserSUM(String city) {
return userMapper.selectCount().toString();
}
@Override
public String getTaskSUM() {
public String getTaskSUM(String city) {
return pipelineHisInstanceDao.selectCount().toString();
}
@Override
public String getVulSUM() {
return vulInfoDao.selectCount().toString();
public String getVulSUM(String city) {
return vulInfoDao.selectCount(new QueryWrapper<VulInfo>().lambda().eq(Objects.nonNull(city),VulInfo::getCity,city)).toString();
}
public String getPerilousVulSUM(){
return vulInfoDao.selectCount("severity", 2).toString();
public String getPerilousVulSUM(String city){
return vulInfoDao.selectCount(new LambdaQueryWrapper<VulInfo>().eq(Objects.nonNull(city),VulInfo::getCity,city).eq(VulInfo::getSeverity,2)).toString();
}
@Override
public DataAnalysisResp getUserAndTaskDataCount() {
return DataAnalysisResp.builder().scanVulSUM("101125698234").useUserSUM(getUseUserSUM())
.taskSUM(getTaskSUM()).vulSUM(getVulSUM()).perilousVulSUM(getPerilousVulSUM()).build();
public DataAnalysisResp getUserAndTaskDataCount(String city) {
return DataAnalysisResp.builder().scanVulSUM("101125698234").useUserSUM(getUseUserSUM(city))
.taskSUM(getTaskSUM(city)).vulSUM(getVulSUM(city)).perilousVulSUM(getPerilousVulSUM(city)).build();
}
@Override
public List<DataAnalysisResp.HoopVulCount> getHoopVulCountList() {
return null;
Integer integer = Integer.decode(getVulSUM(null));
if (integer == 0) {
return null;
}
QueryWrapper<VulInfo> wrapper = new QueryWrapper<>();
wrapper.select("city", "COUNT(*) as value")
.groupBy("city");
List<Map<String, Object>> listMaps = vulInfoDao.selectMaps(wrapper);
List<DataAnalysisResp.HoopVulCount> rest = new ArrayList<>();
listMaps.forEach(map -> {
DataAnalysisResp.HoopVulCount hoopVulCount = BeanUtil.mapToBean(map, DataAnalysisResp.HoopVulCount.class, false);
String format = new DecimalFormat("0.00%").format(hoopVulCount.getValue().divide(new BigDecimal(integer), 4, RoundingMode.HALF_UP));
hoopVulCount.setCount(format);
rest.add(hoopVulCount);
});
return rest;
}
@Override
public List<DataAnalysisResp.ResourceDistCount> getResourceDistCountList() {
public List<DataAnalysisResp.ResourceDistCount> getResourceDistCountList(String city) {
List<MachineInfoDO> machineInfoDOS = machineInfoMapper.selectList();
List<DataAnalysisResp.ResourceDistCount> distCountList = machineInfoDOS.stream().map(obj -> {
DataAnalysisResp.ResourceDistCount distCount = new DataAnalysisResp.ResourceDistCount();
BeanUtil.copyProperties(obj, distCount);
return distCount;
}).collect(Collectors.toList());
if (Objects.nonNull(city)) {
return distCountList.stream().filter(a -> city.equals(a.getCountry())).toList();
}
return distCountList;
}
@Override
public List<VulInfo> getVulMonitorCountList(String city) {
return vulInfoDao.selectList(new QueryWrapper<VulInfo>().lambda().eq(Objects.nonNull(city),VulInfo::getCity,city).orderByDesc(VulInfo::getCreateTime));
}
}

3
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/pipeline/impl/PipelineServiceImpl.java
View File

@ -109,7 +109,8 @@ public class PipelineServiceImpl extends ServiceImpl<PipelineDao, PipPipeline> i
List<PipTask> pipTaskList = new ArrayList<>(0);
PipPipeline pipeline = pipelineConverter.reqToDO(pipelineReq);
Long tenantId = WebFrameworkUtils.getLoginUser().getTenantId();
pipeline.setTenantId(tenantId);
// 随机颜色
int randomNumber = (int)(Math.random() * 5) + 1;
pipeline.setColor(randomNumber);

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/SastService.java
View File

@ -37,4 +37,5 @@ public interface SastService {
List<SastEngineLogResp> engineLog(String applicationId);
SastApplicationBinaryStashResp applicationBinaryStash(SastApplicationBinaryStashReq req) ;
SastApplicationStashResp binaryStashScan(String applicationId);
void saveReportVulInfo(List<SastVulInfoReq> list,String targetType,String targetName,String city,String instanceId,String taskId,String taskType);
}

80
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/sast/impl/SastServiceImpl.java
View File

@ -3,8 +3,16 @@ package cd.casic.ci.process.process.service.sast.impl;
import cd.casic.ci.process.dto.req.sast.*;
import cd.casic.ci.process.dto.resp.report.ReportResp;
import cd.casic.ci.process.dto.resp.sast.*;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.target.TargetVersion;
import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
import cd.casic.ci.process.process.service.pipeline.PipelineService;
import cd.casic.ci.process.process.service.sast.SastService;
import cd.casic.ci.process.process.service.target.TargetVersionService;
import cd.casic.ci.process.process.service.vulInfo.VulInfoService;
import cd.casic.ci.process.properties.SastProperties;
import cd.casic.framework.commons.exception.ServiceException;
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
import cd.casic.framework.commons.pojo.PageParam;
import cd.casic.framework.commons.pojo.PageResult;
import cd.casic.framework.redis.core.RedisTemplateUtils;
@ -20,6 +28,7 @@ import org.luaj.vm2.ast.Str;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.*;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
@ -31,9 +40,7 @@ import java.io.InputStream;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import static cd.casic.ci.process.constant.SastUrlConstant.*;
@ -46,10 +53,17 @@ public class SastServiceImpl implements SastService {
private SastProperties sastProperties;
@Resource
private RestTemplate restTemplate;
@Resource
private PipelineService pipelineService;
@Resource
private TargetVersionService targetVersionService;
@Resource
private VulInfoService vulInfoService;
public static final String TOKEN_PREFIX = "Bearer ";
public static final String TOKEN_HEADER_KEY = "authorization";
public static final String REDIS_SAST_TOKEN_KEY = "REDIS_SAST_TOKEN_KEY";
private SastTokenResp getTokenRemote(){
HttpHeaders httpHeaders = new HttpHeaders();
HttpEntity<SastProperties> httpEntity = new HttpEntity<SastProperties>(sastProperties,httpHeaders);
@ -299,6 +313,19 @@ public class SastServiceImpl implements SastService {
@Override
public SastApplicationBinaryStashResp applicationBinaryStash(SastApplicationBinaryStashReq req) {
HttpHeaders httpHeaders = getHeaders();
req.setProjectId("893ed995-5b81-474a-96a9-2800281421cd");
req.setApplicationName("二进制任务"+ UUID.randomUUID().toString());
req.setCodeSourceFrom("BINARY");
req.setArchAutoIdentify(true);
String pipelineId = req.getPipelineId();
PipPipeline pipeline = pipelineService.getById(pipelineId);
String targetVersionId = pipeline.getTargetVersionId();
TargetVersion targetVersion = targetVersionService.getById(targetVersionId);
File targetFile=new File(targetVersion.getFilePath());
SastFileUploadResp uploadResp = uploadFile(targetFile);
String id = uploadResp.getId();
req.setFileId(Collections.singletonList(id));
log.info("sast二进制上传文件成功，开始调用接口{}",req);
HttpEntity<SastApplicationBinaryStashReq> entity = new HttpEntity<>(req,httpHeaders);
httpHeaders.setContentType(MediaType.APPLICATION_JSON);
ResponseEntity<SastApplicationBinaryStashResp> exchange = restTemplate.exchange(sastProperties.getBaseUrl() +applicationBinaryStash, HttpMethod.POST,entity,SastApplicationBinaryStashResp.class,new HashMap<>());
@ -315,9 +342,56 @@ public class SastServiceImpl implements SastService {
return exchange.getBody();
}
@Override
public void saveReportVulInfo(List<SastVulInfoReq> list,String targetType,String targetName,String city,String instanceId,String taskId,String taskType) {
if (CollectionUtils.isEmpty(list)) {
return;
}
List<VulInfo> vulInfos = new ArrayList<>(list.size());
for (SastVulInfoReq req : list) {
VulInfo vulInfo = new VulInfo();
// 设置安全等级、名称
vulInfo.setVulTitle(req.getName());
String severity = req.getSeverity();
List<SastVulInfoReq.DefectDetail> defectDetails = req.getDefectDetails();
// 设置描述修复措施
if (CollectionUtils.isEmpty(defectDetails)) {
continue;
}
//设置流水线相关属性,因为漏洞描述还有漏洞修复建议列表里面每一项都是一样的，所以直接取第一个
SastVulInfoReq.DefectDetail first = defectDetails.getFirst();
String description = first.getDescription();
String potentialRisk = first.getPotentialRisk();
vulInfo.setSolution(potentialRisk);
vulInfo.setVulDescription(description);
vulInfo.setSeverity(securityLevelToSeverity(severity));
vulInfo.setTargetType(targetType);
vulInfo.setTargetName(targetName);
vulInfo.setCity(city);
vulInfo.setInstanceId(instanceId);
vulInfo.setTaskId(taskId);
vulInfo.setTaskType(taskType);
vulInfos.add(vulInfo);
}
vulInfoService.saveBatch(vulInfos);
}
private HttpHeaders getHeaders(){
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.set(TOKEN_HEADER_KEY,TOKEN_PREFIX+getToken());
return httpHeaders;
}
private Integer securityLevelToSeverity(String securityLevel) {
if (securityLevel == null) {
return 0;
}
// 映射规则：严重 -> 1, 高危 -> 2, 中危 -> 3, 低危 -> 4
return switch (securityLevel) {
case "严重" -> 1;
case "高危" -> 2;
case "中危" -> 3;
case "安全", "低危" -> 4;
default -> 0;
};
}
}

8
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/util/WebFrameworkUtils.java
View File

@ -40,6 +40,14 @@ public class WebFrameworkUtils {
return null;
}
}
public static LoginUser getLoginUser() {
try {
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
return (LoginUser) authentication.getPrincipal();
} catch (Exception e){
return null;
}
}
public static String getLoginUserIdStr(){
Long loginUserId = getLoginUserId();
if (loginUserId!=null) {

10
ops-server/src/test/java/cd/casic/server/VulInfoTest.java
View File

@ -1,13 +1,17 @@
package cd.casic.server;
import cd.casic.ci.process.constant.PipelineTargetTypeConstant;
import cd.casic.ci.process.dto.req.sast.SastVulInfoReq;
import cd.casic.ci.process.dto.req.testCase.TestCaseAITaskCreateReq;
import cd.casic.ci.process.dto.resp.ScaVulInfoResp;
import cd.casic.ci.process.process.converter.VulInfoConverter;
import cd.casic.ci.process.process.dataObject.target.TargetVersion;
import cd.casic.ci.process.process.dataObject.volumnInfo.VulInfo;
import cd.casic.ci.process.process.service.sast.SastService;
import cd.casic.ci.process.process.service.testCase.TestCaseAIGeneratorService;
import cd.casic.ci.process.process.service.vulInfo.VulInfoService;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
@ -34,6 +38,8 @@ public class VulInfoTest {
private VulInfoService vulInfoService;
@Resource
private VulInfoConverter converter;
@Resource
private SastService sastService;
@Test
public void test() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
System.out.println(vulInfoService.scaVulCountGet(681));
@ -61,5 +67,9 @@ public class VulInfoTest {
});
vulInfoService.saveBatch(vulInfos);
}
@Test
public void sastTest(){
}
}