mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of http://1.14.125.6:3000/mianbin/ops-pro

Browse Source
This commit is contained in:
even 2025-08-01 16:14:04 +08:00
commit 4f57e2eb86
20 changed files with 97 additions and 553 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
modules/module-ci-process-api/src/main/java/cd/casic/ci/api/AflManagerController.java
View File

@ -49,15 +49,6 @@ public class AflManagerController {
}
@PostMapping(path="/testSaveAflInfo")
public CommonResult<String> testSaveAflInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
String groupIdentifier = aflInfoService.testSaveAflInfo(req);
return CommonResult.success(groupIdentifier);
}
@PostMapping(path="/findAflInfoNow")
public CommonResult<AflInfoResp> findAflInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
@ -85,15 +76,6 @@ public class AflManagerController {
}
@PostMapping(path="/testSaveAflCrashesInfo")
public CommonResult<String> testSaveAflCrashesInfo(@RequestBody @Valid AflManagerReq req){
String groupIdentifier = aflCrashesInfoService.testSaveAflCrashesInfo(req);
return CommonResult.success(groupIdentifier);
}
@PostMapping(path="/findCrashesCount")
public CommonResult<AflCrashesResp> findCrashesCount(@RequestBody @Valid AflManagerReq req){
@ -140,16 +122,6 @@ public class AflManagerController {
@PostMapping(path="/testSaveAflSeedInfo")
public CommonResult<Void> testSaveAflSeedInfo(@RequestBody @Valid AflManagerReq req){
aflSeedInfoService.testSaveAflSeedInfo(req);
return CommonResult.success();
}
@PostMapping(path="/saveAflPlotInfo")
public CommonResult<String> saveAflPlotInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
@ -159,16 +131,6 @@ public class AflManagerController {
}
@PostMapping(path="/testSaveAflPlotInfo")
public CommonResult<String> testSaveAflPlotInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
String groupIdentifier = aflPlotInfoService.testSaveAflPlotInfo(req);
return CommonResult.success(groupIdentifier);
}
@PostMapping(path="/findAflPlotInfoNow")
public CommonResult<AflPlotInfoResp> findAflPlotInfoNow(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {

10
modules/module-ci-process-api/src/main/java/cd/casic/ci/api/LocalAflCrashesController.java
View File

@ -15,7 +15,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/LocalAflCrashes")
@RequestMapping("/localAflCrashes")
public class LocalAflCrashesController {
@Resource
private LocalAflCrashesService localAflCrashesService;
@ -29,14 +29,6 @@ public class LocalAflCrashesController {
}
@PostMapping(path="/testSaveLocalAflCrashesInfo")
public CommonResult<Void> testSaveLocalAflCrashesInfo(@RequestBody @Valid LocalAflCrashesSaveReq req){
localAflCrashesService.testSaveLocalAflCrashesInfo(req);
return CommonResult.success();
}
/**
*分页模糊查询漏洞信息
*/

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflInfoResp.java
View File

@ -270,6 +270,6 @@ public class AflInfoResp {
/**
* 文件地址路径
*/
private String filePath;
private byte[] fileContent;
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflPlotInfoResp.java
View File

@ -34,7 +34,7 @@ public class AflPlotInfoResp {
/**
* 文件地址路径
*/
private String filePath;
private byte[] fileContent;
/**
* 相对时间

7
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java
View File

@ -3,10 +3,8 @@ package cd.casic.ci.process.engine.worker.afl;
import cd.casic.ci.process.common.WorkAtom;
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
import cd.casic.ci.process.dto.req.localAflCrashes.LocalAflCrashesSaveReq;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.resource.ResourceDetailResp;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.manager.PostHandlerManager;
import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler;
import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker;
@ -24,9 +22,6 @@ import com.alibaba.fastjson.JSON;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.util.Map;
@ -150,7 +145,7 @@ public class AFLWorker extends DockerWorker {
} catch (SftpUploadUtil.SftpUploadException e) {
log.error("保存afl信息失败,入参{}", JSON.toJSONString(req),e);
}
aflSeedInfoService.saveAflSeedInfo(req);
// aflSeedInfoService.saveAflSeedInfo(req);
try {
aflPlotInfoService.saveAflPlotInfo(req);
} catch (SftpUploadUtil.SftpUploadException e) {

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflCrashesInfo.java
View File

@ -24,7 +24,7 @@ public class AflCrashesInfo extends PipBaseElement {
/**
* 文件地址路径
*/
private String filePath;
private byte[] fileContent;
/**
* 崩溃文件名称

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflInfo.java
View File

@ -268,5 +268,5 @@ public class AflInfo extends PipBaseElement {
/**
* 文件地址路径
*/
private String filePath;
private byte[] fileContent;
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflPlotInfo.java
View File

@ -32,7 +32,7 @@ public class AflPlotInfo extends PipBaseElement {
/**
* 文件地址路径
*/
private String filePath;
private byte[] fileContent;
/**
* 相对时间

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflSeedInfo.java
View File

@ -29,7 +29,7 @@ public class AflSeedInfo extends PipBaseElement {
/**
* 文件地址路径
*/
private String filePath;
private byte[] fileContent;
/**

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflCrashesInfoService.java
View File

@ -28,6 +28,4 @@ public interface AflCrashesInfoService extends IService<AflCrashesInfo> {
AflCrashesResp findCrashesCount(@Valid AflManagerReq req);
List<AflCrashesInfoResp> findCrashesNow(@Valid AflManagerReq req);
String testSaveAflCrashesInfo(@Valid AflManagerReq req);
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflInfoService.java
View File

@ -28,6 +28,4 @@ public interface AflInfoService extends IService<AflInfo> {
List<AflInfoResp> findAflInfoListByHistory(@Valid AflManagerReq req);
AflBaseInfoResp findAflBaseInfoList(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException;
String testSaveAflInfo(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException;
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflPlotInfoService.java
View File

@ -25,6 +25,4 @@ public interface AflPlotInfoService extends IService<AflPlotInfo> {
List<AflPlotInfoResp> findAflPlotInfoListByHistory(@Valid AflManagerReq req);
void updateHistoryPipelineIdByAflPlotInfo(@Valid AflManagerReq req);
String testSaveAflPlotInfo(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException;
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflSeedInfoService.java
View File

@ -23,6 +23,4 @@ public interface AflSeedInfoService extends IService<AflSeedInfo> {
void saveAflSeedInfo(@Valid AflManagerReq req);
void updateHistoryPipelineIdByAflSeedInfo(@Valid AflManagerReq req);
void testSaveAflSeedInfo(@Valid AflManagerReq req);
}

197
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflCrashesInfoServiceImpl.java
View File

@ -31,7 +31,6 @@ import org.springframework.util.ObjectUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
* @author HopeLi
@ -91,27 +90,27 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl<AflCrashesInfoDao, Af
resourceMachine.getUsername(),
resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (CollectionUtils.isEmpty(resultList)){
return null;
}
if (!CollectionUtils.isEmpty(resultList)){
//解析文件名和创建时间
for (String o : resultList) {
String[] lineParts = o.split("\\s+创建时间:\\s+");
if (lineParts.length < 2) {
System.err.println("无效格式: " + o);
break;
}
AflCrashesInfo aflCrashesInfo = new AflCrashesInfo();
//解析文件名和创建时间
for (String o : resultList) {
String[] lineParts = o.split("\\s+创建时间:\\s+");
if (lineParts.length < 2) {
System.err.println("无效格式: " + o);
break;
String fileName = lineParts[0].substring(2).trim(); // 文件信息部分
String createTime = lineParts[1].trim(); // 创建时间
aflCrashesInfo.setCrashesName(fileName);
aflCrashesInfo.setCrashesCreateTime(createTime);
aflCrashesInfos.add(aflCrashesInfo);
}
AflCrashesInfo aflCrashesInfo = new AflCrashesInfo();
String fileName = lineParts[0].substring(2).trim(); // 文件信息部分
String createTime = lineParts[1].trim(); // 创建时间
aflCrashesInfo.setCrashesName(fileName);
aflCrashesInfo.setCrashesCreateTime(createTime);
aflCrashesInfos.add(aflCrashesInfo);
}
// 步驟2.查询文件大小
List<String> fileResultList = SftpUploadUtil.findFileByteByCommand(
resourceMachine.getHostIp(),
@ -119,43 +118,47 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl<AflCrashesInfoDao, Af
resourceMachine.getUsername(),
resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (CollectionUtils.isEmpty(fileResultList)){
return null;
}
if (!CollectionUtils.isEmpty(fileResultList) && !CollectionUtils.isEmpty(aflCrashesInfos)){
//解析文件名和文件大小
for (String o : fileResultList) {
// 按空白字符分割
String[] parts = o.trim().split("\\s+");
//解析文件名和文件大小
for (String o : fileResultList) {
// 按空白字符分割
String[] parts = o.trim().split("\\s+");
String fileName = parts[0]; // 文件名称
Integer fileLength = Integer.parseInt(parts[1]); // 文件大小
if (!CollectionUtils.isEmpty(aflCrashesInfos)){
String fileName = parts[0]; // 文件名称
Integer fileLength = Integer.parseInt(parts[1]); // 文件大小
for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(fileName)) {
aflCrashesInfo.setCrashesLength(fileLength);
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
}
aflCrashesInfo.setGroupIdentifier(uuid);
}
}
}
}
// 步骤3：列出源目录下的所有文件
List<String> files = SftpUploadUtil.listFilesInRemoteDirectory(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (!CollectionUtils.isEmpty(files)) {
// 步骤4：批量复制文件到目标目录
Map<String,String> copiedFiles = SftpUploadUtil.copyRemoteFilesToLocalMap(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/crashes_result/" + UUID.randomUUID() + "/");
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
// 步骤4：批量下载文件流与文件绑定
Map<String,byte[]> copiedFiles = SftpUploadUtil.copyRemoteFilesToByteArray(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files);
System.out.println("共下载 " + copiedFiles.size() + " 个文件");
//绑定对应的文件路径
if (!CollectionUtils.isEmpty(copiedFiles)){
for (Map.Entry<String, String> entry : copiedFiles.entrySet()) {
//绑定对应的文件流
if (!ObjectUtils.isEmpty(copiedFiles)){
for (Map.Entry<String, byte[]> entry : copiedFiles.entrySet()) {
if (!CollectionUtils.isEmpty(aflCrashesInfos)){
for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) {
aflCrashesInfo.setFilePath(entry.getValue());
aflCrashesInfo.setFileContent(entry.getValue());
}
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
@ -361,124 +364,4 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl<AflCrashesInfoDao, Af
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"crashes入库失败");
}
}
@Override
public String testSaveAflCrashesInfo(AflManagerReq req) {
List<AflCrashesInfo> aflCrashesInfos = new ArrayList<>(0);
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
String uuid = req.getGroupIdentifier();
ResourceDetailResp resourceById = resourceManagerService.findResourceDetailById("735516490182299648");
if (resourceById == null || resourceById.getMachineInfo() == null || resourceById.getMachineInfo().getAuthenticationType()!=1) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
}
MachineInfoDO resourceMachine = resourceById.getMachineInfo();
String password = CryptogramUtil.doDecrypt(resourceMachine.getPassword());
resourceMachine.setPassword(password);
try {
// 步骤1：使用自定义sql获取崩溃文件的创建时间
//查询创建时间
List<String> resultList = SftpUploadUtil.findCreateTimeByCommand(
resourceMachine.getHostIp(),
resourceMachine.getSshPort(),
resourceMachine.getUsername(),
resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (CollectionUtils.isEmpty(resultList)){
return null;
}
//解析文件名和创建时间
for (String o : resultList) {
String[] lineParts = o.split("\\s+创建时间:\\s+");
if (lineParts.length < 2) {
System.err.println("无效格式: " + o);
break;
}
AflCrashesInfo aflCrashesInfo = new AflCrashesInfo();
String fileName = lineParts[0].substring(2).trim(); // 文件信息部分
String createTime = lineParts[1].trim(); // 创建时间
aflCrashesInfo.setCrashesName(fileName);
aflCrashesInfo.setCrashesCreateTime(createTime);
aflCrashesInfos.add(aflCrashesInfo);
}
// 步驟2.查询文件大小
List<String> fileResultList = SftpUploadUtil.findFileByteByCommand(
resourceMachine.getHostIp(),
resourceMachine.getSshPort(),
resourceMachine.getUsername(),
resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (CollectionUtils.isEmpty(fileResultList)){
return null;
}
//解析文件名和文件大小
for (String o : fileResultList) {
// 按空白字符分割
String[] parts = o.trim().split("\\s+");
String fileName = parts[0]; // 文件名称
Integer fileLength = Integer.parseInt(parts[1]); // 文件大小
if (!CollectionUtils.isEmpty(aflCrashesInfos)){
for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(fileName)) {
aflCrashesInfo.setCrashesLength(fileLength);
}
}
}
}
// 步骤3：列出源目录下的所有文件
List<String> files = SftpUploadUtil.listFilesInRemoteDirectory(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (!CollectionUtils.isEmpty(files)) {
// 步骤4：批量复制文件到目标目录
Map<String,String> copiedFiles = SftpUploadUtil.copyRemoteFilesToLocalMap(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/crashes_result/" + UUID.randomUUID() + "/");
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
//绑定对应的文件路径
if (!CollectionUtils.isEmpty(copiedFiles)){
for (Map.Entry<String, String> entry : copiedFiles.entrySet()) {
if (!CollectionUtils.isEmpty(aflCrashesInfos)){
for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) {
aflCrashesInfo.setFilePath(entry.getValue());
}
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
}
if (!StringUtils.isEmpty(req.getPipelineHistoryId())){
aflCrashesInfo.setPipelineHistoryId(req.getPipelineHistoryId());
}
aflCrashesInfo.setGroupIdentifier(uuid);
}
}
}
//新增数据到crashes表中
baseMapper.insertBatch(aflCrashesInfos);
}
return uuid;
}else {
return null;
}
} catch (Exception e) {
e.printStackTrace();
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"crashes入库失败");
}
}
}

43
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflInfoServiceImpl.java
View File

@ -31,7 +31,7 @@ import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.Map;
/**
* @author HopeLi
@ -76,12 +76,12 @@ public class AflInfoServiceImpl extends ServiceImpl<AflInfoDao, AflInfo> impleme
MachineInfoDO resourceMachine = resourceById.getMachineInfo();
resourceMachine.setPassword(CryptogramUtil.doDecrypt(resourceMachine.getPassword()));
AflInfo aflInfo = SftpUploadUtil.downloadFileSftpForInputStreamAndSetAflInfo(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats");
//在本地创建一个新文件夹，将远程文件复制一份过去
//获取远程文件流，并保存到数据库
List<String> sourceFilePaths = new ArrayList<>();
sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats");
List<String> strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/fuzzer_result/" + UUID.randomUUID() + "/");
Map<String, byte[]> stringMap = SftpUploadUtil.copyRemoteFilesToByteArray(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths);
aflInfo.setFilePath(strings.get(0));
aflInfo.setFileContent(stringMap.get(sourceFilePaths.get(0)));
aflInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflInfo.setTaskId(req.getTaskId());
@ -228,39 +228,4 @@ public class AflInfoServiceImpl extends ServiceImpl<AflInfoDao, AflInfo> impleme
return resp;
}
@Override
public String testSaveAflInfo(AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
ResourceDetailResp resourceById = resourceManagerService.findResourceDetailById("735516490182299648");
if (resourceById == null || resourceById.getMachineInfo() == null) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
}
MachineInfoDO resourceMachine = resourceById.getMachineInfo();
resourceMachine.setPassword(CryptogramUtil.doDecrypt(resourceMachine.getPassword()));
AflInfo aflInfo = SftpUploadUtil.downloadFileSftpForInputStreamAndSetAflInfo(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats");
//在本地创建一个新文件夹，将远程文件复制一份过去
List<String> sourceFilePaths = new ArrayList<>();
sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats");
List<String> strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/fuzzer_result/" + UUID.randomUUID() + "/");
aflInfo.setFilePath(strings.get(0));
aflInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflInfo.setTaskId(req.getTaskId());
}
if (!StringUtils.isEmpty(req.getPipelineHistoryId())){
aflInfo.setPipelineHistoryId(req.getPipelineHistoryId());
}
aflInfo.setGroupIdentifier(req.getGroupIdentifier());
baseMapper.insert(aflInfo);
return aflInfo.getGroupIdentifier();
}
}

44
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflPlotInfoServiceImpl.java
View File

@ -27,7 +27,7 @@ import org.springframework.util.ObjectUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.Map;
/**
* @author HopeLi
@ -72,12 +72,13 @@ public class AflPlotInfoServiceImpl extends ServiceImpl<AflPlotInfoDao, AflPlotI
MachineInfoDO resourceMachine = resourceById.getMachineInfo();
resourceMachine.setPassword(CryptogramUtil.doDecrypt(resourceMachine.getPassword()));
AflPlotInfo aflPlotInfo = SftpUploadUtil.downloadFileSftpForLastLineAndSetAflPlotInfo(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data");
//在本地创建一个新文件夹，将远程文件复制一份过去
//获取远程文件流，并保存到数据库
List<String> sourceFilePaths = new ArrayList<>();
sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data");
List<String> strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/plot_result/" + UUID.randomUUID() + "/");
Map<String, byte[]> stringMap = SftpUploadUtil.copyRemoteFilesToByteArray(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths);
aflPlotInfo.setFilePath(strings.get(0));
aflPlotInfo.setFileContent(stringMap.get(sourceFilePaths.get(0)));
aflPlotInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflPlotInfo.setTaskId(req.getTaskId());
@ -160,39 +161,4 @@ public class AflPlotInfoServiceImpl extends ServiceImpl<AflPlotInfoDao, AflPlotI
return AflManagerConverter.INSTANCE.toAflPlotInfoRespList(aflPlotInfos);
}
@Override
public String testSaveAflPlotInfo(AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
ResourceDetailResp resourceById = resourceManagerService.findResourceDetailById("735516490182299648");
if (resourceById == null || resourceById.getMachineInfo() == null) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
}
MachineInfoDO resourceMachine = resourceById.getMachineInfo();
resourceMachine.setPassword(CryptogramUtil.doDecrypt(resourceMachine.getPassword()));
AflPlotInfo aflPlotInfo = SftpUploadUtil.downloadFileSftpForLastLineAndSetAflPlotInfo(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data");
//在本地创建一个新文件夹，将远程文件复制一份过去
List<String> sourceFilePaths = new ArrayList<>();
sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data");
List<String> strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/plot_result/" + UUID.randomUUID() + "/");
aflPlotInfo.setFilePath(strings.get(0));
aflPlotInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflPlotInfo.setTaskId(req.getTaskId());
}
if (!StringUtils.isEmpty(req.getPipelineHistoryId())){
aflPlotInfo.setPipelineHistoryId(req.getPipelineHistoryId());
}
aflPlotInfo.setGroupIdentifier(req.getGroupIdentifier());
baseMapper.insert(aflPlotInfo);
return aflPlotInfo.getGroupIdentifier();
}
}

120
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflSeedInfoServiceImpl.java
View File

@ -30,7 +30,7 @@ import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.Map;
/**
* @author HopeLi
@ -202,22 +202,22 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + seedFilePath);
if (!CollectionUtils.isEmpty(files)) {
// 步骤2：批量复制文件到目标目录
List<String> copiedFiles = SftpUploadUtil.copyRemoteFilesToLocal(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/seed_result/" + UUID.randomUUID() + "/");
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
// 步骤2：下载文件流并保存到数据库
Map<String,byte[]> copiedFiles = SftpUploadUtil.copyRemoteFilesToByteArray(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files);
System.out.println("共下载 " + copiedFiles.size() + " 个文件");
List<AflSeedInfo> aflSeedInfos = new ArrayList<>(0);
if (!CollectionUtils.isEmpty(copiedFiles)){
if (!ObjectUtils.isEmpty(copiedFiles)){
//新增数据到seed表中
copiedFiles.forEach(o->{
for (Map.Entry<String, byte[]> entry : copiedFiles.entrySet()) {
String key = entry.getKey();
byte[] value = entry.getValue();
AflSeedInfo aflSeedInfo = new AflSeedInfo();
// 提取文件名部分（seed_result/之后的内容）
String fileNamePart = o.substring(o.indexOf("seed_result/") + 12);
//截取最后一个斜杠后的内容，即文件信息部分
fileNamePart = fileNamePart.substring(fileNamePart.lastIndexOf("/") + 1);
// 提取文件名部分（queue/之后的内容）
String fileNamePart = key.substring(key.indexOf("queue/") + 6);
// 使用正则匹配 id: 开头的内容
java.util.regex.Pattern patternId = java.util.regex.Pattern.compile("id:([^,]+)");
@ -237,7 +237,7 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
aflSeedInfo.setSeedId(idValue);
}
aflSeedInfo.setFilePath(o);
aflSeedInfo.setFileContent(value);
if (!StringUtils.isEmpty(req.getTaskId())){
aflSeedInfo.setTaskId(req.getTaskId());
}
@ -248,7 +248,7 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
aflSeedInfo.setGroupIdentifier(uuid);
aflSeedInfos.add(aflSeedInfo);
});
}
}
if (!CollectionUtils.isEmpty(aflSeedInfos)){
@ -286,96 +286,4 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
}
baseMapper.updateBatch(aflSeedInfos);
}
@Override
public void testSaveAflSeedInfo(AflManagerReq req) {
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
String uuid = req.getGroupIdentifier();
ResourceDetailResp resourceById = resourceManagerService.findResourceDetailById("735516490182299648");
if (resourceById == null || resourceById.getMachineInfo() == null) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
}
MachineInfoDO resourceMachine = resourceById.getMachineInfo();
String password = CryptogramUtil.doDecrypt(resourceMachine.getPassword());
resourceMachine.setPassword(password);
try {
// 步骤1：列出源目录下的所有文件
List<String> files = SftpUploadUtil.listFilesInRemoteDirectory(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + seedFilePath);
if (!CollectionUtils.isEmpty(files)) {
// 步骤2：批量复制文件到目标目录
List<String> copiedFiles = SftpUploadUtil.copyRemoteFilesToLocal(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/seed_result/" + UUID.randomUUID() + "/");
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
List<AflSeedInfo> aflSeedInfos = new ArrayList<>(0);
if (!CollectionUtils.isEmpty(copiedFiles)){
//新增数据到seed表中
copiedFiles.forEach(o->{
AflSeedInfo aflSeedInfo = new AflSeedInfo();
// 提取文件名部分（seed_result/之后的内容）
String fileNamePart = o.substring(o.indexOf("seed_result/") + 12);
//截取最后一个斜杠后的内容，即文件信息部分
fileNamePart = fileNamePart.substring(fileNamePart.lastIndexOf("/") + 1);
// 使用正则匹配 id: 开头的内容
java.util.regex.Pattern patternId = java.util.regex.Pattern.compile("id:([^,]+)");
java.util.regex.Matcher matcherId = patternId.matcher(fileNamePart);
// 使用正则匹配 rep: 开头的内容
java.util.regex.Pattern patternRep = java.util.regex.Pattern.compile("rep:(\\d+)");
java.util.regex.Matcher matcherRep = patternRep.matcher(fileNamePart);
if (matcherRep.find()) {
String repValue = matcherRep.group(1);
aflSeedInfo.setSeedAmount(repValue);
}
if (matcherId.find()) {
String idValue = matcherId.group(1);
aflSeedInfo.setSeedId(idValue);
}
aflSeedInfo.setFilePath(o);
if (!StringUtils.isEmpty(req.getTaskId())){
aflSeedInfo.setTaskId(req.getTaskId());
}
if (!StringUtils.isEmpty(req.getPipelineHistoryId())){
aflSeedInfo.setPipelineHistoryId(req.getPipelineHistoryId());
}
aflSeedInfo.setPipelineId(req.getPipelineId());
aflSeedInfo.setGroupIdentifier(uuid);
aflSeedInfos.add(aflSeedInfo);
});
}
if (!CollectionUtils.isEmpty(aflSeedInfos)){
//查询种子运行分钟数，同时入库
List<SeedsCountResp> seedsCount = this.findSeedsCount(req);
if (!CollectionUtils.isEmpty(seedsCount)){
aflSeedInfos.forEach(o->{
seedsCount.forEach(j->{
if (o.getSeedId().equals(j.getSeedId())){
o.setMinutesDifference(j.getMinutesDifference());
}
});
});
}
baseMapper.insertBatch(aflSeedInfos);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/LocalAflCrashesService.java
View File

@ -21,5 +21,4 @@ public interface LocalAflCrashesService extends IService<LocalAflCrashesInfo> {
PageResult<LocalAflCrashesResp> getLocalAflCrashesSearch(LocalAflCrashesQueryReq req);
void testSaveLocalAflCrashesInfo(@Valid LocalAflCrashesSaveReq req);
}

155
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/impl/LocalAflCrashesServiceImpl.java
View File

@ -98,7 +98,7 @@ public class LocalAflCrashesServiceImpl extends ServiceImpl<LocalAflCrashesDao,
}
LocalAflCrashesInfo aflCrashesInfo = new LocalAflCrashesInfo();
String fileName = lineParts[0].substring(2).trim(); // 文件信息部分
String fileName = lineParts[0].substring(0).trim(); // 文件信息部分
String createTime = lineParts[1].trim(); // 创建时间
aflCrashesInfo.setCrashesName(fileName);
aflCrashesInfo.setCrashesCreateTime(createTime);
@ -127,12 +127,13 @@ public class LocalAflCrashesServiceImpl extends ServiceImpl<LocalAflCrashesDao,
for (LocalAflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(fileName)) {
aflCrashesInfo.setCrashesLength(fileLength);
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
}
aflCrashesInfo.setGroupIdentifier(uuid);
}
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
}
aflCrashesInfo.setGroupIdentifier(uuid);
}
}
}
@ -147,25 +148,26 @@ public class LocalAflCrashesServiceImpl extends ServiceImpl<LocalAflCrashesDao,
// 步骤4：批量下载文件流与文件绑定
Map<String,byte[]> copiedFiles = SftpUploadUtil.copyRemoteFilesToByteArray(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files);
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
System.out.println("共下载 " + copiedFiles.size() + " 个文件");
//绑定对应的文件流
if (!CollectionUtils.isEmpty(copiedFiles)){
if (!ObjectUtils.isEmpty(copiedFiles)){
for (Map.Entry<String, byte[]> entry : copiedFiles.entrySet()) {
if (!CollectionUtils.isEmpty(aflCrashesInfos)){
for (LocalAflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) {
aflCrashesInfo.setFileContent(entry.getValue());
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
}
aflCrashesInfo.setTaskType("AFL");
aflCrashesInfo.setInstanceId(req.getInstanceId());
aflCrashesInfo.setTargetType(req.getTargetType());
aflCrashesInfo.setTargetName(req.getTargetName());
aflCrashesInfo.setCity(req.getCity());
}
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
}
aflCrashesInfo.setTaskType("AFL");
aflCrashesInfo.setInstanceId(req.getInstanceId());
aflCrashesInfo.setTargetType(req.getTargetType());
aflCrashesInfo.setTargetName(req.getTargetName());
aflCrashesInfo.setCity(req.getCity());
}
}
}
@ -210,123 +212,4 @@ public class LocalAflCrashesServiceImpl extends ServiceImpl<LocalAflCrashesDao,
return result;
}
@Override
public void testSaveLocalAflCrashesInfo(LocalAflCrashesSaveReq req) {
List<LocalAflCrashesInfo> aflCrashesInfos = new ArrayList<>(0);
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
String uuid = req.getGroupIdentifier();
ResourceDetailResp resourceById = resourceManagerService.findResourceDetailById("735516490182299648");
if (resourceById == null || resourceById.getMachineInfo() == null || resourceById.getMachineInfo().getAuthenticationType()!=1) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
}
MachineInfoDO resourceMachine = resourceById.getMachineInfo();
String password = CryptogramUtil.doDecrypt(resourceMachine.getPassword());
resourceMachine.setPassword(password);
try {
// 步骤1：使用自定义sql获取崩溃文件的创建时间
//查询创建时间
List<String> resultList = SftpUploadUtil.findCreateTimeByCommand(
resourceMachine.getHostIp(),
resourceMachine.getSshPort(),
resourceMachine.getUsername(),
resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (!CollectionUtils.isEmpty(resultList)){
//解析文件名和创建时间
for (String o : resultList) {
String[] lineParts = o.split("\\s+创建时间:\\s+");
if (lineParts.length < 2) {
System.err.println("无效格式: " + o);
break;
}
LocalAflCrashesInfo aflCrashesInfo = new LocalAflCrashesInfo();
String fileName = lineParts[0].substring(2).trim(); // 文件信息部分
String createTime = lineParts[1].trim(); // 创建时间
aflCrashesInfo.setCrashesName(fileName);
aflCrashesInfo.setCrashesCreateTime(createTime);
aflCrashesInfos.add(aflCrashesInfo);
}
}
// 步驟2.查询文件大小
List<String> fileResultList = SftpUploadUtil.findFileByteByCommand(
resourceMachine.getHostIp(),
resourceMachine.getSshPort(),
resourceMachine.getUsername(),
resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (!CollectionUtils.isEmpty(fileResultList) && !CollectionUtils.isEmpty(aflCrashesInfos)){
//解析文件名和文件大小
for (String o : fileResultList) {
// 按空白字符分割
String[] parts = o.trim().split("\\s+");
String fileName = parts[0]; // 文件名称
Integer fileLength = Integer.parseInt(parts[1]); // 文件大小
for (LocalAflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(fileName)) {
aflCrashesInfo.setCrashesLength(fileLength);
}
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
}
aflCrashesInfo.setGroupIdentifier(uuid);
}
}
}
// 步骤3：列出源目录下的所有文件
List<String> files = SftpUploadUtil.listFilesInRemoteDirectory(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (!CollectionUtils.isEmpty(files)) {
// 步骤4：批量下载文件流与文件绑定
Map<String,byte[]> copiedFiles = SftpUploadUtil.copyRemoteFilesToByteArray(
resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files);
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
//绑定对应的文件流
if (!CollectionUtils.isEmpty(copiedFiles)){
for (Map.Entry<String, byte[]> entry : copiedFiles.entrySet()) {
if (!CollectionUtils.isEmpty(aflCrashesInfos)){
for (LocalAflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) {
aflCrashesInfo.setFileContent(entry.getValue());
}
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
}
aflCrashesInfo.setTaskType("AFL");
aflCrashesInfo.setInstanceId(req.getInstanceId());
aflCrashesInfo.setTargetType(req.getTargetType());
aflCrashesInfo.setTargetName(req.getTargetName());
aflCrashesInfo.setCity(req.getCity());
}
}
}
//新增数据到crashes表中
baseMapper.insertBatch(aflCrashesInfos);
}
}
} catch (Exception e) {
e.printStackTrace();
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"crashes入库失败");
}
}
}

15
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/util/SftpUploadUtil.java
View File

@ -1593,15 +1593,14 @@ public class SftpUploadUtil {
channelSftp = (ChannelSftp) channel;
String chmodDir = remoteFilePaths.get(0).substring(0, remoteFilePaths.get(0).indexOf("default"));
log.info("分配权限路径,{}", chmodDir);
// 切换目录并列出内容用于调试
sudoChmodORwx(session, chmodDir, password);
// 遍历远程文件路径并下载
for (String remoteFile : remoteFilePaths) {
String chmodDir = remoteFile.substring(0, remoteFile.indexOf("default"));
log.info("分配权限路径,{}", chmodDir);
// 切换目录并列出内容用于调试
sudoChmodORwx(session, chmodDir, password);
String fileName = new File(remoteFile).getName();
try (InputStream in = channelSftp.get(remoteFile);
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
byte[] buffer = new byte[1024];
@ -1609,7 +1608,7 @@ public class SftpUploadUtil {
while ((bytesRead = in.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
}
fileByteArrays.put(fileName, out.toByteArray()); // 添加文件名和二进制数据
fileByteArrays.put(remoteFile, out.toByteArray()); // 添加文件名和二进制数据
} catch (Exception e) {
throw new SftpUploadException("文件读取失败: " + remoteFile, e);
}