diff --git a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/AflManagerController.java b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/AflManagerController.java index 677ca4a2..63161481 100644 --- a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/AflManagerController.java +++ b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/AflManagerController.java @@ -49,15 +49,6 @@ public class AflManagerController { } - @PostMapping(path="/testSaveAflInfo") - public CommonResult testSaveAflInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException { - - String groupIdentifier = aflInfoService.testSaveAflInfo(req); - - return CommonResult.success(groupIdentifier); - } - - @PostMapping(path="/findAflInfoNow") public CommonResult findAflInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException { @@ -85,15 +76,6 @@ public class AflManagerController { } - @PostMapping(path="/testSaveAflCrashesInfo") - public CommonResult testSaveAflCrashesInfo(@RequestBody @Valid AflManagerReq req){ - - String groupIdentifier = aflCrashesInfoService.testSaveAflCrashesInfo(req); - - return CommonResult.success(groupIdentifier); - } - - @PostMapping(path="/findCrashesCount") public CommonResult findCrashesCount(@RequestBody @Valid AflManagerReq req){ @@ -140,16 +122,6 @@ public class AflManagerController { - @PostMapping(path="/testSaveAflSeedInfo") - public CommonResult testSaveAflSeedInfo(@RequestBody @Valid AflManagerReq req){ - - aflSeedInfoService.testSaveAflSeedInfo(req); - - return CommonResult.success(); - } - - - @PostMapping(path="/saveAflPlotInfo") public CommonResult saveAflPlotInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException { @@ -159,16 +131,6 @@ public class AflManagerController { } - - @PostMapping(path="/testSaveAflPlotInfo") - public CommonResult testSaveAflPlotInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException { - - String groupIdentifier = aflPlotInfoService.testSaveAflPlotInfo(req); - - return CommonResult.success(groupIdentifier); - } - - @PostMapping(path="/findAflPlotInfoNow") public CommonResult findAflPlotInfoNow(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException { diff --git a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/LocalAflCrashesController.java b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/LocalAflCrashesController.java index 65d7d811..a355fc5b 100644 --- a/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/LocalAflCrashesController.java +++ b/modules/module-ci-process-api/src/main/java/cd/casic/ci/api/LocalAflCrashesController.java @@ -15,7 +15,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @RestController -@RequestMapping("/LocalAflCrashes") +@RequestMapping("/localAflCrashes") public class LocalAflCrashesController { @Resource private LocalAflCrashesService localAflCrashesService; @@ -29,14 +29,6 @@ public class LocalAflCrashesController { } - @PostMapping(path="/testSaveLocalAflCrashesInfo") - public CommonResult testSaveLocalAflCrashesInfo(@RequestBody @Valid LocalAflCrashesSaveReq req){ - - localAflCrashesService.testSaveLocalAflCrashesInfo(req); - - return CommonResult.success(); - } - /** *分页模糊查询漏洞信息 */ diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflInfoResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflInfoResp.java index a8b25a5a..dae8f761 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflInfoResp.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflInfoResp.java @@ -270,6 +270,6 @@ public class AflInfoResp { /** * 文件地址路径 */ - private String filePath; + private byte[] fileContent; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflPlotInfoResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflPlotInfoResp.java index 828da62e..2486694e 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflPlotInfoResp.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflPlotInfoResp.java @@ -34,7 +34,7 @@ public class AflPlotInfoResp { /** * 文件地址路径 */ - private String filePath; + private byte[] fileContent; /** * 相对时间 diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java index 816acd82..5d4edafb 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java @@ -3,10 +3,8 @@ package cd.casic.ci.process.engine.worker.afl; import cd.casic.ci.process.common.WorkAtom; import cd.casic.ci.process.dto.req.aflManager.AflManagerReq; import cd.casic.ci.process.dto.req.localAflCrashes.LocalAflCrashesSaveReq; -import cd.casic.ci.process.dto.req.resource.ResourceQueryReq; import cd.casic.ci.process.dto.resp.resource.ResourceDetailResp; import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; -import cd.casic.ci.process.engine.manager.PostHandlerManager; import cd.casic.ci.process.engine.postHandler.ExecuteTaskPostHandler; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.DockerWorker; @@ -24,9 +22,6 @@ import com.alibaba.fastjson.JSON; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.scheduling.annotation.Async; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.util.CollectionUtils; import java.io.File; import java.util.Map; @@ -150,7 +145,7 @@ public class AFLWorker extends DockerWorker { } catch (SftpUploadUtil.SftpUploadException e) { log.error("保存afl信息失败,入参{}", JSON.toJSONString(req),e); } - aflSeedInfoService.saveAflSeedInfo(req); +// aflSeedInfoService.saveAflSeedInfo(req); try { aflPlotInfoService.saveAflPlotInfo(req); } catch (SftpUploadUtil.SftpUploadException e) { diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflCrashesInfo.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflCrashesInfo.java index ea4d4d1f..04e2b2cf 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflCrashesInfo.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflCrashesInfo.java @@ -24,7 +24,7 @@ public class AflCrashesInfo extends PipBaseElement { /** * 文件地址路径 */ - private String filePath; + private byte[] fileContent; /** * 崩溃文件名称 diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflInfo.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflInfo.java index 461738eb..e5debb9b 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflInfo.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflInfo.java @@ -268,5 +268,5 @@ public class AflInfo extends PipBaseElement { /** * 文件地址路径 */ - private String filePath; + private byte[] fileContent; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflPlotInfo.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflPlotInfo.java index e1f737dd..7757f810 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflPlotInfo.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflPlotInfo.java @@ -32,7 +32,7 @@ public class AflPlotInfo extends PipBaseElement { /** * 文件地址路径 */ - private String filePath; + private byte[] fileContent; /** * 相对时间 diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflSeedInfo.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflSeedInfo.java index 285ca533..725aac7b 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflSeedInfo.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflSeedInfo.java @@ -29,7 +29,7 @@ public class AflSeedInfo extends PipBaseElement { /** * 文件地址路径 */ - private String filePath; + private byte[] fileContent; /** diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflCrashesInfoService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflCrashesInfoService.java index 1cf4ea72..5a6f720c 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflCrashesInfoService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflCrashesInfoService.java @@ -28,6 +28,4 @@ public interface AflCrashesInfoService extends IService { AflCrashesResp findCrashesCount(@Valid AflManagerReq req); List findCrashesNow(@Valid AflManagerReq req); - - String testSaveAflCrashesInfo(@Valid AflManagerReq req); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflInfoService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflInfoService.java index 3569f5dc..1aab05ad 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflInfoService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflInfoService.java @@ -28,6 +28,4 @@ public interface AflInfoService extends IService { List findAflInfoListByHistory(@Valid AflManagerReq req); AflBaseInfoResp findAflBaseInfoList(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException; - - String testSaveAflInfo(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflPlotInfoService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflPlotInfoService.java index 3ef10402..eb352786 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflPlotInfoService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflPlotInfoService.java @@ -25,6 +25,4 @@ public interface AflPlotInfoService extends IService { List findAflPlotInfoListByHistory(@Valid AflManagerReq req); void updateHistoryPipelineIdByAflPlotInfo(@Valid AflManagerReq req); - - String testSaveAflPlotInfo(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException; } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflSeedInfoService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflSeedInfoService.java index dd0d54e7..e8c72567 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflSeedInfoService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflSeedInfoService.java @@ -23,6 +23,4 @@ public interface AflSeedInfoService extends IService { void saveAflSeedInfo(@Valid AflManagerReq req); void updateHistoryPipelineIdByAflSeedInfo(@Valid AflManagerReq req); - - void testSaveAflSeedInfo(@Valid AflManagerReq req); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflCrashesInfoServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflCrashesInfoServiceImpl.java index 2fdf0797..c2b8bacd 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflCrashesInfoServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflCrashesInfoServiceImpl.java @@ -31,7 +31,6 @@ import org.springframework.util.ObjectUtils; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.UUID; /** * @author HopeLi @@ -91,27 +90,27 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl fileResultList = SftpUploadUtil.findFileByteByCommand( resourceMachine.getHostIp(), @@ -119,43 +118,47 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl files = SftpUploadUtil.listFilesInRemoteDirectory( resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath); if (!CollectionUtils.isEmpty(files)) { - // 步骤4：批量复制文件到目标目录 - Map copiedFiles = SftpUploadUtil.copyRemoteFilesToLocalMap( - resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/crashes_result/" + UUID.randomUUID() + "/"); - System.out.println("共复制 " + copiedFiles.size() + " 个文件"); + // 步骤4：批量下载文件流与文件绑定 + Map copiedFiles = SftpUploadUtil.copyRemoteFilesToByteArray( + resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files); + System.out.println("共下载 " + copiedFiles.size() + " 个文件"); - //绑定对应的文件路径 - if (!CollectionUtils.isEmpty(copiedFiles)){ - for (Map.Entry entry : copiedFiles.entrySet()) { + //绑定对应的文件流 + if (!ObjectUtils.isEmpty(copiedFiles)){ + for (Map.Entry entry : copiedFiles.entrySet()) { if (!CollectionUtils.isEmpty(aflCrashesInfos)){ for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) { if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) { - aflCrashesInfo.setFilePath(entry.getValue()); + aflCrashesInfo.setFileContent(entry.getValue()); } aflCrashesInfo.setPipelineId(req.getPipelineId()); if (!StringUtils.isEmpty(req.getTaskId())){ @@ -361,124 +364,4 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl aflCrashesInfos = new ArrayList<>(0); - - if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){ - throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空"); - } - String uuid = req.getGroupIdentifier(); - - ResourceDetailResp resourceById = resourceManagerService.findResourceDetailById("735516490182299648"); - if (resourceById == null || resourceById.getMachineInfo() == null || resourceById.getMachineInfo().getAuthenticationType()!=1) { - throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误"); - } - - MachineInfoDO resourceMachine = resourceById.getMachineInfo(); - String password = CryptogramUtil.doDecrypt(resourceMachine.getPassword()); - resourceMachine.setPassword(password); - - try { - // 步骤1：使用自定义sql获取崩溃文件的创建时间 - //查询创建时间 - List resultList = SftpUploadUtil.findCreateTimeByCommand( - resourceMachine.getHostIp(), - resourceMachine.getSshPort(), - resourceMachine.getUsername(), - resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath); - - if (CollectionUtils.isEmpty(resultList)){ - return null; - } - - //解析文件名和创建时间 - for (String o : resultList) { - String[] lineParts = o.split("\\s+创建时间:\\s+"); - if (lineParts.length < 2) { - System.err.println("无效格式: " + o); - break; - } - AflCrashesInfo aflCrashesInfo = new AflCrashesInfo(); - - String fileName = lineParts[0].substring(2).trim(); // 文件信息部分 - String createTime = lineParts[1].trim(); // 创建时间 - aflCrashesInfo.setCrashesName(fileName); - aflCrashesInfo.setCrashesCreateTime(createTime); - - aflCrashesInfos.add(aflCrashesInfo); - } - - // 步驟2.查询文件大小 - List fileResultList = SftpUploadUtil.findFileByteByCommand( - resourceMachine.getHostIp(), - resourceMachine.getSshPort(), - resourceMachine.getUsername(), - resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath); - - if (CollectionUtils.isEmpty(fileResultList)){ - return null; - } - - //解析文件名和文件大小 - for (String o : fileResultList) { - // 按空白字符分割 - String[] parts = o.trim().split("\\s+"); - - String fileName = parts[0]; // 文件名称 - Integer fileLength = Integer.parseInt(parts[1]); // 文件大小 - if (!CollectionUtils.isEmpty(aflCrashesInfos)){ - for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) { - if (aflCrashesInfo.getCrashesName().equals(fileName)) { - aflCrashesInfo.setCrashesLength(fileLength); - } - } - } - } - - // 步骤3：列出源目录下的所有文件 - List files = SftpUploadUtil.listFilesInRemoteDirectory( - resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath); - - if (!CollectionUtils.isEmpty(files)) { - // 步骤4：批量复制文件到目标目录 - Map copiedFiles = SftpUploadUtil.copyRemoteFilesToLocalMap( - resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/crashes_result/" + UUID.randomUUID() + "/"); - System.out.println("共复制 " + copiedFiles.size() + " 个文件"); - - //绑定对应的文件路径 - if (!CollectionUtils.isEmpty(copiedFiles)){ - for (Map.Entry entry : copiedFiles.entrySet()) { - if (!CollectionUtils.isEmpty(aflCrashesInfos)){ - for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) { - if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) { - aflCrashesInfo.setFilePath(entry.getValue()); - } - aflCrashesInfo.setPipelineId(req.getPipelineId()); - if (!StringUtils.isEmpty(req.getTaskId())){ - aflCrashesInfo.setTaskId(req.getTaskId()); - } - if (!StringUtils.isEmpty(req.getPipelineHistoryId())){ - aflCrashesInfo.setPipelineHistoryId(req.getPipelineHistoryId()); - } - aflCrashesInfo.setGroupIdentifier(uuid); - } - } - } - - //新增数据到crashes表中 - baseMapper.insertBatch(aflCrashesInfos); - } - return uuid; - }else { - return null; - } - } catch (Exception e) { - e.printStackTrace(); - throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"crashes入库失败"); - } - } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflInfoServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflInfoServiceImpl.java index c9507b65..8e886acd 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflInfoServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflInfoServiceImpl.java @@ -31,7 +31,7 @@ import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; -import java.util.UUID; +import java.util.Map; /** * @author HopeLi @@ -76,12 +76,12 @@ public class AflInfoServiceImpl extends ServiceImpl impleme MachineInfoDO resourceMachine = resourceById.getMachineInfo(); resourceMachine.setPassword(CryptogramUtil.doDecrypt(resourceMachine.getPassword())); AflInfo aflInfo = SftpUploadUtil.downloadFileSftpForInputStreamAndSetAflInfo(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats"); - //在本地创建一个新文件夹，将远程文件复制一份过去 + //获取远程文件流，并保存到数据库 List sourceFilePaths = new ArrayList<>(); sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats"); - List strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/fuzzer_result/" + UUID.randomUUID() + "/"); + Map stringMap = SftpUploadUtil.copyRemoteFilesToByteArray(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths); - aflInfo.setFilePath(strings.get(0)); + aflInfo.setFileContent(stringMap.get(sourceFilePaths.get(0))); aflInfo.setPipelineId(req.getPipelineId()); if (!StringUtils.isEmpty(req.getTaskId())){ aflInfo.setTaskId(req.getTaskId()); @@ -228,39 +228,4 @@ public class AflInfoServiceImpl extends ServiceImpl impleme return resp; } - - - @Override - public String testSaveAflInfo(AflManagerReq req) throws SftpUploadUtil.SftpUploadException { - if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){ - throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空"); - } - - ResourceDetailResp resourceById = resourceManagerService.findResourceDetailById("735516490182299648"); - if (resourceById == null || resourceById.getMachineInfo() == null) { - throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误"); - } - - MachineInfoDO resourceMachine = resourceById.getMachineInfo(); - resourceMachine.setPassword(CryptogramUtil.doDecrypt(resourceMachine.getPassword())); - AflInfo aflInfo = SftpUploadUtil.downloadFileSftpForInputStreamAndSetAflInfo(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats"); - //在本地创建一个新文件夹，将远程文件复制一份过去 - List sourceFilePaths = new ArrayList<>(); - sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats"); - List strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/fuzzer_result/" + UUID.randomUUID() + "/"); - - aflInfo.setFilePath(strings.get(0)); - aflInfo.setPipelineId(req.getPipelineId()); - if (!StringUtils.isEmpty(req.getTaskId())){ - aflInfo.setTaskId(req.getTaskId()); - } - if (!StringUtils.isEmpty(req.getPipelineHistoryId())){ - aflInfo.setPipelineHistoryId(req.getPipelineHistoryId()); - } - aflInfo.setGroupIdentifier(req.getGroupIdentifier()); - baseMapper.insert(aflInfo); - - return aflInfo.getGroupIdentifier(); - } - } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflPlotInfoServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflPlotInfoServiceImpl.java index 6995149f..52d4a6f5 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflPlotInfoServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflPlotInfoServiceImpl.java @@ -27,7 +27,7 @@ import org.springframework.util.ObjectUtils; import java.util.ArrayList; import java.util.List; -import java.util.UUID; +import java.util.Map; /** * @author HopeLi @@ -72,12 +72,13 @@ public class AflPlotInfoServiceImpl extends ServiceImpl sourceFilePaths = new ArrayList<>(); sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data"); - List strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/plot_result/" + UUID.randomUUID() + "/"); + Map stringMap = SftpUploadUtil.copyRemoteFilesToByteArray(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths); - aflPlotInfo.setFilePath(strings.get(0)); + aflPlotInfo.setFileContent(stringMap.get(sourceFilePaths.get(0))); aflPlotInfo.setPipelineId(req.getPipelineId()); if (!StringUtils.isEmpty(req.getTaskId())){ aflPlotInfo.setTaskId(req.getTaskId()); @@ -160,39 +161,4 @@ public class AflPlotInfoServiceImpl extends ServiceImpl sourceFilePaths = new ArrayList<>(); - sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data"); - List strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/plot_result/" + UUID.randomUUID() + "/"); - - aflPlotInfo.setFilePath(strings.get(0)); - aflPlotInfo.setPipelineId(req.getPipelineId()); - if (!StringUtils.isEmpty(req.getTaskId())){ - aflPlotInfo.setTaskId(req.getTaskId()); - } - if (!StringUtils.isEmpty(req.getPipelineHistoryId())){ - aflPlotInfo.setPipelineHistoryId(req.getPipelineHistoryId()); - } - aflPlotInfo.setGroupIdentifier(req.getGroupIdentifier()); - baseMapper.insert(aflPlotInfo); - - return aflPlotInfo.getGroupIdentifier(); - } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflSeedInfoServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflSeedInfoServiceImpl.java index e23df437..04363c3a 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflSeedInfoServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflSeedInfoServiceImpl.java @@ -30,7 +30,7 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; -import java.util.UUID; +import java.util.Map; /** * @author HopeLi @@ -202,22 +202,22 @@ public class AflSeedInfoServiceImpl extends ServiceImpl copiedFiles = SftpUploadUtil.copyRemoteFilesToLocal( - resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/seed_result/" + UUID.randomUUID() + "/"); - System.out.println("共复制 " + copiedFiles.size() + " 个文件"); + // 步骤2：下载文件流并保存到数据库 + Map copiedFiles = SftpUploadUtil.copyRemoteFilesToByteArray( + resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files); + System.out.println("共下载 " + copiedFiles.size() + " 个文件"); List aflSeedInfos = new ArrayList<>(0); - if (!CollectionUtils.isEmpty(copiedFiles)){ + if (!ObjectUtils.isEmpty(copiedFiles)){ //新增数据到seed表中 - copiedFiles.forEach(o->{ + for (Map.Entry entry : copiedFiles.entrySet()) { + String key = entry.getKey(); + byte[] value = entry.getValue(); + AflSeedInfo aflSeedInfo = new AflSeedInfo(); - // 提取文件名部分（seed_result/之后的内容） - String fileNamePart = o.substring(o.indexOf("seed_result/") + 12); - - //截取最后一个斜杠后的内容，即文件信息部分 - fileNamePart = fileNamePart.substring(fileNamePart.lastIndexOf("/") + 1); + // 提取文件名部分（queue/之后的内容） + String fileNamePart = key.substring(key.indexOf("queue/") + 6); // 使用正则匹配 id: 开头的内容 java.util.regex.Pattern patternId = java.util.regex.Pattern.compile("id:([^,]+)"); @@ -237,7 +237,7 @@ public class AflSeedInfoServiceImpl extends ServiceImpl files = SftpUploadUtil.listFilesInRemoteDirectory( - resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + seedFilePath); - - if (!CollectionUtils.isEmpty(files)) { - // 步骤2：批量复制文件到目标目录 - List copiedFiles = SftpUploadUtil.copyRemoteFilesToLocal( - resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/seed_result/" + UUID.randomUUID() + "/"); - System.out.println("共复制 " + copiedFiles.size() + " 个文件"); - - List aflSeedInfos = new ArrayList<>(0); - if (!CollectionUtils.isEmpty(copiedFiles)){ - //新增数据到seed表中 - copiedFiles.forEach(o->{ - AflSeedInfo aflSeedInfo = new AflSeedInfo(); - - // 提取文件名部分（seed_result/之后的内容） - String fileNamePart = o.substring(o.indexOf("seed_result/") + 12); - - //截取最后一个斜杠后的内容，即文件信息部分 - fileNamePart = fileNamePart.substring(fileNamePart.lastIndexOf("/") + 1); - - // 使用正则匹配 id: 开头的内容 - java.util.regex.Pattern patternId = java.util.regex.Pattern.compile("id:([^,]+)"); - java.util.regex.Matcher matcherId = patternId.matcher(fileNamePart); - - // 使用正则匹配 rep: 开头的内容 - java.util.regex.Pattern patternRep = java.util.regex.Pattern.compile("rep:(\\d+)"); - java.util.regex.Matcher matcherRep = patternRep.matcher(fileNamePart); - - if (matcherRep.find()) { - String repValue = matcherRep.group(1); - aflSeedInfo.setSeedAmount(repValue); - } - - if (matcherId.find()) { - String idValue = matcherId.group(1); - aflSeedInfo.setSeedId(idValue); - } - - aflSeedInfo.setFilePath(o); - if (!StringUtils.isEmpty(req.getTaskId())){ - aflSeedInfo.setTaskId(req.getTaskId()); - } - if (!StringUtils.isEmpty(req.getPipelineHistoryId())){ - aflSeedInfo.setPipelineHistoryId(req.getPipelineHistoryId()); - } - aflSeedInfo.setPipelineId(req.getPipelineId()); - aflSeedInfo.setGroupIdentifier(uuid); - - aflSeedInfos.add(aflSeedInfo); - }); - } - - if (!CollectionUtils.isEmpty(aflSeedInfos)){ - //查询种子运行分钟数，同时入库 - List seedsCount = this.findSeedsCount(req); - if (!CollectionUtils.isEmpty(seedsCount)){ - aflSeedInfos.forEach(o->{ - seedsCount.forEach(j->{ - if (o.getSeedId().equals(j.getSeedId())){ - o.setMinutesDifference(j.getMinutesDifference()); - } - }); - }); - } - baseMapper.insertBatch(aflSeedInfos); - } - } - } catch (Exception e) { - e.printStackTrace(); - } - } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/LocalAflCrashesService.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/LocalAflCrashesService.java index 0774eebd..2e307688 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/LocalAflCrashesService.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/LocalAflCrashesService.java @@ -21,5 +21,4 @@ public interface LocalAflCrashesService extends IService { PageResult getLocalAflCrashesSearch(LocalAflCrashesQueryReq req); - void testSaveLocalAflCrashesInfo(@Valid LocalAflCrashesSaveReq req); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/impl/LocalAflCrashesServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/impl/LocalAflCrashesServiceImpl.java index af952c40..d900635a 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/impl/LocalAflCrashesServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/localAflCrashes/impl/LocalAflCrashesServiceImpl.java @@ -98,7 +98,7 @@ public class LocalAflCrashesServiceImpl extends ServiceImpl copiedFiles = SftpUploadUtil.copyRemoteFilesToByteArray( resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files); - System.out.println("共复制 " + copiedFiles.size() + " 个文件"); + System.out.println("共下载 " + copiedFiles.size() + " 个文件"); //绑定对应的文件流 - if (!CollectionUtils.isEmpty(copiedFiles)){ + if (!ObjectUtils.isEmpty(copiedFiles)){ for (Map.Entry entry : copiedFiles.entrySet()) { if (!CollectionUtils.isEmpty(aflCrashesInfos)){ for (LocalAflCrashesInfo aflCrashesInfo : aflCrashesInfos) { if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) { aflCrashesInfo.setFileContent(entry.getValue()); + + aflCrashesInfo.setPipelineId(req.getPipelineId()); + if (!StringUtils.isEmpty(req.getTaskId())){ + aflCrashesInfo.setTaskId(req.getTaskId()); + } + aflCrashesInfo.setTaskType("AFL"); + aflCrashesInfo.setInstanceId(req.getInstanceId()); + aflCrashesInfo.setTargetType(req.getTargetType()); + aflCrashesInfo.setTargetName(req.getTargetName()); + aflCrashesInfo.setCity(req.getCity()); } - aflCrashesInfo.setPipelineId(req.getPipelineId()); - if (!StringUtils.isEmpty(req.getTaskId())){ - aflCrashesInfo.setTaskId(req.getTaskId()); - } - aflCrashesInfo.setTaskType("AFL"); - aflCrashesInfo.setInstanceId(req.getInstanceId()); - aflCrashesInfo.setTargetType(req.getTargetType()); - aflCrashesInfo.setTargetName(req.getTargetName()); - aflCrashesInfo.setCity(req.getCity()); } } } @@ -210,123 +212,4 @@ public class LocalAflCrashesServiceImpl extends ServiceImpl aflCrashesInfos = new ArrayList<>(0); - - if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){ - throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空"); - } - String uuid = req.getGroupIdentifier(); - - ResourceDetailResp resourceById = resourceManagerService.findResourceDetailById("735516490182299648"); - if (resourceById == null || resourceById.getMachineInfo() == null || resourceById.getMachineInfo().getAuthenticationType()!=1) { - throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误"); - } - - MachineInfoDO resourceMachine = resourceById.getMachineInfo(); - String password = CryptogramUtil.doDecrypt(resourceMachine.getPassword()); - resourceMachine.setPassword(password); - - try { - // 步骤1：使用自定义sql获取崩溃文件的创建时间 - //查询创建时间 - List resultList = SftpUploadUtil.findCreateTimeByCommand( - resourceMachine.getHostIp(), - resourceMachine.getSshPort(), - resourceMachine.getUsername(), - resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath); - - if (!CollectionUtils.isEmpty(resultList)){ - //解析文件名和创建时间 - for (String o : resultList) { - String[] lineParts = o.split("\\s+创建时间:\\s+"); - if (lineParts.length < 2) { - System.err.println("无效格式: " + o); - break; - } - LocalAflCrashesInfo aflCrashesInfo = new LocalAflCrashesInfo(); - - String fileName = lineParts[0].substring(2).trim(); // 文件信息部分 - String createTime = lineParts[1].trim(); // 创建时间 - aflCrashesInfo.setCrashesName(fileName); - aflCrashesInfo.setCrashesCreateTime(createTime); - - aflCrashesInfos.add(aflCrashesInfo); - } - } - - - - // 步驟2.查询文件大小 - List fileResultList = SftpUploadUtil.findFileByteByCommand( - resourceMachine.getHostIp(), - resourceMachine.getSshPort(), - resourceMachine.getUsername(), - resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath); - - if (!CollectionUtils.isEmpty(fileResultList) && !CollectionUtils.isEmpty(aflCrashesInfos)){ - //解析文件名和文件大小 - for (String o : fileResultList) { - // 按空白字符分割 - String[] parts = o.trim().split("\\s+"); - - String fileName = parts[0]; // 文件名称 - Integer fileLength = Integer.parseInt(parts[1]); // 文件大小 - for (LocalAflCrashesInfo aflCrashesInfo : aflCrashesInfos) { - if (aflCrashesInfo.getCrashesName().equals(fileName)) { - aflCrashesInfo.setCrashesLength(fileLength); - } - aflCrashesInfo.setPipelineId(req.getPipelineId()); - if (!StringUtils.isEmpty(req.getTaskId())){ - aflCrashesInfo.setTaskId(req.getTaskId()); - } - aflCrashesInfo.setGroupIdentifier(uuid); - } - } - } - - - - // 步骤3：列出源目录下的所有文件 - List files = SftpUploadUtil.listFilesInRemoteDirectory( - resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath); - - if (!CollectionUtils.isEmpty(files)) { - // 步骤4：批量下载文件流与文件绑定 - Map copiedFiles = SftpUploadUtil.copyRemoteFilesToByteArray( - resourceMachine.getHostIp(), resourceMachine.getSshPort(), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files); - System.out.println("共复制 " + copiedFiles.size() + " 个文件"); - - //绑定对应的文件流 - if (!CollectionUtils.isEmpty(copiedFiles)){ - for (Map.Entry entry : copiedFiles.entrySet()) { - if (!CollectionUtils.isEmpty(aflCrashesInfos)){ - for (LocalAflCrashesInfo aflCrashesInfo : aflCrashesInfos) { - if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) { - aflCrashesInfo.setFileContent(entry.getValue()); - } - aflCrashesInfo.setPipelineId(req.getPipelineId()); - if (!StringUtils.isEmpty(req.getTaskId())){ - aflCrashesInfo.setTaskId(req.getTaskId()); - } - aflCrashesInfo.setTaskType("AFL"); - aflCrashesInfo.setInstanceId(req.getInstanceId()); - aflCrashesInfo.setTargetType(req.getTargetType()); - aflCrashesInfo.setTargetName(req.getTargetName()); - aflCrashesInfo.setCity(req.getCity()); - } - } - } - - //新增数据到crashes表中 - baseMapper.insertBatch(aflCrashesInfos); - } - } - } catch (Exception e) { - e.printStackTrace(); - throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"crashes入库失败"); - } - } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/util/SftpUploadUtil.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/util/SftpUploadUtil.java index c758d510..901f8cf0 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/util/SftpUploadUtil.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/util/SftpUploadUtil.java @@ -1593,15 +1593,14 @@ public class SftpUploadUtil { channelSftp = (ChannelSftp) channel; + + String chmodDir = remoteFilePaths.get(0).substring(0, remoteFilePaths.get(0).indexOf("default")); + log.info("分配权限路径,{}", chmodDir); + // 切换目录并列出内容用于调试 + sudoChmodORwx(session, chmodDir, password); + // 遍历远程文件路径并下载 for (String remoteFile : remoteFilePaths) { - String chmodDir = remoteFile.substring(0, remoteFile.indexOf("default")); - log.info("分配权限路径,{}", chmodDir); - // 切换目录并列出内容用于调试 - sudoChmodORwx(session, chmodDir, password); - - String fileName = new File(remoteFile).getName(); - try (InputStream in = channelSftp.get(remoteFile); ByteArrayOutputStream out = new ByteArrayOutputStream()) { byte[] buffer = new byte[1024]; @@ -1609,7 +1608,7 @@ public class SftpUploadUtil { while ((bytesRead = in.read(buffer)) > 0) { out.write(buffer, 0, bytesRead); } - fileByteArrays.put(fileName, out.toByteArray()); // 添加文件名和二进制数据 + fileByteArrays.put(remoteFile, out.toByteArray()); // 添加文件名和二进制数据 } catch (Exception e) { throw new SftpUploadException("文件读取失败: " + remoteFile, e); }