mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0708 ljc afl模块文件操作以及入库操作以及时序表操作等接口开发，包含分片代码以及文件工具类代码

Browse Source
This commit is contained in:
HopeLi 2025-07-11 17:51:38 +08:00
parent 8b24069dca
commit 44d857db97
11 changed files with 464 additions and 358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
modules/module-ci-process-api/src/main/java/cd/casic/ci/api/AflManagerController.java
View File

@ -49,7 +49,7 @@ public class AflManagerController {
}
@PostMapping(path="/findAflInfo")
@PostMapping(path="/findAflInfoNow")
public CommonResult<AflInfoResp> findAflInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
AflInfoResp resp = aflInfoService.findAflInfo(req);
@ -58,10 +58,10 @@ public class AflManagerController {
}
@PostMapping(path="/findAflInfoList")
public CommonResult<List<AflInfoResp>> findAflInfoList(@RequestBody @Valid AflManagerReq req){
@PostMapping(path="/findAflInfoListByHistory")
public CommonResult<List<AflInfoResp>> findAflInfoListByHistory(@RequestBody @Valid AflManagerReq req){
List<AflInfoResp> respList = aflInfoService.findPipelineList(req);
List<AflInfoResp> respList = aflInfoService.findAflInfoListByHistory(req);
return CommonResult.success(respList);
}
@ -76,10 +76,19 @@ public class AflManagerController {
}
@PostMapping(path="/findAflCrashesInfoList")
public CommonResult<List<AflCrashesInfoResp>> findAflCrashesInfoList(@RequestBody @Valid AflManagerReq req){
@PostMapping(path="/findCrashesCount")
public CommonResult<AflCrashesResp> findCrashCountNow(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
List<AflCrashesInfoResp> respList = aflCrashesInfoService.findPipelineList(req);
AflCrashesResp resp = aflCrashesInfoService.findCrashesCount(req);
return CommonResult.success(resp);
}
@PostMapping(path="/findAflCrashesInfoListByHistory")
public CommonResult<List<AflCrashesInfoResp>> findAflCrashesInfoListByHistory(@RequestBody @Valid AflManagerReq req){
List<AflCrashesInfoResp> respList = aflCrashesInfoService.findAflCrashesInfoListByHistory(req);
return CommonResult.success(respList);
}
@ -94,15 +103,6 @@ public class AflManagerController {
}
@PostMapping(path="/findImperfection")
public CommonResult<AflImperfectionResp> findImperfection(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
AflImperfectionResp resp = aflSeedInfoService.findImperfection(req);
return CommonResult.success(resp);
}
@PostMapping(path="/saveAflSeedInfo")
public CommonResult<Void> saveAflSeedInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {

273
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflCrashesInfoResp.java
View File

@ -11,265 +11,40 @@ import lombok.Data;
@Data
public class AflCrashesInfoResp {
private String id;
/**
* 流水线id
*/
private String pipelineId;
private String pipelineHistoryId;
private String taskId;
//分组唯一标识
private String groupIdentifier;
/**
* 模糊测试进程的开始时间
*/
private String startTime;
/**
* 最后一次更新时间戳
*/
private String lastUpdate;
/**
* 总运行时间
*/
private String runTime;
/**
* 模糊器的进程 ID
*/
private String fuzzerPid;
/**
* 已完成的循环次数
*/
private String cyclesDone;
/**
* 未发现新内容的循环次数
*/
private String cyclesWoFinds;
/**
* 未发现新内容的时间
*/
private String timeWoFinds;
/**
* 模糊测试所用时间
*/
private String fuzzTime;
/**
* 校准时间
*/
private String calibrationTime;
/**
* 比较日志时间
*/
private String cmplogTime;
/**
* 同步时间
*/
private String syncTime;
/**
* 裁剪时间
*/
private String trimTime;
/**
* 已执行的次数
*/
private String execsDone;
/**
* 每秒执行次数
*/
private String execsPerSec;
/**
* 上一分钟内的每秒执行次数
*/
private String execsPsLastMin;
/**
* 语料库中的条目数量
*/
private String corpusCount;
/**
* 语料库中被优先选择的条目
*/
private String corpusFavored;
/**
* 语料库中发现的条目
*/
private String corpusFound;
/**
* 导入到语料库的条目
*/
private String corpusImported;
/**
* 语料库中的可变条目
*/
private String corpusVariable;
/**
* 达到的最大深度
*/
private String maxDepth;
/**
* 当前处理的条目索引
*/
private String curItem;
/**
* 待处理的优先条目数
*/
private String pendingFavs;
/**
* 总待处理条目数
*/
private String pendingTotal;
/**
* 稳定性百分比
*/
private String stability;
/**
* 位图覆盖率百分比
*/
private String bitmapCvg;
/**
* 保存的崩溃次数
*/
private String savedCrashes;
/**
* 保存的挂起次数
*/
private String savedHangs;
/**
* 总超时次数
*/
private String totalTmout;
/**
* 上次发现的时间戳
*/
private String lastFind;
/**
* 上次崩溃的时间戳
*/
private String lastCrash;
/**
* 上次挂起的时间戳
*/
private String lastHang;
/**
* 自上次崩溃以来的执行次数
*/
private String execsSinceCrash;
/**
* 执行超时值
*/
private String execTimeout;
/**
* 最慢执行时间（毫秒）
*/
private String slowestExecMs;
/**
* 峰值 RSS 内存使用量（MB）
*/
private String peakRssMb;
/**
* CPU 亲和性设置
*/
private String cpuAffinity;
/**
* 发现的边数（edges）
*/
private String edgesFound;
/**
* 总可用边数（edges）
*/
private String totalEdges;
/**
* 可变字节计数
*/
private String varByteCount;
/**
* Havoc 扩展因子
*/
private String havocExpansion;
/**
* 自动词典条目数
*/
private String autoDictEntries;
/**
* 测试缓存大小
*/
private String testcacheSize;
/**
* 测试缓存条目数
*/
private String testcacheCount;
/**
* 测试缓存驱逐次数
*/
private String testcacheEvict;
/**
* 目标程序 banner
*/
private String aflBanner;
/**
* 使用的 AFL 版本
*/
private String aflVersion;
/**
* 目标模式配置
*/
private String targetMode;
/**
* 执行使用的命令行
*/
private String commandLine;
/**
* 文件二进制内容
*/
private String contentId;
/**
* 文件地址路径
*/
private String filePath;
/**
* 崩溃文件名称
*/
private String crashesName;
/**
* 崩溃文件生成时间
*/
private String crashesCreateTime;
/**
* 崩溃文件大小（单位：字节）
*/
private Integer crashesLength;
/**
* 崩溃文件描述
*/
private String crashesRemark;
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflImperfectionResp.java → modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/aflManager/AflCrashesResp.java
View File

@ -9,7 +9,7 @@ import lombok.Data;
* @Version 1.0
*/
@Data
public class AflImperfectionResp {
public class AflCrashesResp {
//所有崩溃数
private Integer imperfectionCount = 0;

20
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/aflManager/AflCrashesInfo.java
View File

@ -25,4 +25,24 @@ public class AflCrashesInfo extends PipBaseElement {
* 文件地址路径
*/
private String filePath;
/**
* 崩溃文件名称
*/
private String crashesName;
/**
* 崩溃文件生成时间
*/
private String crashesCreateTime;
/**
* 崩溃文件大小（单位：字节）
*/
private Integer crashesLength;
/**
* 崩溃文件描述
*/
private String crashesRemark;
}

6
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflCrashesInfoService.java
View File

@ -2,6 +2,7 @@ package cd.casic.ci.process.process.service.aflManager;
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
import cd.casic.ci.process.dto.resp.aflManager.AflCrashesInfoResp;
import cd.casic.ci.process.dto.resp.aflManager.AflCrashesResp;
import cd.casic.ci.process.process.dataObject.aflManager.AflCrashesInfo;
import com.baomidou.mybatisplus.extension.service.IService;
import jakarta.validation.Valid;
@ -21,5 +22,8 @@ public interface AflCrashesInfoService extends IService<AflCrashesInfo> {
void updateHistoryPipelineIdByAflCrashesInfo(@Valid AflManagerReq req);
List<AflCrashesInfoResp> findPipelineList(@Valid AflManagerReq req);
List<AflCrashesInfoResp> findAflCrashesInfoListByHistory(@Valid AflManagerReq req);
AflCrashesResp findCrashesCount(@Valid AflManagerReq req);
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflInfoService.java
View File

@ -25,7 +25,7 @@ public interface AflInfoService extends IService<AflInfo> {
AflInfoResp findAflInfo(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException;
List<AflInfoResp> findPipelineList(@Valid AflManagerReq req);
List<AflInfoResp> findAflInfoListByHistory(@Valid AflManagerReq req);
AflBaseInfoResp findAflBaseInfoList(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException;
}

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/AflSeedInfoService.java
View File

@ -1,7 +1,6 @@
package cd.casic.ci.process.process.service.aflManager;
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
import cd.casic.ci.process.dto.resp.aflManager.AflImperfectionResp;
import cd.casic.ci.process.dto.resp.aflManager.SeedsCountResp;
import cd.casic.ci.process.process.dataObject.aflManager.AflSeedInfo;
import cd.casic.ci.process.util.SftpUploadUtil;
@ -25,5 +24,4 @@ public interface AflSeedInfoService extends IService<AflSeedInfo> {
void updateHistoryPipelineIdByAflSeedInfo(@Valid AflManagerReq req);
AflImperfectionResp findImperfection(@Valid AflManagerReq req);
}

135
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflCrashesInfoServiceImpl.java
View File

@ -2,6 +2,7 @@ package cd.casic.ci.process.process.service.aflManager.impl;
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
import cd.casic.ci.process.dto.resp.aflManager.AflCrashesInfoResp;
import cd.casic.ci.process.dto.resp.aflManager.AflCrashesResp;
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
import cd.casic.ci.process.engine.constant.PipelineVariableConstant;
import cd.casic.ci.process.engine.manager.RunContextManager;
@ -9,6 +10,7 @@ import cd.casic.ci.process.engine.runContext.BaseRunContext;
import cd.casic.ci.process.process.converter.AflManagerConverter;
import cd.casic.ci.process.process.dao.aflManager.AflCrashesInfoDao;
import cd.casic.ci.process.process.dataObject.aflManager.AflCrashesInfo;
import cd.casic.ci.process.process.dataObject.aflManager.AflInfo;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.service.aflManager.AflCrashesInfoService;
import cd.casic.ci.process.process.service.resource.ResourceManagerService;
@ -28,6 +30,7 @@ import org.springframework.util.ObjectUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
@ -53,6 +56,8 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl<AflCrashesInfoDao, Af
@Override
public String saveAflCrashesInfo(AflManagerReq req) {
List<AflCrashesInfo> aflCrashesInfos = new ArrayList<>(0);
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
@ -78,22 +83,80 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl<AflCrashesInfoDao, Af
resourceMachine.setPassword(password);
try {
// 步骤1：列出源目录下的所有文件
// 步骤1：使用自定义sql获取崩溃文件的创建时间
//查询创建时间
List<String> resultList = SftpUploadUtil.findCreateTimeByCommand(
resourceMachine.getMachineHost(),
Integer.parseInt(resourceMachine.getSshPort()),
resourceMachine.getUsername(),
resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (CollectionUtils.isEmpty(resultList)){
return null;
}
//解析文件名和创建时间
for (String o : resultList) {
String[] lineParts = o.split("\\s+创建时间:\\s+");
if (lineParts.length < 2) {
System.err.println("无效格式: " + o);
break;
}
AflCrashesInfo aflCrashesInfo = new AflCrashesInfo();
String fileName = lineParts[0].substring(2).trim(); // 文件信息部分
String createTime = lineParts[1].trim(); // 创建时间
aflCrashesInfo.setCrashesName(fileName);
aflCrashesInfo.setCrashesCreateTime(createTime);
aflCrashesInfos.add(aflCrashesInfo);
}
// 步驟2.查询文件大小
List<String> fileResultList = SftpUploadUtil.findFileByteByCommand(
resourceMachine.getMachineHost(),
Integer.parseInt(resourceMachine.getSshPort()),
resourceMachine.getUsername(),
resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (CollectionUtils.isEmpty(fileResultList)){
return null;
}
//解析文件名和文件大小
for (String o : fileResultList) {
// 按空白字符分割
String[] parts = o.trim().split("\\s+");
String fileName = parts[0]; // 文件名称
Integer fileLength = Integer.parseInt(parts[1]); // 文件大小
if (!CollectionUtils.isEmpty(aflCrashesInfos)){
for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(fileName)) {
aflCrashesInfo.setCrashesLength(fileLength);
}
}
}
}
// 步骤3：列出源目录下的所有文件
List<String> files = SftpUploadUtil.listFilesInRemoteDirectory(
resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (!CollectionUtils.isEmpty(files)) {
// 步骤2：批量复制文件到目标目录
List<String> copiedFiles = SftpUploadUtil.copyRemoteFilesToLocal(
// 步骤4：批量复制文件到目标目录
Map<String,String> copiedFiles = SftpUploadUtil.copyRemoteFilesToLocalMap(
resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/crashes_result/" + UUID.randomUUID() + "/");
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
//绑定对应的文件路径
if (!CollectionUtils.isEmpty(copiedFiles)){
List<AflCrashesInfo> aflCrashesInfos = new ArrayList<>(0);
//新增数据到crashes表中
copiedFiles.forEach(o->{
AflCrashesInfo aflCrashesInfo = new AflCrashesInfo();
aflCrashesInfo.setFilePath(o);
for (Map.Entry<String, String> entry : copiedFiles.entrySet()) {
if (!CollectionUtils.isEmpty(aflCrashesInfos)){
for (AflCrashesInfo aflCrashesInfo : aflCrashesInfos) {
if (aflCrashesInfo.getCrashesName().equals(entry.getKey())) {
aflCrashesInfo.setFilePath(entry.getValue());
}
aflCrashesInfo.setPipelineId(req.getPipelineId());
if (!StringUtils.isEmpty(req.getTaskId())){
aflCrashesInfo.setTaskId(req.getTaskId());
@ -102,8 +165,11 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl<AflCrashesInfoDao, Af
aflCrashesInfo.setPipelineHistoryId(req.getPipelineHistoryId());
}
aflCrashesInfo.setGroupIdentifier(uuid);
aflCrashesInfos.add(aflCrashesInfo);
});
}
}
}
//新增数据到crashes表中
baseMapper.insertBatch(aflCrashesInfos);
}
return uuid;
@ -133,7 +199,7 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl<AflCrashesInfoDao, Af
}
@Override
public List<AflCrashesInfoResp> findPipelineList(AflManagerReq req) {
public List<AflCrashesInfoResp> findAflCrashesInfoListByHistory(AflManagerReq req) {
QueryWrapper<AflCrashesInfo> wrapper = new QueryWrapper<>();
if (!ObjectUtils.isEmpty(req.getId())){
wrapper.eq("id",req.getId());
@ -158,4 +224,51 @@ public class AflCrashesInfoServiceImpl extends ServiceImpl<AflCrashesInfoDao, Af
return AflManagerConverter.INSTANCE.toAflCrashesInfoRespList(aflCrashesInfos);
}
@Override
public AflCrashesResp findCrashesCount(AflManagerReq req) {
AflCrashesResp aflCrashesResp = new AflCrashesResp();
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
BaseRunContext runContext = runContextManager.getContext(req.getPipelineId());
if (ObjectUtils.isEmpty(runContext)){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到运行上下文");
}
String resourceId = runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY)) : null;
if (StringUtils.isEmpty(resourceId)){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到资源");
}
ResourceFindResp resourceById = resourceManagerService.findResourceById(resourceId);
if (resourceById == null || resourceById.getResourceMachine() == null) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
}
PipResourceMachine resourceMachine = resourceById.getResourceMachine();
String password = CryptogramUtil.doDecrypt(resourceMachine.getPassword());
resourceMachine.setPassword(password);
try {
// 查询所有崩溃数
List<String> files = SftpUploadUtil.listFilesInRemoteDirectory(
resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + crashesFilePath);
if (!CollectionUtils.isEmpty(files)){
aflCrashesResp.setImperfectionCount(files.size());
}
//查询已验证和未验证缺陷数
AflInfo aflInfo = SftpUploadUtil.downloadFileSftpForInputStreamAndSetAflInfo(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats");
if (!ObjectUtils.isEmpty(aflInfo)){
aflCrashesResp.setVerifiedCount(Integer.parseInt(aflInfo.getSavedCrashes()));
aflCrashesResp.setUnverifiedCount(Integer.parseInt(aflInfo.getPendingFavs()));
}
} catch (Exception e) {
e.printStackTrace();
}
return aflCrashesResp;
}
}

14
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflInfoServiceImpl.java
View File

@ -58,12 +58,12 @@ public class AflInfoServiceImpl extends ServiceImpl<AflInfoDao, AflInfo> impleme
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
// BaseRunContext runContext = runContextManager.getContext(req.getPipelineId());
// if (ObjectUtils.isEmpty(runContext)){
// throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到运行上下文");
// }
// String resourceId = runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY)) : null;
String resourceId = "2";
BaseRunContext runContext = runContextManager.getContext(req.getPipelineId());
if (ObjectUtils.isEmpty(runContext)){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到运行上下文");
}
String resourceId = runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY)) : null;
// String resourceId = "2";
if (StringUtils.isEmpty(resourceId)){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到资源");
}
@ -142,7 +142,7 @@ public class AflInfoServiceImpl extends ServiceImpl<AflInfoDao, AflInfo> impleme
}
@Override
public List<AflInfoResp> findPipelineList(AflManagerReq req) {
public List<AflInfoResp> findAflInfoListByHistory(AflManagerReq req) {
QueryWrapper<AflInfo> wrapper = new QueryWrapper<>();
if (!ObjectUtils.isEmpty(req.getId())){
wrapper.eq("id",req.getId());

52
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/aflManager/impl/AflSeedInfoServiceImpl.java
View File

@ -1,14 +1,12 @@
package cd.casic.ci.process.process.service.aflManager.impl;
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
import cd.casic.ci.process.dto.resp.aflManager.AflImperfectionResp;
import cd.casic.ci.process.dto.resp.aflManager.SeedsCountResp;
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
import cd.casic.ci.process.engine.constant.PipelineVariableConstant;
import cd.casic.ci.process.engine.manager.RunContextManager;
import cd.casic.ci.process.engine.runContext.BaseRunContext;
import cd.casic.ci.process.process.dao.aflManager.AflSeedInfoDao;
import cd.casic.ci.process.process.dataObject.aflManager.AflInfo;
import cd.casic.ci.process.process.dataObject.aflManager.AflSeedInfo;
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
import cd.casic.ci.process.process.service.aflManager.AflSeedInfoService;
@ -55,7 +53,6 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
private static final String seedFilePath = "/ai_afl/default/queue/";
@Override
public List<SeedsCountResp> findSeedsCount(AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
List<SeedsCountResp> seedsCounts = new ArrayList<>(0);
@ -83,7 +80,7 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
String password = CryptogramUtil.doDecrypt(resourceMachine.getPassword());
resourceMachine.setPassword(password);
List<String> resultList = SftpUploadUtil.findSeedCount(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/queue/");
List<String> resultList = SftpUploadUtil.findCreateTimeByCommand(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/queue/");
//解析resultList
@ -291,51 +288,4 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
}
baseMapper.updateBatch(aflSeedInfos);
}
@Override
public AflImperfectionResp findImperfection(AflManagerReq req) {
AflImperfectionResp aflImperfectionResp = new AflImperfectionResp();
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
}
BaseRunContext runContext = runContextManager.getContext(req.getPipelineId());
if (ObjectUtils.isEmpty(runContext)){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到运行上下文");
}
String resourceId = runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY)) : null;
if (StringUtils.isEmpty(resourceId)){
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到资源");
}
ResourceFindResp resourceById = resourceManagerService.findResourceById(resourceId);
if (resourceById == null || resourceById.getResourceMachine() == null) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
}
PipResourceMachine resourceMachine = resourceById.getResourceMachine();
String password = CryptogramUtil.doDecrypt(resourceMachine.getPassword());
resourceMachine.setPassword(password);
try {
// 查询所有缺陷数
List<String> files = SftpUploadUtil.listFilesInRemoteDirectory(
resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, remoteFilePath + "PIP_" + req.getPipelineId() + seedFilePath);
if (!CollectionUtils.isEmpty(files)){
aflImperfectionResp.setImperfectionCount(files.size());
}
//查询已验证和未验证缺陷数
AflInfo aflInfo = SftpUploadUtil.downloadFileSftpForInputStreamAndSetAflInfo(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats");
if (!ObjectUtils.isEmpty(aflInfo)){
aflImperfectionResp.setVerifiedCount(Integer.parseInt(aflInfo.getSavedCrashes()));
aflImperfectionResp.setUnverifiedCount(Integer.parseInt(aflInfo.getPendingFavs()));
}
} catch (Exception e) {
e.printStackTrace();
}
return aflImperfectionResp;
}
}

248
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/util/SftpUploadUtil.java
View File

@ -1039,7 +1039,7 @@ public class SftpUploadUtil {
}
public static List<String> findSeedCount(String remoteHost,
public static List<String> findCreateTimeByCommand(String remoteHost,
Integer remotePort,
String username,
String password,
@ -1125,6 +1125,7 @@ public class SftpUploadUtil {
}
// 转成字符串并按行拆分
if (outputStream.size() > 0){
String outputStr = outputStream.toString();
String[] lines = outputStr.split("\\r?\\n");
@ -1133,6 +1134,126 @@ public class SftpUploadUtil {
execChannel.disconnect();
return result;
}else {
return new ArrayList<>();
}
} catch (JSchException e) {
throw new SftpUploadException("SFTP 连接或认证失败: " + e.getMessage(), e);
} catch (SftpUploadException e) {
// 重新抛出自定义异常
throw e;
} catch (Exception e) {
// 捕获其他未知异常
throw new SftpUploadException("SFTP 上传过程中发生未知异常: " + e.getMessage(), e);
} finally {
if (session != null) {
session.disconnect();
System.out.println("SFTP Session 已断开.");
}
}
}
public static List<String> findFileByteByCommand(String remoteHost,
Integer remotePort,
String username,
String password,
String sshKeyPath,
String remoteDir) throws SftpUploadException {
Session session = null;
try {
JSch jsch = new JSch();
// 1. 添加身份认证信息 (密码或密钥)
if (sshKeyPath != null && !sshKeyPath.trim().isEmpty()) {
// 使用 SSH Key 认证
File sshKeyFile = new File(sshKeyPath);
if (!sshKeyFile.exists() || !sshKeyFile.isFile()) {
throw new SftpUploadException("SSH Key 文件不存在或不是一个有效文件: " + sshKeyPath);
}
jsch.addIdentity(sshKeyPath);
System.out.println("使用 SSH Key 认证: " + sshKeyPath);
} else if (password == null || password.trim().isEmpty()) {
// 如果没有提供密码或密钥路径，则认证信息不全
throw new SftpUploadException("必须提供密码或 SSH Key 路径进行 SFTP 认证.");
}
// 如果提供了密码，将在 getSession 后设置，因为 getSession 需要用户名、主机和端口先建立连接意图
// 2. 获取 Session
int port = (remotePort != null && remotePort > 0) ? remotePort : DEFAULT_SFTP_PORT;
session = jsch.getSession(username, remoteHost, port);
System.out.println("尝试连接 SFTP 服务器: " + username + "@" + remoteHost + ":" + port);
// 如果使用密码认证且提供了密码
if (password != null && !password.trim().isEmpty() && (sshKeyPath == null || sshKeyPath.trim().isEmpty())) {
session.setPassword(password);
System.out.println("使用密码认证.");
}
// 设置连接不进行主机密钥检查 (生产环境不推荐，应该配置 known_hosts)
// 在实际应用中，应该引导用户信任主机密钥或提前将主机密钥加入 known_hosts
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no"); // !!! 生产环境请谨慎使用或配置正确的主机密钥检查 !!!
session.setConfig(config);
// 3. 连接 Session
session.connect();
System.out.println("SFTP Session 连接成功.");
String chmodDir = remoteDir.substring(0, remoteDir.indexOf("default"));
log.info("分配权限路径,{}",chmodDir);
// 切换目录并列出内容用于调试
sudoChmodORwx(session,chmodDir,password);
// 开始执行 find 命令
ChannelExec execChannel = (ChannelExec) session.openChannel("exec");
// 设置要执行的命令
String command = "find " + remoteDir + " -maxdepth 1 -type f -exec stat -c \"%n %s bytes\" {} \\;";
execChannel.setCommand(command);
InputStream in = execChannel.getInputStream();
execChannel.connect();
byte[] tmp = new byte[1024];
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
while (true) {
while (in.available() > 0) {
int i = in.read(tmp, 0, 1024);
if (i < 0) break;
outputStream.write(tmp, 0, i);
}
if (execChannel.isClosed()) {
if (in.available() > 0) continue;
break;
}
try {
Thread.sleep(1000);
} catch (Exception ignored) {}
}
// 转成字符串并按行拆分
if (outputStream.size() > 0){
String outputStr = outputStream.toString();
String[] lines = outputStr.split("\\r?\\n");
// 添加进 list
List<String> result = new ArrayList<>(Arrays.asList(lines));
execChannel.disconnect();
return result;
}else {
return new ArrayList<>();
}
} catch (JSchException e) {
throw new SftpUploadException("SFTP 连接或认证失败: " + e.getMessage(), e);
} catch (SftpUploadException e) {
@ -1276,6 +1397,131 @@ public class SftpUploadUtil {
}
public static Map<String,String> copyRemoteFilesToLocalMap(String remoteHost,
Integer remotePort,
String username,
String password,
String sshKeyPath,
List<String> remoteFilePaths,
String localDir) throws SftpUploadException {
Map<String,String> copiedLocalPaths = new HashMap<>();
Session session = null;
Channel channel = null;
ChannelSftp channelSftp = null;
InputStream inputStream = null;
try {
JSch jsch = new JSch();
// 1. 添加身份认证信息 (密码或密钥)
if (sshKeyPath != null && !sshKeyPath.trim().isEmpty()) {
// 使用 SSH Key 认证
File sshKeyFile = new File(sshKeyPath);
if (!sshKeyFile.exists() || !sshKeyFile.isFile()) {
throw new SftpUploadException("SSH Key 文件不存在或不是一个有效文件: " + sshKeyPath);
}
jsch.addIdentity(sshKeyPath);
System.out.println("使用 SSH Key 认证: " + sshKeyPath);
} else if (password == null || password.trim().isEmpty()) {
// 如果没有提供密码或密钥路径，则认证信息不全
throw new SftpUploadException("必须提供密码或 SSH Key 路径进行 SFTP 认证.");
}
// 如果提供了密码，将在 getSession 后设置，因为 getSession 需要用户名、主机和端口先建立连接意图
// 2. 获取 Session
int port = (remotePort != null && remotePort > 0) ? remotePort : DEFAULT_SFTP_PORT;
session = jsch.getSession(username, remoteHost, port);
System.out.println("尝试连接 SFTP 服务器: " + username + "@" + remoteHost + ":" + port);
// 如果使用密码认证且提供了密码
if (password != null && !password.trim().isEmpty() && (sshKeyPath == null || sshKeyPath.trim().isEmpty())) {
session.setPassword(password);
System.out.println("使用密码认证.");
}
// 设置连接不进行主机密钥检查 (生产环境不推荐，应该配置 known_hosts)
// 在实际应用中，应该引导用户信任主机密钥或提前将主机密钥加入 known_hosts
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no"); // !!! 生产环境请谨慎使用或配置正确的主机密钥检查 !!!
session.setConfig(config);
// 3. 连接 Session
session.connect();
System.out.println("SFTP Session 连接成功.");
// 4. 打开 SFTP Channel
channel = session.openChannel("sftp");
channel.connect();
System.out.println("SFTP Channel 打开成功.");
channelSftp = (ChannelSftp) channel;
// 创建本地目录
File localDirectory = new File(localDir);
if (!localDirectory.exists()) {
localDirectory.mkdirs();
}
// 遍历远程文件路径并下载
for (String remoteFile : remoteFilePaths) {
String chmodDir = remoteFile.substring(0, remoteFile.indexOf("default"));
log.info("分配权限路径,{}",chmodDir);
// 切换目录并列出内容用于调试
sudoChmodORwx(session,chmodDir,password);
String fileName = new File(remoteFile).getName();
String localFilePath = localDir + fileName;
try (InputStream in = channelSftp.get(remoteFile);
FileOutputStream out = new FileOutputStream(localFilePath)) {
byte[] buffer = new byte[1024];
int bytesRead;
while ((bytesRead = in.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
}
copiedLocalPaths.put(fileName,localFilePath); // 添加本地路径
} catch (Exception e) {
throw new RuntimeException("文件复制失败: " + remoteFile, e);
}
}
return copiedLocalPaths;
} catch (JSchException e) {
throw new SftpUploadException("SFTP 连接或认证失败: " + e.getMessage(), e);
} catch (SftpUploadException e) {
// 重新抛出自定义异常
throw e;
} catch (Exception e) {
// 捕获其他未知异常
throw new SftpUploadException("SFTP 上传过程中发生未知异常: " + e.getMessage(), e);
} finally {
// 9. 关闭资源 (确保在任何情况下都关闭)
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
System.err.println("关闭本地文件流失败: " + e.getMessage());
e.printStackTrace(); // 打印堆栈以便调试
}
}
if (channelSftp != null) {
channelSftp.disconnect();
System.out.println("SFTP Channel 已断开.");
}
if (channel != null) {
channel.disconnect();
System.out.println("SFTP Channel 资源已释放.");
}
if (session != null) {
session.disconnect();
System.out.println("SFTP Session 已断开.");
}
}
}
/**
* 从远程服务器下载文件并返回文件输入流，并设置AflInfo对象