0705 ljc afl模块文件操作以及入库操作以及时序表操作等接口开发，包含分片代码以及文件工具类代码
This commit is contained in:
HopeLi
parent
42321b4528
commit
db5479cdd0
@ -3,9 +3,11 @@ package cd.casic.ci.api;
|
|||||||
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
|
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
|
||||||
import cd.casic.ci.process.dto.resp.aflManager.AflCrashesInfoResp;
|
import cd.casic.ci.process.dto.resp.aflManager.AflCrashesInfoResp;
|
||||||
import cd.casic.ci.process.dto.resp.aflManager.AflInfoResp;
|
import cd.casic.ci.process.dto.resp.aflManager.AflInfoResp;
|
||||||
|
import cd.casic.ci.process.dto.resp.aflManager.AflPlotInfoResp;
|
||||||
import cd.casic.ci.process.dto.resp.aflManager.SeedsCountResp;
|
import cd.casic.ci.process.dto.resp.aflManager.SeedsCountResp;
|
||||||
import cd.casic.ci.process.process.service.aflManager.AflCrashesInfoService;
|
import cd.casic.ci.process.process.service.aflManager.AflCrashesInfoService;
|
||||||
import cd.casic.ci.process.process.service.aflManager.AflInfoService;
|
import cd.casic.ci.process.process.service.aflManager.AflInfoService;
|
||||||
|
import cd.casic.ci.process.process.service.aflManager.AflPlotInfoService;
|
||||||
import cd.casic.ci.process.process.service.aflManager.AflSeedInfoService;
|
import cd.casic.ci.process.process.service.aflManager.AflSeedInfoService;
|
||||||
import cd.casic.ci.process.util.SftpUploadUtil;
|
import cd.casic.ci.process.util.SftpUploadUtil;
|
||||||
import cd.casic.framework.commons.pojo.CommonResult;
|
import cd.casic.framework.commons.pojo.CommonResult;
|
||||||
@ -38,6 +40,9 @@ public class AflManagerController {
|
|||||||
@Resource
|
@Resource
|
||||||
private AflSeedInfoService aflSeedInfoService;
|
private AflSeedInfoService aflSeedInfoService;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private AflPlotInfoService aflPlotInfoService;
|
||||||
|
|
||||||
@PostMapping(path="/saveAflInfo")
|
@PostMapping(path="/saveAflInfo")
|
||||||
public CommonResult<String> saveAflInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
|
public CommonResult<String> saveAflInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
|
||||||
|
|
||||||
@ -99,4 +104,32 @@ public class AflManagerController {
|
|||||||
|
|
||||||
return CommonResult.success();
|
return CommonResult.success();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@PostMapping(path="/saveAflPlotInfo")
|
||||||
|
public CommonResult<String> saveAflPlotInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
|
||||||
|
|
||||||
|
String groupIdentifier = aflPlotInfoService.saveAflPlotInfo(req);
|
||||||
|
|
||||||
|
return CommonResult.success(groupIdentifier);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@PostMapping(path="/findAflPlotInfo")
|
||||||
|
public CommonResult<AflPlotInfoResp> findAflPlotInfo(@RequestBody @Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
|
||||||
|
|
||||||
|
AflPlotInfoResp resp = aflPlotInfoService.findAflPlotInfo(req);
|
||||||
|
|
||||||
|
return CommonResult.success(resp);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@PostMapping(path="/findAflPlotInfoList")
|
||||||
|
public CommonResult<List<AflPlotInfoResp>> findAflPlotInfoList(@RequestBody @Valid AflManagerReq req){
|
||||||
|
|
||||||
|
List<AflPlotInfoResp> respList = aflPlotInfoService.findAflPlotInfoList(req);
|
||||||
|
|
||||||
|
return CommonResult.success(respList);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,114 @@
|
|||||||
|
package cd.casic.ci.process.dto.resp.aflManager;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @ClassName ReportResp
|
||||||
|
* @Author hopeli
|
||||||
|
* @Date 2025/5/10 10:53
|
||||||
|
* @Version 1.0
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
public class AflPlotInfoResp {
|
||||||
|
private String id;
|
||||||
|
/**
|
||||||
|
* 流水线ID
|
||||||
|
*/
|
||||||
|
private String pipelineId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 流水线历史记录ID
|
||||||
|
*/
|
||||||
|
private String pipelineHistoryId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务节点ID
|
||||||
|
*/
|
||||||
|
private String taskId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分组标识
|
||||||
|
*/
|
||||||
|
private String groupIdentifier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文件地址路径
|
||||||
|
*/
|
||||||
|
private String filePath;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 相对时间
|
||||||
|
*/
|
||||||
|
private String relativeTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 完成的周期数
|
||||||
|
*/
|
||||||
|
private String cyclesDone;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 当前项目
|
||||||
|
*/
|
||||||
|
private String curItem;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 语料库数量
|
||||||
|
*/
|
||||||
|
private String corpusCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 挂起总数
|
||||||
|
*/
|
||||||
|
private String pendingTotal;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 挂起的重点项
|
||||||
|
*/
|
||||||
|
private String pendingFavs;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 映射大小
|
||||||
|
*/
|
||||||
|
private String mapSize;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 已保存的崩溃数
|
||||||
|
*/
|
||||||
|
private String savedCrashes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 已保存的挂起数
|
||||||
|
*/
|
||||||
|
private String savedHangs;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 最大深度
|
||||||
|
*/
|
||||||
|
private String maxDepth;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 每秒执行次数
|
||||||
|
*/
|
||||||
|
private String execsPerSec;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 总执行次数
|
||||||
|
*/
|
||||||
|
private String totalExecs;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 发现的边数
|
||||||
|
*/
|
||||||
|
private String edgesFound;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 总崩溃次数
|
||||||
|
*/
|
||||||
|
private String totalCrashes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 服务器数量
|
||||||
|
*/
|
||||||
|
private String serversCount;
|
||||||
|
|
||||||
|
}
|
||||||
@ -2,8 +2,10 @@ package cd.casic.ci.process.process.converter;
|
|||||||
|
|
||||||
import cd.casic.ci.process.dto.resp.aflManager.AflCrashesInfoResp;
|
import cd.casic.ci.process.dto.resp.aflManager.AflCrashesInfoResp;
|
||||||
import cd.casic.ci.process.dto.resp.aflManager.AflInfoResp;
|
import cd.casic.ci.process.dto.resp.aflManager.AflInfoResp;
|
||||||
|
import cd.casic.ci.process.dto.resp.aflManager.AflPlotInfoResp;
|
||||||
import cd.casic.ci.process.process.dataObject.aflManager.AflCrashesInfo;
|
import cd.casic.ci.process.process.dataObject.aflManager.AflCrashesInfo;
|
||||||
import cd.casic.ci.process.process.dataObject.aflManager.AflInfo;
|
import cd.casic.ci.process.process.dataObject.aflManager.AflInfo;
|
||||||
|
import cd.casic.ci.process.process.dataObject.aflManager.AflPlotInfo;
|
||||||
import org.mapstruct.Mapper;
|
import org.mapstruct.Mapper;
|
||||||
import org.mapstruct.NullValuePropertyMappingStrategy;
|
import org.mapstruct.NullValuePropertyMappingStrategy;
|
||||||
import org.mapstruct.factory.Mappers;
|
import org.mapstruct.factory.Mappers;
|
||||||
@ -26,4 +28,8 @@ public interface AflManagerConverter {
|
|||||||
List<AflInfoResp> toAflInfoRespList(List<AflInfo> aflInfos);
|
List<AflInfoResp> toAflInfoRespList(List<AflInfo> aflInfos);
|
||||||
|
|
||||||
List<AflCrashesInfoResp> toAflCrashesInfoRespList(List<AflCrashesInfo> aflCrashesInfos);
|
List<AflCrashesInfoResp> toAflCrashesInfoRespList(List<AflCrashesInfo> aflCrashesInfos);
|
||||||
|
|
||||||
|
AflPlotInfoResp toAflPlotInfoResp(AflPlotInfo aflPlotInfo);
|
||||||
|
|
||||||
|
List<AflPlotInfoResp> toAflPlotInfoRespList(List<AflPlotInfo> aflPlotInfos);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,14 @@
|
|||||||
|
package cd.casic.ci.process.process.dao.aflManager;
|
||||||
|
|
||||||
|
import cd.casic.ci.process.process.dataObject.aflManager.AflPlotInfo;
|
||||||
|
import cd.casic.framework.mybatis.core.mapper.BaseMapperX;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author HopeLi
|
||||||
|
* @version v1.0
|
||||||
|
* @ClassName AflInfoDao
|
||||||
|
* @Date: 2025/5/13 14:39
|
||||||
|
* @Description:
|
||||||
|
*/
|
||||||
|
public interface AflPlotInfoDao extends BaseMapperX<AflPlotInfo> {
|
||||||
|
}
|
||||||
@ -0,0 +1,112 @@
|
|||||||
|
package cd.casic.ci.process.process.dataObject.aflManager;
|
||||||
|
|
||||||
|
import cd.casic.ci.process.process.dataObject.base.PipBaseElement;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.EqualsAndHashCode;
|
||||||
|
|
||||||
|
@EqualsAndHashCode(callSuper = true)
|
||||||
|
@Data
|
||||||
|
@TableName("afl_plot_info")
|
||||||
|
public class AflPlotInfo extends PipBaseElement {
|
||||||
|
/**
|
||||||
|
* 流水线ID
|
||||||
|
*/
|
||||||
|
private String pipelineId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 流水线历史记录ID
|
||||||
|
*/
|
||||||
|
private String pipelineHistoryId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务节点ID
|
||||||
|
*/
|
||||||
|
private String taskId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分组标识
|
||||||
|
*/
|
||||||
|
private String groupIdentifier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文件地址路径
|
||||||
|
*/
|
||||||
|
private String filePath;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 相对时间
|
||||||
|
*/
|
||||||
|
private String relativeTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 完成的周期数
|
||||||
|
*/
|
||||||
|
private String cyclesDone;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 当前项目
|
||||||
|
*/
|
||||||
|
private String curItem;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 语料库数量
|
||||||
|
*/
|
||||||
|
private String corpusCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 挂起总数
|
||||||
|
*/
|
||||||
|
private String pendingTotal;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 挂起的重点项
|
||||||
|
*/
|
||||||
|
private String pendingFavs;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 映射大小
|
||||||
|
*/
|
||||||
|
private String mapSize;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 已保存的崩溃数
|
||||||
|
*/
|
||||||
|
private String savedCrashes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 已保存的挂起数
|
||||||
|
*/
|
||||||
|
private String savedHangs;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 最大深度
|
||||||
|
*/
|
||||||
|
private String maxDepth;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 每秒执行次数
|
||||||
|
*/
|
||||||
|
private String execsPerSec;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 总执行次数
|
||||||
|
*/
|
||||||
|
private String totalExecs;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 发现的边数
|
||||||
|
*/
|
||||||
|
private String edgesFound;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 总崩溃次数
|
||||||
|
*/
|
||||||
|
private String totalCrashes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 服务器数量
|
||||||
|
*/
|
||||||
|
private String serversCount;
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
package cd.casic.ci.process.process.service.aflManager;
|
||||||
|
|
||||||
|
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
|
||||||
|
import cd.casic.ci.process.dto.resp.aflManager.AflPlotInfoResp;
|
||||||
|
import cd.casic.ci.process.process.dataObject.aflManager.AflPlotInfo;
|
||||||
|
import cd.casic.ci.process.util.SftpUploadUtil;
|
||||||
|
import com.baomidou.mybatisplus.extension.service.IService;
|
||||||
|
import jakarta.validation.Valid;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author HopeLi
|
||||||
|
* @version v1.0
|
||||||
|
* @ClassName AflInfoService
|
||||||
|
* @Date: 2025/5/17 10:20
|
||||||
|
* @Description:
|
||||||
|
*/
|
||||||
|
public interface AflPlotInfoService extends IService<AflPlotInfo> {
|
||||||
|
|
||||||
|
String saveAflPlotInfo(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException;
|
||||||
|
|
||||||
|
AflPlotInfoResp findAflPlotInfo(@Valid AflManagerReq req) throws SftpUploadUtil.SftpUploadException;
|
||||||
|
|
||||||
|
List<AflPlotInfoResp> findAflPlotInfoList(@Valid AflManagerReq req);
|
||||||
|
|
||||||
|
void updateHistoryPipelineIdByAflPlotInfo(@Valid AflManagerReq req);
|
||||||
|
}
|
||||||
@ -55,6 +55,7 @@ public class AflInfoServiceImpl extends ServiceImpl<AflInfoDao, AflInfo> impleme
|
|||||||
|
|
||||||
BaseRunContext runContext = runContextManager.getContext(req.getPipelineId());
|
BaseRunContext runContext = runContextManager.getContext(req.getPipelineId());
|
||||||
String resourceId = runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY)) : null;
|
String resourceId = runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY)) : null;
|
||||||
|
// String resourceId = "1";
|
||||||
if (StringUtils.isEmpty(resourceId)){
|
if (StringUtils.isEmpty(resourceId)){
|
||||||
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到资源");
|
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到资源");
|
||||||
}
|
}
|
||||||
@ -65,6 +66,7 @@ public class AflInfoServiceImpl extends ServiceImpl<AflInfoDao, AflInfo> impleme
|
|||||||
}
|
}
|
||||||
|
|
||||||
PipResourceMachine resourceMachine = resourceById.getResourceMachine();
|
PipResourceMachine resourceMachine = resourceById.getResourceMachine();
|
||||||
|
// resourceMachine.setPassword("hnidc0628cn!@#lld");
|
||||||
|
|
||||||
AflInfo aflInfo = SftpUploadUtil.downloadFileSftpForInputStreamAndSetAflInfo(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats");
|
AflInfo aflInfo = SftpUploadUtil.downloadFileSftpForInputStreamAndSetAflInfo(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/fuzzer_stats");
|
||||||
//在本地创建一个新文件夹，将远程文件复制一份过去
|
//在本地创建一个新文件夹，将远程文件复制一份过去
|
||||||
|
|||||||
@ -0,0 +1,154 @@
|
|||||||
|
package cd.casic.ci.process.process.service.aflManager.impl;
|
||||||
|
|
||||||
|
import cd.casic.ci.process.dto.req.aflManager.AflManagerReq;
|
||||||
|
import cd.casic.ci.process.dto.resp.aflManager.AflPlotInfoResp;
|
||||||
|
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
|
||||||
|
import cd.casic.ci.process.engine.constant.PipelineVariableConstant;
|
||||||
|
import cd.casic.ci.process.engine.manager.RunContextManager;
|
||||||
|
import cd.casic.ci.process.engine.runContext.BaseRunContext;
|
||||||
|
import cd.casic.ci.process.process.converter.AflManagerConverter;
|
||||||
|
import cd.casic.ci.process.process.dao.aflManager.AflPlotInfoDao;
|
||||||
|
import cd.casic.ci.process.process.dataObject.aflManager.AflPlotInfo;
|
||||||
|
import cd.casic.ci.process.process.dataObject.resource.PipResourceMachine;
|
||||||
|
import cd.casic.ci.process.process.service.aflManager.AflPlotInfoService;
|
||||||
|
import cd.casic.ci.process.process.service.resource.ResourceManagerService;
|
||||||
|
import cd.casic.ci.process.util.SftpUploadUtil;
|
||||||
|
import cd.casic.framework.commons.exception.ServiceException;
|
||||||
|
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
||||||
|
import de.danielbechler.util.Collections;
|
||||||
|
import jakarta.annotation.Resource;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.ObjectUtils;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author HopeLi
|
||||||
|
* @version v1.0
|
||||||
|
* @ClassName TargetVersionServiceImpl
|
||||||
|
* @Date: 2025/5/17 15:20
|
||||||
|
* @Description:
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@Slf4j
|
||||||
|
public class AflPlotInfoServiceImpl extends ServiceImpl<AflPlotInfoDao, AflPlotInfo> implements AflPlotInfoService {
|
||||||
|
private static final String remoteFilePath = "/home/casic/706/yunqi/";
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private RunContextManager runContextManager;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private ResourceManagerService resourceManagerService;
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String saveAflPlotInfo(AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
|
||||||
|
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
|
||||||
|
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
|
||||||
|
}
|
||||||
|
|
||||||
|
BaseRunContext runContext = runContextManager.getContext(req.getPipelineId());
|
||||||
|
String resourceId = runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY)) : null;
|
||||||
|
if (StringUtils.isEmpty(resourceId)){
|
||||||
|
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到资源");
|
||||||
|
}
|
||||||
|
|
||||||
|
ResourceFindResp resourceById = resourceManagerService.findResourceById(resourceId);
|
||||||
|
if (resourceById == null || resourceById.getResourceMachine() == null) {
|
||||||
|
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
|
||||||
|
}
|
||||||
|
|
||||||
|
PipResourceMachine resourceMachine = resourceById.getResourceMachine();
|
||||||
|
|
||||||
|
AflPlotInfo aflPlotInfo = SftpUploadUtil.downloadFileSftpForLastLineAndSetAflPlotInfo(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data");
|
||||||
|
//在本地创建一个新文件夹，将远程文件复制一份过去
|
||||||
|
List<String> sourceFilePaths = new ArrayList<>();
|
||||||
|
sourceFilePaths.add(remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data");
|
||||||
|
List<String> strings = SftpUploadUtil.copyRemoteFilesToLocal(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, sourceFilePaths, "/home/ops/opsFile/plot_result/" + UUID.randomUUID() + "/");
|
||||||
|
|
||||||
|
aflPlotInfo.setFilePath(strings.get(0));
|
||||||
|
aflPlotInfo.setPipelineId(req.getPipelineId());
|
||||||
|
if (!StringUtils.isEmpty(req.getTaskId())){
|
||||||
|
aflPlotInfo.setTaskId(req.getTaskId());
|
||||||
|
}
|
||||||
|
if (!StringUtils.isEmpty(req.getPipelineHistoryId())){
|
||||||
|
aflPlotInfo.setPipelineHistoryId(req.getPipelineHistoryId());
|
||||||
|
}
|
||||||
|
aflPlotInfo.setGroupIdentifier(UUID.randomUUID().toString());
|
||||||
|
baseMapper.insert(aflPlotInfo);
|
||||||
|
|
||||||
|
return aflPlotInfo.getGroupIdentifier();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateHistoryPipelineIdByAflPlotInfo(AflManagerReq req) {
|
||||||
|
if (!StringUtils.isEmpty(req.getGroupIdentifier())){
|
||||||
|
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"groupIdentifier不能为空");
|
||||||
|
}
|
||||||
|
QueryWrapper<AflPlotInfo> wrapper = new QueryWrapper<>();
|
||||||
|
wrapper.eq("group_identifier", req.getGroupIdentifier());
|
||||||
|
List<AflPlotInfo> aflPlotInfos = baseMapper.selectList(wrapper);
|
||||||
|
if (!Collections.isEmpty(aflPlotInfos)){
|
||||||
|
aflPlotInfos.forEach(o->{
|
||||||
|
o.setPipelineHistoryId(req.getPipelineHistoryId());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
baseMapper.updateBatch(aflPlotInfos);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AflPlotInfoResp findAflPlotInfo(AflManagerReq req) throws SftpUploadUtil.SftpUploadException {
|
||||||
|
if (StringUtils.isEmpty(req.getPipelineId()) || StringUtils.isEmpty(req.getTaskId())){
|
||||||
|
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"pipelineId或者taskId不能为空");
|
||||||
|
}
|
||||||
|
|
||||||
|
BaseRunContext runContext = runContextManager.getContext(req.getPipelineId());
|
||||||
|
String resourceId = runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY) instanceof String ? ((String) runContext.getGlobalVariables().get(PipelineVariableConstant.AFL_RESOURCE_MANAGER_ID_KEY)) : null;
|
||||||
|
if (StringUtils.isEmpty(resourceId)){
|
||||||
|
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"未找到资源");
|
||||||
|
}
|
||||||
|
|
||||||
|
ResourceFindResp resourceById = resourceManagerService.findResourceById(resourceId);
|
||||||
|
if (resourceById == null || resourceById.getResourceMachine() == null) {
|
||||||
|
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"资源信息错误");
|
||||||
|
}
|
||||||
|
|
||||||
|
PipResourceMachine resourceMachine = resourceById.getResourceMachine();
|
||||||
|
AflPlotInfo aflPlotInfo = SftpUploadUtil.downloadFileSftpForLastLineAndSetAflPlotInfo(resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(),null, remoteFilePath + "PIP_" + req.getPipelineId() + "/ai_afl/default/plot_data");
|
||||||
|
|
||||||
|
return AflManagerConverter.INSTANCE.toAflPlotInfoResp(aflPlotInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<AflPlotInfoResp> findAflPlotInfoList(AflManagerReq req) {
|
||||||
|
QueryWrapper<AflPlotInfo> wrapper = new QueryWrapper<>();
|
||||||
|
if (!ObjectUtils.isEmpty(req.getId())){
|
||||||
|
wrapper.eq("id",req.getId());
|
||||||
|
}
|
||||||
|
if (!ObjectUtils.isEmpty(req.getPipelineId())){
|
||||||
|
wrapper.eq("pipeline_id",req.getPipelineId());
|
||||||
|
}
|
||||||
|
if (!ObjectUtils.isEmpty(req.getPipelineHistoryId())){
|
||||||
|
wrapper.eq("pipeline_history_id",req.getPipelineHistoryId());
|
||||||
|
}
|
||||||
|
if (!ObjectUtils.isEmpty(req.getTaskId())){
|
||||||
|
wrapper.eq("task_id",req.getTaskId());
|
||||||
|
}
|
||||||
|
if (!ObjectUtils.isEmpty(req.getGroupIdentifier())){
|
||||||
|
wrapper.eq("group_identifier",req.getGroupIdentifier());
|
||||||
|
}
|
||||||
|
|
||||||
|
List<AflPlotInfo> aflPlotInfos = baseMapper.selectList(wrapper);
|
||||||
|
if (ObjectUtils.isEmpty(aflPlotInfos)){
|
||||||
|
return new ArrayList<>(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
return AflManagerConverter.INSTANCE.toAflPlotInfoRespList(aflPlotInfos);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -186,8 +186,8 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
|
|||||||
resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/seed_result/" + UUID.randomUUID() + "/");
|
resourceMachine.getMachineHost(), Integer.parseInt(resourceMachine.getSshPort()), resourceMachine.getUsername(), resourceMachine.getPassword(), null, files, "/home/ops/opsFile/seed_result/" + UUID.randomUUID() + "/");
|
||||||
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
|
System.out.println("共复制 " + copiedFiles.size() + " 个文件");
|
||||||
|
|
||||||
|
List<AflSeedInfo> aflSeedInfos = new ArrayList<>(0);
|
||||||
if (!CollectionUtils.isEmpty(copiedFiles)){
|
if (!CollectionUtils.isEmpty(copiedFiles)){
|
||||||
List<AflSeedInfo> aflSeedInfos = new ArrayList<>(0);
|
|
||||||
//新增数据到crashes表中
|
//新增数据到crashes表中
|
||||||
copiedFiles.forEach(o->{
|
copiedFiles.forEach(o->{
|
||||||
AflSeedInfo aflSeedInfo = new AflSeedInfo();
|
AflSeedInfo aflSeedInfo = new AflSeedInfo();
|
||||||
@ -228,9 +228,22 @@ public class AflSeedInfoServiceImpl extends ServiceImpl<AflSeedInfoDao, AflSeedI
|
|||||||
|
|
||||||
aflSeedInfos.add(aflSeedInfo);
|
aflSeedInfos.add(aflSeedInfo);
|
||||||
});
|
});
|
||||||
baseMapper.insertBatch(aflSeedInfos);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!CollectionUtils.isEmpty(aflSeedInfos)){
|
||||||
|
//查询种子运行分钟数，同时入库
|
||||||
|
List<SeedsCountResp> seedsCount = this.findSeedsCount(req);
|
||||||
|
if (!CollectionUtils.isEmpty(seedsCount)){
|
||||||
|
aflSeedInfos.forEach(o->{
|
||||||
|
seedsCount.forEach(j->{
|
||||||
|
if (o.getSeedId().equals(j.getSeedId())){
|
||||||
|
o.setMinutesDifference(j.getMinutesDifference());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
baseMapper.insertBatch(aflSeedInfos);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
package cd.casic.ci.process.util;
|
package cd.casic.ci.process.util;
|
||||||
|
|
||||||
import cd.casic.ci.process.process.dataObject.aflManager.AflInfo;
|
import cd.casic.ci.process.process.dataObject.aflManager.AflInfo;
|
||||||
|
import cd.casic.ci.process.process.dataObject.aflManager.AflPlotInfo;
|
||||||
import cd.casic.framework.commons.exception.ServiceException;
|
import cd.casic.framework.commons.exception.ServiceException;
|
||||||
import com.amazonaws.util.IOUtils;
|
import com.amazonaws.util.IOUtils;
|
||||||
import com.jcraft.jsch.*;
|
import com.jcraft.jsch.*;
|
||||||
@ -11,6 +12,8 @@ import org.springframework.web.multipart.MultipartFile;
|
|||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
|
import static com.fhs.common.utils.StringUtil.trim;
|
||||||
|
|
||||||
public class SftpUploadUtil {
|
public class SftpUploadUtil {
|
||||||
|
|
||||||
private static final int DEFAULT_SFTP_PORT = 22;
|
private static final int DEFAULT_SFTP_PORT = 22;
|
||||||
@ -1245,6 +1248,168 @@ public class SftpUploadUtil {
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从远程服务器下载文件并返回文件输入流，并设置AflInfo对象
|
||||||
|
*
|
||||||
|
* @param remoteHost 远程服务器地址
|
||||||
|
* @param remotePort 远程服务器端口
|
||||||
|
* @param username 远程服务器用户名
|
||||||
|
* @param password 远程服务器密码
|
||||||
|
* @param sshKeyPath SSH密钥路径
|
||||||
|
* @param remoteFilePath 远程文件路径
|
||||||
|
* @return AflInfo对象，包含文件输入流和文件信息
|
||||||
|
* @throws SftpUploadException 如果下载文件或设置AflInfo对象时发生错误
|
||||||
|
*/
|
||||||
|
public static AflPlotInfo downloadFileSftpForLastLineAndSetAflPlotInfo(String remoteHost,
|
||||||
|
Integer remotePort,
|
||||||
|
String username,
|
||||||
|
String password,
|
||||||
|
String sshKeyPath,
|
||||||
|
String remoteFilePath) throws SftpUploadException {
|
||||||
|
|
||||||
|
Session session = null;
|
||||||
|
Channel channel = null;
|
||||||
|
ChannelSftp channelSftp = null;
|
||||||
|
InputStream inputStream = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
JSch jsch = new JSch();
|
||||||
|
|
||||||
|
// 1. 添加身份认证信息 (密码或密钥)
|
||||||
|
if (sshKeyPath != null && !sshKeyPath.trim().isEmpty()) {
|
||||||
|
// 使用 SSH Key 认证
|
||||||
|
File sshKeyFile = new File(sshKeyPath);
|
||||||
|
if (!sshKeyFile.exists() || !sshKeyFile.isFile()) {
|
||||||
|
throw new SftpUploadException("SSH Key 文件不存在或不是一个有效文件: " + sshKeyPath);
|
||||||
|
}
|
||||||
|
jsch.addIdentity(sshKeyPath);
|
||||||
|
System.out.println("使用 SSH Key 认证: " + sshKeyPath);
|
||||||
|
} else if (password == null || password.trim().isEmpty()) {
|
||||||
|
// 如果没有提供密码或密钥路径，则认证信息不全
|
||||||
|
throw new SftpUploadException("必须提供密码或 SSH Key 路径进行 SFTP 认证.");
|
||||||
|
}
|
||||||
|
// 如果提供了密码，将在 getSession 后设置，因为 getSession 需要用户名、主机和端口先建立连接意图
|
||||||
|
|
||||||
|
|
||||||
|
// 2. 获取 Session
|
||||||
|
int port = (remotePort != null && remotePort > 0) ? remotePort : DEFAULT_SFTP_PORT;
|
||||||
|
session = jsch.getSession(username, remoteHost, port);
|
||||||
|
System.out.println("尝试连接 SFTP 服务器: " + username + "@" + remoteHost + ":" + port);
|
||||||
|
|
||||||
|
|
||||||
|
// 如果使用密码认证且提供了密码
|
||||||
|
if (password != null && !password.trim().isEmpty() && (sshKeyPath == null || sshKeyPath.trim().isEmpty())) {
|
||||||
|
session.setPassword(password);
|
||||||
|
System.out.println("使用密码认证.");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 设置连接不进行主机密钥检查 (生产环境不推荐，应该配置 known_hosts)
|
||||||
|
// 在实际应用中，应该引导用户信任主机密钥或提前将主机密钥加入 known_hosts
|
||||||
|
java.util.Properties config = new java.util.Properties();
|
||||||
|
config.put("StrictHostKeyChecking", "no"); // !!! 生产环境请谨慎使用或配置正确的主机密钥检查 !!!
|
||||||
|
session.setConfig(config);
|
||||||
|
|
||||||
|
// 3. 连接 Session
|
||||||
|
session.connect();
|
||||||
|
System.out.println("SFTP Session 连接成功.");
|
||||||
|
|
||||||
|
// 4. 打开 SFTP Channel
|
||||||
|
channel = session.openChannel("sftp");
|
||||||
|
channel.connect();
|
||||||
|
System.out.println("SFTP Channel 打开成功.");
|
||||||
|
|
||||||
|
channelSftp = (ChannelSftp) channel;
|
||||||
|
|
||||||
|
String remoteDir = remoteFilePath.substring(0, remoteFilePath.lastIndexOf('/'));
|
||||||
|
String fileName = remoteFilePath.substring(remoteFilePath.lastIndexOf('/') + 1);
|
||||||
|
|
||||||
|
channelSftp.cd(remoteDir);
|
||||||
|
inputStream = channelSftp.get(fileName);
|
||||||
|
|
||||||
|
if (inputStream == null) {
|
||||||
|
throw new SftpUploadException("无法获取远程文件输入流，请检查文件是否存在或被其他进程占用: " + fileName);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 读取文件，找到最后一行
|
||||||
|
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
|
||||||
|
String line;
|
||||||
|
String lastLine = "";
|
||||||
|
|
||||||
|
while ((line = reader.readLine()) != null) {
|
||||||
|
if (!line.startsWith("#")) { // 忽略注释行
|
||||||
|
lastLine = line;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lastLine.isEmpty()) {
|
||||||
|
throw new SftpUploadException("文件内容为空或没有有效数据行");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 按逗号分割字段
|
||||||
|
String[] values = lastLine.split(",");
|
||||||
|
|
||||||
|
if (values.length < 15) {
|
||||||
|
throw new SftpUploadException("文件格式不正确，字段数量不足");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 构建 AflPlotInfo 对象
|
||||||
|
AflPlotInfo aflPlotInfo = new AflPlotInfo();
|
||||||
|
|
||||||
|
aflPlotInfo.setRelativeTime(trim(values[0]));
|
||||||
|
aflPlotInfo.setCyclesDone(trim(values[1]));
|
||||||
|
aflPlotInfo.setCurItem(trim(values[2]));
|
||||||
|
aflPlotInfo.setCorpusCount(trim(values[3]));
|
||||||
|
aflPlotInfo.setPendingTotal(trim(values[4]));
|
||||||
|
aflPlotInfo.setPendingFavs(trim(values[5]));
|
||||||
|
aflPlotInfo.setMapSize(trim(values[6]));
|
||||||
|
aflPlotInfo.setSavedCrashes(trim(values[7]));
|
||||||
|
aflPlotInfo.setSavedHangs(trim(values[8]));
|
||||||
|
aflPlotInfo.setMaxDepth(trim(values[9]));
|
||||||
|
aflPlotInfo.setExecsPerSec(trim(values[10]));
|
||||||
|
aflPlotInfo.setTotalExecs(trim(values[11]));
|
||||||
|
aflPlotInfo.setEdgesFound(trim(values[12]));
|
||||||
|
aflPlotInfo.setTotalCrashes(trim(values[13]));
|
||||||
|
aflPlotInfo.setServersCount(trim(values[14]));
|
||||||
|
|
||||||
|
return aflPlotInfo;
|
||||||
|
} catch (JSchException e) {
|
||||||
|
throw new SftpUploadException("SFTP 连接或认证失败: " + e.getMessage(), e);
|
||||||
|
} catch (SftpException e) {
|
||||||
|
throw new SftpUploadException("SFTP 操作失败 (如切换目录或上传文件): " + e.getMessage(), e);
|
||||||
|
} catch (SftpUploadException e) {
|
||||||
|
// 重新抛出自定义异常
|
||||||
|
throw e;
|
||||||
|
} catch (Exception e) {
|
||||||
|
// 捕获其他未知异常
|
||||||
|
throw new SftpUploadException("SFTP 上传过程中发生未知异常: " + e.getMessage(), e);
|
||||||
|
} finally {
|
||||||
|
// 9. 关闭资源 (确保在任何情况下都关闭)
|
||||||
|
if (inputStream != null) {
|
||||||
|
try {
|
||||||
|
inputStream.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
System.err.println("关闭本地文件流失败: " + e.getMessage());
|
||||||
|
e.printStackTrace(); // 打印堆栈以便调试
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (channelSftp != null) {
|
||||||
|
channelSftp.disconnect();
|
||||||
|
System.out.println("SFTP Channel 已断开.");
|
||||||
|
}
|
||||||
|
if (channel != null) {
|
||||||
|
channel.disconnect();
|
||||||
|
System.out.println("SFTP Channel 资源已释放.");
|
||||||
|
}
|
||||||
|
if (session != null) {
|
||||||
|
session.disconnect();
|
||||||
|
System.out.println("SFTP Session 已断开.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private static AflInfo setAflinfo(Map<String, String> statsMap) {
|
private static AflInfo setAflinfo(Map<String, String> statsMap) {
|
||||||
//映射到 AflInfo 实体
|
//映射到 AflInfo 实体
|
||||||
AflInfo aflInfo = new AflInfo();
|
AflInfo aflInfo = new AflInfo();
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user