mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

接口优化。以及 AFL相关work修改

Browse Source
This commit is contained in:
even 2025-06-25 14:41:33 +08:00
parent 21b2319093
commit 22ec4646e8
8 changed files with 160 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
framework/spring-boot-starter-biz-data-permission/src/main/java/cd/casic/framework/datapermission/service/user/AdminUserService.java
View File

@ -138,6 +138,8 @@ public interface AdminUserService {
*/
AdminUserDO getUser(Long id);
public List<AdminUserDO> getUserListByIds(Collection<Long> idList);
/**
* 获得指定部门的用户数组
*

15
framework/spring-boot-starter-biz-tenant/src/main/java/cd/casic/framework/tenant/core/service/AdminUserServiceImpl.java
View File

@ -23,7 +23,6 @@ import cn.hutool.core.util.StrUtil;
import cd.casic.framework.commons.enums.CommonStatusEnum;
import cd.casic.framework.commons.exception.ServiceException;
import cd.casic.framework.commons.pojo.PageResult;
import cd.casic.framework.commons.util.collection.CollectionUtils;
import cd.casic.framework.commons.util.object.BeanUtils;
import cd.casic.framework.commons.util.validation.ValidationUtils;
@ -43,10 +42,13 @@ import org.springframework.context.annotation.Lazy;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import java.io.InputStream;
import java.time.LocalDateTime;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import static cd.casic.framework.commons.exception.util.ServiceExceptionUtil.exception;
import static cd.casic.framework.commons.util.collection.CollectionUtils.*;
@ -286,6 +288,13 @@ public class AdminUserServiceImpl implements AdminUserService {
public AdminUserDO getUser(Long id) {
return userMapper.selectById(id);
}
@Override
public List<AdminUserDO> getUserListByIds(Collection<Long> idList){
if (CollectionUtils.isEmpty(idList)) {
return Collections.emptyList();
}
return userMapper.selectByIds(idList);
}
@Override
public List<AdminUserDO> getUserListByDeptIds(Collection<Long> deptIds) {
@ -322,7 +331,7 @@ public class AdminUserServiceImpl implements AdminUserService {
}
// 获得岗位信息
List<AdminUserDO> users = userMapper.selectBatchIds(ids);
Map<Long, AdminUserDO> userMap = CollectionUtils.convertMap(users, AdminUserDO::getId);
Map<Long, AdminUserDO> userMap = users.stream().collect(Collectors.toMap(AdminUserDO::getId,Function.identity()));
// 校验
ids.forEach(id -> {
AdminUserDO user = userMap.get(id);
@ -368,7 +377,7 @@ public class AdminUserServiceImpl implements AdminUserService {
// 校验邮箱唯一
validateEmailUnique(id, email);
// 校验部门处于开启状态
deptService.validateDeptList(CollectionUtils.singleton(deptId));
deptService.validateDeptList(Collections.singleton(deptId));
// 校验岗位处于开启状态
postService.validatePostList(postIds);
return user;

11
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/docker/DockerImageNameResp.java Normal file
View File

@ -0,0 +1,11 @@
package cd.casic.ci.process.dto.resp.docker;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class DockerImageNameResp {
private String id;
private String value;
}

33
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java
View File

@ -15,6 +15,7 @@ import cd.casic.ci.process.process.dataObject.target.TargetVersion;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.service.resource.ResourceManagerService;
import cd.casic.ci.process.process.service.target.TargetVersionService;
import cd.casic.ci.process.util.CryptogramUtil;
import cd.casic.ci.process.util.SftpUploadUtil;
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo;
@ -62,6 +63,8 @@ public class AFLSlotCompileWorker extends DockerWorker {
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(task.getPipelineId()).getContextDef();
// 获取目标文件
TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId());
String fileName = targetVersion.getFileName();
append(context,"读取到当前文件名称:"+fileName);
String filePath = targetVersion.getFilePath();
File file = new File(filePath);
if (!file.exists() || !file.canRead()) {
@ -74,25 +77,49 @@ public class AFLSlotCompileWorker extends DockerWorker {
String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir;
// 上传目标文件 到指定资源服务器
try {
append(context,"AFL编译，上传文件路径:"+realPath);
SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,realPath,file.getName());
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(), CryptogramUtil.doDecrypt(machineInfo.getPassword()),"",filePath,realPath,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) {
append(context,"上传文件失败，请确认资源信息是否有误:"+JSON.toJSONString(machineInfo));
log.error("上传文件报错",e);
toBadEnding();
}
// 执行预设命令 ,进入目录
String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" +
"cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录
"mkdir -p "+workDir+"\n"+
"cd "+workDir+"\n";
"cd "+workDir+"\n"+handleZipFile(fileName);
Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
allCommand += commandScript+"\nll -s";
allCommand += commandScript+"\n";
log.info("AFL插装编译容器内执行的命令：{}",allCommand);
dockerRun(allCommand,dockerInfo,context);
// 更新全局变量
// /test目录下当前流水线工作目录
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir);
}
private String handleZipFile(String fileName){
if (StringUtils.isEmpty(fileName)) {
return "";
}
int i = fileName.lastIndexOf(".");
if (i==-1||i>=fileName.length()-1) {
return "";
}
String prefix = fileName.substring(0, i);
String suffix = fileName.substring(i + 1);
if ("zip".equals(suffix)) {
String cmd ="unzip -o "+fileName+"\n" +
"cd "+prefix;
return cmd;
}
if ("tar".equals(suffix)) {
String cmd ="tar -zxvf "+fileName+"\n" +
"cd "+prefix;
return cmd;
}
return "";
}
}

28
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java
View File

@ -8,7 +8,11 @@ import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.target.TargetVersion;
import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.service.target.TargetVersionService;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@ -19,8 +23,8 @@ import java.util.Map;
@WorkAtom(taskType = "AFL")
@Slf4j
public class AFLWorker extends DockerWorker {
@Resource
private TargetVersionService targetVersionService;
@Override
public void execute(TaskRunContext context) {
int statusCode = -1;
@ -54,13 +58,18 @@ public class AFLWorker extends DockerWorker {
append(context,"当前机器不支持docker");
return;
}
String output = workDir+PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT;
String output = workDir + PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT;
//
String volumeWorkDirPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX;
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef();
TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId());
String fileName = targetVersion.getFileName();
String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" +
"cd /test\n" +
"mkdir -p"+output+"\n" +
"afl-fuzz -i "+seedPath+" -o "+output+" "+workDir+ File.separator+commandEnd;
"cd " +PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+ getSourceName(fileName) +
"mkdir -p "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+File.separator+output+"\n" +
"afl-fuzz -i "+seedPath+" -o "+output+" "+commandEnd;
try {
//将节点的配置信息反编译成对象
log.info("AFL模糊测试执行脚本：{}",commandScript);
@ -75,4 +84,11 @@ public class AFLWorker extends DockerWorker {
localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode);
}
}
public String getSourceName(String fileName){
int dotIndex = fileName.lastIndexOf(".");
if (dotIndex!=-1) {
return "cd "+fileName.substring(0, dotIndex)+"\n";
}
return "";
}
}

8
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java
View File

@ -36,7 +36,7 @@ public class TestCaseGenerationWorker extends DockerWorker {
String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED;
Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY);
String binaryName = taskProperties.get("binaryName").toString();
String binaryName = taskProperties.get("binaryName") instanceof String ? ((String) taskProperties.get("binaryName")) : null;
String managerId = taskProperties.get("managerId") instanceof String ? ((String) taskProperties.get("managerId")) : null;
ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId);
String machineId = resourceById.getMachineId();
@ -73,17 +73,19 @@ public class TestCaseGenerationWorker extends DockerWorker {
toBadEnding();
}
File file = new File(filePath);
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+workDir+File.separator+seedPath;
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath;
// 将文件上传到服务器的 目录底下
log.info("测试用例选用上传模式,种子文件路径：{},种子上传路径:{}",filePath,seedTarget);
append(context,"测试用例选用上传模式,种子文件路径："+filePath+",种子上传路径:"+seedPath);
try {
SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,seedTarget,file.getName());
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),CryptogramUtil.doDecrypt(machineInfo.getPassword()),"",filePath,seedTarget,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) {
append(context,"seed文件上传失败");
log.error("seed文件上传失败",e);
}
append(context,"上传文件成功！");
}
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY,seedPath);
}

27
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/docker/impl/DockerServiceImpl.java
View File

@ -17,6 +17,7 @@ import org.assertj.core.util.Lists;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@ -48,7 +49,9 @@ public class DockerServiceImpl implements DockerService {
}
ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder().dockerHost(uri).build();
DockerClient build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build();
DockerClient build = null;
try{
build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build();
List<Image> exec = build.listImagesCmd().exec();
List<String> res = new LinkedList<>();
for (Image image : exec) {
@ -58,6 +61,15 @@ public class DockerServiceImpl implements DockerService {
}
}
return res;
} finally {
if (build!=null) {
try {
build.close();
} catch (IOException e) {
}
}
}
}
@Override
@ -72,7 +84,9 @@ public class DockerServiceImpl implements DockerService {
}
ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder().dockerHost(uri).build();
DockerClient build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build();
DockerClient build = null;
try{
build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build();
List<Image> exec = build.listImagesCmd().exec();
List<String> res = new LinkedList<>();
for (Image image : exec) {
@ -82,5 +96,14 @@ public class DockerServiceImpl implements DockerService {
}
}
return res;
} finally {
if (build!=null) {
try {
build.close();
} catch (IOException e) {
}
}
}
}
}

36
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/pipeline/impl/PipelineServiceImpl.java
View File

@ -47,6 +47,8 @@ import org.springframework.util.ObjectUtils;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* @author HopeLi
@ -231,7 +233,8 @@ public class PipelineServiceImpl extends ServiceImpl<PipelineDao, PipPipeline> i
//对流水线进行流水线信息赋值
respList.forEach(this::setStageAndTask);
//对用户姓名进行赋值
respList.forEach(this::setUserName);
// respList.forEach(this::setUserName);
setUserName(respList);
return respList;
}
@ -316,35 +319,52 @@ public class PipelineServiceImpl extends ServiceImpl<PipelineDao, PipPipeline> i
List<PipelineFindResp> respList = PipelineConverter.INSTANCE.toRespList(pipPipelinePage.getRecords());
//对流水线进行流水线信息赋值
respList.forEach(this::setStageAndTask);
respList.forEach(this::setUserName);
// respList.forEach(this::setStageAndTask);
// respList.forEach(this::setUserName);
setUserName(respList);
PageResult<PipelineFindResp> pageResult = new PageResult<>(respList,pipPipelinePage.getTotal(),pipPipelinePage.getCurrent(),pipPipelinePage.getSize());
return pageResult;
}
private void setUserName(PipelineFindResp pipelineFindResp) {
private void setUserName(List<PipelineFindResp> pipelineFindRespList) {
Set<Long> userIdSet = new HashSet<>(pipelineFindRespList.size()*3);
for (PipelineFindResp pipelineFindResp : pipelineFindRespList) {
if (!StringUtils.isEmpty(pipelineFindResp.getCreator())){
AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getCreator()));
userIdSet.add(Long.valueOf(pipelineFindResp.getCreator()));
}
if (!StringUtils.isEmpty(pipelineFindResp.getUpdater())){
userIdSet.add(Long.valueOf(pipelineFindResp.getUpdater()));
}
if (!StringUtils.isEmpty(pipelineFindResp.getExecuteUserId())){
userIdSet.add(Long.valueOf(pipelineFindResp.getExecuteUserId()));
}
}
Map<Long, AdminUserDO> userIdMap = adminUserService.getUserListByIds(userIdSet).stream().collect(Collectors.toMap(AdminUserDO::getId, Function.identity()));
for (PipelineFindResp pipelineFindResp : pipelineFindRespList) {
if (!StringUtils.isEmpty(pipelineFindResp.getCreator())){
AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getCreator()));
if (!ObjectUtils.isEmpty(user)){
pipelineFindResp.setCreatorName(user.getUsername());
}
}
if (!StringUtils.isEmpty(pipelineFindResp.getUpdater())){
AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getUpdater()));
AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getUpdater()));
if (!ObjectUtils.isEmpty(user)){
pipelineFindResp.setUpdaterName(user.getUsername());
}
}
if (!StringUtils.isEmpty(pipelineFindResp.getExecuteUserId())){
AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getExecuteUserId()));
AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getExecuteUserId()));
if (!ObjectUtils.isEmpty(user)){
pipelineFindResp.setExecuteUserName(user.getUsername());
}
}
}
}
@Override
public void pipelineClone(PipelineReq req) {