From 22ec4646e842fedc0bedcc53cde97a355dcc59f6 Mon Sep 17 00:00:00 2001 From: even <827656971@qq.com> Date: Wed, 25 Jun 2025 14:41:33 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E4=BC=98=E5=8C=96=E3=80=82?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=20AFL=E7=9B=B8=E5=85=B3work=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/user/AdminUserService.java | 2 + .../core/service/AdminUserServiceImpl.java | 15 ++++- .../dto/resp/docker/DockerImageNameResp.java | 11 ++++ .../worker/afl/AFLSlotCompileWorker.java | 33 ++++++++++- .../process/engine/worker/afl/AFLWorker.java | 28 +++++++-- .../worker/afl/TestCaseGenerationWorker.java | 8 ++- .../docker/impl/DockerServiceImpl.java | 55 +++++++++++++----- .../pipeline/impl/PipelineServiceImpl.java | 58 +++++++++++++------ 8 files changed, 160 insertions(+), 50 deletions(-) create mode 100644 modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/docker/DockerImageNameResp.java diff --git a/framework/spring-boot-starter-biz-data-permission/src/main/java/cd/casic/framework/datapermission/service/user/AdminUserService.java b/framework/spring-boot-starter-biz-data-permission/src/main/java/cd/casic/framework/datapermission/service/user/AdminUserService.java index 86fe41aa..30d61479 100644 --- a/framework/spring-boot-starter-biz-data-permission/src/main/java/cd/casic/framework/datapermission/service/user/AdminUserService.java +++ b/framework/spring-boot-starter-biz-data-permission/src/main/java/cd/casic/framework/datapermission/service/user/AdminUserService.java @@ -138,6 +138,8 @@ public interface AdminUserService { */ AdminUserDO getUser(Long id); + public List getUserListByIds(Collection idList); + /** * 获得指定部门的用户数组 * diff --git a/framework/spring-boot-starter-biz-tenant/src/main/java/cd/casic/framework/tenant/core/service/AdminUserServiceImpl.java b/framework/spring-boot-starter-biz-tenant/src/main/java/cd/casic/framework/tenant/core/service/AdminUserServiceImpl.java index 6b42aa77..52c82ca1 100644 --- a/framework/spring-boot-starter-biz-tenant/src/main/java/cd/casic/framework/tenant/core/service/AdminUserServiceImpl.java +++ b/framework/spring-boot-starter-biz-tenant/src/main/java/cd/casic/framework/tenant/core/service/AdminUserServiceImpl.java @@ -23,7 +23,6 @@ import cn.hutool.core.util.StrUtil; import cd.casic.framework.commons.enums.CommonStatusEnum; import cd.casic.framework.commons.exception.ServiceException; import cd.casic.framework.commons.pojo.PageResult; -import cd.casic.framework.commons.util.collection.CollectionUtils; import cd.casic.framework.commons.util.object.BeanUtils; import cd.casic.framework.commons.util.validation.ValidationUtils; @@ -43,10 +42,13 @@ import org.springframework.context.annotation.Lazy; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; import java.io.InputStream; import java.time.LocalDateTime; import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; import static cd.casic.framework.commons.exception.util.ServiceExceptionUtil.exception; import static cd.casic.framework.commons.util.collection.CollectionUtils.*; @@ -286,6 +288,13 @@ public class AdminUserServiceImpl implements AdminUserService { public AdminUserDO getUser(Long id) { return userMapper.selectById(id); } + @Override + public List getUserListByIds(Collection idList){ + if (CollectionUtils.isEmpty(idList)) { + return Collections.emptyList(); + } + return userMapper.selectByIds(idList); + } @Override public List getUserListByDeptIds(Collection deptIds) { @@ -322,7 +331,7 @@ public class AdminUserServiceImpl implements AdminUserService { } // 获得岗位信息 List users = userMapper.selectBatchIds(ids); - Map userMap = CollectionUtils.convertMap(users, AdminUserDO::getId); + Map userMap = users.stream().collect(Collectors.toMap(AdminUserDO::getId,Function.identity())); // 校验 ids.forEach(id -> { AdminUserDO user = userMap.get(id); @@ -368,7 +377,7 @@ public class AdminUserServiceImpl implements AdminUserService { // 校验邮箱唯一 validateEmailUnique(id, email); // 校验部门处于开启状态 - deptService.validateDeptList(CollectionUtils.singleton(deptId)); + deptService.validateDeptList(Collections.singleton(deptId)); // 校验岗位处于开启状态 postService.validatePostList(postIds); return user; diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/docker/DockerImageNameResp.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/docker/DockerImageNameResp.java new file mode 100644 index 00000000..65721c1a --- /dev/null +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/docker/DockerImageNameResp.java @@ -0,0 +1,11 @@ +package cd.casic.ci.process.dto.resp.docker; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class DockerImageNameResp { + private String id; + private String value; +} diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java index 45d72501..9e8d9be5 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java @@ -15,6 +15,7 @@ import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.service.resource.ResourceManagerService; import cd.casic.ci.process.process.service.target.TargetVersionService; +import cd.casic.ci.process.util.CryptogramUtil; import cd.casic.ci.process.util.SftpUploadUtil; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo; @@ -62,6 +63,8 @@ public class AFLSlotCompileWorker extends DockerWorker { PipPipeline pipeline = (PipPipeline) getContextManager().getContext(task.getPipelineId()).getContextDef(); // 获取目标文件 TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId()); + String fileName = targetVersion.getFileName(); + append(context,"读取到当前文件名称:"+fileName); String filePath = targetVersion.getFilePath(); File file = new File(filePath); if (!file.exists() || !file.canRead()) { @@ -74,25 +77,49 @@ public class AFLSlotCompileWorker extends DockerWorker { String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir; // 上传目标文件 到指定资源服务器 try { + append(context,"AFL编译，上传文件路径:"+realPath); SftpUploadUtil.uploadFileViaSftp( machineInfo.getMachineHost() - ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,realPath,file.getName()); + ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(), CryptogramUtil.doDecrypt(machineInfo.getPassword()),"",filePath,realPath,file.getName()); } catch (SftpUploadUtil.SftpUploadException e) { append(context,"上传文件失败，请确认资源信息是否有误:"+JSON.toJSONString(machineInfo)); + log.error("上传文件报错",e); toBadEnding(); } // 执行预设命令 ,进入目录 String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" + "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录 "mkdir -p "+workDir+"\n"+ - "cd "+workDir+"\n"; + "cd "+workDir+"\n"+handleZipFile(fileName); Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT); String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null; - allCommand += commandScript+"\nll -s"; + allCommand += commandScript+"\n"; log.info("AFL插装编译容器内执行的命令：{}",allCommand); dockerRun(allCommand,dockerInfo,context); // 更新全局变量 // /test目录下当前流水线工作目录 globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); } + private String handleZipFile(String fileName){ + if (StringUtils.isEmpty(fileName)) { + return ""; + } + int i = fileName.lastIndexOf("."); + if (i==-1||i>=fileName.length()-1) { + return ""; + } + String prefix = fileName.substring(0, i); + String suffix = fileName.substring(i + 1); + if ("zip".equals(suffix)) { + String cmd ="unzip -o "+fileName+"\n" + + "cd "+prefix; + return cmd; + } + if ("tar".equals(suffix)) { + String cmd ="tar -zxvf "+fileName+"\n" + + "cd "+prefix; + return cmd; + } + return ""; + } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java index 1f8fe4d7..fd0c7023 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java @@ -8,7 +8,11 @@ import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant; import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.worker.base.DockerWorker; +import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline; +import cd.casic.ci.process.process.dataObject.target.TargetVersion; import cd.casic.ci.process.process.dataObject.task.PipTask; +import cd.casic.ci.process.process.service.target.TargetVersionService; +import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -19,8 +23,8 @@ import java.util.Map; @WorkAtom(taskType = "AFL") @Slf4j public class AFLWorker extends DockerWorker { - - + @Resource + private TargetVersionService targetVersionService; @Override public void execute(TaskRunContext context) { int statusCode = -1; @@ -54,13 +58,18 @@ public class AFLWorker extends DockerWorker { append(context,"当前机器不支持docker"); return; } - String output = workDir+PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT; + String output = workDir + PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT; // String volumeWorkDirPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX; + PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef(); + TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId()); + String fileName = targetVersion.getFileName(); + + String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" + - "cd /test\n" + - "mkdir -p"+output+"\n" + - "afl-fuzz -i "+seedPath+" -o "+output+" "+workDir+ File.separator+commandEnd; + "cd " +PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+ getSourceName(fileName) + + "mkdir -p "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+File.separator+output+"\n" + + "afl-fuzz -i "+seedPath+" -o "+output+" "+commandEnd; try { //将节点的配置信息反编译成对象 log.info("AFL模糊测试执行脚本：{}",commandScript); @@ -75,4 +84,11 @@ public class AFLWorker extends DockerWorker { localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode); } } + public String getSourceName(String fileName){ + int dotIndex = fileName.lastIndexOf("."); + if (dotIndex!=-1) { + return "cd "+fileName.substring(0, dotIndex)+"\n"; + } + return ""; + } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java index 1be8f64f..862767e1 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java @@ -36,7 +36,7 @@ public class TestCaseGenerationWorker extends DockerWorker { String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED; Map taskProperties = taskDef.getTaskProperties(); Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY); - String binaryName = taskProperties.get("binaryName").toString(); + String binaryName = taskProperties.get("binaryName") instanceof String ? ((String) taskProperties.get("binaryName")) : null; String managerId = taskProperties.get("managerId") instanceof String ? ((String) taskProperties.get("managerId")) : null; ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId); String machineId = resourceById.getMachineId(); @@ -73,17 +73,19 @@ public class TestCaseGenerationWorker extends DockerWorker { toBadEnding(); } File file = new File(filePath); - String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+workDir+File.separator+seedPath; + String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; // 将文件上传到服务器的 目录底下 log.info("测试用例选用上传模式,种子文件路径：{},种子上传路径:{}",filePath,seedTarget); + append(context,"测试用例选用上传模式,种子文件路径："+filePath+",种子上传路径:"+seedPath); try { SftpUploadUtil.uploadFileViaSftp( machineInfo.getMachineHost() - ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,seedTarget,file.getName()); + ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),CryptogramUtil.doDecrypt(machineInfo.getPassword()),"",filePath,seedTarget,file.getName()); } catch (SftpUploadUtil.SftpUploadException e) { append(context,"seed文件上传失败"); log.error("seed文件上传失败",e); } + append(context,"上传文件成功！"); } globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY,seedPath); } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/docker/impl/DockerServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/docker/impl/DockerServiceImpl.java index fe5b61d2..fd89e426 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/docker/impl/DockerServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/docker/impl/DockerServiceImpl.java @@ -17,6 +17,7 @@ import org.assertj.core.util.Lists; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -48,16 +49,27 @@ public class DockerServiceImpl implements DockerService { } ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder().dockerHost(uri).build(); - DockerClient build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build(); - List exec = build.listImagesCmd().exec(); - List res = new LinkedList<>(); - for (Image image : exec) { - String[] repoTags = image.getRepoTags(); - if (image.getRepoTags() != null && repoTags.length > 0) { - res.add(repoTags[0]); + DockerClient build = null; + try{ + build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build(); + List exec = build.listImagesCmd().exec(); + List res = new LinkedList<>(); + for (Image image : exec) { + String[] repoTags = image.getRepoTags(); + if (image.getRepoTags() != null && repoTags.length > 0) { + res.add(repoTags[0]); + } + } + return res; + } finally { + if (build!=null) { + try { + build.close(); + } catch (IOException e) { + + } } } - return res; } @Override @@ -72,15 +84,26 @@ public class DockerServiceImpl implements DockerService { } ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder().dockerHost(uri).build(); - DockerClient build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build(); - List exec = build.listImagesCmd().exec(); - List res = new LinkedList<>(); - for (Image image : exec) { - String[] repoTags = image.getRepoTags(); - if (image.getRepoTags() != null && repoTags.length > 0) { - res.add(repoTags[0]); + DockerClient build = null; + try{ + build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build(); + List exec = build.listImagesCmd().exec(); + List res = new LinkedList<>(); + for (Image image : exec) { + String[] repoTags = image.getRepoTags(); + if (image.getRepoTags() != null && repoTags.length > 0) { + res.add(repoTags[0]); + } + } + return res; + } finally { + if (build!=null) { + try { + build.close(); + } catch (IOException e) { + + } } } - return res; } } diff --git a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/pipeline/impl/PipelineServiceImpl.java b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/pipeline/impl/PipelineServiceImpl.java index a2fcaf81..e28d1078 100644 --- a/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/pipeline/impl/PipelineServiceImpl.java +++ b/modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/pipeline/impl/PipelineServiceImpl.java @@ -47,6 +47,8 @@ import org.springframework.util.ObjectUtils; import java.time.Duration; import java.time.LocalDateTime; import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; /** * @author HopeLi @@ -231,7 +233,8 @@ public class PipelineServiceImpl extends ServiceImpl i //对流水线进行流水线信息赋值 respList.forEach(this::setStageAndTask); //对用户姓名进行赋值 - respList.forEach(this::setUserName); +// respList.forEach(this::setUserName); + setUserName(respList); return respList; } @@ -316,32 +319,49 @@ public class PipelineServiceImpl extends ServiceImpl i List respList = PipelineConverter.INSTANCE.toRespList(pipPipelinePage.getRecords()); //对流水线进行流水线信息赋值 - respList.forEach(this::setStageAndTask); - respList.forEach(this::setUserName); - +// respList.forEach(this::setStageAndTask); +// respList.forEach(this::setUserName); + setUserName(respList); PageResult pageResult = new PageResult<>(respList,pipPipelinePage.getTotal(),pipPipelinePage.getCurrent(),pipPipelinePage.getSize()); return pageResult; } - private void setUserName(PipelineFindResp pipelineFindResp) { - if (!StringUtils.isEmpty(pipelineFindResp.getCreator())){ - AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getCreator())); - if (!ObjectUtils.isEmpty(user)){ - pipelineFindResp.setCreatorName(user.getUsername()); + private void setUserName(List pipelineFindRespList) { + Set userIdSet = new HashSet<>(pipelineFindRespList.size()*3); + for (PipelineFindResp pipelineFindResp : pipelineFindRespList) { + if (!StringUtils.isEmpty(pipelineFindResp.getCreator())){ + userIdSet.add(Long.valueOf(pipelineFindResp.getCreator())); + } + + if (!StringUtils.isEmpty(pipelineFindResp.getUpdater())){ + userIdSet.add(Long.valueOf(pipelineFindResp.getUpdater())); + } + + if (!StringUtils.isEmpty(pipelineFindResp.getExecuteUserId())){ + userIdSet.add(Long.valueOf(pipelineFindResp.getExecuteUserId())); } } - - if (!StringUtils.isEmpty(pipelineFindResp.getUpdater())){ - AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getUpdater())); - if (!ObjectUtils.isEmpty(user)){ - pipelineFindResp.setUpdaterName(user.getUsername()); + Map userIdMap = adminUserService.getUserListByIds(userIdSet).stream().collect(Collectors.toMap(AdminUserDO::getId, Function.identity())); + for (PipelineFindResp pipelineFindResp : pipelineFindRespList) { + if (!StringUtils.isEmpty(pipelineFindResp.getCreator())){ + AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getCreator())); + if (!ObjectUtils.isEmpty(user)){ + pipelineFindResp.setCreatorName(user.getUsername()); + } } - } - if (!StringUtils.isEmpty(pipelineFindResp.getExecuteUserId())){ - AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getExecuteUserId())); - if (!ObjectUtils.isEmpty(user)){ - pipelineFindResp.setExecuteUserName(user.getUsername()); + if (!StringUtils.isEmpty(pipelineFindResp.getUpdater())){ + AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getUpdater())); + if (!ObjectUtils.isEmpty(user)){ + pipelineFindResp.setUpdaterName(user.getUsername()); + } + } + + if (!StringUtils.isEmpty(pipelineFindResp.getExecuteUserId())){ + AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getExecuteUserId())); + if (!ObjectUtils.isEmpty(user)){ + pipelineFindResp.setExecuteUserName(user.getUsername()); + } } } }