mianbin/ops-pro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: mianbin:22ec4646e842fedc0bedcc53cde97a355dcc59f6
Branches Tags
mianbin:master
mianbin:jenkins-engin
mianbin:temp
..
compare: mianbin:24c2e9acac8eae5a0a1410d3b835d9cf278a9373
Branches Tags
mianbin:master
mianbin:jenkins-engin
mianbin:temp

No commits in common. "22ec4646e842fedc0bedcc53cde97a355dcc59f6" and "24c2e9acac8eae5a0a1410d3b835d9cf278a9373" have entirely different histories.
22ec4646e8 ... 24c2e9acac

15 changed files with 67 additions and 332 deletions
Show Stats Expand all files Collapse all files

2
framework/spring-boot-starter-biz-data-permission/src/main/java/cd/casic/framework/datapermission/service/user/AdminUserService.java
View File

@ -138,8 +138,6 @@ public interface AdminUserService {
*/ */
AdminUserDO getUser(Long id); AdminUserDO getUser(Long id);
public List<AdminUserDO> getUserListByIds(Collection<Long> idList);
/** /**
* 获得指定部门的用户数组 * 获得指定部门的用户数组
* *

15
framework/spring-boot-starter-biz-tenant/src/main/java/cd/casic/framework/tenant/core/service/AdminUserServiceImpl.java
View File

@ -23,6 +23,7 @@ import cn.hutool.core.util.StrUtil;
import cd.casic.framework.commons.enums.CommonStatusEnum; import cd.casic.framework.commons.enums.CommonStatusEnum;
import cd.casic.framework.commons.exception.ServiceException; import cd.casic.framework.commons.exception.ServiceException;
import cd.casic.framework.commons.pojo.PageResult; import cd.casic.framework.commons.pojo.PageResult;
import cd.casic.framework.commons.util.collection.CollectionUtils;
import cd.casic.framework.commons.util.object.BeanUtils; import cd.casic.framework.commons.util.object.BeanUtils;
import cd.casic.framework.commons.util.validation.ValidationUtils; import cd.casic.framework.commons.util.validation.ValidationUtils;
@ -42,13 +43,10 @@ import org.springframework.context.annotation.Lazy;
import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import java.io.InputStream; import java.io.InputStream;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.*; import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import static cd.casic.framework.commons.exception.util.ServiceExceptionUtil.exception; import static cd.casic.framework.commons.exception.util.ServiceExceptionUtil.exception;
import static cd.casic.framework.commons.util.collection.CollectionUtils.*; import static cd.casic.framework.commons.util.collection.CollectionUtils.*;
@ -288,13 +286,6 @@ public class AdminUserServiceImpl implements AdminUserService {
public AdminUserDO getUser(Long id) { public AdminUserDO getUser(Long id) {
return userMapper.selectById(id); return userMapper.selectById(id);
} }
@Override
public List<AdminUserDO> getUserListByIds(Collection<Long> idList){
if (CollectionUtils.isEmpty(idList)) {
return Collections.emptyList();
}
return userMapper.selectByIds(idList);
}
@Override @Override
public List<AdminUserDO> getUserListByDeptIds(Collection<Long> deptIds) { public List<AdminUserDO> getUserListByDeptIds(Collection<Long> deptIds) {
@ -331,7 +322,7 @@ public class AdminUserServiceImpl implements AdminUserService {
} }
// 获得岗位信息 // 获得岗位信息
List<AdminUserDO> users = userMapper.selectBatchIds(ids); List<AdminUserDO> users = userMapper.selectBatchIds(ids);
Map<Long, AdminUserDO> userMap = users.stream().collect(Collectors.toMap(AdminUserDO::getId,Function.identity())); Map<Long, AdminUserDO> userMap = CollectionUtils.convertMap(users, AdminUserDO::getId);
// 校验 // 校验
ids.forEach(id -> { ids.forEach(id -> {
AdminUserDO user = userMap.get(id); AdminUserDO user = userMap.get(id);
@ -377,7 +368,7 @@ public class AdminUserServiceImpl implements AdminUserService {
// 校验邮箱唯一 // 校验邮箱唯一
validateEmailUnique(id, email); validateEmailUnique(id, email);
// 校验部门处于开启状态 // 校验部门处于开启状态
deptService.validateDeptList(Collections.singleton(deptId)); deptService.validateDeptList(CollectionUtils.singleton(deptId));
// 校验岗位处于开启状态 // 校验岗位处于开启状态
postService.validatePostList(postIds); postService.validatePostList(postIds);
return user; return user;

26
modules/module-ci-process-api/src/main/java/cd/casic/ci/api/DockerController.java
View File

@ -1,26 +0,0 @@
package cd.casic.ci.api;
import cd.casic.ci.process.process.service.docker.DockerService;
import cd.casic.framework.commons.pojo.CommonResult;
import jakarta.annotation.Resource;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
@RestController
@RequestMapping("/docker")
public class DockerController {
@Resource
private DockerService dockerService;
@GetMapping("/imageNameListByResourceDetailId/{resourceDetailId}")
public CommonResult<List<String>> imageList(@PathVariable String resourceDetailId){
return CommonResult.success(dockerService.imageNameList(resourceDetailId));
}
@GetMapping("/imageListByResourceId/{resourceId}")
public CommonResult<List<String>> imageListByResourceId(@PathVariable String resourceId){
return CommonResult.success(dockerService.imageListByResourceId(resourceId));
}
}

11
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/docker/DockerImageNameResp.java
View File

@ -1,11 +0,0 @@
package cd.casic.ci.process.dto.resp.docker;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class DockerImageNameResp {
private String id;
private String value;
}

1
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/AFLSlotCompileConstant.java
View File

@ -2,6 +2,5 @@ package cd.casic.ci.process.engine.constant;
public class AFLSlotCompileConstant { public class AFLSlotCompileConstant {
public static final String MANAGER_ID = "managerId"; public static final String MANAGER_ID = "managerId";
public static final String IMAGE_NAME = "imageName";
public static final String COMMAND_SCRIPT ="buildScript"; public static final String COMMAND_SCRIPT ="buildScript";
} }

6
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/constant/PipelineGlobalVariableConstant.java
View File

@ -21,11 +21,9 @@ public class PipelineGlobalVariableConstant {
* */ * */
public static final String AFL_DOCKER_SLOT_COMPILE_PATH_KEY = "AFL_DOCKER_SLOT_COMPILE_PATH_KEY"; public static final String AFL_DOCKER_SLOT_COMPILE_PATH_KEY = "AFL_DOCKER_SLOT_COMPILE_PATH_KEY";
/** /**
* 测试用例生成，ai生成或者直接文件上传的文件夹名称：SEED * 测试用例生成，ai生成或者直接文件上传的路径，工作目录下 SEED
* */ * */
public static final String AFL_DOCKER_SEED ="SEED"; public static final String AFL_DOCKER_SEED_PATH="SEED";
public static final String AFL_DOCKER_SEED_PATH_KEY="AFL_DOCKER_SEED_PATH_KEY";
/** /**
* AFL输出目录 * AFL输出目录
* */ * */

51
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLSlotCompileWorker.java
View File

@ -15,7 +15,6 @@ import cd.casic.ci.process.process.dataObject.target.TargetVersion;
import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.service.resource.ResourceManagerService; import cd.casic.ci.process.process.service.resource.ResourceManagerService;
import cd.casic.ci.process.process.service.target.TargetVersionService; import cd.casic.ci.process.process.service.target.TargetVersionService;
import cd.casic.ci.process.util.CryptogramUtil;
import cd.casic.ci.process.util.SftpUploadUtil; import cd.casic.ci.process.util.SftpUploadUtil;
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants; import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo; import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo;
@ -42,12 +41,11 @@ public class AFLSlotCompileWorker extends DockerWorker {
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir);
} }
Map<String, Object> taskProperties = task.getTaskProperties(); Map<String, Object> taskProperties = task.getTaskProperties();
String managerId = taskProperties.get(AFLSlotCompileConstant.MANAGER_ID) instanceof String ? ((String) taskProperties.get(AFLSlotCompileConstant.MANAGER_ID)) : null; String managerId = taskProperties.get(AFLSlotCompileConstant.MANAGER_ID) instanceof String ? ((String) taskProperties.get("managerId")) : null;
// ssh 上传目标文件 // ssh 上传目标文件
ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId); ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId);
String machineId = resourceById.getMachineId(); String machineId = resourceById.getMachineId();
String dockerId = resourceById.getDockerId(); String dockerId = resourceById.getDockerId();
String imageName = taskProperties.get(AFLSlotCompileConstant.IMAGE_NAME) instanceof String ? ((String) taskProperties.get(AFLSlotCompileConstant.IMAGE_NAME)) : null;
if (StringUtils.isEmpty(machineId)||StringUtils.isEmpty(dockerId)) { if (StringUtils.isEmpty(machineId)||StringUtils.isEmpty(dockerId)) {
append(context,"该资源不支持docker或者ssh"); append(context,"该资源不支持docker或者ssh");
} }
@ -63,8 +61,6 @@ public class AFLSlotCompileWorker extends DockerWorker {
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(task.getPipelineId()).getContextDef(); PipPipeline pipeline = (PipPipeline) getContextManager().getContext(task.getPipelineId()).getContextDef();
// 获取目标文件 // 获取目标文件
TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId()); TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId());
String fileName = targetVersion.getFileName();
append(context,"读取到当前文件名称:"+fileName);
String filePath = targetVersion.getFilePath(); String filePath = targetVersion.getFilePath();
File file = new File(filePath); File file = new File(filePath);
if (!file.exists() || !file.canRead()) { if (!file.exists() || !file.canRead()) {
@ -73,53 +69,34 @@ public class AFLSlotCompileWorker extends DockerWorker {
toBadEnding(); toBadEnding();
} }
// 工作目录下的相对路径,/test 作为根路径 // 工作目录下的相对路径,/test 作为根路径
String compilePath = PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_PREFIX
+ task.getPipelineId()
+ File.separator
+ PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_PREFIX+task.getId();
// 卷挂载目录+流水线目录 + task编译目录 // 卷挂载目录+流水线目录 + task编译目录
String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ workDir; String realPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+ compilePath;
// 上传目标文件 到指定资源服务器 // 上传目标文件 到指定资源服务器
try { try {
append(context,"AFL编译，上传文件路径:"+realPath);
SftpUploadUtil.uploadFileViaSftp( SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost() machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(), CryptogramUtil.doDecrypt(machineInfo.getPassword()),"",filePath,realPath,file.getName()); ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",realPath,filePath,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) { } catch (SftpUploadUtil.SftpUploadException e) {
append(context,"上传文件失败，请确认资源信息是否有误:"+JSON.toJSONString(machineInfo)); append(context,"上传文件失败，请确认资源信息是否有误:"+JSON.toJSONString(machineInfo));
log.error("上传文件报错",e);
toBadEnding(); toBadEnding();
} }
// 执行预设命令 ,进入目录 // 执行预设命令 ,进入目录
String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it "+imageName+" bash\n" + String allCommand = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" +
"cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录 "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n"+// 进入到容器中的卷挂载目录
"mkdir -p "+workDir+"\n"+ "mkdir -p "+compilePath+"\n"+
"cd "+workDir+"\n"+handleZipFile(fileName); "cd "+compilePath+"\n";
Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT); Object commandScriptObj = taskProperties.get(AFLSlotCompileConstant.COMMAND_SCRIPT);
String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null; String commandScript = commandScriptObj instanceof String ? ((String) commandScriptObj) : null;
allCommand += commandScript+"\n"; allCommand += commandScript+"\nll -s";
log.info("AFL插装编译容器内执行的命令：{}",allCommand);
dockerRun(allCommand,dockerInfo,context); dockerRun(allCommand,dockerInfo,context);
// 更新全局变量 // 更新全局变量
// /test目录下当前流水线工作目录 // /test目录下编译文件与目标所在目录,当前pipelineId+taskId组合
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_KEY,compilePath);
// /test目录下当前流水线工作目录（没有task部分）
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir); globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY,workDir);
} }
private String handleZipFile(String fileName){
if (StringUtils.isEmpty(fileName)) {
return "";
}
int i = fileName.lastIndexOf(".");
if (i==-1||i>=fileName.length()-1) {
return "";
}
String prefix = fileName.substring(0, i);
String suffix = fileName.substring(i + 1);
if ("zip".equals(suffix)) {
String cmd ="unzip -o "+fileName+"\n" +
"cd "+prefix;
return cmd;
}
if ("tar".equals(suffix)) {
String cmd ="tar -zxvf "+fileName+"\n" +
"cd "+prefix;
return cmd;
}
return "";
}
} }

52
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java
View File

@ -5,47 +5,41 @@ import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp; import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.engine.constant.AFLConstant; import cd.casic.ci.process.engine.constant.AFLConstant;
import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant; import cd.casic.ci.process.engine.constant.DIYImageExecuteCommandConstant;
import cd.casic.ci.process.engine.constant.PipelineGlobalVariableConstant;
import cd.casic.ci.process.engine.runContext.TaskRunContext; import cd.casic.ci.process.engine.runContext.TaskRunContext;
import cd.casic.ci.process.engine.worker.base.DockerWorker; import cd.casic.ci.process.engine.worker.base.DockerWorker;
import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
import cd.casic.ci.process.process.dataObject.target.TargetVersion;
import cd.casic.ci.process.process.dataObject.task.PipTask; import cd.casic.ci.process.process.dataObject.task.PipTask;
import cd.casic.ci.process.process.service.target.TargetVersionService;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.io.File;
import java.util.Map; import java.util.Map;
@WorkAtom(taskType = "AFL") @WorkAtom(taskType = "AFL")
@Slf4j @Slf4j
public class AFLWorker extends DockerWorker { public class AFLWorker extends DockerWorker {
@Resource
private TargetVersionService targetVersionService;
@Override @Override
public void execute(TaskRunContext context) { public void execute(TaskRunContext context) {
int statusCode = -1; int statusCode = -1;
Map<String, Object> localVariables = context.getLocalVariables(); Map<String, Object> localVariables = context.getLocalVariables();
Map<String, Object> globalVariables = context.getGlobalVariables();
if (context.getContextDef() instanceof PipTask taskDef) { if (context.getContextDef() instanceof PipTask taskDef) {
log.info(taskDef.getTaskName()); log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties(); Map<String, Object> taskProperties = taskDef.getTaskProperties();
//从taskProperties中获取资源id //从taskProperties中获取资源id
String resourceType = taskProperties.get("resourceType").toString(); String resourceType = taskProperties.get("resourceType").toString();
String resourceId = taskProperties.get("resourceId").toString(); String resourceId = taskProperties.get("resourceId").toString();
String commandEnd = taskProperties.get("commandEnd").toString(); // 待测试文件路径
String imageName = taskProperties.get("imageName").toString(); String binaryPath = taskProperties.get(AFLConstant.BINARY) instanceof String ? ((String) taskProperties.get(AFLConstant.BINARY)) : null;
// 代码编译目录 String output = taskProperties.get(AFLConstant.OUTPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.OUTPUT)) : null;
String workDir = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY); String input = taskProperties.get(AFLConstant.INPUT) instanceof String ? ((String) taskProperties.get(AFLConstant.INPUT)) : null;
// 入参目录 String workDir = taskProperties.get(AFLConstant.WORK_DIR) instanceof String ? ((String) taskProperties.get(AFLConstant.WORK_DIR)) : null;
String seedPath = (String)globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY);
if (StringUtils.isEmpty(seedPath) || if (StringUtils.isEmpty(binaryPath) ||
StringUtils.isEmpty(output) ||
StringUtils.isEmpty(input) ||
StringUtils.isEmpty(workDir) || StringUtils.isEmpty(workDir) ||
StringUtils.isEmpty(resourceId) || StringUtils.isEmpty(resourceId) ||
StringUtils.isEmpty(commandEnd) ||
StringUtils.isEmpty(resourceType)) { StringUtils.isEmpty(resourceType)) {
// 缺少参数 // 缺少参数
toBadEnding(); toBadEnding();
@ -58,21 +52,12 @@ public class AFLWorker extends DockerWorker {
append(context,"当前机器不支持docker"); append(context,"当前机器不支持docker");
return; return;
} }
String output = workDir + PipelineGlobalVariableConstant.AFL_DOCKER_OUTPUT; String commandScript = "docker run -v "+workDir+":/test -it aflplusplus/aflplusplus bash\n" +
// "cd /test\n" +
String volumeWorkDirPath = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX; "afl-fuzz -i "+input+" -o "+output+" -t 3000 -Q "+binaryPath+" @@";
PipPipeline pipeline = (PipPipeline) getContextManager().getContext(taskDef.getPipelineId()).getContextDef();
TargetVersion targetVersion = targetVersionService.getById(pipeline.getTargetVersionId());
String fileName = targetVersion.getFileName();
String commandScript = "docker run -v "+volumeWorkDirPath+":/test -it "+imageName+" bash\n" +
"cd " +PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+ getSourceName(fileName) +
"mkdir -p "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+File.separator+workDir+File.separator+output+"\n" +
"afl-fuzz -i "+seedPath+" -o "+output+" "+commandEnd;
try { try {
//将节点的配置信息反编译成对象 //将节点的配置信息反编译成对象
log.info("AFL模糊测试执行脚本：{}",commandScript); log.info("构建脚本" + commandScript);
// 获取docker 暂时先写固定值 // 获取docker 暂时先写固定值
dockerRun(commandScript,resourceListByType.getDockerEndpointList().get(0),context); dockerRun(commandScript,resourceListByType.getDockerEndpointList().get(0),context);
} catch (Exception e) { } catch (Exception e) {
@ -84,11 +69,4 @@ public class AFLWorker extends DockerWorker {
localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode); localVariables.put(DIYImageExecuteCommandConstant.STATUS_CODE, statusCode);
} }
} }
public String getSourceName(String fileName){
int dotIndex = fileName.lastIndexOf(".");
if (dotIndex!=-1) {
return "cd "+fileName.substring(0, dotIndex)+"\n";
}
return "";
}
} }

25
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/TestCaseGenerationWorker.java
View File

@ -33,10 +33,11 @@ public class TestCaseGenerationWorker extends DockerWorker {
Map<String, Object> globalVariables = context.getGlobalVariables(); Map<String, Object> globalVariables = context.getGlobalVariables();
String workDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null; String workDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null;
if (context.getContextDef() instanceof PipTask taskDef) { if (context.getContextDef() instanceof PipTask taskDef) {
String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED; String seedPath= workDir+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_PREFIX+taskDef.getId()+File.separator+PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH;
log.info(taskDef.getTaskName());
Map<String, Object> taskProperties = taskDef.getTaskProperties(); Map<String, Object> taskProperties = taskDef.getTaskProperties();
Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY); Object caseType = taskProperties.get(TestCaseGenerationConstant.CASE_TYPE_KEY);
String binaryName = taskProperties.get("binaryName") instanceof String ? ((String) taskProperties.get("binaryName")) : null; String binaryName = taskProperties.get("binaryName").toString();
String managerId = taskProperties.get("managerId") instanceof String ? ((String) taskProperties.get("managerId")) : null; String managerId = taskProperties.get("managerId") instanceof String ? ((String) taskProperties.get("managerId")) : null;
ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId); ResourceFindResp resourceById = resourceManagerService.findResourceById(managerId);
String machineId = resourceById.getMachineId(); String machineId = resourceById.getMachineId();
@ -56,15 +57,15 @@ public class TestCaseGenerationWorker extends DockerWorker {
//如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器 //如果machineId为0，则说明该节点没有配置机器，则使用开始节点的机器
if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) { if (TestCaseGenerationConstant.CASE_TYPE_AI.equals(caseType)) {
// 容器内部test 目录, 获取编译后的文件 // 容器内部test 目录, 获取编译后的文件
String runScript = "docker run -v "+ PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX +":"+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+" -it aflplusplus bash\n" // 目前测试用例ai生成只有这个路径能跑 String compileDir = globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_SLOT_COMPILE_PATH_KEY) instanceof String ? ((String) globalVariables.get(PipelineGlobalVariableConstant.AFL_DOCKER_WORK_DIR_KEY)) : null;
+ "cd "+PipelineGlobalVariableConstant.AFL_DOCKER_BASE_DIR+"\n" +
String commandScript = "cd /test\n" +
"mkdir -p "+seedPath +"\n"+ "mkdir -p "+seedPath +"\n"+
"cd "+seedPath +"\n"+ "PYTHONPATH=/test/CaseGenerator/src python3 /test/CaseGenerator/src/CaseGenerator/main.py --work-dir "+compileDir+" --binary "+compileDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100";
"PYTHONPATH=CaseGenerator/src python3 CaseGenerator/src/CaseGenerator/main.py --work-dir "+workDir+" --binary "+workDir+File.separator+binaryName+" --output-dir "+seedPath+" --count 100";
//将节点的配置信息反编译成对象 //将节点的配置信息反编译成对象
log.info("测试用例生成-AI生成，实际执行命令:{}" , runScript); log.info("构建脚本" + commandScript);
//获取机器 //获取机器
dockerRun(runScript,dockerInfo,context); dockerRun(commandScript,dockerInfo,context);
} else { } else {
// 文件上传 // 文件上传
String filePath = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE)) : null; String filePath = taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE) instanceof String ? ((String) taskProperties.get(TestCaseGenerationConstant.SEED_SOURCE)) : null;
@ -75,19 +76,15 @@ public class TestCaseGenerationWorker extends DockerWorker {
File file = new File(filePath); File file = new File(filePath);
String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath; String seedTarget = PipelineGlobalVariableConstant.AFL_VOLUME_WORK_DIR_PREFIX+File.separator+seedPath;
// 将文件上传到服务器的 目录底下 // 将文件上传到服务器的 目录底下
log.info("测试用例选用上传模式,种子文件路径：{},种子上传路径:{}",filePath,seedTarget); try {
append(context,"测试用例选用上传模式,种子文件路径："+filePath+",种子上传路径:"+seedPath);
try {
SftpUploadUtil.uploadFileViaSftp( SftpUploadUtil.uploadFileViaSftp(
machineInfo.getMachineHost() machineInfo.getMachineHost()
,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),CryptogramUtil.doDecrypt(machineInfo.getPassword()),"",filePath,seedTarget,file.getName()); ,Integer.valueOf(machineInfo.getSshPort()),machineInfo.getUsername(),machineInfo.getPassword(),"",filePath,seedTarget,file.getName());
} catch (SftpUploadUtil.SftpUploadException e) { } catch (SftpUploadUtil.SftpUploadException e) {
append(context,"seed文件上传失败"); append(context,"seed文件上传失败");
log.error("seed文件上传失败",e); log.error("seed文件上传失败",e);
} }
append(context,"上传文件成功！");
} }
globalVariables.put(PipelineGlobalVariableConstant.AFL_DOCKER_SEED_PATH_KEY,seedPath);
} }
} }
} }

2
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/DockerWorker.java
View File

@ -126,7 +126,7 @@ public abstract class DockerWorker extends BaseWorker{
log.warn("未知流类型: {}", frame.getStreamType()); log.warn("未知流类型: {}", frame.getStreamType());
} }
} }
}).awaitCompletion(); }).awaitCompletion(60, TimeUnit.SECONDS);
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.error("执行异常",e); log.error("执行异常",e);
append(context,"执行异常,容器id："+containerId); append(context,"执行异常,容器id："+containerId);

10
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/docker/DockerService.java
View File

@ -1,10 +0,0 @@
package cd.casic.ci.process.process.service.docker;
import org.springframework.web.bind.annotation.PathVariable;
import java.util.List;
public interface DockerService {
List<String> imageNameList(String resourceDetailId);
List<String> imageListByResourceId( String resourceId);
}

109
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/docker/impl/DockerServiceImpl.java
View File

@ -1,109 +0,0 @@
package cd.casic.ci.process.process.service.docker.impl;
import cd.casic.ci.process.dto.req.resource.ResourceQueryReq;
import cd.casic.ci.process.dto.resp.resource.ResourceFindResp;
import cd.casic.ci.process.dto.resp.taskResource.TaskResourceFindResp;
import cd.casic.ci.process.process.service.docker.DockerService;
import cd.casic.ci.process.process.service.resource.ResourceManagerService;
import cd.casic.framework.commons.exception.ServiceException;
import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
import cd.casic.module.execute.docker.dataobject.dto.DockerEndpointDo;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.model.Image;
import com.github.dockerjava.core.DockerClientBuilder;
import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
import jakarta.annotation.Resource;
import org.assertj.core.util.Lists;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import static java.lang.String.format;
@Service
public class DockerServiceImpl implements DockerService {
@Resource
private ResourceManagerService resourceManagerService;
@Override
public List<String> imageNameList(String resourceDetailId) {
ResourceQueryReq req = new ResourceQueryReq();
req.setType("docker");
req.setId(resourceDetailId);
TaskResourceFindResp resourceListByType = resourceManagerService.findResourceListByType(req);
List<DockerEndpointDo> dockerEndpointList = resourceListByType.getDockerEndpointList();
if (CollectionUtils.isEmpty(dockerEndpointList)) {
return new ArrayList<>();
}
DockerEndpointDo dockerInfo = dockerEndpointList.get(0);
URI uri = null;
try {
uri = new URI(format("tcp://%s:%s", dockerInfo.getHost(), dockerInfo.getPort()));
} catch (URISyntaxException e) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR);
}
ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder().dockerHost(uri).build();
DockerClient build = null;
try{
build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build();
List<Image> exec = build.listImagesCmd().exec();
List<String> res = new LinkedList<>();
for (Image image : exec) {
String[] repoTags = image.getRepoTags();
if (image.getRepoTags() != null && repoTags.length > 0) {
res.add(repoTags[0]);
}
}
return res;
} finally {
if (build!=null) {
try {
build.close();
} catch (IOException e) {
}
}
}
}
@Override
public List<String> imageListByResourceId(String resourceId) {
ResourceFindResp resourceById = resourceManagerService.findResourceById(resourceId);
DockerEndpointDo dockerInfo = resourceById.getDockerEndpoint();
URI uri = null;
try {
uri = new URI(format("tcp://%s:%s", dockerInfo.getHost(), dockerInfo.getPort()));
} catch (URISyntaxException e) {
throw new ServiceException(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR);
}
ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder().dockerHost(uri).build();
DockerClient build = null;
try{
build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build();
List<Image> exec = build.listImagesCmd().exec();
List<String> res = new LinkedList<>();
for (Image image : exec) {
String[] repoTags = image.getRepoTags();
if (image.getRepoTags() != null && repoTags.length > 0) {
res.add(repoTags[0]);
}
}
return res;
} finally {
if (build!=null) {
try {
build.close();
} catch (IOException e) {
}
}
}
}
}

56
modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/pipeline/impl/PipelineServiceImpl.java
View File

@ -47,8 +47,6 @@ import org.springframework.util.ObjectUtils;
import java.time.Duration; import java.time.Duration;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.*; import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
/** /**
* @author HopeLi * @author HopeLi
@ -233,8 +231,7 @@ public class PipelineServiceImpl extends ServiceImpl<PipelineDao, PipPipeline> i
//对流水线进行流水线信息赋值 //对流水线进行流水线信息赋值
respList.forEach(this::setStageAndTask); respList.forEach(this::setStageAndTask);
//对用户姓名进行赋值 //对用户姓名进行赋值
// respList.forEach(this::setUserName); respList.forEach(this::setUserName);
setUserName(respList);
return respList; return respList;
} }
@ -319,49 +316,32 @@ public class PipelineServiceImpl extends ServiceImpl<PipelineDao, PipPipeline> i
List<PipelineFindResp> respList = PipelineConverter.INSTANCE.toRespList(pipPipelinePage.getRecords()); List<PipelineFindResp> respList = PipelineConverter.INSTANCE.toRespList(pipPipelinePage.getRecords());
//对流水线进行流水线信息赋值 //对流水线进行流水线信息赋值
// respList.forEach(this::setStageAndTask); respList.forEach(this::setStageAndTask);
// respList.forEach(this::setUserName); respList.forEach(this::setUserName);
setUserName(respList);
PageResult<PipelineFindResp> pageResult = new PageResult<>(respList,pipPipelinePage.getTotal(),pipPipelinePage.getCurrent(),pipPipelinePage.getSize()); PageResult<PipelineFindResp> pageResult = new PageResult<>(respList,pipPipelinePage.getTotal(),pipPipelinePage.getCurrent(),pipPipelinePage.getSize());
return pageResult; return pageResult;
} }
private void setUserName(List<PipelineFindResp> pipelineFindRespList) { private void setUserName(PipelineFindResp pipelineFindResp) {
Set<Long> userIdSet = new HashSet<>(pipelineFindRespList.size()*3); if (!StringUtils.isEmpty(pipelineFindResp.getCreator())){
for (PipelineFindResp pipelineFindResp : pipelineFindRespList) { AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getCreator()));
if (!StringUtils.isEmpty(pipelineFindResp.getCreator())){ if (!ObjectUtils.isEmpty(user)){
userIdSet.add(Long.valueOf(pipelineFindResp.getCreator())); pipelineFindResp.setCreatorName(user.getUsername());
}
if (!StringUtils.isEmpty(pipelineFindResp.getUpdater())){
userIdSet.add(Long.valueOf(pipelineFindResp.getUpdater()));
}
if (!StringUtils.isEmpty(pipelineFindResp.getExecuteUserId())){
userIdSet.add(Long.valueOf(pipelineFindResp.getExecuteUserId()));
} }
} }
Map<Long, AdminUserDO> userIdMap = adminUserService.getUserListByIds(userIdSet).stream().collect(Collectors.toMap(AdminUserDO::getId, Function.identity()));
for (PipelineFindResp pipelineFindResp : pipelineFindRespList) {
if (!StringUtils.isEmpty(pipelineFindResp.getCreator())){
AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getCreator()));
if (!ObjectUtils.isEmpty(user)){
pipelineFindResp.setCreatorName(user.getUsername());
}
}
if (!StringUtils.isEmpty(pipelineFindResp.getUpdater())){ if (!StringUtils.isEmpty(pipelineFindResp.getUpdater())){
AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getUpdater())); AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getUpdater()));
if (!ObjectUtils.isEmpty(user)){ if (!ObjectUtils.isEmpty(user)){
pipelineFindResp.setUpdaterName(user.getUsername()); pipelineFindResp.setUpdaterName(user.getUsername());
}
} }
}
if (!StringUtils.isEmpty(pipelineFindResp.getExecuteUserId())){ if (!StringUtils.isEmpty(pipelineFindResp.getExecuteUserId())){
AdminUserDO user = userIdMap.get(Long.valueOf(pipelineFindResp.getExecuteUserId())); AdminUserDO user = adminUserService.getUser(Long.valueOf(pipelineFindResp.getExecuteUserId()));
if (!ObjectUtils.isEmpty(user)){ if (!ObjectUtils.isEmpty(user)){
pipelineFindResp.setExecuteUserName(user.getUsername()); pipelineFindResp.setExecuteUserName(user.getUsername());
}
} }
} }
} }

7
ops-server/src/main/resources/application-local.yaml
View File

@ -70,9 +70,9 @@ spring:
# Redis 配置。Redisson 默认的配置足够使用，一般不需要进行调优 # Redis 配置。Redisson 默认的配置足够使用，一般不需要进行调优
data: data:
redis: redis:
# host: 127.0.0.1 # 地址 host: 127.0.0.1 # 地址
# port: 16379 # 端口 # port: 16379 # 端口
host: 192.168.1.120 # 地址 # host: 192.168.1.120 # 地址
port: 6379 # 端口 port: 6379 # 端口
database: 0 # 数据库索引 database: 0 # 数据库索引
# password: dev # 密码，建议生产环境开启 # password: dev # 密码，建议生产环境开启
@ -174,5 +174,4 @@ tartet:
remotePort: 22 remotePort: 22
username: roots username: roots
password: hnidc0327cn!@#xhh password: hnidc0327cn!@#xhh
remoteDir: /home/ops/ops-pro/file/ remoteDir: /home/ops/ops-pro/file/

26
ops-server/src/test/java/cd/casic/server/DockerTest.java
View File

@ -5,7 +5,6 @@ import com.github.dockerjava.api.command.ExecCreateCmdResponse;
import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Frame; import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.HostConfig; import com.github.dockerjava.api.model.HostConfig;
import com.github.dockerjava.api.model.Image;
import com.github.dockerjava.core.DockerClientBuilder; import com.github.dockerjava.core.DockerClientBuilder;
import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -16,11 +15,7 @@ import org.springframework.test.context.ActiveProfiles;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static java.lang.String.format;
@SpringBootTest(classes = {OpsServerApplication.class}) @SpringBootTest(classes = {OpsServerApplication.class})
@ActiveProfiles("local") @ActiveProfiles("local")
public class DockerTest { public class DockerTest {
@ -197,26 +192,5 @@ public class DockerTest {
System.err.println("---------------------------"); System.err.println("---------------------------");
} }
} }
@Test
public void imageList() throws URISyntaxException {
URI uri = new URI(format("tcp://%s:%s", "175.6.27.228", "22375"));
ApacheDockerHttpClient httpClient = new ApacheDockerHttpClient.Builder().dockerHost(uri).build();
DockerClient build = DockerClientBuilder.getInstance().withDockerHttpClient(httpClient).build();
List<Image> exec = build.listImagesCmd().exec();
for (Image image : exec) {
String[] repoTags = image.getRepoTags();
if (repoTags ==null) {
System.out.println("虚悬镜像");
} else if (repoTags.length>1){
System.out.println("多标签1镜像");
} else {
System.out.println("");
}
}
System.out.println(exec);
}
} }