Merge branch 'master' of http://1.14.125.6:3000/mianbin/ops-pro

modules/module-ci-process-api/src/main/java/cd/casic/ci/api/PipHistoryController.java

@@ -3,24 +3,28 @@ package cd.casic.ci.api;
 import cd.casic.ci.process.dto.req.history.PipelineHistoryQueryReq;
 import cd.casic.ci.process.process.dataObject.history.PipPipelineHisInstance;
 import cd.casic.ci.process.process.dataObject.log.PipTaskLog;
+import cd.casic.ci.process.process.dataObject.log.PipTaskLogLine;
 import cd.casic.ci.process.process.service.history.PipelineHistoryService;
+import cd.casic.ci.process.process.service.taskLog.PipTaskLogLineService;
 import cd.casic.ci.process.process.service.taskLog.TaskLogService;
 import cd.casic.framework.commons.pojo.CommonResult;
 import cd.casic.framework.commons.pojo.PageResult;
 import jakarta.annotation.Resource;
 import org.springframework.web.bind.annotation.*;
 
+import java.util.List;
+
 @RestController
 @RequestMapping("/history")
 public class PipHistoryController {
     @Resource
-    TaskLogService taskLogService;
+    private PipelineHistoryService pipelineHistoryService;
     @Resource
-    PipelineHistoryService pipelineHistoryService;
+    private PipTaskLogLineService logLineService;
     @GetMapping("/getLogById")
-    public CommonResult<PipTaskLog> getLogById(@RequestParam String id){
-        PipTaskLog byId = taskLogService.getById(id);
-        return CommonResult.success(byId);
+    public CommonResult<String> getLogById(@RequestParam String id){
+        String content = logLineService.getLineContentByLogId(id, 10000);
+        return CommonResult.success(content);
     }
     @PostMapping("/list")
     public CommonResult<PageResult<PipPipelineHisInstance>> list(@RequestBody PipelineHistoryQueryReq req){

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/constant/ShuiMuUrlConstant.java

@@ -0,0 +1,17 @@
+package cd.casic.ci.process.constant;
+
+public class ShuiMuUrlConstant {
+    // 获取检测镜像
+    public static final String imageList = "/api/image/list";
+    public static final String getToken = "/oauth/token";
+    // 创建项目
+    public static final String createProject = "/api/driverGeneration/createDriverProject";
+    // 获取检测配置
+    public static final String readWfuzzJsonFile = "/api/driverGeneration/readWfuzzJsonFile";
+    // 获取检测 构建配置需要填写的信息
+    public static final String readHelpersJsonFile = "/api/driverGeneration/readHelpersJsonFile";
+    public static final String getProjectVersion = "/api/project/code/getProjectVersion";
+    public static final String buildAndSaveConfig = "/api/driverGeneration/buildAndSaveConfig";
+    public static final String getProjectById = "/api/project/";
+    public static final String uploadWfuzzJson = "/api/project/code/uploadWfuzzJson";
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/shuimu/ShuiMuBuildAndSaveReq.java

@@ -0,0 +1,10 @@
+package cd.casic.ci.process.dto.req.shuimu;
+
+import lombok.Data;
+
+@Data
+public class ShuiMuBuildAndSaveReq {
+    private String projectId;
+    private String env;
+    private String fileName;
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/shuimu/ShuiMuProjectCreateReq.java

@@ -0,0 +1,23 @@
+package cd.casic.ci.process.dto.req.shuimu;
+
+import lombok.Data;
+
+@Data
+public class ShuiMuProjectCreateReq {
+    /**
+     *  语言 例如 java
+     * */
+    private String language;
+    /**
+     *  应用分析资源
+     * */
+    private String env;
+    /**
+     *  2c4g
+     * */
+    private String envSpecs;
+    /**
+     *   流水线id
+     * */
+    private String pipelineId;
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/req/shuimu/ShuiMuUpdateWFuzzJsonReq.java

@@ -0,0 +1,9 @@
+package cd.casic.ci.process.dto.req.shuimu;
+
+import lombok.Data;
+
+@Data
+public class ShuiMuUpdateWFuzzJsonReq {
+    private String projectId;
+    private String build;
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/shuimu/ShuiMuProjectInfoResp.java

@@ -0,0 +1,18 @@
+package cd.casic.ci.process.dto.resp.shuimu;
+
+import lombok.Data;
+
+@Data
+public class ShuiMuProjectInfoResp {
+    private Integer id;
+    private String name;
+    private String language;
+    private String description;
+    private String product;
+    private String ownerId;
+    private Integer createBy;
+    private Long createAt;
+    private Long updateAt;
+    private Long deleteAt;
+    private Integer projectFlag;
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/shuimu/ShuiMuProjectVersionResp.java

@@ -0,0 +1,68 @@
+package cd.casic.ci.process.dto.resp.shuimu;
+
+import com.alibaba.fastjson.JSON;
+import lombok.Data;
+
+@Data
+public class ShuiMuProjectVersionResp {
+    private Integer id;
+    private String vcs;
+    private String versionCode;
+    private Repo repo;
+    private Integer projectId;
+    private Integer createBy;
+    private Long createAt;
+    private Long updateAt;
+    private Long deleteAt;
+    private String ownerId;
+    private String workSpaceState;
+    private BuildState buildState;
+    private Boolean containsNeedFile;
+    private String buildResult;
+    private String buildLog;
+    private VerifyState verifyState;
+    private String verifyLog;
+    public void setRepo(Object repo) {
+        if (repo instanceof String) {
+            this.repo = JSON.parseObject((String) repo, Repo.class);
+        } else if (repo instanceof Repo) {
+            this.repo = (Repo) repo;
+        }
+    }
+
+    public void setBuildState(Object buildState) {
+        if (buildState instanceof String) {
+            this.buildState = JSON.parseObject((String) buildState, BuildState.class);
+        } else if (buildState instanceof BuildState) {
+            this.buildState = (BuildState) buildState;
+        }
+    }
+
+    public void setVerifyState(Object verifyState) {
+        if (verifyState instanceof String) {
+            this.verifyState = JSON.parseObject((String) verifyState, VerifyState.class);
+        } else if (verifyState instanceof VerifyState) {
+            this.verifyState = (VerifyState) verifyState;
+        }
+    }
+
+    @Data
+    public static class Repo {
+        private String env;
+        private String envSpecs;
+        private String filename;
+        private String filesize;
+        private String hash;
+        private String totalCount;
+    }
+
+    @Data
+    public static class BuildState {
+        private Integer state;
+    }
+
+    @Data
+    public static class VerifyState {
+        private Integer state;
+    }
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/dto/resp/shuimu/ShuiMuTokenResp.java

@@ -0,0 +1,12 @@
+package cd.casic.ci.process.dto.resp.shuimu;
+
+import lombok.Data;
+
+@Data
+public class ShuiMuTokenResp {
+    private String accessToken;
+    private Integer expiresIn;
+    private Integer refreshExpiresIn;
+    private String refreshToken;
+    private String tokenType;
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/manager/impl/MemoryLogManager.java

@@ -4,12 +4,16 @@ import cd.casic.ci.process.engine.constant.EngineRuntimeConstant;
 import cd.casic.ci.process.engine.manager.LoggerManager;
 import cd.casic.ci.process.engine.runContext.TaskRunContext;
 import cd.casic.ci.process.process.dao.pipeline.PipTaskLogDao;
+import cd.casic.ci.process.process.dao.pipeline.PipTaskLogLineDao;
 import cd.casic.ci.process.process.dataObject.log.PipTaskLog;
+import cd.casic.ci.process.process.dataObject.log.PipTaskLogLine;
 import cd.casic.ci.process.process.dataObject.task.PipTask;
 import cd.casic.ci.process.process.service.task.TaskService;
+import cd.casic.ci.process.process.service.taskLog.PipTaskLogLineService;
 import cd.casic.ci.process.util.snowflake.SnowflakeIdentifierGenerator;
 import cd.casic.framework.commons.util.network.IpUtil;
 import cd.casic.ci.process.util.WebFrameworkUtils;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import jakarta.annotation.Resource;
 import jakarta.servlet.http.HttpServletRequest;
 import lombok.extern.slf4j.Slf4j;
@@ -35,15 +39,8 @@ public class MemoryLogManager implements LoggerManager {
      *  第一级 是taskId，可能同一个账号、IP、建立多个连接所以保存为list
      * */
     private final  Map<String, List<SseEmitter>> taskIdSSEMap = new ConcurrentHashMap<>();
-
-    private final  Map<String,StringBuffer> taskIdMemoryLogMap = new ConcurrentHashMap<>();
-    @Resource
-    private SnowflakeIdentifierGenerator identifierGenerator;
     @Resource
     private TaskService taskService;
-
-    public final Integer FLUSH_DB_SIZE=2*1024*1024;
-//    public final Integer FLUSH_DB_SIZE=1000;
     private ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
     /**
      *  缓存最近一次执行的日志，key是taskId，val是数据库id（已入库的情况下）用于buffer满了增加日志内容
@@ -52,6 +49,8 @@ public class MemoryLogManager implements LoggerManager {
     private final static Map<String,String> taskIdDbMap = new ConcurrentHashMap<>();
     @Resource
     private PipTaskLogDao logDao;
+    @Resource
+    private PipTaskLogLineService logLineService;
     @Override
     public SseEmitter subscribe(String taskId,HttpServletRequest request) {
         if (request!=null) {
@@ -77,35 +76,27 @@ public class MemoryLogManager implements LoggerManager {
          *  内存满4mb入库，查询日志和入库操作用同一把锁
          *  然后新内容推sse
          * */
-        StringBuffer logCache = taskIdMemoryLogMap.computeIfAbsent(taskId, k -> new StringBuffer());
-        int length = logCache.length();
-        if (length>=FLUSH_DB_SIZE) {
-            synchronized (this){
-                if (length>=FLUSH_DB_SIZE) {
-                    StringBuffer stringBuffer = new StringBuffer();
-                    //            清空缓存
-                    taskIdMemoryLogMap.put(taskId, stringBuffer);
-                    // 入库或者更新
-                    if (taskIdDbMap.containsKey(taskId)) {
-                        // 之前已经入库过
-                        // 存在则更新
-                        String id = taskIdDbMap.get(taskId);
-                        PipTaskLog pipTaskLog = logDao.selectById(id);
-                        pipTaskLog.setContent(pipTaskLog.getContent()+logCache.toString());
-                        logDao.updateById(pipTaskLog);
-                    } else {
-                        // 不存在就新增
-                        PipTaskLog pipTaskLog = new PipTaskLog();
-                        pipTaskLog.setTaskId(taskId);
-                        pipTaskLog.setContent(logCache.toString());
-                        logDao.insert(pipTaskLog);
-                        taskIdDbMap.put(taskId,pipTaskLog.getId());
-                    }
-                    logCache = stringBuffer;
-                }
-            }
+        // 入库或者更新
+        if (taskIdDbMap.containsKey(taskId)) {
+            // 之前已经入库过
+            // 存在则更新
+            String id = taskIdDbMap.get(taskId);
+            PipTaskLogLine line = new PipTaskLogLine();
+            line.setLogId(id);
+            line.setLineContent(logContent);
+            logLineService.save(line);
+        } else {
+            // 不存在就新增
+            PipTaskLog pipTaskLog = new PipTaskLog();
+            pipTaskLog.setTaskId(taskId);
+//            pipTaskLog.setContent(logCache.toString());
+            logDao.insert(pipTaskLog);
+            taskIdDbMap.put(taskId,pipTaskLog.getId());
+            PipTaskLogLine line = new PipTaskLogLine();
+            line.setLogId(pipTaskLog.getTaskId());
+            line.setLineContent(logContent);
+            logLineService.save(line);
         }
-        logCache.append(logContent);
         List<SseEmitter> sseEmitters = taskIdSSEMap.get(taskId);
         if (!CollectionUtils.isEmpty(sseEmitters)) {
             for (SseEmitter sseEmitter : sseEmitters) {
@@ -120,63 +111,31 @@ public class MemoryLogManager implements LoggerManager {
     @Transactional(rollbackFor = Exception.class)
     public void flushMemory(List<TaskRunContext> taskContextList){
         log.info("流水线日志开始入库");
-        List<PipTaskLog> insertList = new ArrayList<>();
-        List<PipTaskLog> updateList = new ArrayList<>();
         for (TaskRunContext taskRunContext : taskContextList) {
             String taskId = taskRunContext.getContextDef().getId();
-            StringBuffer logCache = taskIdMemoryLogMap.get(taskId);
             if (taskIdDbMap.containsKey(taskId)) {
-                // 之前已经入库过
-                // 存在则更新
                 String id = taskIdDbMap.get(taskId);
-                PipTaskLog pipTaskLog = logDao.selectById(id);
-                if (logCache!=null) {
-                    pipTaskLog.setContent(pipTaskLog.getContent()+logCache);
-                }
-                updateList.add(pipTaskLog);
-                taskIdMemoryLogMap.remove(taskId);
-                taskRunContext.getLocalVariables().put(EngineRuntimeConstant.HIS_LOG_KEY,pipTaskLog.getId());
-                List<SseEmitter> sseEmitters = taskIdSSEMap.get(taskId);
-                if (!CollectionUtils.isEmpty(sseEmitters)) {
-                    sseEmitters.forEach(ResponseBodyEmitter::complete);
-                }
-                logDao.updateById(pipTaskLog);
-            } else {
-                // 不存在就新增
-                PipTaskLog pipTaskLog = new PipTaskLog();
-                pipTaskLog.setTaskId(taskId);
-                if (logCache!=null) {
-                    pipTaskLog.setContent(logCache.toString());
-                }
-                pipTaskLog.setId(identifierGenerator.nextUUID(null));
-                insertList.add(pipTaskLog);
-                taskRunContext.getLocalVariables().put(EngineRuntimeConstant.HIS_LOG_KEY,pipTaskLog.getId());
-                taskIdMemoryLogMap.remove(taskId);
-                taskIdDbMap.put(taskId,pipTaskLog.getId());
+                taskRunContext.getLocalVariables().put(EngineRuntimeConstant.HIS_LOG_KEY,id);
                 List<SseEmitter> sseEmitters = taskIdSSEMap.get(taskId);
                 if (!CollectionUtils.isEmpty(sseEmitters)) {
                     sseEmitters.forEach(ResponseBodyEmitter::complete);
                 }
             }
         }
-        if (!CollectionUtils.isEmpty(insertList)) {
-            logDao.insertBatch(insertList);
-        }
-        if (!CollectionUtils.isEmpty(updateList)) {
-            logDao.updateBatch(updateList);
-        }
-
     }
 
     @Override
     public String getLogContent(String taskId) {
-        StringBuffer logCache = taskIdMemoryLogMap.getOrDefault(taskId, new StringBuffer());
         String id = taskIdDbMap.get(taskId);
         if (id != null) {
-            PipTaskLog pipTaskLog = logDao.selectById(id);
-            return pipTaskLog.getContent()+logCache.toString();
+            List<PipTaskLogLine> lineList = logLineService.getLineListByLogId(id,10000);
+            StringBuilder stringBuilder = new StringBuilder();
+            for (PipTaskLogLine line : lineList) {
+                stringBuilder.append(line.getLineContent());
+            }
+            return stringBuilder.toString();
         }
-        return logCache.toString();
+        return "";
     }
     public void clear(String pipelineId){
         PipTask query = new PipTask();
@@ -210,4 +169,6 @@ public class MemoryLogManager implements LoggerManager {
             log.error("===================超时，taskId:{}断开连接===============",taskId);
         });
     }
+
+
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/afl/AFLWorker.java

@@ -79,11 +79,14 @@ public class AFLWorker extends DockerWorker {
             append(context,"AFL 输出路径："+output);
             Object timeObj = taskProperties.get(RUNNING_TIME);
             Double timeHours = Double.valueOf(String.valueOf(timeObj));
+            String outputPath = AFL_DOCKER_BASE_DIR + File.separator + output;
             String commandScript = "docker run -v "+volumeWorkDirPath+":"+AFL_DOCKER_BASE_DIR+"  -it "+imageName+" bash\n" +
                     "cd " + AFL_DOCKER_BASE_DIR+File.separator+workDir+ "\n"+
                     cdSourceName(fileName) +
-                    "mkdir -p "+ AFL_DOCKER_BASE_DIR+File.separator+output+"\n" +
-                    "afl-fuzz -i "+ AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+ AFL_DOCKER_BASE_DIR+File.separator+output+"  ./"+ executableName + " " +commandEnd;
+                    "mkdir -p "+ outputPath +"\n" +
+                    "rm -rf " + outputPath + "\n" +
+                    "chmod o+rwx -p "+outputPath+"\n"+
+                    "afl-fuzz -i "+ AFL_DOCKER_BASE_DIR+File.separator+seedPath+" -o "+ outputPath+"  ./"+ executableName + " " +commandEnd;
             try {
                 //将节点的配置信息反编译成对象
                 log.info("AFL模糊测试执行脚本：{}",commandScript);
@@ -93,8 +96,8 @@ public class AFLWorker extends DockerWorker {
                 // 获取docker 暂时先写固定值
                 dockerRun(commandScript,resourceListByType.getDockerEndpointList().get(0),context, runningTime);
             } catch (Exception e) {
-                String errorMessage = "该节点配置信息为空，请先配置该节点信息" + "\r\n";
-                log.error("执行ssh失败:", e);
+                String errorMessage = "执行afl失败"+e.getMessage() + "\r\n";
+                log.error("执行afl失败", e);
                 append(context, errorMessage);
                 toBadEnding();
             }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/engine/worker/base/BaseWorker.java

@@ -31,6 +31,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 
 import javax.swing.text.StringContent;
+import java.io.File;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dao/pipeline/PipTaskLogDao.java

@@ -2,6 +2,7 @@ package cd.casic.ci.process.process.dao.pipeline;
 
 import cd.casic.ci.process.process.dataObject.log.PipTaskLog;
 import cd.casic.framework.mybatis.core.mapper.BaseMapperX;
+import org.apache.ibatis.annotations.Param;
 
 public interface PipTaskLogDao extends BaseMapperX<PipTaskLog> {
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dao/pipeline/PipTaskLogLineDao.java

@@ -0,0 +1,8 @@
+package cd.casic.ci.process.process.dao.pipeline;
+
+import cd.casic.ci.process.process.dataObject.log.PipTaskLog;
+import cd.casic.ci.process.process.dataObject.log.PipTaskLogLine;
+import cd.casic.framework.mybatis.core.mapper.BaseMapperX;
+
+public interface PipTaskLogLineDao extends BaseMapperX<PipTaskLogLine> {
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/log/PipTaskLog.java

@@ -11,11 +11,6 @@ import lombok.EqualsAndHashCode;
 @Data
 public class PipTaskLog extends BaseDO {
     private String taskId;
-    private String content;
     @TableId(type = IdType.ASSIGN_ID)
     private String id;
-    public void append(String content){
-        this.content += CommandConstant.ENTER;
-        this.content +=content;
-    }
 }

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/dataObject/log/PipTaskLogLine.java

@@ -0,0 +1,13 @@
+package cd.casic.ci.process.process.dataObject.log;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import lombok.Data;
+
+@Data
+public class PipTaskLogLine {
+    @TableId(type = IdType.ASSIGN_ID)
+    private String id;
+    private String logId;
+    private String lineContent;
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/ShuiMuService.java

@@ -0,0 +1,32 @@
+package cd.casic.ci.process.process.service.shuimu;
+
+import cd.casic.ci.process.dto.req.shuimu.ShuiMuBuildAndSaveReq;
+import cd.casic.ci.process.dto.req.shuimu.ShuiMuProjectCreateReq;
+import cd.casic.ci.process.dto.req.shuimu.ShuiMuUpdateWFuzzJsonReq;
+import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectInfoResp;
+import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectVersionResp;
+
+import java.util.List;
+
+public interface ShuiMuService {
+    String getToken();
+    /**
+     *  获取编译环境
+     * */
+    List<String> imageList(String label);
+    void createProject(ShuiMuProjectCreateReq req);
+    /**
+     *  获取检测配置
+     * */
+    String readWfuzzJsonFile(String projectId);
+    /**
+     *  获取需要填写的表单项信息
+     * */
+    String readHelpersJsonFile(String projectId);
+
+    ShuiMuProjectVersionResp getProjectVersion(String projectId);
+    ShuiMuProjectInfoResp getProjectInfo(String projectId);
+
+    void buildAndSaveConfig(ShuiMuBuildAndSaveReq req);
+    void uploadWfuzzJson(ShuiMuUpdateWFuzzJsonReq req);
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/shuimu/impl/ShuiMuServiceImpl.java

@@ -0,0 +1,231 @@
+package cd.casic.ci.process.process.service.shuimu.impl;
+
+import cd.casic.ci.process.dto.req.shuimu.ShuiMuBuildAndSaveReq;
+import cd.casic.ci.process.dto.req.shuimu.ShuiMuProjectCreateReq;
+import cd.casic.ci.process.dto.req.shuimu.ShuiMuUpdateWFuzzJsonReq;
+import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectInfoResp;
+import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectVersionResp;
+import cd.casic.ci.process.dto.resp.shuimu.ShuiMuTokenResp;
+import cd.casic.ci.process.engine.manager.RunContextManager;
+import cd.casic.ci.process.process.dataObject.pipeline.PipPipeline;
+import cd.casic.ci.process.process.dataObject.target.TargetVersion;
+import cd.casic.ci.process.process.service.pipeline.PipelineService;
+import cd.casic.ci.process.process.service.shuimu.ShuiMuService;
+import cd.casic.ci.process.process.service.target.TargetVersionService;
+import cd.casic.ci.process.properties.ShuiMuProperties;
+import cd.casic.framework.commons.exception.ServiceException;
+import cd.casic.framework.commons.exception.enums.GlobalErrorCodeConstants;
+import cd.casic.framework.redis.core.RedisTemplateUtils;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import jakarta.annotation.Resource;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.http.*;
+import org.springframework.stereotype.Service;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.io.*;
+import java.util.*;
+
+import static cd.casic.ci.process.constant.ShuiMuUrlConstant.*;
+
+@Service
+public class ShuiMuServiceImpl implements ShuiMuService {
+    @Resource
+    private ShuiMuProperties shuiMuProperties;
+    @Resource
+    private RestTemplate restTemplate;
+    @Resource
+    private RedisTemplateUtils redisTemplateUtils;
+    private static final String REDIS_SHUI_MU_TOKEN_KEY = "REDIS_SHUI_MU_TOKEN_KEY";
+    private static final String SHUI_MU_TOKEN_PREFIX = "Bearer ";
+    private static final String SHUI_MU_IMAGE_LIST_FILTER = "wfuzz-built";
+    @Resource
+    private TargetVersionService targetVersionService;
+    @Resource
+    private PipelineService pipelineService;
+    private ShuiMuTokenResp getTokenRemote(){
+        HttpHeaders httpHeaders = new HttpHeaders();
+        httpHeaders.setContentType(MediaType.MULTIPART_FORM_DATA);
+        HttpEntity<MultiValueMap<String,String>> httpEntity = new HttpEntity<>(buildMapReq(),httpHeaders);
+        ResponseEntity<String> exchange = restTemplate.exchange(shuiMuProperties.getBaseUrl() +getToken, HttpMethod.POST, httpEntity, String.class, new HashMap<>());
+        String body = exchange.getBody();
+        JSONObject bodyObject = JSON.parseObject(body);
+        ShuiMuTokenResp tokenResp = new ShuiMuTokenResp();
+        tokenResp.setAccessToken(bodyObject.getString("access_token"));
+        tokenResp.setExpiresIn(bodyObject.getInteger("expires_in"));
+        tokenResp.setRefreshToken(bodyObject.getString("refresh_token"));
+        tokenResp.setTokenType(bodyObject.getString("token_type"));
+        return tokenResp;
+    }
+    private MultiValueMap<String,String> buildMapReq(){
+        MultiValueMap<String,String> req = new LinkedMultiValueMap<>();
+        req.add("username",shuiMuProperties.getUsername());
+        req.add("password",shuiMuProperties.getPassword());
+        req.add("grant_type",shuiMuProperties.getGrantType());
+        req.add("client_id",shuiMuProperties.getClientId());
+        return req;
+    }
+
+    @Override
+    public String getToken() {
+        Object o = redisTemplateUtils.get(REDIS_SHUI_MU_TOKEN_KEY);
+        String token = o instanceof String ? ((String) o) : "";
+        if (StringUtils.isEmpty(token)) {
+            synchronized(this){
+                o = redisTemplateUtils.get(REDIS_SHUI_MU_TOKEN_KEY);
+                token = o instanceof String ? ((String) o) : "";
+                if (StringUtils.isEmpty(token)) {
+                    ShuiMuTokenResp tokenRemote = getTokenRemote();
+                    String accessToken = tokenRemote.getAccessToken();
+                    redisTemplateUtils.set(REDIS_SHUI_MU_TOKEN_KEY,accessToken,tokenRemote.getExpiresIn());
+                    token = accessToken;
+                }
+            }
+        }
+        return SHUI_MU_TOKEN_PREFIX+token;
+    }
+
+    @Override
+    public List<String> imageList(String label) {
+
+        String uriString = UriComponentsBuilder.fromUriString(shuiMuProperties.getBaseUrl()+imageList).queryParam("label", label).toUriString();
+        HttpEntity<String> entity = new HttpEntity<>(null,getHeaders());
+        ResponseEntity<String> exchange = restTemplate.exchange(uriString, HttpMethod.GET, entity, String.class, new HashMap<>());
+        JSONObject jsonObject = JSONObject.parseObject(exchange.getBody());
+        JSONArray objects = JSONArray.parseArray(jsonObject.getString("data"));
+        List<String> res = new ArrayList<>(objects.size());
+        for (int i = 0; i < objects.size(); i++) {
+            JSONObject item = objects.getJSONObject(i);
+            String name = item.getString("name");
+            if (StringUtils.isNotEmpty(name)) {
+                res.add(name);
+            }
+        }
+        return res.stream().filter(item->item.startsWith(SHUI_MU_IMAGE_LIST_FILTER)).toList();
+    }
+
+    @Override
+    public void createProject(ShuiMuProjectCreateReq req) {
+        MultiValueMap<String,Object> body = new LinkedMultiValueMap<>();
+        body.add("projectName","应用检测_"+ UUID.randomUUID().toString().replaceAll("-",""));
+        body.add("language",req.getLanguage());
+        body.add("env",req.getEnv());
+        body.add("envSpecs",req.getEnvSpecs());
+        PipPipeline pipeline = pipelineService.getById(req.getPipelineId());
+        String targetVersionId = pipeline.getTargetVersionId();
+        TargetVersion targetVersion = targetVersionService.getById(targetVersionId);
+        File targetFile = new File(targetVersion.getFilePath());
+        if (!targetFile.exists()) {
+            throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"找不到目标文件");
+        }
+        FileSystemResource resource = new FileSystemResource(targetFile);
+        body.add("file",resource);
+        HttpEntity<MultiValueMap<String,Object>> entity = new HttpEntity<>(body,getHeaders());
+        ResponseEntity<String> exchange = restTemplate.exchange(shuiMuProperties.getBaseUrl() + createProject, HttpMethod.POST, entity, String.class, new HashMap<>());
+        HttpStatusCode statusCode = exchange.getStatusCode();
+        if (statusCode.isError()) {
+            throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"创建项目失败");
+        }
+    }
+
+    @Override
+    public String readWfuzzJsonFile(String projectId) {
+        HttpEntity<String> entity = new HttpEntity<>(null,getHeaders());
+        String uriString = UriComponentsBuilder.fromUriString(shuiMuProperties.getBaseUrl() + readWfuzzJsonFile)
+                .queryParam("projectId", projectId).toUriString();
+        ResponseEntity<String> exchange = restTemplate.exchange(uriString, HttpMethod.GET, entity, String.class, new HashMap<>());
+        return exchange.getBody();
+    }
+
+    @Override
+    public String readHelpersJsonFile(String projectId) {
+        HttpEntity<String> entity = new HttpEntity<>(null,getHeaders());
+        String uriString = UriComponentsBuilder.fromUriString(shuiMuProperties.getBaseUrl() + readHelpersJsonFile)
+                .queryParam("projectId", projectId).toUriString();
+        ResponseEntity<String> exchange = restTemplate.exchange(uriString, HttpMethod.GET, entity, String.class, new HashMap<>());
+        return exchange.getBody();
+    }
+
+    @Override
+    public ShuiMuProjectVersionResp getProjectVersion(String projectId) {
+        HttpEntity<String> entity = new HttpEntity<>(null,getHeaders());
+        String uriString = UriComponentsBuilder.fromUriString(shuiMuProperties.getBaseUrl() + getProjectVersion)
+                .queryParam("projectId", projectId).toUriString();
+        ResponseEntity<String> exchange = restTemplate.exchange(uriString, HttpMethod.GET, entity, String.class, new HashMap<>());
+        if (exchange.getStatusCode().isError()) {
+            throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"保存并且构建失败");
+        }
+        String data = JSON.parseObject(exchange.getBody()).getString("data");
+        return JSON.parseObject(data, ShuiMuProjectVersionResp.class);
+    }
+
+    @Override
+    public ShuiMuProjectInfoResp getProjectInfo(String projectId) {
+        HttpEntity<String> entity = new HttpEntity<>(null,getHeaders());
+        String uriString = UriComponentsBuilder.fromUriString(shuiMuProperties.getBaseUrl() + getProjectById +projectId)
+                .toUriString();
+        ResponseEntity<String> exchange = restTemplate.exchange(uriString, HttpMethod.GET, entity, String.class, new HashMap<>());
+        if (exchange.getStatusCode().isError()) {
+            throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"保存并且构建失败");
+        }
+        return JSON.parseObject(JSON.parseObject(exchange.getBody()).getString("data"), ShuiMuProjectInfoResp.class);
+    }
+
+    @Override
+    public void buildAndSaveConfig(ShuiMuBuildAndSaveReq req) {
+        MultiValueMap<String,String> body = new LinkedMultiValueMap<>();
+        body.add("projectId",req.getProjectId());
+        body.add("env",req.getEnv());
+        body.add("fileName",req.getFileName());
+        HttpEntity<MultiValueMap<String,String>> entity = new HttpEntity<>(body,getHeaders());
+        ResponseEntity<String> exchange = restTemplate.exchange(shuiMuProperties.getBaseUrl() + buildAndSaveConfig, HttpMethod.POST, entity, String.class, new HashMap<>());
+        if (exchange.getStatusCode().isError()) {
+            throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"保存并且构建失败");
+        }
+    }
+
+    @Override
+    public void uploadWfuzzJson(ShuiMuUpdateWFuzzJsonReq req) {
+        ShuiMuProjectInfoResp projectInfo = getProjectInfo(req.getProjectId());
+        String projectName = projectInfo.getName();
+        Map<String,String> restReq = new HashMap<>();
+        restReq.put("project",projectName);
+        restReq.put("product",projectInfo.getProduct());
+        restReq.put("language",projectInfo.getLanguage());
+        restReq.put("build",req.getBuild());
+        ByteArrayOutputStream bis = new ByteArrayOutputStream();
+        try {
+            ObjectOutputStream oos =new ObjectOutputStream(bis);
+            oos.writeObject(restReq);
+            File tempFile = File.createTempFile(UUID.randomUUID().toString().replaceAll("-", ""), "");
+            FileSystemResource resource = new FileSystemResource(tempFile);
+            MultiValueMap<String,Object> body = new LinkedMultiValueMap<>();
+            body.add("projectId",req.getProjectId());
+            body.add("file",resource);
+            HttpEntity<MultiValueMap<String,Object>> entity = new HttpEntity<>(body,getHeaders());
+            ResponseEntity<String> exchange = restTemplate.exchange(shuiMuProperties.getBaseUrl() + uploadWfuzzJson, HttpMethod.POST, entity, String.class, new HashMap<>());
+            if (exchange.getStatusCode().isError()) {
+                throw new ServiceException(GlobalErrorCodeConstants.PIPELINE_ERROR.getCode(),"更新配置文件失败");
+            }
+            tempFile.deleteOnExit();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+
+    private HttpHeaders getHeaders(){
+        HttpHeaders httpHeaders = new HttpHeaders();
+        httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+        httpHeaders.set("authorization",getToken());
+        return httpHeaders;
+    }
+
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/taskLog/PipTaskLogLineService.java

@@ -0,0 +1,12 @@
+package cd.casic.ci.process.process.service.taskLog;
+
+import cd.casic.ci.process.process.dataObject.log.PipTaskLog;
+import cd.casic.ci.process.process.dataObject.log.PipTaskLogLine;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+import java.util.List;
+
+public interface PipTaskLogLineService extends IService<PipTaskLogLine> {
+    String getLineContentByLogId(String logId,Integer count);
+    List<PipTaskLogLine> getLineListByLogId(String logId, Integer count);
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/process/service/taskLog/impl/PipTaskLogLineServiceImpl.java

@@ -0,0 +1,39 @@
+package cd.casic.ci.process.process.service.taskLog.impl;
+
+import cd.casic.ci.process.process.dao.pipeline.PipTaskLogLineDao;
+import cd.casic.ci.process.process.dataObject.log.PipTaskLogLine;
+import cd.casic.ci.process.process.service.taskLog.PipTaskLogLineService;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.extension.service.IService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import jakarta.annotation.Resource;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.util.Collections;
+import java.util.List;
+
+@Service
+public class PipTaskLogLineServiceImpl extends ServiceImpl<PipTaskLogLineDao,PipTaskLogLine> implements PipTaskLogLineService {
+    @Resource
+    private PipTaskLogLineDao logLineDao;
+    @Override
+    public String getLineContentByLogId(String logId,Integer count){
+        List<PipTaskLogLine> lineList = getLineListByLogId(logId, count);
+        if (CollectionUtils.isEmpty(lineList)) {
+            return "";
+        }
+        StringBuilder stringBuilder = new StringBuilder();
+        for (PipTaskLogLine line : lineList) {
+            stringBuilder.append(line.getLineContent());
+        }
+        return stringBuilder.toString();
+    }
+    @Override
+    public List<PipTaskLogLine> getLineListByLogId(String logId, Integer count){
+        LambdaQueryWrapper<PipTaskLogLine> wrapper = new LambdaQueryWrapper<>();
+        wrapper.eq(PipTaskLogLine::getLogId,logId);
+        wrapper.last("LIMIT "+count);
+        return logLineDao.selectList(wrapper);
+    }
+}

modules/module-ci-process-biz/src/main/java/cd/casic/ci/process/properties/ShuiMuProperties.java

@@ -0,0 +1,23 @@
+package cd.casic.ci.process.properties;
+
+import lombok.Data;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties(prefix = "shuimu")
+@Component
+@Data
+public class ShuiMuProperties {
+    @Value("username")
+    private String username;
+    @Value("password")
+    private String password;
+    @Value("baseUrl")
+    private String baseUrl;
+    @Value("grant_type")
+    private String grantType;
+    @Value("client_id")
+    private String clientId;
+
+}

ops-server/src/main/resources/application-local.yaml

@@ -70,10 +70,10 @@ spring:
   # Redis 配置。Redisson 默认的配置足够使用，一般不需要进行调优
   data:
     redis:
-      host: 127.0.0.1 # 地址
-      port: 16379 # 端口
-#      host: 192.168.1.120 # 地址
-#      port: 6379 # 端口
+#      host: 127.0.0.1 # 地址
+#      port: 16379 # 端口
+      host: 192.168.1.120 # 地址
+      port: 6379 # 端口
       database: 0 # 数据库索引
 #    password: dev # 密码，建议生产环境开启
 
@@ -175,4 +175,10 @@ tartet:
     username: roots
     password: hnidc0327cn!@#xhh
     remoteDir: /home/ops/ops-pro/file/
+shuimu:
+  base-url: http://175.6.27.155:10200
+  password: saas@2022
+  username: admin
+  grant-type: password
+  client-id: client
 

ops-server/src/test/java/cd/casic/server/ShuiMuTest.java

@@ -0,0 +1,43 @@
+package cd.casic.server;
+
+import cd.casic.ci.process.dto.resp.shuimu.ShuiMuProjectVersionResp;
+import cd.casic.ci.process.process.service.shuimu.ShuiMuService;
+import jakarta.annotation.Resource;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+
+@SpringBootTest(classes = {OpsServerApplication.class})
+@ActiveProfiles("local")
+@Slf4j
+public class ShuiMuTest {
+    @Resource
+    ShuiMuService shuiMuService;
+    @Test
+    public void getToken(){
+        System.out.println(shuiMuService.getToken());
+    }
+    @Test
+    public void imageList(){
+        System.out.println(shuiMuService.imageList("java"));
+    }
+    @Test
+    public void readWfuzzJsonFile(){
+        System.out.println(shuiMuService.readWfuzzJsonFile("6"));
+    }
+    @Test
+    public void readHelpersJsonFile(){
+        System.out.println(shuiMuService.readHelpersJsonFile("6"));
+    }
+    @Test
+    public void getProjectVersion(){
+        ShuiMuProjectVersionResp projectVersion = shuiMuService.getProjectVersion("6");
+        System.out.println(projectVersion);
+        System.out.println(projectVersion.getRepo());
+    }
+    @Test
+    public void getProjectInfo(){
+        System.out.println(shuiMuService.getProjectInfo("4"));
+    }
+}