修改:hpc执行流程从上个节点获取文件
This commit is contained in:
@@ -5,6 +5,11 @@ import lombok.Data;
|
||||
@Data
|
||||
public class HPCExecuteConfig extends BaseExecuteConfig {
|
||||
|
||||
private String currentNodeId;
|
||||
private String beforeNodeId;
|
||||
// 先默认写死一个,后面前端配置传递
|
||||
private String masterFileRegularStr="^aa\\.xml$";
|
||||
// 先默认写死一个,后面前端配置传递
|
||||
private String inputFilesRegularStr="^.+\\.json$";
|
||||
|
||||
}
|
||||
|
||||
@@ -49,22 +49,13 @@ public class SubmitHpcTaskRemoteReq {
|
||||
@Schema(description = "计算任务所属算力名称")
|
||||
public String runName;
|
||||
|
||||
@Schema(description = "执行的命令")
|
||||
@Schema(description = "执行的命令,无需前端传递,根据soft查询")
|
||||
public String command;
|
||||
|
||||
@Schema(description = "命令执行输出文件名xx.out")
|
||||
public String stdout;
|
||||
@Schema(description = "命令执行输出文件名xx.out,默认stdout.out")
|
||||
public String stdout="stdout.out";
|
||||
|
||||
@Schema(description = "任务所属项目")
|
||||
public String projectname;
|
||||
|
||||
// @Schema(description = "获取文件的方式:上一节点:beforeNode,hpc节点(文件提前上传工作目录):hpcNode")
|
||||
// public String featchFileType;
|
||||
//
|
||||
// @Schema(description = "上一节点Id,featchFileType:beforeNode时传递 ")
|
||||
// public String beforeNodeId;
|
||||
|
||||
// @Schema(description= "自定义占位符,只有列表展示使用,key 就是占位符")
|
||||
// private Map<String,SimulationCommandPlaceholderReq> commandExpand;
|
||||
|
||||
}
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
package com.sdm.common.utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.MappedByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Vector;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@@ -311,4 +317,97 @@ public class FilesUtil {
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从工作目录中递归查找文件:
|
||||
* 1. master 文件:匹配 masterFileRegularStr,按修改时间倒序取最新一个
|
||||
* 2. input 文件:匹配 inputFilesRegularStr 的全部文件
|
||||
*/
|
||||
public static void collectFiles(
|
||||
String jobWorkDir,
|
||||
String masterFileRegularStr,
|
||||
String inputFilesRegularStr,
|
||||
AtomicReference<String> masterFilePath,
|
||||
List<String> inputFilePaths) {
|
||||
Objects.requireNonNull(jobWorkDir, "jobWorkDir 不能为空");
|
||||
boolean hasMasterRule =
|
||||
masterFileRegularStr != null && !masterFileRegularStr.isBlank();
|
||||
boolean hasInputRule =
|
||||
inputFilesRegularStr != null && !inputFilesRegularStr.isBlank();
|
||||
// 如果两个规则都没有,直接返回,避免无意义扫描
|
||||
if (!hasMasterRule && !hasInputRule) {
|
||||
return;
|
||||
}
|
||||
Pattern masterPattern = hasMasterRule ? Pattern.compile(masterFileRegularStr) : null;
|
||||
Pattern inputPattern = hasInputRule ? Pattern.compile(inputFilesRegularStr) : null;
|
||||
Path root = Paths.get(jobWorkDir);
|
||||
if (!Files.exists(root) || !Files.isDirectory(root)) {
|
||||
throw new IllegalArgumentException("工作目录不存在或不是目录:" + jobWorkDir);
|
||||
}
|
||||
|
||||
// 只有需要 master 文件时才创建
|
||||
AtomicReference<FileTime> latestMasterTime =
|
||||
hasMasterRule ? new AtomicReference<>() : null;
|
||||
|
||||
try {
|
||||
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
||||
if (!attrs.isRegularFile()) {
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
String fileName = file.getFileName().toString();
|
||||
// master 文件逻辑
|
||||
if (hasMasterRule && masterPattern.matcher(fileName).matches()) {
|
||||
FileTime lastModified = attrs.lastModifiedTime();
|
||||
FileTime currentMax = latestMasterTime.get();
|
||||
|
||||
if (currentMax == null || lastModified.compareTo(currentMax) > 0) {
|
||||
latestMasterTime.set(lastModified);
|
||||
masterFilePath.set(file.toAbsolutePath().toString());
|
||||
}
|
||||
}
|
||||
|
||||
// input 文件逻辑
|
||||
if (hasInputRule && inputPattern.matcher(fileName).matches()) {
|
||||
inputFilePaths.add(file.toAbsolutePath().toString());
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFileFailed(Path file, IOException exc) {
|
||||
// 安全策略:跳过不可访问文件
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("遍历目录失败:" + jobWorkDir, e);
|
||||
}
|
||||
}
|
||||
|
||||
public static MultipartFile toMultipartFile(String absoluteFilePath) throws IOException {
|
||||
Path path = Path.of(absoluteFilePath);
|
||||
|
||||
if (!Files.exists(path) || !Files.isRegularFile(path)) {
|
||||
throw new IllegalArgumentException("文件不存在或不是普通文件:" + absoluteFilePath);
|
||||
}
|
||||
|
||||
String fileName = path.getFileName().toString();
|
||||
String contentType = Files.probeContentType(path);
|
||||
|
||||
try (FileInputStream fis = new FileInputStream(path.toFile())) {
|
||||
return new MockMultipartFile(
|
||||
"file", // form field name
|
||||
fileName, // original filename
|
||||
contentType != null ? contentType : "application/octet-stream",
|
||||
fis
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user