修改:hpc执行流程从上个节点获取文件

This commit is contained in:
yangyang01000846
2025-12-05 09:11:55 +08:00
parent e3284bb718
commit 5f61d9c796
8 changed files with 229 additions and 49 deletions

View File

@@ -5,6 +5,11 @@ import lombok.Data;
@Data
public class HPCExecuteConfig extends BaseExecuteConfig {
private String currentNodeId;
private String beforeNodeId;
// 先默认写死一个,后面前端配置传递
private String masterFileRegularStr="^aa\\.xml$";
// 先默认写死一个,后面前端配置传递
private String inputFilesRegularStr="^.+\\.json$";
}

View File

@@ -49,22 +49,13 @@ public class SubmitHpcTaskRemoteReq {
@Schema(description = "计算任务所属算力名称")
public String runName;
@Schema(description = "执行的命令")
@Schema(description = "执行的命令无需前端传递根据soft查询")
public String command;
@Schema(description = "命令执行输出文件名xx.out")
public String stdout;
@Schema(description = "命令执行输出文件名xx.out,默认stdout.out")
public String stdout="stdout.out";
@Schema(description = "任务所属项目")
public String projectname;
// @Schema(description = "获取文件的方式上一节点beforeNode,hpc节点文件提前上传工作目录hpcNode")
// public String featchFileType;
//
// @Schema(description = "上一节点Id,featchFileType:beforeNode时传递 ")
// public String beforeNodeId;
// @Schema(description= "自定义占位符,只有列表展示使用key 就是占位符")
// private Map<String,SimulationCommandPlaceholderReq> commandExpand;
}

View File

@@ -1,15 +1,21 @@
package com.sdm.common.utils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.nio.MappedByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.util.List;
import java.util.Objects;
import java.util.Vector;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -311,4 +317,97 @@ public class FilesUtil {
in.close();
}
}
/**
* 从工作目录中递归查找文件:
* 1. master 文件:匹配 masterFileRegularStr按修改时间倒序取最新一个
* 2. input 文件:匹配 inputFilesRegularStr 的全部文件
*/
public static void collectFiles(
String jobWorkDir,
String masterFileRegularStr,
String inputFilesRegularStr,
AtomicReference<String> masterFilePath,
List<String> inputFilePaths) {
Objects.requireNonNull(jobWorkDir, "jobWorkDir 不能为空");
boolean hasMasterRule =
masterFileRegularStr != null && !masterFileRegularStr.isBlank();
boolean hasInputRule =
inputFilesRegularStr != null && !inputFilesRegularStr.isBlank();
// 如果两个规则都没有,直接返回,避免无意义扫描
if (!hasMasterRule && !hasInputRule) {
return;
}
Pattern masterPattern = hasMasterRule ? Pattern.compile(masterFileRegularStr) : null;
Pattern inputPattern = hasInputRule ? Pattern.compile(inputFilesRegularStr) : null;
Path root = Paths.get(jobWorkDir);
if (!Files.exists(root) || !Files.isDirectory(root)) {
throw new IllegalArgumentException("工作目录不存在或不是目录:" + jobWorkDir);
}
// 只有需要 master 文件时才创建
AtomicReference<FileTime> latestMasterTime =
hasMasterRule ? new AtomicReference<>() : null;
try {
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
if (!attrs.isRegularFile()) {
return FileVisitResult.CONTINUE;
}
String fileName = file.getFileName().toString();
// master 文件逻辑
if (hasMasterRule && masterPattern.matcher(fileName).matches()) {
FileTime lastModified = attrs.lastModifiedTime();
FileTime currentMax = latestMasterTime.get();
if (currentMax == null || lastModified.compareTo(currentMax) > 0) {
latestMasterTime.set(lastModified);
masterFilePath.set(file.toAbsolutePath().toString());
}
}
// input 文件逻辑
if (hasInputRule && inputPattern.matcher(fileName).matches()) {
inputFilePaths.add(file.toAbsolutePath().toString());
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
// 安全策略:跳过不可访问文件
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
throw new RuntimeException("遍历目录失败:" + jobWorkDir, e);
}
}
public static MultipartFile toMultipartFile(String absoluteFilePath) throws IOException {
Path path = Path.of(absoluteFilePath);
if (!Files.exists(path) || !Files.isRegularFile(path)) {
throw new IllegalArgumentException("文件不存在或不是普通文件:" + absoluteFilePath);
}
String fileName = path.getFileName().toString();
String contentType = Files.probeContentType(path);
try (FileInputStream fis = new FileInputStream(path.toFile())) {
return new MockMultipartFile(
"file", // form field name
fileName, // original filename
contentType != null ? contentType : "application/octet-stream",
fis
);
}
}
}