toolchaintechnologycenter/spdm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改：hpc提交，下载，上传求解文件优化

Browse Source
This commit is contained in:
yangyang01000846
2025-12-04 09:41:17 +08:00
parent c1f9a6065e
commit f97a6dfee8
12 changed files with 313 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pbs/src/main/java/com/sdm/pbs/controller/TaskController.java
View File

@@ -1,6 +1,7 @@
package com.sdm.pbs.controller;
import com.sdm.common.common.SdmResponse;
import com.sdm.common.entity.req.pbs.HpcTaskFileDownReq;
import com.sdm.common.entity.req.pbs.HpcTaskFileReq;
import com.sdm.common.entity.req.pbs.SubmitHpcTaskRemoteReq;
import com.sdm.common.entity.req.pbs.hpc.*;
@@ -28,6 +29,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@@ -71,9 +73,9 @@ public class TaskController implements ITaskFeignClient {
return pbsService.queryHpcResource();
}
@PostMapping("/submitHpcJob")
@PostMapping(value = "/submitHpcJob", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@Operation(summary = "作业提交")
public SdmResponse<String> submitHpcJob(@RequestBody SubmitHpcTaskRemoteReq req) {
public SdmResponse<String> submitHpcJob(SubmitHpcTaskRemoteReq req) {
SubmitHpcTaskReq submitHpcTaskReq = new SubmitHpcTaskReq();
BeanUtils.copyProperties(req,submitHpcTaskReq);
return pbsService.submitHpcJob(submitHpcTaskReq);
@@ -97,10 +99,10 @@ public class TaskController implements ITaskFeignClient {
return pbsService.getJobResultFiles(req.getJobId(),req.getTargetDir());
}
@GetMapping("/hpcDownloadFile")
@PostMapping("/hpcDownloadFile")
@Operation(summary = "作业下文件下载")
ResponseEntity<StreamingResponseBody> hpcDownloadFile(@RequestParam String jobId,@RequestParam String fileName,@RequestParam Long fileSize) {
return pbsService.downloadFile(jobId,fileName,fileSize);
ResponseEntity<StreamingResponseBody> hpcDownloadFile(@RequestBody HpcTaskFileDownReq req) {
return pbsService.downloadFile(req.getJobId(),req.getFileName(),req.getFileSize());
}
@PostMapping("/queryJobs")

20
pbs/src/main/java/com/sdm/pbs/model/req/SubmitHpcTaskReq.java
View File

@@ -1,7 +1,9 @@
package com.sdm.pbs.model.req;
import com.alibaba.fastjson2.annotation.JSONField;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import org.springframework.web.multipart.MultipartFile;
import java.util.ArrayList;
import java.util.List;
@@ -24,10 +26,18 @@ public class SubmitHpcTaskReq {
public boolean independence;
@Schema(description = "求解文件")
public List<String> inputFiles = new ArrayList<>();
@JSONField(serialize = false)
public List<MultipartFile> inputFiles = new ArrayList<>();
@Schema(description = "求解文件路径")
public List<String> inputFilePaths = new ArrayList<>();
@Schema(description = "计算主文件")
public String masterFile;
@JSONField(serialize = false)
public MultipartFile masterFile;
@Schema(description = "主文件上传后的路径")
public String masterFilePath;
@Schema(description = "计算任务所属任务ID")
public String taskId;
@@ -44,6 +54,12 @@ public class SubmitHpcTaskReq {
@Schema(description = "执行的命令")
public String command;
@Schema(description = "命令执行输出文件名xx.out")
public String stdout;
@Schema(description = "工作目录，代码逻辑生成，和求解主文件平级")
public String workDir;
@Schema(description = "任务所属项目")
public String projectname;

3
pbs/src/main/java/com/sdm/pbs/service/impl/HpcInstructionServiceImpl.java
View File

@@ -118,7 +118,8 @@ public class HpcInstructionServiceImpl implements HpcInstructionService {
String prefixStr = HpcCommandBuilderUtil.initAddJobPrefixStr(req.getJobId());
AddJobParam addJobParam = new AddJobParam();
BeanUtils.copyProperties(req, addJobParam);
String targetWorkDir = addJobParam.getWorkdir() + "\\" + req.getJobId();
// String targetWorkDir = addJobParam.getWorkdir() + "\\" + req.getJobId();
String targetWorkDir = addJobParam.getWorkdir();
Pair<Boolean, String> workDirPair = createDirIfNotExist(targetWorkDir);
if(!workDirPair.getLeft()){
AddJobResp addJobResp=new AddJobResp();

7
pbs/src/main/java/com/sdm/pbs/service/impl/IPbsHpcServiceImpl.java
View File

@@ -65,11 +65,8 @@ public class IPbsHpcServiceImpl implements IPbsService {
newJobReq.setProjectname(req.getProjectname());
AddJobReq addJobReq = new AddJobReq();
addJobReq.setName(req.getRunName());
// todo
addJobReq.setStdout("1126.out");
// todo
addJobReq.setWorkdir("\\\\CARSAFE\\share\\spdm");
// todo
addJobReq.setStdout(req.getStdout());
addJobReq.setWorkdir(req.getWorkDir());
addJobReq.setCommand(req.getCommand());
SubmitHpcJobReq submitHpcJobReq = new SubmitHpcJobReq();
mergeSubmitHpcJobReq.setNewJobReq(newJobReq);

100
pbs/src/main/java/com/sdm/pbs/service/impl/PbsServiceDecorator.java
View File

@@ -7,6 +7,7 @@ import com.github.pagehelper.PageInfo;
import com.sdm.common.common.SdmResponse;
import com.sdm.common.entity.resp.PageDataResp;
import com.sdm.common.entity.resp.pbs.hpc.FileNodeInfo;
import com.sdm.common.utils.HpcCommandExcuteUtil;
import com.sdm.common.utils.PageUtils;
import com.sdm.pbs.model.bo.HpcJobStatusInfo;
import com.sdm.pbs.model.bo.HpcResouceInfo;
@@ -17,12 +18,14 @@ import com.sdm.pbs.model.req.JobFileCallBackReq;
import com.sdm.pbs.model.req.QueryJobReq;
import com.sdm.pbs.model.req.SubmitHpcTaskReq;
import com.sdm.pbs.service.*;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.time.LocalDateTime;
@@ -36,6 +39,9 @@ import java.util.stream.Collectors;
@ConditionalOnProperty(name = "pbs.task.impl", havingValue = "hpc")
public class PbsServiceDecorator implements IPbsServiceDecorator {
@Autowired
private HpcCommandExcuteUtil hpcCommandExcuteUtil;
// 正则匹配%后的单词（\w+ 匹配字母、数字、下划线）
private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("%(\\w+)");
@@ -67,9 +73,24 @@ public class PbsServiceDecorator implements IPbsServiceDecorator {
@Override
public SdmResponse<String> submitHpcJob(SubmitHpcTaskReq req) {
// SdmResponse<String> response = pbsService.submitHpcJob(req);
// if(response.isSuccess()&&StringUtils.isNotEmpty(response.getData())) {
String jobId = "8848";
//1. 上传hpc主文件 及 其他文件
MultipartFile masterFile = req.getMasterFile();
String subDir = req.getJobName()+"\\"+System.currentTimeMillis();
// webClient 调用上传，这个是主文件，求解算出的文件，及stdout文件都指定这个文件夹下面
String masterFilePath = hpcCommandExcuteUtil.uploaHpcFile(masterFile,subDir);
dealInputFiles(req,subDir);
// 任务输出的文件夹
String hpcOutPutDir = extractDirectory(masterFilePath);
req.setWorkDir(hpcOutPutDir);
// 前置处理 替换求解文件
String formatCommand = String.format(req.getCommand(), masterFilePath);
req.setCommand(formatCommand);
req.setMasterFilePath(masterFilePath);
SdmResponse<String> response = pbsService.submitHpcJob(req);
String jobId="";
if(response.isSuccess()&&StringUtils.isNotEmpty(response.getData())) {
jobId = response.getData();
}
if(StringUtils.isNotEmpty(jobId)) {
// 数据入库
SimulationJob simulationJob = new SimulationJob();
@@ -80,13 +101,15 @@ public class PbsServiceDecorator implements IPbsServiceDecorator {
simulationJob.setSoftware(req.getSoftware());
simulationJob.setJobType(req.getJobType());
simulationJob.setIndependence(req.isIndependence());
simulationJob.setInputFiles(JSONObject.toJSONString(req.getInputFiles()));
simulationJob.setMasterFile(req.getMasterFile());
// simulationJob.setInputFiles(JSONObject.toJSONString(req.getInputFiles()));
// 主文件位置 todo
simulationJob.setMasterFile(req.getMasterFilePath());
// 求解文件集合
simulationJob.setInputFiles(JSONObject.toJSONString(req.getInputFilePaths()));
simulationJob.setTaskId(req.getTaskId());
simulationJob.setTaskName(req.getTaskName());
simulationJob.setRunId(req.getRunId());
simulationJob.setRunName(req.getRunName());
// 软件及文件关联
simulationJob.setSoftwareId(req.getSoftwareId());
// 下面的待定 todo
@@ -94,11 +117,9 @@ public class PbsServiceDecorator implements IPbsServiceDecorator {
simulationJob.setJobId(jobId);
// 没必要要
simulationJob.setJobDetailId("todo");
// 文件路径 todo 共享目录+jobName（文件回传)+uuid，下面可能有多个文件
simulationJob.setStdoutHpcFilePath("/hpc/shared/job001/uuid-123");
simulationJob.setStdoutHpcFilePath(hpcOutPutDir);
simulationJob.setStdoutSpdmFilePath("/minio/base/job001/uuid-123");
// todo 执行信息 定时任务回传的时候修改
simulationJob.setNodeName("todo");
simulationJob.setExecutCommand("ansys -b -input input.dat -output output.log");
@@ -108,11 +129,10 @@ public class PbsServiceDecorator implements IPbsServiceDecorator {
simulationJob.setJobStatus("Configuring");
// ? todo 没比要
simulationJob.setSolverName("LS-DYNA");
// todo 执行信息 定时任务回传的时候修改
simulationJob.setTotalKernelTime(3600000L);
simulationJob.setTotalUserTime(7200000L);
simulationJob.setTotalElapsedTime(9000L);
simulationJob.setTotalKernelTime(null);
simulationJob.setTotalUserTime(null);
simulationJob.setTotalElapsedTime(null);
// 标识及状态
simulationJob.setUuid("f81d4fae7dec11d0a76500a0c91e6bf6");
@@ -129,6 +149,41 @@ public class PbsServiceDecorator implements IPbsServiceDecorator {
return SdmResponse.success(jobId);
}
private void dealInputFiles(SubmitHpcTaskReq req, String subDir) {
if(req.getInputFiles()==null|| CollectionUtils.isEmpty(req.getInputFiles())) {
return;
}
List<MultipartFile> inputFiles = req.getInputFiles();
List<String> list = new ArrayList<>();
for (MultipartFile inputFile : inputFiles) {
String inputFilePath = hpcCommandExcuteUtil.uploaHpcFile(inputFile,subDir);
list.add(inputFilePath);
}
req.setInputFilePaths(list);
}
/**
* 从文件路径中提取目录部分（包含最后一个路径分隔符）
* @param fullPath 完整的文件路径
* @return 目录路径（包含最后一个反斜杠），若路径为空或无分隔符则返回原路径
*/
private String extractDirectory(String fullPath) {
// 校验参数
if (fullPath == null || fullPath.isEmpty()) {
return fullPath;
}
// 找到最后一个反斜杠的位置
int lastSeparatorIndex = fullPath.lastIndexOf("\\");
// 若没有找到分隔符，返回原路径；否则截取到最后一个分隔符（包含）
if (lastSeparatorIndex == -1) {
return fullPath;
}
return fullPath.substring(0, lastSeparatorIndex + 1);
}
@Override
public SdmResponse<Boolean> stopHpcJob(String jobId) {
return pbsService.stopHpcJob(jobId);
@@ -142,9 +197,22 @@ public class PbsServiceDecorator implements IPbsServiceDecorator {
@Override
public SdmResponse<List<FileNodeInfo>> getJobResultFiles(String jobId,String targetDir) {
// todo 根据jobId 获取工作目录，共享目录+jobName（文件回传)+uuid，下面可能有多个文件
String workDir = StringUtils.isNotBlank(targetDir) ? targetDir :"D:\\需求";
SdmResponse<List<FileNodeInfo>> nodeInfos = pbsService.getJobResultFiles("", workDir);
// 根据 jobId 获取工作信息
SimulationJob simulationJob = simulationJobService.lambdaQuery()
.eq(SimulationJob::getJobId, jobId)
.one();
// 选择 queryPath
String queryPath;
if (targetDir != null && !targetDir.isEmpty()) {
queryPath = targetDir;
} else if (simulationJob!=null&&simulationJob.getStdoutHpcFilePath() != null && !simulationJob.getStdoutHpcFilePath().isEmpty()) {
queryPath = simulationJob.getStdoutHpcFilePath();
} else {
return SdmResponse.failed("查询路径为空，无法获取文件");
}
// 调用 PBS 服务获取文件列表
SdmResponse<List<FileNodeInfo>> nodeInfos = pbsService.getJobResultFiles("", queryPath);
return nodeInfos;
}

2
pbs/src/main/resources/application-dev.yml
View File

@@ -108,6 +108,8 @@ hpc:
remoteCreateDirUrl: http://192.168.65.55:9097/createDir
remoteScanDirUrl: http://192.168.65.55:9097/scanDir
remoteDownLoadFileUrl: http://192.168.65.55:9097/hpcDownload
# remoteDownLoadFileUrl: http://127.0.0.1:9097/hpcDownload
remoteUploadFileUrl: http://192.168.65.55:9097/uploadHpcFile
callHpcUpload: http://192.168.65.55:9097/addJobQueue