toolchaintechnologycenter/spdm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改：minio分片上传，增加有条件保存本地工作目录

Browse Source
This commit is contained in:
yangyang01000846
2026-01-20 23:27:17 +08:00
parent c38f19f6a1
commit a0df889714
4 changed files with 208 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
data/src/main/java/com/sdm/data/service/impl/LocalFileService.java Normal file
View File

@@ -0,0 +1,152 @@
package com.sdm.data.service.impl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@Service
@Slf4j
public class LocalFileService {
/**
* 将MultipartFile文件保存到指定的本地路径
* @param file 待保存的文件
* @param absoluteLocalFilePath 文件要保存的绝对路径（包含文件名）
* @return 保存是否成功
*/
public boolean saveSingleFileToLocal(MultipartFile file, String absoluteLocalFilePath) {
// 1. 入参校验
if (file == null || file.isEmpty()) {
throw new IllegalArgumentException("待保存的文件不能为空");
}
if (absoluteLocalFilePath == null || absoluteLocalFilePath.trim().isEmpty()) {
throw new IllegalArgumentException("文件保存路径不能为空");
}
// 2. 创建文件目录（如果不存在）
Path filePath = Paths.get(absoluteLocalFilePath);
File parentDir = filePath.getParent().toFile();
if (!parentDir.exists()) {
// 递归创建目录，mkdirs()会创建所有不存在的父目录
boolean dirCreated = parentDir.mkdirs();
if (!dirCreated) {
log.error("saveSingleFileToLocal 无法创建文件目录:{}",absoluteLocalFilePath);
return false;
}
}
// 3. 将MultipartFile写入到指定路径
// StandardCopyOption.REPLACE_EXISTING 表示如果文件已存在则覆盖
try {
Files.copy(file.getInputStream(), filePath, StandardCopyOption.REPLACE_EXISTING);
} catch (Exception e) {
log.error("saveSingleFileToLocal 文件保存失败:{}",absoluteLocalFilePath);
return false;
}
// 4. 验证文件是否保存成功
File savedFile = filePath.toFile();
return savedFile.exists();
}
/**
* 合并本地分片文件为完整文件
* @param tempDirPath 分片文件所在临时目录
* @param finalLocalFilePath 合并后的完整文件路径
* @param chunkTotal 分片总数
* @param chunkSuffix 分片文件后缀（如 .chunk）
* @return 合并是否成功
* 合并过程中IO异常
*/
public boolean mergeLocalChunks(String tempDirPath, String finalLocalFilePath,
int chunkTotal, String chunkSuffix) {
// 1. 校验临时目录和分片总数
File tempDir = new File(tempDirPath);
if (!tempDir.exists() || !tempDir.isDirectory()) {
log.error("mergeLocalChunks 地临时分片目录不存在:{}",tempDirPath);
return false;
}
if (chunkTotal <= 0) {
log.error("mergeLocalChunks 分片总数必须大于0:{}",chunkTotal);
return false;
}
// 2. 收集所有分片文件（按分片编号排序）
List<File> chunkFiles = new ArrayList<>();
for (int i = 1; i <= chunkTotal; i++) {
String chunkFileName = tempDirPath + i + chunkSuffix;
File chunkFile = new File(chunkFileName);
if (!chunkFile.exists()) {
log.error("mergeLocalChunks 分片文件缺失:{}",chunkFileName);
}
chunkFiles.add(chunkFile);
}
// 3. 创建合并后的文件目录
Path finalFilePath = Paths.get(finalLocalFilePath);
File finalFileParent = finalFilePath.getParent().toFile();
if (!finalFileParent.exists()) {
boolean dirCreated = finalFileParent.mkdirs();
if (!dirCreated) {
log.error("mergeLocalChunks 无法创建最终文件目录:{}",finalLocalFilePath);
return false;
}
}
// 4. 合并分片文件到最终文件
try (FileOutputStream fos = new FileOutputStream(finalLocalFilePath, true);
BufferedOutputStream bos = new BufferedOutputStream(fos)) {
byte[] buffer = new byte[1024 * 1024]; // 1MB 缓冲区
int bytesRead;
for (File chunkFile : chunkFiles) {
try (FileInputStream fis = new FileInputStream(chunkFile);
BufferedInputStream bis = new BufferedInputStream(fis)) {
while ((bytesRead = bis.read(buffer)) != -1) {
bos.write(buffer, 0, bytesRead);
}
}
}
bos.flush();
}catch (Exception e){
log.error("mergeLocalChunks 合并文件异常:{}",e.getMessage());
}
// 5. 验证合并后的文件是否有效
File finalFile = new File(finalLocalFilePath);
boolean mergeSuccess = finalFile.exists();
// 6. 合并成功后删除临时分片文件和目录
deleteLocalDirectory(tempDirPath);
return mergeSuccess;
}
/**
* 删除本地目录及其中所有文件
* @param dirPath 目录路径
*
*/
public void deleteLocalDirectory(String dirPath) {
log.info("deleteLocalDirectory 开始删除本地临时文件:{}",dirPath);
try {
Files.walk( Paths.get(dirPath))
.sorted(Comparator.reverseOrder())
.forEach(path -> {
try { Files.delete(path); } catch (Exception ignored) {
log.error("deleteLocalDirectory 文件删除失败：{}",path.getFileName());
}
});
} catch (Exception e) {
log.error("deleteLocalDirectory 文件删除异常：{}",e.getMessage());
}
}
}

51
data/src/main/java/com/sdm/data/service/impl/MinioFileIDataFileServiceImpl.java
View File

@@ -16,16 +16,12 @@ import com.sdm.common.entity.constants.PermConstants;
import com.sdm.common.entity.enums.*;
import com.sdm.common.entity.pojo.task.TaskBaseInfo;
import com.sdm.common.entity.req.data.*;
import com.sdm.common.entity.req.data.CopyFileToTaskReq;
import com.sdm.common.entity.req.project.SpdmNodeListReq;
import com.sdm.common.entity.req.system.LaunchApproveReq;
import com.sdm.common.entity.req.system.UserListReq;
import com.sdm.common.entity.req.system.UserQueryReq;
import com.sdm.common.entity.resp.PageDataResp;
import com.sdm.common.entity.resp.data.BatchAddFileInfoResp;
import com.sdm.common.entity.resp.data.ChunkUploadMinioFileResp;
import com.sdm.common.entity.resp.data.FileMetadataInfoResp;
import com.sdm.common.entity.resp.data.FileSimulationMappingResp;
import com.sdm.common.entity.resp.data.*;
import com.sdm.common.entity.resp.project.SimulationNodeResp;
import com.sdm.common.entity.resp.system.CIDUserResp;
import com.sdm.common.feign.impl.project.SimulationNodeFeignClientImpl;
@@ -40,8 +36,6 @@ import com.sdm.data.aop.PermissionCheckAspect;
import com.sdm.data.model.bo.ApprovalFileDataContentsModel;
import com.sdm.data.model.dto.ExportKnowledgeDto;
import com.sdm.data.model.entity.*;
import com.sdm.common.entity.resp.data.PoolInfo;
import com.sdm.data.model.enums.ApproveFileActionENUM;
import com.sdm.data.model.req.*;
import com.sdm.data.model.resp.KKFileViewURLFromMinioResp;
@@ -101,6 +95,8 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService {
// fileData 知识库文件列表可见的数据
private final List<Integer> fileDatdList = ApproveFileDataTypeEnum.getVisibleInFileList();
private static final String FLOWABLE_SIMULATION_BASEDIR = "/home/simulation/";
@Autowired
private IFileMetadataInfoService fileMetadataInfoService;
@@ -156,6 +152,9 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService {
@Qualifier(value = "nonSensitiveTaskPool")
private Executor nonSensitiveTaskPool;
@Autowired
private LocalFileService localFileService;
// @Override
// public String getType() {
// return type;
@@ -1145,12 +1144,25 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService {
String timestamp = req.getUploadTaskId();
// 合并目录
String filePath = getMinioFolderPath(req.getObjectKey());
// 本地磁盘保存父级路径
String absoluteLocalFileDirPath = FLOWABLE_SIMULATION_BASEDIR + filePath;
log.info("saveSingleFileToLocal 本地保存文件父级路径：{}",absoluteLocalFileDirPath);
// 0. 一个文件直接传
if(Objects.equals(req.getChunkTotal(),NumberConstants.ONE)&&
Objects.equals(req.getChunk(),NumberConstants.ONE)){
// 第一步业务数据入表已经规定好这个文件的路径了
String finalFileName = req.getObjectKey();
Boolean b = minioService.chunkUpload(chunkBucket, req.getFile(), finalFileName,null);
// 保存本地
if(Objects.equals("Y",req.getIsSaveLocal())){
// 单一文件的绝对名称
String absoluteLocalFilePath = absoluteLocalFileDirPath+req.getSourceFileName();
MultipartFile file = req.getFile();
boolean b1 = localFileService.saveSingleFileToLocal(file, absoluteLocalFilePath);
log.info("saveSingleFileToLocal 保存单一文件：{}，结果：{}",absoluteLocalFilePath,b1);
}
if(!b){
return buildFailedResponse(resp,"单一文件上传失败",req);
}
@@ -1165,6 +1177,17 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService {
Map<String, String> tags = new HashMap<>();
tags.put("auto-expire", "1d");
Boolean b = minioService.chunkUpload(chunkBucket, req.getFile(), chunkFileName,tags);
// 保存本地碎片,本地失败了，先忽略
String localTempDirPath ="";
if(Objects.equals("Y",req.getIsSaveLocal())){
localTempDirPath = FLOWABLE_SIMULATION_BASEDIR + tempDirPath;
log.info("saveChunkFileToLocal 保存本地碎片路径:{}",localTempDirPath);
String localChunkFileName = localTempDirPath + req.getChunk() + PermConstants.CHUNK_TEMPFILE_SUFFIX;
boolean localChunkSaveSuccess = localFileService.saveSingleFileToLocal(req.getFile(), localChunkFileName);
log.info("saveChunkFileToLocal 保存本地碎片:{},结果：{}",localTempDirPath,localChunkSaveSuccess);
}
if(!b){
deleteTempFileAfterFailed(tempDirPath,chunkBucket);
return buildFailedResponse(resp,"chunkUpload第"+req.getChunk()+"次失败",req);
@@ -1176,6 +1199,20 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService {
// 3. 全部分片已经上传 => 自动合并
String finalFileName = req.getObjectKey();
Boolean merge = minioService.merge(chunkBucket, tempDirPath, chunkBucket, finalFileName);
// 是否本地保存
if(Objects.equals("Y",req.getIsSaveLocal())){
String finalLocalFilePath = absoluteLocalFileDirPath+req.getSourceFileName();
boolean localMergeSuccess = localFileService.mergeLocalChunks(
localTempDirPath,
finalLocalFilePath,
req.getChunkTotal(),
PermConstants.CHUNK_TEMPFILE_SUFFIX
);
log.info("saveChunkFileToLocal 合并本地文件finalLocalFilePath:{},localTempDirPath:{},getChunkTotal:{}结果:{}",
finalLocalFilePath,localTempDirPath,req.getChunkTotal(),localMergeSuccess);
}
if(!merge){
deleteTempFileAfterFailed(tempDirPath,chunkBucket);
return buildFailedResponse(resp,req.getSourceFileName()+"合并分片失败",req);