diff --git a/common/src/main/java/com/sdm/common/entity/req/data/ChunkUploadMinioFileReq.java b/common/src/main/java/com/sdm/common/entity/req/data/ChunkUploadMinioFileReq.java index 3c9d7fd5..c0edad49 100644 --- a/common/src/main/java/com/sdm/common/entity/req/data/ChunkUploadMinioFileReq.java +++ b/common/src/main/java/com/sdm/common/entity/req/data/ChunkUploadMinioFileReq.java @@ -33,5 +33,7 @@ public class ChunkUploadMinioFileReq { // 单一文件维度。第一片请求不传，后面的请求必传，第一次请求成功后后端会返回,本次文件的父目录 private String fileTempPath; + // 分片上传的同时 是否报存到本地磁盘 Y：需要保存；N：不需要保存 + private String isSaveLocal; } diff --git a/data/src/main/java/com/sdm/data/service/impl/LocalFileService.java b/data/src/main/java/com/sdm/data/service/impl/LocalFileService.java new file mode 100644 index 00000000..ceabc716 --- /dev/null +++ b/data/src/main/java/com/sdm/data/service/impl/LocalFileService.java @@ -0,0 +1,152 @@ +package com.sdm.data.service.impl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +@Service +@Slf4j +public class LocalFileService { + + /** + * 将MultipartFile文件保存到指定的本地路径 + * @param file 待保存的文件 + * @param absoluteLocalFilePath 文件要保存的绝对路径（包含文件名） + * @return 保存是否成功 + */ + public boolean saveSingleFileToLocal(MultipartFile file, String absoluteLocalFilePath) { + // 1. 入参校验 + if (file == null || file.isEmpty()) { + throw new IllegalArgumentException("待保存的文件不能为空"); + } + if (absoluteLocalFilePath == null || absoluteLocalFilePath.trim().isEmpty()) { + throw new IllegalArgumentException("文件保存路径不能为空"); + } + + // 2. 创建文件目录（如果不存在） + Path filePath = Paths.get(absoluteLocalFilePath); + File parentDir = filePath.getParent().toFile(); + if (!parentDir.exists()) { + // 递归创建目录，mkdirs()会创建所有不存在的父目录 + boolean dirCreated = parentDir.mkdirs(); + if (!dirCreated) { + log.error("saveSingleFileToLocal 无法创建文件目录:{}",absoluteLocalFilePath); + return false; + } + } + + // 3. 将MultipartFile写入到指定路径 + // StandardCopyOption.REPLACE_EXISTING 表示如果文件已存在则覆盖 + try { + Files.copy(file.getInputStream(), filePath, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { + log.error("saveSingleFileToLocal 文件保存失败:{}",absoluteLocalFilePath); + return false; + } + // 4. 验证文件是否保存成功 + File savedFile = filePath.toFile(); + return savedFile.exists(); + } + + /** + * 合并本地分片文件为完整文件 + * @param tempDirPath 分片文件所在临时目录 + * @param finalLocalFilePath 合并后的完整文件路径 + * @param chunkTotal 分片总数 + * @param chunkSuffix 分片文件后缀（如 .chunk） + * @return 合并是否成功 + * 合并过程中IO异常 + */ + public boolean mergeLocalChunks(String tempDirPath, String finalLocalFilePath, + int chunkTotal, String chunkSuffix) { + // 1. 校验临时目录和分片总数 + File tempDir = new File(tempDirPath); + if (!tempDir.exists() || !tempDir.isDirectory()) { + log.error("mergeLocalChunks 地临时分片目录不存在:{}",tempDirPath); + return false; + } + if (chunkTotal <= 0) { + log.error("mergeLocalChunks 分片总数必须大于0:{}",chunkTotal); + return false; + } + + // 2. 收集所有分片文件（按分片编号排序） + List chunkFiles = new ArrayList<>(); + for (int i = 1; i <= chunkTotal; i++) { + String chunkFileName = tempDirPath + i + chunkSuffix; + File chunkFile = new File(chunkFileName); + if (!chunkFile.exists()) { + log.error("mergeLocalChunks 分片文件缺失:{}",chunkFileName); + } + chunkFiles.add(chunkFile); + } + + // 3. 创建合并后的文件目录 + Path finalFilePath = Paths.get(finalLocalFilePath); + File finalFileParent = finalFilePath.getParent().toFile(); + if (!finalFileParent.exists()) { + boolean dirCreated = finalFileParent.mkdirs(); + if (!dirCreated) { + log.error("mergeLocalChunks 无法创建最终文件目录:{}",finalLocalFilePath); + return false; + } + } + + // 4. 合并分片文件到最终文件 + try (FileOutputStream fos = new FileOutputStream(finalLocalFilePath, true); + BufferedOutputStream bos = new BufferedOutputStream(fos)) { + byte[] buffer = new byte[1024 * 1024]; // 1MB 缓冲区 + int bytesRead; + for (File chunkFile : chunkFiles) { + try (FileInputStream fis = new FileInputStream(chunkFile); + BufferedInputStream bis = new BufferedInputStream(fis)) { + while ((bytesRead = bis.read(buffer)) != -1) { + bos.write(buffer, 0, bytesRead); + } + } + } + bos.flush(); + }catch (Exception e){ + log.error("mergeLocalChunks 合并文件异常:{}",e.getMessage()); + } + + // 5. 验证合并后的文件是否有效 + File finalFile = new File(finalLocalFilePath); + boolean mergeSuccess = finalFile.exists(); + // 6. 合并成功后删除临时分片文件和目录 + deleteLocalDirectory(tempDirPath); + return mergeSuccess; + } + + /** + * 删除本地目录及其中所有文件 + * @param dirPath 目录路径 + * + */ + public void deleteLocalDirectory(String dirPath) { + log.info("deleteLocalDirectory 开始删除本地临时文件:{}",dirPath); + try { + Files.walk( Paths.get(dirPath)) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { Files.delete(path); } catch (Exception ignored) { + log.error("deleteLocalDirectory 文件删除失败：{}",path.getFileName()); + } + }); + } catch (Exception e) { + log.error("deleteLocalDirectory 文件删除异常：{}",e.getMessage()); + } + + } + + +} diff --git a/data/src/main/java/com/sdm/data/service/impl/MinioFileIDataFileServiceImpl.java b/data/src/main/java/com/sdm/data/service/impl/MinioFileIDataFileServiceImpl.java index 62322ba9..595f1d7b 100644 --- a/data/src/main/java/com/sdm/data/service/impl/MinioFileIDataFileServiceImpl.java +++ b/data/src/main/java/com/sdm/data/service/impl/MinioFileIDataFileServiceImpl.java @@ -16,16 +16,12 @@ import com.sdm.common.entity.constants.PermConstants; import com.sdm.common.entity.enums.*; import com.sdm.common.entity.pojo.task.TaskBaseInfo; import com.sdm.common.entity.req.data.*; -import com.sdm.common.entity.req.data.CopyFileToTaskReq; import com.sdm.common.entity.req.project.SpdmNodeListReq; import com.sdm.common.entity.req.system.LaunchApproveReq; import com.sdm.common.entity.req.system.UserListReq; import com.sdm.common.entity.req.system.UserQueryReq; import com.sdm.common.entity.resp.PageDataResp; -import com.sdm.common.entity.resp.data.BatchAddFileInfoResp; -import com.sdm.common.entity.resp.data.ChunkUploadMinioFileResp; -import com.sdm.common.entity.resp.data.FileMetadataInfoResp; -import com.sdm.common.entity.resp.data.FileSimulationMappingResp; +import com.sdm.common.entity.resp.data.*; import com.sdm.common.entity.resp.project.SimulationNodeResp; import com.sdm.common.entity.resp.system.CIDUserResp; import com.sdm.common.feign.impl.project.SimulationNodeFeignClientImpl; @@ -40,8 +36,6 @@ import com.sdm.data.aop.PermissionCheckAspect; import com.sdm.data.model.bo.ApprovalFileDataContentsModel; import com.sdm.data.model.dto.ExportKnowledgeDto; import com.sdm.data.model.entity.*; -import com.sdm.common.entity.resp.data.PoolInfo; - import com.sdm.data.model.enums.ApproveFileActionENUM; import com.sdm.data.model.req.*; import com.sdm.data.model.resp.KKFileViewURLFromMinioResp; @@ -101,6 +95,8 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService { // fileData 知识库文件列表可见的数据 private final List fileDatdList = ApproveFileDataTypeEnum.getVisibleInFileList(); + private static final String FLOWABLE_SIMULATION_BASEDIR = "/home/simulation/"; + @Autowired private IFileMetadataInfoService fileMetadataInfoService; @@ -156,6 +152,9 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService { @Qualifier(value = "nonSensitiveTaskPool") private Executor nonSensitiveTaskPool; + @Autowired + private LocalFileService localFileService; + // @Override // public String getType() { // return type; @@ -1145,12 +1144,25 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService { String timestamp = req.getUploadTaskId(); // 合并目录 String filePath = getMinioFolderPath(req.getObjectKey()); + // 本地磁盘保存父级路径 + String absoluteLocalFileDirPath = FLOWABLE_SIMULATION_BASEDIR + filePath; + log.info("saveSingleFileToLocal 本地保存文件父级路径：{}",absoluteLocalFileDirPath); + // 0. 一个文件直接传 if(Objects.equals(req.getChunkTotal(),NumberConstants.ONE)&& Objects.equals(req.getChunk(),NumberConstants.ONE)){ // 第一步业务数据入表已经规定好这个文件的路径了 String finalFileName = req.getObjectKey(); Boolean b = minioService.chunkUpload(chunkBucket, req.getFile(), finalFileName,null); + // 保存本地 + if(Objects.equals("Y",req.getIsSaveLocal())){ + // 单一文件的绝对名称 + String absoluteLocalFilePath = absoluteLocalFileDirPath+req.getSourceFileName(); + MultipartFile file = req.getFile(); + boolean b1 = localFileService.saveSingleFileToLocal(file, absoluteLocalFilePath); + log.info("saveSingleFileToLocal 保存单一文件：{}，结果：{}",absoluteLocalFilePath,b1); + } + if(!b){ return buildFailedResponse(resp,"单一文件上传失败",req); } @@ -1165,6 +1177,17 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService { Map tags = new HashMap<>(); tags.put("auto-expire", "1d"); Boolean b = minioService.chunkUpload(chunkBucket, req.getFile(), chunkFileName,tags); + + // 保存本地碎片,本地失败了，先忽略 + String localTempDirPath =""; + if(Objects.equals("Y",req.getIsSaveLocal())){ + localTempDirPath = FLOWABLE_SIMULATION_BASEDIR + tempDirPath; + log.info("saveChunkFileToLocal 保存本地碎片路径:{}",localTempDirPath); + String localChunkFileName = localTempDirPath + req.getChunk() + PermConstants.CHUNK_TEMPFILE_SUFFIX; + boolean localChunkSaveSuccess = localFileService.saveSingleFileToLocal(req.getFile(), localChunkFileName); + log.info("saveChunkFileToLocal 保存本地碎片:{},结果：{}",localTempDirPath,localChunkSaveSuccess); + } + if(!b){ deleteTempFileAfterFailed(tempDirPath,chunkBucket); return buildFailedResponse(resp,"chunkUpload第"+req.getChunk()+"次失败",req); @@ -1176,6 +1199,20 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService { // 3. 全部分片已经上传 => 自动合并 String finalFileName = req.getObjectKey(); Boolean merge = minioService.merge(chunkBucket, tempDirPath, chunkBucket, finalFileName); + + // 是否本地保存 + if(Objects.equals("Y",req.getIsSaveLocal())){ + String finalLocalFilePath = absoluteLocalFileDirPath+req.getSourceFileName(); + boolean localMergeSuccess = localFileService.mergeLocalChunks( + localTempDirPath, + finalLocalFilePath, + req.getChunkTotal(), + PermConstants.CHUNK_TEMPFILE_SUFFIX + ); + log.info("saveChunkFileToLocal 合并本地文件finalLocalFilePath:{},localTempDirPath:{},getChunkTotal:{}结果:{}", + finalLocalFilePath,localTempDirPath,req.getChunkTotal(),localMergeSuccess); + } + if(!merge){ deleteTempFileAfterFailed(tempDirPath,chunkBucket); return buildFailedResponse(resp,req.getSourceFileName()+"合并分片失败",req); diff --git a/project/src/main/java/com/sdm/project/service/impl/SimulationRunServiceImpl.java b/project/src/main/java/com/sdm/project/service/impl/SimulationRunServiceImpl.java index 4e927831..68f323d2 100644 --- a/project/src/main/java/com/sdm/project/service/impl/SimulationRunServiceImpl.java +++ b/project/src/main/java/com/sdm/project/service/impl/SimulationRunServiceImpl.java @@ -748,15 +748,16 @@ public class SimulationRunServiceImpl extends ServiceImpl> listSdmResponse = dataFeignClient.batchAddFileInfo(req); - if(listSdmResponse.isSuccess() && CollectionUtils.isNotEmpty(listSdmResponse.getData())) { - // 需要将minio文件下载到服务器的文件夹下 - listSdmResponse.getData().forEach(data -> { - String fileObjectKey = data.getObjectKey(); - String dirObjectKey = fileObjectKey.substring(0, fileObjectKey.lastIndexOf("/") + 1); - log.info("开始下载文件：{},目录文件建:{}", dirObjectKey, dirObjectKey); - dataFeignClient.downloadFileToLocal(data.getBusinessId(), FlowableConfig.FLOWABLE_SIMULATION_BASEDIR + dirObjectKey); - }); - } + // 神仙代码，废弃 +// if(listSdmResponse.isSuccess() && CollectionUtils.isNotEmpty(listSdmResponse.getData())) { +// // 需要将minio文件下载到服务器的文件夹下 +// listSdmResponse.getData().forEach(data -> { +// String fileObjectKey = data.getObjectKey(); +// String dirObjectKey = fileObjectKey.substring(0, fileObjectKey.lastIndexOf("/") + 1); +// log.info("开始下载文件：{},目录文件建:{}", dirObjectKey, dirObjectKey); +// dataFeignClient.downloadFileToLocal(data.getBusinessId(), FlowableConfig.FLOWABLE_SIMULATION_BASEDIR + dirObjectKey); +// }); +// } return listSdmResponse; }