toolchaintechnologycenter/spdm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of http://192.168.65.198:3000/toolchaintechnologycenter/spdm-backend

Browse Source
This commit is contained in:
lidongyang
2026-01-22 16:15:09 +08:00
parent c4093f8c20 e104fb2ab8
commit 45d2c82a52
13 changed files with 723 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
common/src/main/java/com/sdm/common/entity/req/data/BatchChildDirNodeInfo.java Normal file
View File

@@ -0,0 +1,27 @@
package com.sdm.common.entity.req.data;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
/**
* 子节点文件夹信息
*/
@Data
@Schema(description = "子节点文件夹信息")
public class BatchChildDirNodeInfo {
@Schema(description = "节点的业务UUID", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank(message = "节点UUID不能为空")
private String uuId;
@Schema(description = "父节点UUID（第一层可为空，表示挂在系统根目录下）")
private String parentUuId;
@Schema(description = "UUID所属类型:node,task,run,performance", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank(message = "UUID类型不能为空")
private String uuIdOwnType;
@Schema(description = "文件夹名称（如果不传则使用uuid作为名称）")
private String dirName;
}

37
common/src/main/java/com/sdm/common/entity/req/data/BatchCreateDirItem.java Normal file
View File

@@ -0,0 +1,37 @@
package com.sdm.common.entity.req.data;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
/**
* 批量创建文件夹单项 - 表示一组父子节点的创建信息
* <p>
* 示例：
* <pre>
* {
* "parentDirNodeInfo": {"uuId": "proj-001", "uuIdOwnType": "project", "dirName": "项目A"},
* "childDirNodeInfos": [
* {"uuId": "task-001", "parentUuId": "proj-001", "uuIdOwnType": "task", "dirName": "任务1"},
* {"uuId": "task-002", "parentUuId": "proj-001", "uuIdOwnType": "task", "dirName": "任务2"}
* ]
* }
* </pre>
*/
@Data
@Schema(description = "批量创建文件夹单项")
public class BatchCreateDirItem {
@Schema(description = "父节点信息（第一层为项目节点，也需要创建）", requiredMode = Schema.RequiredMode.REQUIRED)
@NotNull(message = "父节点信息不能为空")
@Valid
private DirNodeInfo parentDirNodeInfo;
@Schema(description = "子节点文件夹信息列表")
@Valid
private List<DirNodeInfo> childDirNodeInfos;
}

77
common/src/main/java/com/sdm/common/entity/req/data/BatchCreateDirReq.java Normal file
View File

@@ -0,0 +1,77 @@
package com.sdm.common.entity.req.data;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
/**
* 批量创建文件夹请求 - 支持多级树形结构批量创建
*<p>
* [
* {
* // 第一个 Item：定义项目根节点，并横向创建两个任务
* "parentDirNodeInfo": {
* "uuId": "proj-协同-001",
* "parentUuId": null,
* "uuIdOwnType": "project",
* "dirName": "2026年度协同设计项目"
* },
* "childDirNodeInfos": [
* {
* "uuId": "task-101",
* "parentUuId": "proj-协同-001",
* "uuIdOwnType": "task",
* "dirName": "任务1：前端界面开发"
* },
* {
* "uuId": "task-102",
* "parentUuId": "proj-协同-001",
* "uuIdOwnType": "task",
* "dirName": "任务2：后端接口实现"
* }
* ]
* },
* {
* // 第二个 Item：针对“任务1”进一步纵向创建其子目录
* "parentDirNodeInfo": {
* "uuId": "task-101",
* "parentUuId": "proj-协同-001",
* "uuIdOwnType": "task",
* "dirName": "任务1：前端界面开发"
* },
* "childDirNodeInfos": [
* {
* "uuId": "run-201",
* "parentUuId": "task-101",
* "uuIdOwnType": "run",
* "dirName": "UI原型评审"
* },
* {
* "uuId": "run-202",
* "parentUuId": "task-101",
* "uuIdOwnType": "run",
* "dirName": "交互逻辑验证"
* }
* ]
* }
* ]
*</p>
*/
@Data
@Schema(description = "批量创建文件夹请求")
public class BatchCreateDirReq {
@Schema(description = "批量创建项列表", requiredMode = Schema.RequiredMode.REQUIRED)
@NotEmpty(message = "创建项列表不能为空")
@Valid
private List<BatchCreateDirItem> items;
@Schema(description = "文件夹类型DirType：1 知识库文件夹，2 项目节点文件夹，3 头像库文件夹，4 仿真参数库文件夹,5 训练模型文件夹",
requiredMode = Schema.RequiredMode.REQUIRED)
@NotNull(message = "文件夹类型不能为空")
private Integer dirType;
}

34
common/src/main/java/com/sdm/common/entity/req/data/DirNodeInfo.java Normal file
View File

@@ -0,0 +1,34 @@
package com.sdm.common.entity.req.data;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
/**
* 目录节点信息（统一表示父节点和子节点）
* <p>
* 示例：
* <pre>
* - 项目节点：{uuId="proj-001", parentUuId=null, uuIdOwnType="project", dirName="项目A"}
* - 任务节点：{uuId="task-001", parentUuId="proj-001", uuIdOwnType="task", dirName="任务1"}
* - 运行节点：{uuId="run-001", parentUuId="task-001", uuIdOwnType="run", dirName="运行1"}
* </pre>
*/
@Data
@Schema(description = "目录节点信息")
public class DirNodeInfo {
@Schema(description = "节点的业务UUID", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank(message = "节点UUID不能为空")
private String uuId;
@Schema(description = "父节点UUID（第一层项目节点的parentUuId为空）")
private String parentUuId;
@Schema(description = "UUID所属类型: project, task, run, performance", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank(message = "UUID类型不能为空")
private String uuIdOwnType;
@Schema(description = "文件夹名称（不传则使用uuid作为名称）")
private String dirName;
}

13
common/src/main/java/com/sdm/common/feign/impl/data/DataClientFeignClientImpl.java
View File

@@ -44,6 +44,19 @@ public class DataClientFeignClientImpl implements IDataFeignClient {
}
}
@Override
public SdmResponse<List<Long>> batchCreateDir(BatchCreateDirReq req) {
SdmResponse<List<Long>> response;
try {
response = dataClient.batchCreateDir(req);
log.info("批量创建文件夹响应："+ response);
return response;
} catch (Exception e) {
log.error("批量创建文件夹失败", e);
return SdmResponse.failed("批量创建文件夹失败");
}
}
@Override
public SdmResponse<PageDataResp<List<FileMetadataInfoResp>>> queryDir(QueryDirReq req) {
SdmResponse<PageDataResp<List<FileMetadataInfoResp>>> response;

3
common/src/main/java/com/sdm/common/feign/inter/data/IDataFeignClient.java
View File

@@ -27,6 +27,9 @@ public interface IDataFeignClient {
@PostMapping("/data/createDir")
SdmResponse createDir(@RequestBody @Validated CreateDirReq req);
@PostMapping("/data/batchCreateDir")
SdmResponse<List<Long>> batchCreateDir(@RequestBody @Validated BatchCreateDirReq req);
@PostMapping("/data/queryDir")
SdmResponse<PageDataResp<List<FileMetadataInfoResp>>> queryDir(@RequestBody @Validated QueryDirReq req);

9
data/src/main/java/com/sdm/data/config/MinioConfig.java
View File

@@ -3,10 +3,14 @@ package com.sdm.data.config;
import io.minio.MinioClient;
import io.minio.admin.MinioAdminClient;
import lombok.Data;
import okhttp3.ConnectionPool;
import okhttp3.OkHttpClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.concurrent.TimeUnit;
@Configuration
@ConfigurationProperties(prefix = "minio")
@Data
@@ -41,9 +45,14 @@ public class MinioConfig {
@Bean
public MinioClient minioClient() {
// 自定义OkHttpClient，设置连接池参数,复用连接池中的长连接,提高性能避免频繁创建、销毁
OkHttpClient httpClient = new OkHttpClient.Builder()
.connectionPool(new ConnectionPool(100, 5, TimeUnit.MINUTES)) // 允许100个空闲连接
.build();
return MinioClient.builder()
.endpoint(endpoint, port, secure)
.credentials(accessKey, secretKey)
.httpClient(httpClient)
.build();
}
}

37
data/src/main/java/com/sdm/data/config/thread/MinioTaskThreadPool.java Normal file
View File

@@ -0,0 +1,37 @@
package com.sdm.data.config.thread;
import com.sdm.common.mdc.MdcTaskDecorator;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.Executor;
import java.util.concurrent.ThreadPoolExecutor;
/**
* 专门用于 MinIO 批量 IO 操作的线程池
*/
@Configuration
public class MinioTaskThreadPool {
@Bean(name = "minioBatchExecutor")
public Executor minioBatchExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 核心线程数：IO密集型建议 (CPU核数 * 2) 或更高。针对2500个请求，建议 20-50
executor.setCorePoolSize(30);
// 最大线程数：根据 MinIO 服务器承受能力定
executor.setMaxPoolSize(100);
// 队列容量：2500个任务，设置5000足够
executor.setQueueCapacity(5000);
executor.setKeepAliveSeconds(60);
executor.setThreadNamePrefix("minio-batch-");
// 拒绝策略：关键！不能用 Discard，否则目录会创建丢失。
// 使用 CallerRunsPolicy：如果线程池满了，让提交任务的线程（主线程）自己执行，起到“限流”作用。
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
executor.setTaskDecorator(new MdcTaskDecorator());
executor.initialize();
return executor;
}
}

12
data/src/main/java/com/sdm/data/controller/DataFileController.java
View File

@@ -62,6 +62,18 @@ public class DataFileController implements IDataFeignClient {
return IDataFileService.createDir(req);
}
/**
* 批量创建文件夹（支持多级树形结构）
*
* @param req 批量创建请求
* @return 创建成功的文件夹ID列表
*/
@PostMapping("/batchCreateDir")
@Operation(summary = "批量创建文件夹", description = "支持多级树形结构的批量创建，采用BFS分层批量插入策略")
public SdmResponse<List<Long>> batchCreateDir(@RequestBody @Validated BatchCreateDirReq req) {
return IDataFileService.batchCreateDir(req);
}
/**
* 因为审批的都是文件 所以这个审批回调方法可以公用
* @param req

10
data/src/main/java/com/sdm/data/service/IDataFileService.java
View File

@@ -30,6 +30,16 @@ public interface IDataFileService {
*/
SdmResponse createDir(CreateDirReq req);
/**
* 批量创建目录（支持多级树形结构）
* 采用BFS分层批量插入策略，每层只执行一次批量数据库操作
* @param req 批量创建目录请求参数
* @return 创建成功的目录ID列表
*/
default SdmResponse<List<Long>> batchCreateDir(BatchCreateDirReq req) {
return null;
}
/**
* 删除目录
* @param req 删除目录请求参数

14
data/src/main/java/com/sdm/data/service/IMinioService.java
View File

@@ -33,6 +33,20 @@ public interface IMinioService {
*/
void createDirectoryByObjectKey(String objectKey, String bucketName);
/**
* 批量创建目录
* @param objectKeys
* @param bucketName
*/
void batchCreateDirectories(List<String> objectKeys, String bucketName);
/**
* 批量删除文件/目录
* @param objectKeys 文件/目录的objectKey列表
* @param bucketName 桶名称
*/
void batchDeleteFiles(List<String> objectKeys, String bucketName);
/**
* 递归删除指定目录下的所有对象。
*

396
data/src/main/java/com/sdm/data/service/impl/MinioFileIDataFileServiceImpl.java
View File

@@ -49,6 +49,7 @@ import com.sdm.data.service.impl.dataFileHandle.ApproveStrategyFactory;
import io.minio.Result;
import io.minio.messages.Item;
import jakarta.servlet.http.HttpServletResponse;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
@@ -3283,4 +3284,399 @@ public class MinioFileIDataFileServiceImpl implements IDataFileService {
return SdmResponse.success(fileMetadataInfoResp);
}
// ========================== 批量创建目录实现 ==========================
/**
* 批量创建目录上下文：用于在各私有方法间传递状态
* <p>
* 示例数据结构（创建项目节点目录树）：
* <pre>
* projectNode/ ← 系统根目录 (id=1, objectKey="projectNode/")
* ├── 项目A/ ← 第1层-项目节点 (uuid="proj-001", parentUuId=null, uuIdOwnType="project")
* │ ├── 任务1/ ← 第2层-任务节点 (uuid="task-001", parentUuId="proj-001", uuIdOwnType="task")
* │ │ └── 运行1/ ← 第3层-运行节点 (uuid="run-001", parentUuId="task-001", uuIdOwnType="run")
* │ └── 任务2/ ← 第2层-任务节点 (uuid="task-002", parentUuId="proj-001", uuIdOwnType="task")
* └── 项目B/ ← 第1层-项目节点 (uuid="proj-002", parentUuId=null, uuIdOwnType="project")
* </pre>
*/
@Data
private static class BatchCreateDirContext {
/** 租户ID，从线程上下文获取 */
Long tenantId;
/** 目录类型：1-知识库, 2-项目节点, 3-头像库... */
Integer dirType;
/** 创建成功的目录ID列表，最终返回给调用方 */
List<Long> createdDirIds = new ArrayList<>();
/** 已创建的MinIO目录key列表，用于异常时补偿删除 */
List<String> createdMinioKeys = new ArrayList<>();
/**
* 父子关系映射：parentUuId -> 子节点列表
* <pre>
* 示例：
* "proj-001" -> [task-001, task-002]
* "task-001" -> [run-001]
* </pre>
*/
Map<String, List<DirNodeInfo>> parentToChildren;
/**
* 所有要创建的节点UUID集合（包括项目节点、任务节点、运行节点等）
* <pre>
* 示例：{"proj-001", "proj-002", "task-001", "task-002", "run-001"}
* </pre>
*/
Set<String> allNodeUuIds;
/**
* 第一层父节点UUID集合（这些节点的父目录是系统根目录）
* <pre>
* 判断规则：parentUuId为空
* </pre>
*/
Set<String> firstLevelParentUuIds;
/**
* UUID到数据库ID的映射，随着分层插入逐步填充
* <pre>
* 初始化："" -> 1 (系统根目录ID)
* 第1层插入后："proj-001" -> 100, "proj-002" -> 101
* 第2层插入后："task-001" -> 200, "task-002" -> 201
* 第3层插入后："run-001" -> 300
* </pre>
*/
Map<String, Long> uuidToDbIdMap = new HashMap<>();
/**
* UUID到MinIO ObjectKey的映射，用于构建子目录路径
* <pre>
* 初始化："" -> "projectNode/" (系统根目录objectKey)
* 第1层插入后："proj-001" -> "projectNode/项目A/"
* 第2层插入后："task-001" -> "projectNode/项目A/任务1/"
* 第3层插入后："run-001" -> "projectNode/项目A/任务1/运行1/"
* </pre>
*/
Map<String, String> uuidToObjectKeyMap = new HashMap<>();
}
/**
* 批量创建目录（支持多级树形结构）
* 采用BFS分层批量插入策略，每层只执行一次批量数据库操作
*/
@Override
@Transactional(rollbackFor = Exception.class)
public SdmResponse<List<Long>> batchCreateDir(BatchCreateDirReq req) {
log.info("开始执行批量创建目录，dirType: {}, items数量: {}", req.getDirType(),
req.getItems() == null ? 0 : req.getItems().size());
long startTime = System.currentTimeMillis();
// 1. 参数校验
SdmResponse<Void> validateResult = validateBatchCreateDirRequest(req);
if (!validateResult.isSuccess()) {
log.error("批量创建目录参数校验失败: {}", validateResult.getMessage());
return SdmResponse.failed(validateResult.getMessage());
}
BatchCreateDirContext ctx = new BatchCreateDirContext();
ctx.setTenantId(ThreadLocalContext.getTenantId());
ctx.setDirType(req.getDirType());
try {
// 2. 构建节点分组数据
buildNodeGroupingData(req, ctx);
log.info("节点分组数据构建完成，总节点数: {}, 顶层父节点数: {}",
ctx.getAllNodeUuIds().size(), ctx.getFirstLevelParentUuIds().size());
// 3. 检查重复创建
SdmResponse<Void> duplicateCheck = checkDuplicateNodes(ctx);
if (!duplicateCheck.isSuccess()) {
log.error("批量创建目录失败，检测到重复节点: {}", duplicateCheck.getMessage());
return SdmResponse.failed(duplicateCheck.getMessage());
}
// 5. 初始化UUID映射
// 获取系统根目录
SdmResponse<FileMetadataInfo> rootDirResult = getSystemRootDir(ctx.dirType);
if (!rootDirResult.isSuccess()) {
log.error("获取系统根目录失败: {}", rootDirResult.getMessage());
return SdmResponse.failed(rootDirResult.getMessage());
}
FileMetadataInfo systemRootDir = rootDirResult.getData();
initUuidMappings(ctx, systemRootDir);
log.info("初始化UUID映射完成，系统根路径: {}", systemRootDir.getObjectKey());
// 6. BFS层级遍历，按层分组
List<List<DirNodeInfo>> levelNodes = buildLevelNodesByBFS(ctx.getParentToChildren(), ctx.getFirstLevelParentUuIds());
log.info("BFS层级分析完成，共计{}层，总节点数：{}", levelNodes.size(),
levelNodes.stream().mapToInt(List::size).sum());
// 7. 分层批量插入
for (int levelIndex = 0; levelIndex < levelNodes.size(); levelIndex++) {
processLevelNodes(levelNodes.get(levelIndex), levelIndex, ctx);
}
long duration = System.currentTimeMillis() - startTime;
log.info("批量创建目录成功，耗时: {}ms, 共创建{}个目录", duration, ctx.createdDirIds.size());
return SdmResponse.success(ctx.createdDirIds);
} catch (Exception e) {
log.error("批量创建目录异常，已创建MinIO Key数量: {}, 错误信息: {}",
ctx.createdMinioKeys.size(), e.getMessage(), e);
compensateDeleteMinioKeys(ctx.createdMinioKeys);
throw new RuntimeException("批量创建目录失败: " + e.getMessage(), e);
}
}
/**
* 校验批量创建目录请求参数
*/
private SdmResponse<Void> validateBatchCreateDirRequest(BatchCreateDirReq req) {
if (req == null || CollectionUtils.isEmpty(req.getItems())) {
log.error("批量创建目录失败，请求参数为空");
return SdmResponse.failed("请求参数不能为空");
}
if (req.getDirType() == null) {
log.error("批量创建目录失败，文件夹类型为空");
return SdmResponse.failed("文件夹类型不能为空");
}
return SdmResponse.success(null);
}
/**
* 构建节点分组数据：展开所有节点（包括父节点和子节点）并按parentUuId分组
*/
private void buildNodeGroupingData(BatchCreateDirReq req, BatchCreateDirContext ctx) {
// 展开所有节点：包括父节点（项目节点）和子节点
List<DirNodeInfo> allNodes = new ArrayList<>();
Set<String> allNodeUuIds =new HashSet<>();
List<BatchCreateDirItem> batchCreateDirItems = req.getItems();
for (BatchCreateDirItem item : batchCreateDirItems) {
// 添加父节点（项目节点也需要创建）
// 如果父节点已经添加过了，则跳过
if(allNodeUuIds.contains(item.getParentDirNodeInfo().getUuId())){
continue;
}
allNodeUuIds.add(item.getParentDirNodeInfo().getUuId());
allNodes.add(item.getParentDirNodeInfo());
// 添加子节点
if (CollectionUtils.isNotEmpty(item.getChildDirNodeInfos())) {
item.getChildDirNodeInfos().stream()
.filter(child -> !allNodeUuIds.contains(child.getUuId()))
.forEach(child -> {
allNodeUuIds.add(child.getUuId());
allNodes.add(child);
});
}
}
// 收集所有节点UUID
ctx.setAllNodeUuIds(allNodeUuIds);
// 按parentUuId分组，构建父子关系图
Map<String, List<DirNodeInfo>> parentToChildren = batchCreateDirItems.stream().collect(Collectors.toMap(
// key：父节点UUID
item -> item.getParentDirNodeInfo().getUuId(),
// value：当前item的子节点列表
BatchCreateDirItem::getChildDirNodeInfos
));
ctx.setParentToChildren(parentToChildren);
// 找出第一层父节点UUID集合（这些节点的父目录是系统根目录）
ctx.setFirstLevelParentUuIds(allNodes.stream()
.filter(element -> element.getParentUuId() == null)
.map(DirNodeInfo::getUuId)
.collect(Collectors.toSet()));
}
/**
* 检查要创建的节点是否已存在（重复导入检测）
*/
private SdmResponse<Void> checkDuplicateNodes(BatchCreateDirContext ctx) {
List<FileMetadataInfo> existingNodes = fileMetadataInfoService.lambdaQuery()
.in(FileMetadataInfo::getRelatedResourceUuid, ctx.allNodeUuIds)
.eq(FileMetadataInfo::getTenantId, ctx.tenantId)
.list();
if (CollectionUtils.isNotEmpty(existingNodes)) {
log.error("批量创建目录失败，部分目录已存在（非新项目），已存在的uuid: {}",
existingNodes.stream().map(FileMetadataInfo::getRelatedResourceUuid).collect(Collectors.toList()));
return SdmResponse.failed("目录已存在，请勿重复创建");
}
return SdmResponse.success(null);
}
/**
* 根据dirType获取系统根目录
*/
private SdmResponse<FileMetadataInfo> getSystemRootDir(Integer dirType) {
DirTypeEnum dirTypeEnum = DirTypeEnum.getDirTypeByValue(dirType);
if (dirTypeEnum == null) {
log.error("批量创建目录失败，无效的目录类型: {}", dirType);
return SdmResponse.failed("无效的目录类型");
}
String rootDirObjectKey = getDirMinioObjectKey(dirTypeEnum.getDirName());
Optional<FileMetadataInfo> rootDirOpt = getFileMetadataInfoByObjectKey(rootDirObjectKey, null);
if (!rootDirOpt.isPresent()) {
log.error("批量创建目录失败，系统根目录不存在");
return SdmResponse.failed("系统根目录不存在，请先初始化系统目录");
}
return SdmResponse.success(rootDirOpt.get());
}
/**
* 初始化UUID到数据库ID/ObjectKey的映射
*/
private void initUuidMappings(BatchCreateDirContext ctx, FileMetadataInfo systemRootDir) {
// 空字符串表示第一层节点的父目录（系统根目录）
ctx.uuidToDbIdMap.put("", systemRootDir.getId());
ctx.uuidToObjectKeyMap.put("", systemRootDir.getObjectKey());
}
/**
* 处理单层节点的创建：创建MinIO目录、数据库记录、权限记录
*/
private void processLevelNodes(List<DirNodeInfo> nodes, int levelIndex, BatchCreateDirContext ctx) {
long levelStartTime = System.currentTimeMillis();
log.info("开始处理第{}层，节点数：{}", levelIndex + 1, nodes.size());
// 1. 构建该层所有节点的元数据对象，并收集需要创建的MinIO目录路径
List<FileMetadataInfo> entities = new ArrayList<>();
List<String> minioObjectKeys = new ArrayList<>();
for (DirNodeInfo node : nodes) {
FileMetadataInfo entity = buildDirNodeMetadata(node, ctx);
entities.add(entity);
minioObjectKeys.add(entity.getObjectKey());
}
// 2. 批量创建MinIO目录
long minioStartTime = System.currentTimeMillis();
minioService.batchCreateDirectories(minioObjectKeys, null);
ctx.getCreatedMinioKeys().addAll(minioObjectKeys);
log.info("第{}层 MinIO目录创建完成，耗时: {}ms", levelIndex + 1, System.currentTimeMillis() - minioStartTime);
// 3. 批量插入数据库
long dbStartTime = System.currentTimeMillis();
fileMetadataInfoService.saveBatch(entities, 500);
log.info("第{}层 数据库元数据插入完成，耗时: {}ms", levelIndex + 1, System.currentTimeMillis() - dbStartTime);
// 4. 插入后更新映射（MyBatis-Plus会回填ID）
for (FileMetadataInfo entity : entities) {
ctx.getUuidToDbIdMap().put(entity.getRelatedResourceUuid(), entity.getId());
ctx.getCreatedDirIds().add(entity.getId());
}
// 5. 批量创建权限记录
createBatchPermissions(entities, ctx.tenantId);
log.info("第{}层处理完成，创建了{}个目录，总耗时: {}ms",
levelIndex + 1, entities.size(), System.currentTimeMillis() - levelStartTime);
}
/**
* 构建单个目录节点的元数据（不创建MinIO目录）
*/
private FileMetadataInfo buildDirNodeMetadata(DirNodeInfo node, BatchCreateDirContext ctx) {
String parentUuId = node.getParentUuId() == null ? "" : node.getParentUuId();
Long parentDbId = ctx.getUuidToDbIdMap().get(parentUuId);
String parentObjectKey = ctx.getUuidToObjectKeyMap().get(parentUuId);
if (parentDbId == null || parentObjectKey == null) {
log.error("批量创建目录失败，父目录信息不存在，parentUuId={}", node.getParentUuId());
throw new RuntimeException("父目录信息不存在: " + node.getParentUuId());
}
// 构造子目录完整路径
String childDirMinioObjectKey = getDirMinioObjectKey(parentObjectKey + node.getDirName());
// 创建目录元数据
FileMetadataInfo dirInfo = createDirectoryMetadata(
childDirMinioObjectKey, node.getDirName(), false, parentDbId,
node.getUuId(), node.getUuIdOwnType(), ctx.dirType);
// 预先更新ObjectKey映射（供下一层使用）
ctx.getUuidToObjectKeyMap().put(node.getUuId(), childDirMinioObjectKey);
return dirInfo;
}
/**
* 批量创建权限记录
*/
private void createBatchPermissions(List<FileMetadataInfo> entities, Long tenantId) {
List<FileUserPermission> permissions = entities.stream()
.map(entity -> {
FileUserPermission permission = new FileUserPermission();
permission.setTFilemetaId(entity.getId());
permission.setPermission(FilePermissionEnum.ALL.getValue());
permission.setUserId(ThreadLocalContext.getUserId());
permission.setTenantId(tenantId);
return permission;
})
.collect(Collectors.toList());
fileUserPermissionService.saveBatch(permissions, 500);
}
/**
* 补偿删除已创建的MinIO目录
*/
private void compensateDeleteMinioKeys(List<String> createdMinioKeys) {
if (createdMinioKeys == null || createdMinioKeys.isEmpty()) {
return;
}
try {
minioService.batchDeleteFiles(createdMinioKeys, null);
} catch (Exception e) {
log.warn("补偿删除MinIO目录失败，数量：{}", createdMinioKeys.size(), e);
}
}
/**
* BFS层级遍历，按层分组节点
*
* @param parentToChildren 父子关系映射
* @param firstLevelParentUuIds 第一层父节点UUID集合（挂在系统根目录下）
* @return 按层分组的节点列表
*/
private List<List<DirNodeInfo>> buildLevelNodesByBFS(
Map<String, List<DirNodeInfo>> parentToChildren,
Set<String> firstLevelParentUuIds) {
List<List<DirNodeInfo>> levelNodes = new ArrayList<>();
Set<String> currentLevelParents = new HashSet<>(firstLevelParentUuIds);
while (!currentLevelParents.isEmpty()) {
List<DirNodeInfo> currentLevelChildren = new ArrayList<>();
Set<String> nextLevelParents = new HashSet<>();
for (String parentUuId : currentLevelParents) {
List<DirNodeInfo> children = parentToChildren.get(parentUuId);
if (children != null) {
for (DirNodeInfo child : children) {
currentLevelChildren.add(child);
// 如果这个子节点还有子节点，则加入下一层的父节点集合
if (parentToChildren.containsKey(child.getUuId())) {
nextLevelParents.add(child.getUuId());
}
}
}
}
if (!currentLevelChildren.isEmpty()) {
levelNodes.add(currentLevelChildren);
}
currentLevelParents = nextLevelParents;
}
return levelNodes;
}
}

54
data/src/main/java/com/sdm/data/service/minio/MinioService.java
View File

@@ -15,6 +15,7 @@ import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
@@ -34,7 +35,9 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.text.DecimalFormat;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -44,6 +47,11 @@ public class MinioService implements IMinioService {
private final MinioClient minioClient;
private final MinioConfig minioConfig;
@Autowired
@Qualifier("minioBatchExecutor")
private Executor minioBatchExecutor;
// 可配置缓冲区大小） 16KB
private static final int DEFAULT_BUFFER_SIZE = 16384;
@@ -203,6 +211,52 @@ public class MinioService implements IMinioService {
}
}
@Override
public void batchCreateDirectories(List<String> objectKeys, String bucketName) {
List<CompletableFuture<Void>> futures = objectKeys.stream()
.map(key -> CompletableFuture.runAsync(() -> {
createDirectoryByObjectKey(key, bucketName);
}, minioBatchExecutor))
.toList();
// 等待所有任务完成
try {
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
.get(2, TimeUnit.MINUTES); // 设置总超时时间
} catch (Exception e) {
log.error("批量创建目录部分失败或超时", e);
}
}
@Override
public void batchDeleteFiles(List<String> objectKeys, String bucketName) {
if (objectKeys == null || objectKeys.isEmpty()) {
return;
}
try {
// 构建删除对象列表
List<DeleteObject> deleteObjects = objectKeys.stream()
.map(key -> new DeleteObject(dealDirPath(key)))
.toList();
// 批量删除
Iterable<Result<DeleteError>> deleteErrors = minioClient.removeObjects(
RemoveObjectsArgs.builder()
.bucket(getBucketName(bucketName))
.objects(deleteObjects)
.build());
// 处理删除错误（如果有）
for (Result<DeleteError> deleteErrorResult : deleteErrors) {
DeleteError deleteError = deleteErrorResult.get();
log.warn("批量删除对象失败: {}, 错误: {}", deleteError.objectName(), deleteError.message());
}
} catch (Exception e) {
log.error("批量删除文件失败", e);
throw new RuntimeException("批量删除文件失败", e);
}
}
private static String dealDirPath(String dirPath) {
if (dirPath.startsWith("/")) {