toolchaintechnologycenter/spdm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化数据存储展示

Browse Source
This commit is contained in:
gulongcheng
2025-12-23 16:30:24 +08:00
parent bc8e5e771d
commit 742b77c10c
4 changed files with 136 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
data/src/main/java/com/sdm/data/controller/DataStorageAnalysisController.java
View File

@@ -1,6 +1,7 @@
package com.sdm.data.controller;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.ObjectUtils;
import com.sdm.common.common.SdmResponse;
import com.sdm.common.entity.resp.PageDataResp;
@@ -34,7 +35,7 @@ public class DataStorageAnalysisController {
/**
* 根据nodeId(项目nodeId)获取指定查询类型（queryNodeType）文件空间占用(近几个月、增量查询)
* 查询项目存储空间占用：项目Uuid+项目queryNodeType
* 查询学科存储空间占用：项目Uuid+学科queryNodeType
* 查询学科存储空间占用：学科字典名：MVH+学科queryNodeType
*
* @param queryNodeType 需要统计的节点类型
* @param uuids 节点uuid
@@ -44,7 +45,7 @@ public class DataStorageAnalysisController {
@Operation(summary = "根据nodeId(项目nodeId)获取指定查询类型（queryNodeType）文件空间占用")
public SdmResponse getNodeSizeByNodeType(
@Parameter(description = "查询节点类型：project,discipline") @RequestParam(value = "queryNodeType", required = false) String queryNodeType,
@Parameter(description = "节点id:项目Uuid列表") @RequestParam(value = "uuids", required = false) List<String> uuids,
@Parameter(description = "节点id:项目Uuid列表或者学科字典名") @RequestParam(value = "uuids", required = false) List<String> uuids,
@Parameter(description = "查询时间间隔（月）") @RequestParam(value = "intervalMonths", required = false) Integer intervalMonths,
@Parameter(description = "增量查询指定的月：2025-06") @RequestParam(value = "targetYm", required = false) String targetYm
) {
@@ -56,7 +57,7 @@ public class DataStorageAnalysisController {
for (String uuid : uuids) {
try {
SdmResponse<List<JSONObject>> sdmResponse = dataStorageAnalysis.getNodeSizeByNodeType(queryNodeType, uuid, intervalMonths, targetYm);
if (sdmResponse.getData() != null) {
if (CollectionUtils.isNotEmpty(sdmResponse.getData())) {
result.add(sdmResponse.getData());
}
} catch (Exception e) {

125
data/src/main/java/com/sdm/data/service/impl/DataAnalysisServiceImpl.java
View File

@@ -29,8 +29,8 @@ import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
@Service
@Slf4j
@@ -46,76 +46,103 @@ public class DataAnalysisServiceImpl implements IDataAnalysisService {
@Override
public SdmResponse<PageDataResp<List<SimulationTaskResultCurveResp>>> getSimulationTaskFile(GetSimulationTaskFileReq getSimulationTaskFileReq) {
// 1. 构造查询条件
QueryBigFileReq queryBigFileReq = new QueryBigFileReq();
BeanUtils.copyProperties(getSimulationTaskFileReq, queryBigFileReq);
queryBigFileReq.setFileBizType(List.of(getSimulationTaskFileReq.getFileBizType()));
if(ObjectUtils.isNotEmpty(getSimulationTaskFileReq.getUuid())){
// 获取特定 UUID 对应的目录 ID
if (ObjectUtils.isNotEmpty(getSimulationTaskFileReq.getUuid())) {
FileMetadataInfo fileMetadataInfo = fileMetadataInfoService.lambdaQuery()
.eq(FileMetadataInfo::getRelatedResourceUuid, getSimulationTaskFileReq.getUuid())
.eq(FileMetadataInfo::getTenantId, ThreadLocalContext.getTenantId())
.one();
if(ObjectUtils.isNotEmpty(fileMetadataInfo)) {
if (ObjectUtils.isNotEmpty(fileMetadataInfo)) {
queryBigFileReq.setDirId(fileMetadataInfo.getId());
}
}
// 2. 调用大文件查询服务
SdmResponse<PageDataResp<List<FileStorage>>> searchResult = dataStorageAnalysis.listBigFile(queryBigFileReq);
PageDataResp<List<FileStorage>> pageDataResp = searchResult.getData();
List<FileStorage> data = pageDataResp.getData();
if(CollectionUtils.isEmpty(data)) {
List<SimulationTaskResultCurveResp> emptyData = new ArrayList<>();
PageInfo<FileMetadataInfo> emptyPageInfo =new PageInfo<>();
emptyPageInfo.setTotal(0);
emptyPageInfo.setPageNum(0);
emptyPageInfo.setPageSize(0);
return PageUtils.getJsonObjectSdmResponse(emptyData,emptyPageInfo);
}
List<Long> fileIdList = data.stream().map(FileStorage::getFileId).toList();
List<FileStorage> fileStorages = pageDataResp.getData();
List<FileMetadataInfo> simulationTaskFileMetadataInfos = fileMetadataInfoService
.lambdaQuery()
// 如果数据为空，提前返回
if (CollectionUtils.isEmpty(fileStorages)) {
return PageUtils.getJsonObjectSdmResponse(new ArrayList<>(), new PageInfo<>());
}
// 3. 获取当前页文件的详细元数据
List<Long> fileIdList = fileStorages.stream().map(FileStorage::getFileId).toList();
List<FileMetadataInfo> currentFiles = fileMetadataInfoService.lambdaQuery()
.eq(FileMetadataInfo::getTenantId, ThreadLocalContext.getTenantId())
.in(FileMetadataInfo::getId, fileIdList)
.list();
PageInfo<FileMetadataInfo> pageInfo =new PageInfo<>();
// 批量分层获取所有相关的父目录
// key 是 ID，value 是对应的元数据实体。用于在内存中快速查找。
Map<Long, FileMetadataInfo> parentCacheMap = new HashMap<>();
// 当前需要去数据库查的父级 ID 集合
Set<Long> nextFetchIds = currentFiles.stream()
.map(FileMetadataInfo::getParentId)
.filter(pid -> pid != null && pid != 0)
.collect(Collectors.toSet());
int safetyDepth = 0; // 防死循环计数器
// 只要还有没查过的父 ID，且深度在合理范围内（10层），就继续批量查
while (CollectionUtils.isNotEmpty(nextFetchIds) && safetyDepth < 10) {
// 一次性查出当前这一层所有的父节点信息
List<FileMetadataInfo> parents = fileMetadataInfoService.listByIds(nextFetchIds);
if (CollectionUtils.isEmpty(parents)) break;
nextFetchIds = new HashSet<>(); // 重置，准备收集下一层 ID
for (FileMetadataInfo p : parents) {
parentCacheMap.put(p.getId(), p);
// 如果这个父节点还有上级，且我们之前没查过这个上级，就加进下一次查询列表
if (p.getParentId() != null && p.getParentId() != 0 && !parentCacheMap.containsKey(p.getParentId())) {
nextFetchIds.add(p.getParentId());
}
}
safetyDepth++;
}
// 内存组装数据：将 FileMetadata 转换为 Response，并回溯层级信息
List<SimulationTaskResultCurveResp> finalResultList = currentFiles.stream().map(file -> {
SimulationTaskResultCurveResp resp = new SimulationTaskResultCurveResp();
BeanUtils.copyProperties(file, resp);
resp.setFormatFileSize(FileSizeUtils.formatFileSize(BigDecimal.valueOf(file.getFileSize())));
// 从 parentCacheMap 中回溯，设置项目、阶段、专业信息
Long pid = file.getParentId();
int limit = 0;
// 这里的循环完全在内存中进行，速度极快且不产生日志
while (pid != null && parentCacheMap.containsKey(pid) && limit < 15) {
FileMetadataInfo folder = parentCacheMap.get(pid);
String ownType = folder.getRelatedResourceUuidOwnType();
if (NodeTypeEnum.PROJECT.getValue().equals(ownType)) {
resp.setProjectName(folder.getOriginalName());
resp.setProjectId(folder.getRelatedResourceUuid());
} else if (NodeTypeEnum.PHASE.getValue().equals(ownType)) {
resp.setPhaseName(folder.getOriginalName());
resp.setPhaseId(folder.getRelatedResourceUuid());
} else if (NodeTypeEnum.DISCIPLINE.getValue().equals(ownType)) {
resp.setDisciplineName(folder.getOriginalName());
resp.setDisciplineId(folder.getRelatedResourceUuid());
}
pid = folder.getParentId();
limit++;
}
return resp;
}).toList();
// 6. 构造分页信息并返回
PageInfo<FileMetadataInfo> pageInfo = new PageInfo<>();
pageInfo.setTotal(pageDataResp.getTotal());
pageInfo.setPageNum(pageDataResp.getCurrentPage());
pageInfo.setPageSize(pageDataResp.getPageSize());
List<SimulationTaskResultCurveResp> taskResultCurveResps = simulationTaskFileMetadataInfos.stream().map(simulationTaskFile -> {
SimulationTaskResultCurveResp taskResultCurveResp = new SimulationTaskResultCurveResp();
BeanUtils.copyProperties(simulationTaskFile, taskResultCurveResp);
taskResultCurveResp.setFormatFileSize(FileSizeUtils.formatFileSize(BigDecimal.valueOf(simulationTaskFile.getFileSize())));
Long parentId = simulationTaskFile.getParentId();
while (ObjectUtils.isNotEmpty(parentId)){
FileMetadataInfo parDir = fileMetadataInfoService.lambdaQuery()
.eq(FileMetadataInfo::getId, parentId).one();
if(ObjectUtils.isNotEmpty(parDir)){
if(NodeTypeEnum.PROJECT.getValue().equals(parDir.getRelatedResourceUuidOwnType())){
taskResultCurveResp.setProjectName(parDir.getOriginalName());
taskResultCurveResp.setProjectId(parDir.getRelatedResourceUuid());
}
if(NodeTypeEnum.PHASE.getValue().equals(parDir.getRelatedResourceUuidOwnType())){
taskResultCurveResp.setPhaseName(parDir.getOriginalName());
taskResultCurveResp.setPhaseId(parDir.getRelatedResourceUuid());
}
if(NodeTypeEnum.DISCIPLINE.getValue().equals(parDir.getRelatedResourceUuidOwnType())){
taskResultCurveResp.setDisciplineName(parDir.getOriginalName());
taskResultCurveResp.setDisciplineId(parDir.getRelatedResourceUuid());
}
parentId = parDir.getParentId();
}
}
return taskResultCurveResp;
}).toList();
return PageUtils.getJsonObjectSdmResponse(taskResultCurveResps,pageInfo);
return PageUtils.getJsonObjectSdmResponse(finalResultList, pageInfo);
}
@Override

70
data/src/main/java/com/sdm/data/service/impl/DataStorageAnalysisImpl.java
View File

@@ -6,12 +6,11 @@ import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.sdm.common.common.SdmResponse;
import com.sdm.common.common.ThreadLocalContext;
import com.sdm.common.entity.enums.NodeTypeEnum;
import com.sdm.common.entity.req.system.UserListReq;
import com.sdm.common.entity.req.system.UserQueryReq;
import com.sdm.common.entity.resp.AllNodeByProjectIdAndTypeResp;
import com.sdm.common.entity.resp.PageDataResp;
import com.sdm.common.entity.resp.system.CIDUserResp;
import com.sdm.common.feign.impl.project.SimulationNodeFeignClientImpl;
import com.sdm.common.feign.impl.system.SysUserFeignClientImpl;
import com.sdm.common.utils.FileSizeUtils;
import com.sdm.common.utils.PageUtils;
@@ -29,6 +28,7 @@ import com.sdm.data.model.resp.FileStorageQuotaResp;
import com.sdm.data.service.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@@ -52,9 +52,6 @@ public class DataStorageAnalysisImpl implements DataStorageAnalysis {
@Autowired
IFileStorageService fileStorageService;
@Autowired
SimulationNodeFeignClientImpl simuluationNodeFeignClient;
@Autowired
SysUserFeignClientImpl sysUserFeignClient;
@@ -66,26 +63,48 @@ public class DataStorageAnalysisImpl implements DataStorageAnalysis {
IFileStorageQuotaService fileStorageQuotaService;
public SdmResponse<List<JSONObject>> getNodeSizeByNodeType(String queryNodeType, String uuid, Integer intervalMonths, String targetYm) {
SdmResponse<List<AllNodeByProjectIdAndTypeResp>> response = simuluationNodeFeignClient.getAllNodeByProjectIdAndType(List.of(uuid), queryNodeType);
Long tenantId = ThreadLocalContext.getTenantId();
if (!response.isSuccess()) {
log.error("获取节点信息失败");
return SdmResponse.success();
List<FileMetadataInfo> nodeList = null;
String actualNodeName = uuid; // 存储实际的节点名称
if(NodeTypeEnum.PROJECT.getValue().equals(queryNodeType)){
// uuid是项目的uuid
nodeList = fileMetadataInfoService.lambdaQuery()
.eq(FileMetadataInfo::getRelatedResourceUuidOwnType, queryNodeType)
.eq(FileMetadataInfo::getRelatedResourceUuid, uuid)
.list();
actualNodeName = nodeList.get(0).getOriginalName(); // 对于项目，节点名称是项目的名字
}else if(NodeTypeEnum.DISCIPLINE.getValue().equals(queryNodeType)){
// uuid就是字典中配置的学科名：MHC，不再是uuid了
nodeList = fileMetadataInfoService.lambdaQuery()
.eq(FileMetadataInfo::getRelatedResourceUuidOwnType, queryNodeType)
.eq(FileMetadataInfo::getOriginalName, uuid)
.list();
actualNodeName = uuid; // 对于学科，节点名称就是传入的uuid值
}else {
log.error("不支持的节点类型: {}", queryNodeType);
return SdmResponse.success(new ArrayList<>());
}
List<AllNodeByProjectIdAndTypeResp> nodeLists = response.getData();
Long tenantId = ThreadLocalContext.getTenantId();
if (ObjectUtils.isEmpty(nodeList)) {
log.error("获取节点信息失败，节点类型: {}, 标识符: {}", queryNodeType, uuid);
return getEmptyNodeSize(actualNodeName);
}
// List<AllNodeByProjectIdAndTypeResp> nodeLists = response.getData();
// uuid1-学科1 uuid2-学科1 转换成 valueToKeysMap: 学科1->[uuid1,uuid2] 学科2->[uuid3,uuid4]
Map<String, List<String>> nodeNameToUuidListMap = nodeLists.stream()
.collect(Collectors.groupingBy(
AllNodeByProjectIdAndTypeResp::getNodeName,
Collectors.mapping(AllNodeByProjectIdAndTypeResp::getUuid, Collectors.toList())
//将 nodeList的 relatedResourceUuid1-学科1 relatedResourceUuid2-学科1 转换成 valueToKeysMap: 学科1->[uuid1,uuid2] 学科2->[uuid3,uuid4]
Map<String, List<String>> nodeNameToUuidListMap = nodeList.stream()
.collect(Collectors.groupingBy(FileMetadataInfo::getOriginalName,
Collectors.mapping(FileMetadataInfo::getRelatedResourceUuid, Collectors.toList())
));
List<JSONObject> result = new ArrayList<>();
//nodeIds获取所有节点id: [nodeid1,nodeid2,nodeid3,nodeid4]
List<String> uuids = nodeLists.stream().map(AllNodeByProjectIdAndTypeResp::getUuid).collect(Collectors.toList());
List<String> uuids = nodeList.stream().map(FileMetadataInfo::getRelatedResourceUuid).collect(Collectors.toList());
if (ObjectUtils.isNotEmpty(uuids)) {
// uuidToDirIdMap :获取uuid->fileID映射
Map<String, Long> uuidToDirIdMap = fileMetadataInfoService.lambdaQuery().select(FileMetadataInfo::getId, FileMetadataInfo::getRelatedResourceUuid)
@@ -93,8 +112,8 @@ public class DataStorageAnalysisImpl implements DataStorageAnalysis {
.eq(FileMetadataInfo::getTenantId, tenantId)
.list().stream().collect(Collectors.toMap(FileMetadataInfo::getRelatedResourceUuid, FileMetadataInfo::getId));
if (CollectionUtils.isEmpty(uuidToDirIdMap)) {
log.error("获取节点信息失败");
return SdmResponse.success();
log.error("获取节点ID映射失败，节点类型: {}, 标识符: {}", queryNodeType, uuid);
return getEmptyNodeSize(actualNodeName);
}
// fileMetadIds: uuid对应的fileid结合
@@ -113,10 +132,7 @@ public class DataStorageAnalysisImpl implements DataStorageAnalysis {
if (ObjectUtils.isEmpty(nodeSizeDTOS)) {
// 空间为空 也要返回nodeName
JSONObject jsonObject = new JSONObject();
jsonObject.put("nodeName", nodeLists.get(0).getNodeName());
jsonObject.put("totalSize", 0);
return SdmResponse.success(Arrays.asList(jsonObject));
return getEmptyNodeSize(actualNodeName);
}
@@ -141,6 +157,14 @@ public class DataStorageAnalysisImpl implements DataStorageAnalysis {
return SdmResponse.success(result);
}
@NotNull
private static SdmResponse<List<JSONObject>> getEmptyNodeSize(String actualNodeName) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("nodeName", actualNodeName);
jsonObject.put("totalSize", 0);
return SdmResponse.success(Arrays.asList(jsonObject));
}
@Override
public SdmResponse getDirectorySizeByUserId(List<Long> userIds, Integer intervalMonths, String targetYm) {
Long tenantId = ThreadLocalContext.getTenantId();

16
data/src/main/java/com/sdm/data/service/impl/DimensionTemplateServiceImpl.java
View File

@@ -257,15 +257,17 @@ public class DimensionTemplateServiceImpl extends ServiceImpl<DimensionTemplateM
}
}
if (CollectionUtils.isEmpty(uuids)) {
return SdmResponse.success(Collections.emptyList());
if (CollectionUtils.isNotEmpty(uuids)) {
List<FileMetadataInfo> nodeDirInfos = fileMetadataInfoService.lambdaQuery()
.in(FileMetadataInfo::getRelatedResourceUuid, uuids)
.eq(FileMetadataInfo::getTenantId, tenantId)
.orderByDesc(FileMetadataInfo::getCreateTime).list();
resultDir.addAll(nodeDirInfos);
}
List<FileMetadataInfo> nodeDirInfos = fileMetadataInfoService.lambdaQuery()
.in(FileMetadataInfo::getRelatedResourceUuid, uuids)
.eq(FileMetadataInfo::getTenantId, tenantId)
.orderByDesc(FileMetadataInfo::getCreateTime).list();
resultDir.addAll(nodeDirInfos);
if(CollectionUtils.isEmpty(resultDir)){
return SdmResponse.success(Collections.emptyList());
}
// 对nodeDirInfos按照originalName进行合并，并处理bucketName生成totalName
List<FileMetadataChildrenDTO> mergedResult = mergeNodeDirInfos(resultDir);