toolchaintechnologycenter/spdm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据统计

Browse Source
This commit is contained in:
gulongcheng
2025-10-17 16:44:46 +08:00
parent 5bb77c2a51
commit 7f84de7d2b
9 changed files with 191 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
data/src/main/java/com/sdm/data/controller/ModelTraningController.java Normal file
View File

@@ -0,0 +1,27 @@
package com.sdm.data.controller;
import com.sdm.data.service.IModelService;
import io.swagger.v3.oas.annotations.Operation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* 模型训练控制器
*/
@RestController
@RequestMapping("/modelTraning")
public class ModelTraningController {
@Autowired
private IModelService modelService;
/**
* 调用python脚本处理导入数据
*/
@GetMapping("/handleLoadData")
@Operation(summary = "调用python脚本处理导入数据", description = "调用python脚本处理导入数据")
public void handleLoadData(Integer fileId, String localFilePath, String pythonScriptPath, String paramJsonPath) {
modelService.handleLoadData(fileId, localFilePath, pythonScriptPath,paramJsonPath );
}
}

11
data/src/main/java/com/sdm/data/service/IModelService.java Normal file
View File

@@ -0,0 +1,11 @@
package com.sdm.data.service;
/**
* 模型训练服务接口
*/
public interface IModelService {
/**
* 调用python脚本处理导入数据
*/
void handleLoadData(Integer fileId, String localFilePath, String pythonScriptPath, String paramJsonPath);
}

81
data/src/main/java/com/sdm/data/service/impl/ModelServiceImpl.java Normal file
View File

@@ -0,0 +1,81 @@
package com.sdm.data.service.impl;
import com.sdm.data.service.IDataFileService;
import com.sdm.data.service.IModelService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.*;
import java.io.FileOutputStream;
import java.io.OutputStream;
@Service
@Slf4j
public class ModelServiceImpl implements IModelService {
@Autowired
IDataFileService dataFileService;
@Override
public void handleLoadData(Integer fileId, String localFilePath, String pythonScriptPath,String paramJsonPath) {
try {
// 从MinIO下载文件到本地路径
downloadFromMinIO(fileId,localFilePath);
callPythonScript(pythonScriptPath, paramJsonPath);
} catch (Exception e) {
log.error("调用python脚本处理导入数据", e);
throw new RuntimeException(e);
}
}
private void downloadFromMinIO(Integer fileId, String localFilePath) {
try {
InputStream in = dataFileService.getMinioInputStream(fileId);
OutputStream out = new FileOutputStream(localFilePath);
// 将输入流写入本地文件
byte[] buffer = new byte[1024];
int len;
while ((len = in.read(buffer)) != -1) {
out.write(buffer, 0, len);
}
} catch (IOException e) {
log.error("下载文件到本地失败", e);
throw new RuntimeException(e);
}
}
public void callPythonScript(String pythonScriptPath, String paramJsonPath) throws IOException, InterruptedException {
try {
// 构建执行 Python 脚本的命令
ProcessBuilder processBuilder = new ProcessBuilder(
"python3",
pythonScriptPath,
paramJsonPath
);
// 重定向进程的输入、输出流，便于查看执行日志
processBuilder.redirectErrorStream(true);
Process process = processBuilder.start();
// 读取 Python 脚本的输出
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(process.getInputStream()))) {
String line;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
}
// 等待 Python 脚本执行完成，并获取退出码
int exitCode = process.waitFor();
if (exitCode == 0) {
System.out.println("Python 脚本执行成功");
} else {
System.out.println("Python 脚本执行失败，退出码: " + exitCode);
}
} catch (Exception e) {
log.error("调用Python脚本失败", e);
throw new RuntimeException(e);
}
}
}

4
gateway2/src/main/resources/application-dev.yml
View File

@@ -16,8 +16,8 @@ spring:
enabled: true
gateway:
httpclient:
connect-timeout: 5000
response-timeout: 5000
connect-timeout: 10000
response-timeout: 10000
routes:
- id: approve-service
uri: lb://approve

4
gateway2/src/main/resources/application-local.yml
View File

@@ -16,8 +16,8 @@ spring:
enabled: true
gateway:
httpclient:
connect-timeout: 5000
response-timeout: 5000
connect-timeout: 10000
response-timeout: 10000
routes:
- id: approve-service
uri: lb://approve

4
gateway2/src/main/resources/application-prod.yml
View File

@@ -16,8 +16,8 @@ spring:
enabled: true
gateway:
httpclient:
connect-timeout: 5000
response-timeout: 5000
connect-timeout: 10000
response-timeout: 10000
routes:
- id: approve-service
uri: lb://approve

2
gateway2/src/main/resources/application.yml
View File

@@ -1,6 +1,6 @@
spring:
profiles:
active: local
active: dev
main:
web-application-type: reactive
cloud:

35
project/src/main/java/com/sdm/project/service/impl/NodeServiceImpl.java
View File

@@ -727,11 +727,15 @@ public class NodeServiceImpl extends ServiceImpl<SimulationNodeMapper, Simulatio
// 按用户分组统计任务状态
Map<Integer, UserGroupTaskCompleteStatisticsVo> userStatisticsMap = new HashMap<>();
// 所有任务执行状态
Set<String> allExeStatus = new HashSet<>();
// 统计每个用户的各种状态任务数量
for (UserGroupTaskCompleteVo item : userGroupTaskCompleteStatistics) {
Integer userId = item.getUserId();
String exeStatus = item.getExeStatus();
allExeStatus.add(exeStatus);
UserGroupTaskCompleteStatisticsVo userStat = userStatisticsMap.getOrDefault(userId, new UserGroupTaskCompleteStatisticsVo());
userStat.setUserId(userId);
userStat.setUserName(item.getNickname());
@@ -750,8 +754,10 @@ public class NodeServiceImpl extends ServiceImpl<SimulationNodeMapper, Simulatio
// 转换为列表返回
List<UserGroupTaskCompleteStatisticsVo> result = new ArrayList<>(userStatisticsMap.values());
return SdmResponse.success(result);
JSONObject resultResponse = new JSONObject();
resultResponse.put("allExeStatus", allExeStatus);
resultResponse.put("result", result);
return SdmResponse.success(resultResponse);
}
@Override
@@ -761,10 +767,14 @@ public class NodeServiceImpl extends ServiceImpl<SimulationNodeMapper, Simulatio
// 按用户分组统计任务状态
Map<Integer, UserGroupDifficultyStatisticsVo> userStatisticsMap = new HashMap<>();
// 所有难度值
Set<Float> alldifficultyValue = new HashSet<>();
// 统计每个用户的各种状态任务数量
for (UserGroupDifficultyVo item : userGroupDifficultyStatistics) {
Integer userId = item.getUserId();
Float difficulty = item.getDifficulty();
alldifficultyValue.add(difficulty);
UserGroupDifficultyStatisticsVo userStat = userStatisticsMap.getOrDefault(userId, new UserGroupDifficultyStatisticsVo());
userStat.setUserId(userId);
@@ -784,12 +794,17 @@ public class NodeServiceImpl extends ServiceImpl<SimulationNodeMapper, Simulatio
// 转换为列表返回
List<UserGroupDifficultyStatisticsVo> result = new ArrayList<>(userStatisticsMap.values());
return SdmResponse.success(result);
JSONObject resultResponse = new JSONObject();
resultResponse.put("alldifficultyValue", alldifficultyValue);
resultResponse.put("result", result);
return SdmResponse.success(resultResponse);
}
@Override
public SdmResponse getCommonCompleteStatistics(CommonGetCompleteStatisticsReq req) {
// 所有任务执行状态
Set<String> allExeStatus = new HashSet<>();
if ("task".equals(req.getQueryType())) {
// 处理任务完成情况统计
List<CommonGetCompleteFromTaskVo> commonCompleteStatisticsFromTask = this.baseMapper.getCommonCompleteStatisticsFromTask(req);
@@ -811,13 +826,17 @@ public class NodeServiceImpl extends ServiceImpl<SimulationNodeMapper, Simulatio
}
String exeStatus = item.getExeStatus();
allExeStatus.add(exeStatus);
statusCount.put(exeStatus, statusCount.getOrDefault(exeStatus, 0) + 1);
taskStatisticsMap.put(name, stat);
}
// 转换为列表返回
List<CommonCompleteStatisticsVo> taskResult = new ArrayList<>(taskStatisticsMap.values());
return SdmResponse.success(taskResult);
JSONObject resultResponse = new JSONObject();
resultResponse.put("allExeStatus", allExeStatus);
resultResponse.put("result", taskResult);
return SdmResponse.success(resultResponse);
} else if ("performance".equals(req.getQueryType())) {
// 处理指标完成情况统计
List<CommonGetCompleteFromPerformanceVo> commonCompleteStatisticsFromPerformance = this.baseMapper.getCommonCompleteStatisticsFromPerformance(req);
@@ -839,6 +858,7 @@ public class NodeServiceImpl extends ServiceImpl<SimulationNodeMapper, Simulatio
}
String completeStatus = item.getCompleteStatus();
allExeStatus.add(completeStatus);
statusCount.put(completeStatus, statusCount.getOrDefault(completeStatus, 0) + 1);
performanceStatisticsMap.put(nodeName, stat);
@@ -846,7 +866,10 @@ public class NodeServiceImpl extends ServiceImpl<SimulationNodeMapper, Simulatio
// 转换为列表返回
List<CommonCompleteStatisticsVo> performanceResult = new ArrayList<>(performanceStatisticsMap.values());
return SdmResponse.success(performanceResult);
JSONObject resultResponse = new JSONObject();
resultResponse.put("allExeStatus", allExeStatus);
resultResponse.put("result", performanceResult);
return SdmResponse.success(resultResponse);
}
return SdmResponse.success(new ArrayList<>());
}

72
project/src/main/resources/mapper/SimulationNodeMapper.xml
View File

@@ -366,31 +366,31 @@
<if test="req.tag1 != null and req.tag1 !='' ">
and task.tag1 = #{req.tag1}
</if>
<if test="req.tag2 != null">
<if test="req.tag2 != null and req.tag2 !='' ">
and task.tag2 = #{req.tag2}
</if>
<if test="req.tag3 != null">
<if test="req.tag3 != null and req.tag3 !='' ">
and task.tag3 = #{req.tag3}
</if>
<if test="req.tag4 != null">
<if test="req.tag4 != null and req.tag4 !='' ">
and task.tag4 = #{req.tag4}
</if>
<if test="req.tag5 != null">
<if test="req.tag5 != null and req.tag5 !='' ">
and task.tag5 = #{req.tag5}
</if>
<if test="req.tag6 != null">
<if test="req.tag6 != null and req.tag6 !='' ">
and task.tag6 = #{req.tag6}
</if>
<if test="req.tag7 != null">
<if test="req.tag7 != null and req.tag7 !='' ">
and task.tag7 = #{req.tag7}
</if>
<if test="req.tag8 != null">
<if test="req.tag8 != null and req.tag8 !='' ">
and task.tag8 = #{req.tag8}
</if>
<if test="req.tag9 != null">
<if test="req.tag9 != null and req.tag9 !='' ">
and task.tag9 = #{req.tag9}
</if>
<if test="req.tag10 != null">
<if test="req.tag10 != null and req.tag10 !='' ">
and task.tag10 = #{req.tag10}
</if>
</where>
@@ -419,31 +419,31 @@
<if test="req.tag1 != null and req.tag1 !='' ">
and task.tag1 = #{req.tag1}
</if>
<if test="req.tag2 != null">
<if test="req.tag2 != null and req.tag2 !='' ">
and task.tag2 = #{req.tag2}
</if>
<if test="req.tag3 != null">
<if test="req.tag3 != null and req.tag3 !='' ">
and task.tag3 = #{req.tag3}
</if>
<if test="req.tag4 != null">
<if test="req.tag4 != null and req.tag4 !='' ">
and task.tag4 = #{req.tag4}
</if>
<if test="req.tag5 != null">
<if test="req.tag5 != null and req.tag5 !='' ">
and task.tag5 = #{req.tag5}
</if>
<if test="req.tag6 != null">
<if test="req.tag6 != null and req.tag6 !='' ">
and task.tag6 = #{req.tag6}
</if>
<if test="req.tag7 != null">
<if test="req.tag7 != null and req.tag7 !='' ">
and task.tag7 = #{req.tag7}
</if>
<if test="req.tag8 != null">
<if test="req.tag8 != null and req.tag8 !='' ">
and task.tag8 = #{req.tag8}
</if>
<if test="req.tag9 != null">
<if test="req.tag9 != null and req.tag9 !='' ">
and task.tag9 = #{req.tag9}
</if>
<if test="req.tag10 != null">
<if test="req.tag10 != null and req.tag10 !='' ">
and task.tag10 = #{req.tag10}
</if>
</where>
@@ -463,31 +463,31 @@
<if test="req.tag1 != null and req.tag1 !='' ">
and task.tag1 = #{req.tag1}
</if>
<if test="req.tag2 != null">
<if test="req.tag2 != null and req.tag2 !='' ">
and task.tag2 = #{req.tag2}
</if>
<if test="req.tag3 != null">
<if test="req.tag3 != null and req.tag3 !='' ">
and task.tag3 = #{req.tag3}
</if>
<if test="req.tag4 != null">
<if test="req.tag4 != null and req.tag4 !='' ">
and task.tag4 = #{req.tag4}
</if>
<if test="req.tag5 != null">
<if test="req.tag5 != null and req.tag5 !='' ">
and task.tag5 = #{req.tag5}
</if>
<if test="req.tag6 != null">
<if test="req.tag6 != null and req.tag6 !='' ">
and task.tag6 = #{req.tag6}
</if>
<if test="req.tag7 != null">
<if test="req.tag7 != null and req.tag7 !='' ">
and task.tag7 = #{req.tag7}
</if>
<if test="req.tag8 != null">
<if test="req.tag8 != null and req.tag8 !='' ">
and task.tag8 = #{req.tag8}
</if>
<if test="req.tag9 != null">
<if test="req.tag9 != null and req.tag9 !='' ">
and task.tag9 = #{req.tag9}
</if>
<if test="req.tag10 != null">
<if test="req.tag10 != null and req.tag10 !='' ">
and task.tag10 = #{req.tag10}
</if>
</where>
@@ -509,31 +509,31 @@
<if test="req.tag1 != null and req.tag1 !='' ">
and task.tag1 = #{req.tag1}
</if>
<if test="req.tag2 != null">
<if test="req.tag2 != null and req.tag2 !='' ">
and task.tag2 = #{req.tag2}
</if>
<if test="req.tag3 != null">
<if test="req.tag3 != null and req.tag3 !='' ">
and task.tag3 = #{req.tag3}
</if>
<if test="req.tag4 != null">
<if test="req.tag4 != null and req.tag4 !='' ">
and task.tag4 = #{req.tag4}
</if>
<if test="req.tag5 != null">
<if test="req.tag5 != null and req.tag5 !='' ">
and task.tag5 = #{req.tag5}
</if>
<if test="req.tag6 != null">
<if test="req.tag6 != null and req.tag6 !='' ">
and task.tag6 = #{req.tag6}
</if>
<if test="req.tag7 != null">
<if test="req.tag7 != null and req.tag7 !='' ">
and task.tag7 = #{req.tag7}
</if>
<if test="req.tag8 != null">
<if test="req.tag8 != null and req.tag8 !='' ">
and task.tag8 = #{req.tag8}
</if>
<if test="req.tag9 != null">
<if test="req.tag9 != null and req.tag9 !='' ">
and task.tag9 = #{req.tag9}
</if>
<if test="req.tag10 != null">
<if test="req.tag10 != null and req.tag10 !='' ">
and task.tag10 = #{req.tag10}
</if>
</where>