toolchaintechnologycenter/ModelTrainingPython
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

修复预测脚本和训练脚本的执行bug

Browse Source
This commit is contained in:
liyong
2025-10-21 19:49:21 +08:00
parent b9cce1d733
commit 4fb2da1366
10 changed files with 87 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/misc.xml generated
View File

@@ -3,4 +3,5 @@
<component name="Black"> <component name="Black">
<option name="sdkName" value="tvs_dl" /> <option name="sdkName" value="tvs_dl" />
</component> </component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
</project> </project>

2
FC_ML_Baseline/FC_ML_Baseline_Data_Handler/Data_Load.py
View File

@@ -55,7 +55,7 @@ from FC_ML_Tool.Serialization import parse_json_file
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='导入数据文件参数') parser = argparse.ArgumentParser(description='导入数据文件参数')
parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\param.json', parser.add_argument('--param', default='D:\liyong\project\ModelTrainingPython\FC_ML_Baseline\FC_ML_Baseline_Test\param.json',
help='配置参数文件绝对路径') help='配置参数文件绝对路径')
parser.add_argument('--export', default='source.json', parser.add_argument('--export', default='source.json',
help='导出JSON文件名') help='导出JSON文件名')

18
FC_ML_Baseline/FC_ML_Baseline_Predict/Model_Pred.py
View File

@@ -2,7 +2,6 @@ import argparse
import json import json
import torch import torch
from openpyxl.styles.builtins import output
from FC_ML_Data.FC_ML_Data_Process.Data_Process_Normalization import Normalizer from FC_ML_Data.FC_ML_Data_Process.Data_Process_Normalization import Normalizer
from FC_ML_NN_Model.Poly_Model import PolyModel from FC_ML_NN_Model.Poly_Model import PolyModel
@@ -10,18 +9,19 @@ from FC_ML_Tool.Serialization import parse_json_file
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description='代理模型训练参数输入') parser = argparse.ArgumentParser(description='代理模型训练参数输入')
parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json', parser.add_argument('--param', default='D:\liyong\project\ModelTrainingPython\FC_ML_Baseline\FC_ML_Baseline_Test\pred\param.json',
help='配置参数文件绝对路径') help='配置参数文件绝对路径')
args = parser.parse_args() args = parser.parse_args()
params = parse_json_file(args.param) params = parse_json_file(args.param)
print(params)
source_dir = params["path"] + "/" source_dir = params["path"] + "/"
model_file = source_dir + params["modelFile"] model_file = source_dir + params["modelFile"]
inputs = [] inputs = []
names = [] names = params["output"]["names"]
#获取输入特征 #获取输入特征
for input_value in params["input"]: for input_value in params["input"]:
inputs.append(input_value["value"]) inputs.append(input_value["value"])
names.append(input_value["name"]) # names.append(input_value["name"])
#记载模型进行预测 #记载模型进行预测
input_size = params["modelParams"]["inputSize"] input_size = params["modelParams"]["inputSize"]
output_size = params["modelParams"]["outputSize"] output_size = params["modelParams"]["outputSize"]
@@ -36,21 +36,23 @@ if __name__ == "__main__":
normalization_max = params["modelParams"]["normalizerMax"] normalization_max = params["modelParams"]["normalizerMax"]
normalization_min = params["modelParams"]["normalizerMin"] normalization_min = params["modelParams"]["normalizerMin"]
normalizer = Normalizer(method=normalization_type) normalizer = Normalizer(method=normalization_type)
normalizer.load_params(normalization_type,normalization_min,normalization_max) normalizer.load_params(normalization_type,normalization_min[0:input_size],normalization_max[0:input_size])
input_data = normalizer.transform(torch.tensor(inputs)) input_data = normalizer.transform(torch.tensor(inputs))
#执行模型预测 #执行模型预测
with torch.no_grad(): with torch.no_grad():
output_data = model(input_data) output_data = model(input_data)
print(f"Prediction result: {output_data.item():.4f}") # print(f"Prediction result: {output_data.item().tolist():.4f}")
normalizer.load_params(normalization_type, normalization_min[-output_size:], normalization_max[-output_size:])
output_data_ori = normalizer.inverse_transform(output_data) output_data_ori = normalizer.inverse_transform(output_data)
print(f"Prediction real result: {output_data_ori.item():.4f}") # print(f"Prediction real result: {output_data_ori.item().tolist():.4f}")
#输出预测结果到文件中 #输出预测结果到文件中
output_datas = output_data_ori.tolist() output_datas = output_data_ori.tolist()
json_str = {} json_str = {}
if len(output_datas) == len(names): if len(output_datas) == len(names):
for i in range(len(names)): for i in range(len(names)):
json_str[names[i]] = output_datas[i] json_str[names[i]] = output_datas[i]
with open(source_dir + "forecast.json", ) as f: with open(source_dir + "forecast.json","w") as f:
f.write(json.dumps(json_str, indent=None, ensure_ascii=False)) f.write(json.dumps(json_str, indent=None, ensure_ascii=False))

2
FC_ML_Baseline/FC_ML_Baseline_Test/Data_Handle/param.json
View File

@@ -1,4 +1,4 @@
{ {
"files": ["sample1.CSV"], "files": ["sample1.CSV"],
"path": "D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle" "path": "D:\\liyong\\project\\ModelTrainingPython\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle"
} }

15
FC_ML_Baseline/FC_ML_Baseline_Test/Train/param.json
View File

@@ -1,20 +1,23 @@
{ {
"path": ["D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Train"], "files": ["sample1.CSV"],
"path": "D:\\liyong\\project\\ModelTrainingPython\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle",
"algorithmParam": { "algorithmParam": {
"inputSize": 9,
"outputSize": 8,
"algorithm": "多项式拟合", "algorithm": "多项式拟合",
"activateFun": "sigmod", "activateFun": "sigmod",
"lossFun": "l1", "lossFun": "l1",
"optimizeFun": "sgd", "optimizeFun": "sgd",
"exportFormat": ".onnx", "exportFormat": "bin",
"trainingRatio": 80, "trainingRatio": 80,
"loadSize": 32, "loadSize": 32,
"studyPercent": 0.001, "studyPercent": 0.001,
"stepCounts": 3, "stepCounts": 3,
"roundPrint": 11, "roundPrint": 10,
"round": 1001, "round": 300,
"preDisposeData": false, "preDisposeData": true,
"disposeMethod": "minmax", "disposeMethod": "minmax",
"dataNoOrder": false "dataNoOrder": true
}, },
"algorithm": "基础神经网络NN" "algorithm": "基础神经网络NN"
} }

53
FC_ML_Baseline/FC_ML_Baseline_Test/pred/param.json
View File

@@ -1,20 +1,51 @@
{ {
"modelFile": "model.onnx", "modelFile": "model.bin",
"path": "D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\pred", "path": "D:\\liyong\\project\\ModelTrainingPython\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle",
"modelParams": { "modelParams": {
"inputSize": 3, "inputSize": 9,
"outputSize": 3, "outputSize": 8,
"normalizerType": "minmax", "normalizerType": "minmax",
"normalizerMax": 100, "normalizerMax": [2000.0,575.9771118164062,5999.64208984375,5806.2333984375,6711.77880859375,99.99962615966797,99.99884796142578,-29.81661605834961,59.998504638671875,27.299999237060547,5.230000019073486,131.0,8.170000076293945,11.899999618530273,-0.8949999809265137,27.100000381469727,17.899999618530273],
"normalizerMin": 10 "normalizerMin": [0.10000000149011612,5.022274971008301,2.0320935249328613,0.21287846565246582,3853.6533203125,0.019815441220998764,0.0033870770130306482,-29.81661605834961,0.0007396229775622487,-37.900001525878906,0.06520000100135803,-9.699999809265137,2.0299999713897705,-32.900001525878906,-32.70000076293945,-29.0,-29.600000381469727]
}, },
"input": [ "input": [
{ {
"name": "质量", "name": "param1",
"value": 1 "value": 0.1
}, { }, {
"name": "系数", "name": "param1",
"value": 2 "value": 371.6669936
},
{
"name": "param1",
"value": 3483.012088
},
{
"name": "param1",
"value": 4333.292092
},
{
"name": "param1",
"value": 5582.788747
},
{
"name": "param1",
"value": 22.33362393
},
{
"name": "param1",
"value": 74.76711286
},
{
"name": "param1",
"value": -29.816617
},
{
"name": "param1",
"value": 17.14707502
} }
] ],
"output": {
"names": ["label1","label2","label3","label4","label5","label6","label7","label8"]
}
} }

21
FC_ML_Baseline/FC_ML_Baseline_Train/Train_Proxy_Model.py
View File

@@ -35,26 +35,25 @@
*/ */
''' '''
import argparse import argparse
from locale import normalize from copy import deepcopy
import torch import torch
from torch.utils.data import TensorDataset, DataLoader from torch.utils.data import TensorDataset, DataLoader
from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_data_from_csv_feature, get_train_data_from_csv from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_train_data_from_csv
from FC_ML_Data.FC_ML_Data_Output.Data_Output_Pytorch import export_model from FC_ML_Data.FC_ML_Data_Output.Data_Output_Pytorch import export_model
from FC_ML_Loss_Function.Loss_Function_Selector import LossFunctionSelector from FC_ML_Loss_Function.Loss_Function_Selector import LossFunctionSelector
from FC_ML_Model.Model_Train_Data import TrainData
from FC_ML_NN_Model.Poly_Model import PolyModel from FC_ML_NN_Model.Poly_Model import PolyModel
from FC_ML_Optim_Function.Optimizer_Selector import OptimizerSelector from FC_ML_Optim_Function.Optimizer_Selector import OptimizerSelector
from FC_ML_Tool.Serialization import parse_json_file from FC_ML_Tool.Serialization import parse_json_file
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='代理模型训练参数输入') parser = argparse.ArgumentParser(description='代理模型训练参数输入')
parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json', parser.add_argument('--param', default='D:\liyong\project\ModelTrainingPython\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
help='配置参数文件绝对路径') help='配置参数文件绝对路径')
args = parser.parse_args() args = parser.parse_args()
params = parse_json_file(args.param) params = parse_json_file(args.param)
train_data = torch.tensor() print(params)
# print(params) # print(params)
#获取训练参数 #获取训练参数
input_Size = params["algorithmParam"]["inputSize"]#输入特征维度 input_Size = params["algorithmParam"]["inputSize"]#输入特征维度
@@ -73,11 +72,15 @@ if __name__ == '__main__':
dispose_method = params["algorithmParam"]["disposeMethod"] # 数据预处理方法 dispose_method = params["algorithmParam"]["disposeMethod"] # 数据预处理方法
data_no_order = params["algorithmParam"]["dataNoOrder"] # 训练数据是否乱序处理 data_no_order = params["algorithmParam"]["dataNoOrder"] # 训练数据是否乱序处理
#加载所有训练数据 #加载所有训练数据
train_data = []
source_dir = params["path"] + "/" source_dir = params["path"] + "/"
for data_file in params["files"]: for data_file in params["files"]:
data_file_path = source_dir + data_file data_file_path = source_dir + data_file
ori_data,normalize = get_train_data_from_csv(data_file_path,pre_dispose_data,dispose_method) ori_data,normalize = get_train_data_from_csv(data_file_path,pre_dispose_data,dispose_method)
torch.cat((train_data,ori_data),dim=0)#按行拼接 if len(train_data) == 0:
train_data = deepcopy(ori_data)
else:
train_data = torch.cat((train_data,ori_data),dim=0)#按行拼接
#拆分测试集和训练集 #拆分测试集和训练集
split = int(training_ratio / 100 * len(train_data)) split = int(training_ratio / 100 * len(train_data))
train_dataset = TensorDataset(train_data[:split,0:input_Size], train_data[:split,input_Size:]) train_dataset = TensorDataset(train_data[:split,0:input_Size], train_data[:split,input_Size:])
@@ -140,11 +143,11 @@ if __name__ == '__main__':
#每100次迭代输出一次损失数值 #每100次迭代输出一次损失数值
if epoch % round_print == 0: if epoch % round_print == 0:
print( print(
f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1") f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | Loss Factor: {avg_train_loss / avg_test_loss:.2f}:1")
with open(source_dir + "training.log", "a") as f: with open(source_dir + "training.log", "a") as f:
f.write(f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1\n") # 自动换行追加 f.write(f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | Loss Factor: {avg_train_loss / avg_test_loss:.2f}:1\n") # 自动换行追加
#导出训练后的模型 #导出训练后的模型
export_model(model,source_dir,"model",export_format) export_model(model,source_dir,"model",export_format,torch.randn(1, input_Size))

4
FC_ML_Data/FC_ML_Data_Load/Data_Load_Excel.py
View File

@@ -182,7 +182,7 @@ def get_data_from_csv_feature(data_path,skip_rows = 100,sample_rows = 100,normal
sampled_indices = torch.arange(0, len(data_ori), skip_rows) # 记录行号 sampled_indices = torch.arange(0, len(data_ori), skip_rows) # 记录行号
return label_name,source_data,normalizer.params["min"],normalizer.params["max"],normalizer.params["mean"],sampled_indices,data_sample return label_name,source_data,normalizer.params["min"],normalizer.params["max"],normalizer.params["mean"],sampled_indices,data_sample
def get_train_data_from_csv(data_path,normalization = false,normalization_type = 'minmax'): def get_train_data_from_csv(data_path,normalization = True,normalization_type = 'minmax'):
"""读取csv数据文件并生成标准化训练数据 """读取csv数据文件并生成标准化训练数据
Args: Args:
data_path (str): 文件绝对路径 data_path (str): 文件绝对路径
@@ -196,6 +196,8 @@ def get_train_data_from_csv(data_path,normalization = false,normalization_type =
Examples: Examples:
get_data_from_csv_feature("D://test.excel") get_data_from_csv_feature("D://test.excel")
:param normalization_type:
:param normalization:
""" """
# 读取前xx行数据 # 读取前xx行数据
df = pd.read_csv(data_path,encoding='gbk') df = pd.read_csv(data_path,encoding='gbk')

10
FC_ML_Data/FC_ML_Data_Output/Data_Output_Pytorch.py
View File

@@ -3,7 +3,7 @@ import torch
def export_model_pt(model,target,name = "model"): def export_model_pt(model,target,name = "model"):
script_model = torch.jit.script(model) # 或 torch.jit.trace(model, input) script_model = torch.jit.script(model) # 或 torch.jit.trace(model, input)
script_model.save(target + name + ".pt") script_model.save(target + name + ".pth")
#2 通用格式导出 #2 通用格式导出
def export_model_onnx(model,input_tensor,target,name="model"): def export_model_onnx(model,input_tensor,target,name="model"):
torch.onnx.export(model, input_tensor, target+ name + ".onnx") torch.onnx.export(model, input_tensor, target+ name + ".onnx")
@@ -11,12 +11,12 @@ def export_model_onnx(model,input_tensor,target,name="model"):
def export_model_bin(model,target,name = "weights"): def export_model_bin(model,target,name = "weights"):
torch.save(model.state_dict(), target + name + ".bin") torch.save(model.state_dict(), target + name + ".bin")
def export_model(model,target,file_name,name): def export_model(model,target,file_name,name,input_tensor):
if name == 'bin': if name == 'bin':
return export_model_bin(model,target,file_name) return export_model_bin(model,target,file_name)
if name == 'onnx': if name == 'onnx':
return export_model_onnx(model,target,file_name) return export_model_onnx(model,input_tensor,target,file_name)
if name == 'pt': if name == 'pth':
return export_model_bin(model,target,file_name) return export_model_pt(model,target,file_name)
else: else:
raise ValueError(f"不支持的导出类型") raise ValueError(f"不支持的导出类型")

6
FC_ML_Data/FC_ML_Data_Process/Data_Process_Normalization.py
View File

@@ -20,10 +20,10 @@ class Normalizer:
self.params['max_abs'] = data.abs().max(dim=0)[0] self.params['max_abs'] = data.abs().max(dim=0)[0]
return self return self
def load_params(self,method = "minmax",min_in = 0,max_in = 0,mean_in =0,std=0,max_abs=0): def load_params(self,method = "minmax",min_in = [],max_in = [],mean_in =[],std=[],max_abs=[]):
self.method = method self.method = method
self.params['min'] = min_in self.params['min'] = torch.tensor(min_in)
self.params['max'] = max_in self.params['max'] = torch.tensor(max_in)
self.params['mean'] = mean_in self.params['mean'] = mean_in
self.params['std'] = std self.params['std'] = std
self.params['max_abs'] = max_abs self.params['max_abs'] = max_abs