@@ -1,148 +0,0 @@
|
||||
'''
|
||||
/**
|
||||
* 导入数据文件脚本
|
||||
* 传参：python xxx.py /xx/xx/param.json
|
||||
* param.json 包含：
|
||||
* {
|
||||
* files:[xxx.excel,xxx.excel,xxx.excel],
|
||||
* path: '/xx/xx/xx',//源文件路径
|
||||
* }
|
||||
* 返回：source.json source.json文件放到 源文件路径下
|
||||
* source.json文件包含：
|
||||
* 表格标题：sourceTitle
|
||||
* 表格数据：sourceData
|
||||
* 稳态必传：averageData
|
||||
* 动态必传：sourceCurveData
|
||||
*/
|
||||
|
||||
//源数据表头
|
||||
export const sourceTitle = [
|
||||
{ value: 'param1', label: '整备质量' },
|
||||
{ value: 'param2', label: '迎风面积' },
|
||||
{ value: 'param3', label: '风阻系数' },
|
||||
{ value: 'param4', label: '转动惯量' },
|
||||
{ value: 'param5', label: '最大转速时拖拽扭矩' },
|
||||
{ value: 'param6', label: '速比' },
|
||||
];
|
||||
//源数据
|
||||
export const sourceData = [{ param1: 1, param2: 32, param3: 432, param4: 643, param5: 34, param6: 22 }];
|
||||
//最大平均最小值
|
||||
export const averageData = [
|
||||
{ property: '最大值', param1: 1, param2: 32, param3: 432, param4: 643, param5: 34, param6: 22 },
|
||||
{ property: '平均值', param1: 1, param2: 32, param3: 432, param4: 643, param5: 34, param6: 22 },
|
||||
{ property: '最小值', param1: 1, param2: 32, param3: 432, param4: 643, param5: 34, param6: 22 },
|
||||
];
|
||||
|
||||
//源数据曲线
|
||||
export const sourceCurveData = {
|
||||
legend: ['整备质量', '迎风面积', '风阻系数', '转动惯量', '最大转速时拖拽扭矩', '速比'],
|
||||
xAxis: ['0', '200', '400', '600', '800', '1000', '1200'],
|
||||
series: [
|
||||
{ name: '整备质量', data: [120, 132, 101, 134, 90, 230, 210] },
|
||||
{ name: '迎风面积', data: [220, 182, 191, 234, 290, 330, 310] },
|
||||
{ name: '风阻系数', data: [150, 232, 201, 154, 190, 330, 410] },
|
||||
{ name: '滑行阻力曲线', data: [320, 332, 301, 334, 390, 330, 320] },
|
||||
{ name: '转动惯量', data: [820, 932, 901, 934, 1290, 1330, 1320] },
|
||||
],
|
||||
};
|
||||
|
||||
'''
|
||||
import argparse
|
||||
|
||||
from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_data_from_csv_feature
|
||||
from FC_ML_Model.Model_Train_Data import TrainData
|
||||
from FC_ML_Tool.Serialization import parse_json_file
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='导入数据文件参数')
|
||||
parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\param.json',
|
||||
help='配置参数文件绝对路径')
|
||||
parser.add_argument('--export', default='source.json',
|
||||
help='导出JSON文件名')
|
||||
args = parser.parse_args()
|
||||
params = parse_json_file(args.param)
|
||||
train_data = TrainData()
|
||||
print(params)
|
||||
source_dir = params["path"] + "/"
|
||||
#需要保证同批数据特征一致，否则无法抛出异常
|
||||
for data_file in params["files"]:
|
||||
data_file_path = source_dir + data_file
|
||||
label_name,source_data,data_min,data_max,data_mean,sampled_indices,data_sample = get_data_from_csv_feature(data_file_path)
|
||||
#拾取标签名
|
||||
if len(train_data.source_title) == 0:
|
||||
param_label = 1
|
||||
for label_name in label_name:
|
||||
dic_name = {"param" + str(param_label): label_name}
|
||||
param_label += 1
|
||||
train_data.source_title.append(dic_name)
|
||||
#拾取源数据
|
||||
if len(train_data.source_data) == 0:
|
||||
for rows in source_data:
|
||||
cols = rows.tolist()
|
||||
param_label = 1
|
||||
dic_source_data = {}
|
||||
for col in cols:
|
||||
dic_source_data["param" + str(param_label)] = col
|
||||
param_label += 1
|
||||
train_data.source_data.append(dic_source_data)
|
||||
#拾取特征数据
|
||||
if len(train_data.average_data) == 0:
|
||||
#首次直接记录
|
||||
property_max = {}
|
||||
property_max["property"] = "最大值"
|
||||
property_min = {}
|
||||
property_min["property"] = "最小值"
|
||||
property_mean = {}
|
||||
property_mean["property"] = "平均值"
|
||||
param_label = 1
|
||||
for value in data_max.tolist():
|
||||
property_max["param" + str(param_label)] = value
|
||||
param_label += 1
|
||||
param_label = 1
|
||||
for value in data_mean.tolist():
|
||||
property_mean["param" + str(param_label)] = value
|
||||
param_label += 1
|
||||
param_label = 1
|
||||
for value in data_min.tolist():
|
||||
property_min["param" + str(param_label)] = value
|
||||
param_label += 1
|
||||
train_data.average_data.append(property_max)
|
||||
train_data.average_data.append(property_mean)
|
||||
train_data.average_data.append(property_min)
|
||||
else:
|
||||
#非首次对数据进行合并操作
|
||||
param_label = 1
|
||||
for value in data_max.tolist():
|
||||
if train_data.average_data[0]["param" + str(param_label)] < value:
|
||||
train_data.average_data[0]["param" + str(param_label)] = value
|
||||
param_label += 1
|
||||
param_label = 1
|
||||
for value in data_mean.tolist():
|
||||
train_data.average_data[0]["param" + str(param_label)] = (train_data.average_data[0]["param" + str(param_label)] + value) / 2
|
||||
param_label += 1
|
||||
param_label = 1
|
||||
for value in data_min.tolist():
|
||||
if train_data.average_data[0]["param" + str(param_label)] > value:
|
||||
train_data.average_data[0]["param" + str(param_label)] = value
|
||||
param_label += 1
|
||||
#拾取曲线数据
|
||||
if len(train_data.source_curve_data["legend"]) == 0:
|
||||
train_data.source_curve_data["legend"] = train_data.source_title
|
||||
train_data.source_curve_data["xAxis"] = sampled_indices.tolist()
|
||||
param_label = 1
|
||||
for col_idx in range(data_sample.size(1)): # 获取列数
|
||||
serie = {}
|
||||
serie["name"] = "param1" + str(param_label)
|
||||
param_label += 1
|
||||
serie["data"] = []
|
||||
column = data_sample[:, col_idx] # 获取第col_idx列
|
||||
serie["data"].append(column.tolist())
|
||||
train_data.source_curve_data["series"].append(serie)
|
||||
else: #补充曲线数据
|
||||
param_label = 0
|
||||
for col_idx in range(data_sample.size(1)): # 获取列数
|
||||
column = data_sample[:, col_idx] # 获取第col_idx列
|
||||
train_data.source_curve_data["series"][param_label].append(column.tolist())
|
||||
print(train_data)
|
||||
with open(params["path"] + "/" + args.export, 'w') as f:
|
||||
f.write(train_data.to_json()) # 写入后会覆盖原有内容
|
||||
@@ -1 +0,0 @@
|
||||
#代理模型数据预处独立功能包
|
||||
@@ -1,56 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
|
||||
import torch
|
||||
from openpyxl.styles.builtins import output
|
||||
|
||||
from FC_ML_Data.FC_ML_Data_Process.Data_Process_Normalization import Normalizer
|
||||
from FC_ML_NN_Model.Poly_Model import PolyModel
|
||||
from FC_ML_Tool.Serialization import parse_json_file
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='代理模型训练参数输入')
|
||||
parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
|
||||
help='配置参数文件绝对路径')
|
||||
args = parser.parse_args()
|
||||
params = parse_json_file(args.param)
|
||||
source_dir = params["path"] + "/"
|
||||
model_file = source_dir + params["modelFile"]
|
||||
inputs = []
|
||||
names = []
|
||||
#获取输入特征
|
||||
for input_value in params["input"]:
|
||||
inputs.append(input_value["value"])
|
||||
names.append(input_value["name"])
|
||||
#记载模型进行预测
|
||||
input_size = params["modelParams"]["inputSize"]
|
||||
output_size = params["modelParams"]["outputSize"]
|
||||
model_path = params["path"] + "/" + params["modelFile"]
|
||||
device = torch.device('cpu')
|
||||
model = PolyModel(input_size,output_size).to(device)
|
||||
model.load_state_dict(torch.load(model_file))
|
||||
model.eval()
|
||||
|
||||
#加载数据处理器
|
||||
normalization_type = params["modelParams"]["normalizerType"]
|
||||
normalization_max = params["modelParams"]["normalizerMax"]
|
||||
normalization_min = params["modelParams"]["normalizerMin"]
|
||||
normalizer = Normalizer(method=normalization_type)
|
||||
normalizer.load_params(normalization_type,normalization_min,normalization_max)
|
||||
input_data = normalizer.transform(torch.tensor(inputs))
|
||||
|
||||
#执行模型预测
|
||||
with torch.no_grad():
|
||||
output_data = model(input_data)
|
||||
print(f"Prediction result: {output_data.item():.4f}")
|
||||
output_data_ori = normalizer.inverse_transform(output_data)
|
||||
print(f"Prediction real result: {output_data_ori.item():.4f}")
|
||||
#输出预测结果到文件中
|
||||
output_datas = output_data_ori.tolist()
|
||||
json_str = {}
|
||||
if len(output_datas) == len(names):
|
||||
for i in range(len(names)):
|
||||
json_str[names[i]] = output_datas[i]
|
||||
|
||||
with open(source_dir + "forecast.json", ) as f:
|
||||
f.write(json.dumps(json_str, indent=None, ensure_ascii=False))
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"files": ["sample1.CSV"],
|
||||
"path": "D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"path": ["D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Train"],
|
||||
"algorithmParam": {
|
||||
"algorithm": "多项式拟合",
|
||||
"activateFun": "sigmod",
|
||||
"lossFun": "l1",
|
||||
"optimizeFun": "sgd",
|
||||
"exportFormat": ".onnx",
|
||||
"trainingRatio": 80,
|
||||
"loadSize": 32,
|
||||
"studyPercent": 0.001,
|
||||
"stepCounts": 3,
|
||||
"roundPrint": 11,
|
||||
"round": 1001,
|
||||
"preDisposeData": false,
|
||||
"disposeMethod": "minmax",
|
||||
"dataNoOrder": false
|
||||
},
|
||||
"algorithm": "基础神经网络NN"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"modelFile": "model.onnx",
|
||||
"path": "D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\pred",
|
||||
"modelParams": {
|
||||
"inputSize": 3,
|
||||
"outputSize": 3,
|
||||
"normalizerType": "minmax",
|
||||
"normalizerMax": 100,
|
||||
"normalizerMin": 10
|
||||
},
|
||||
"input": [
|
||||
{
|
||||
"name": "质量",
|
||||
"value": 1
|
||||
}, {
|
||||
"name": "系数",
|
||||
"value": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
'''
|
||||
/**
|
||||
* 开始训练脚本
|
||||
* 传参：python xxx.py 源文件路径 文件名 /xx/xx/param.json
|
||||
* param.json 包含：
|
||||
* {
|
||||
*
|
||||
* path: '/xx/xx/xx',//源文件路径
|
||||
* algorithmParam:{
|
||||
algorithm: '多项式拟合',
|
||||
activateFun: 'sigmod',
|
||||
lossFun: 'l1',
|
||||
optimizeFun: 'sgd',
|
||||
exportFormat: '.onnx',
|
||||
trainingRatio: 80,
|
||||
loadSize: 32,
|
||||
studyPercent: 0.001,
|
||||
stepCounts: 3,
|
||||
roundPrint: 11,
|
||||
round: 1001,
|
||||
preDisposeData: false,
|
||||
disposeMethod: 'minmax',
|
||||
dataNoOrder: false,
|
||||
},
|
||||
* algorithm: '基础神经网络NN',//线性拟合 多项式拟合 基础神经网络NN 循环神经网络RNN 长短记忆神经网络LSTM
|
||||
* }
|
||||
* 生成training.json文件放到 源文件路径下
|
||||
* training.json文件包含 ：
|
||||
* 日志曲线：logCurveData
|
||||
* 表格数据：sourceData
|
||||
* 稳态必传：averageData
|
||||
* 动态必传：sourceCurveData
|
||||
*
|
||||
* 生成 training.log日志文件放到 源文件路径下
|
||||
*/
|
||||
'''
|
||||
import argparse
|
||||
from locale import normalize
|
||||
|
||||
import torch
|
||||
from torch.utils.data import TensorDataset, DataLoader
|
||||
|
||||
from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_data_from_csv_feature, get_train_data_from_csv
|
||||
from FC_ML_Data.FC_ML_Data_Output.Data_Output_Pytorch import export_model
|
||||
from FC_ML_Loss_Function.Loss_Function_Selector import LossFunctionSelector
|
||||
from FC_ML_Model.Model_Train_Data import TrainData
|
||||
from FC_ML_NN_Model.Poly_Model import PolyModel
|
||||
from FC_ML_Optim_Function.Optimizer_Selector import OptimizerSelector
|
||||
from FC_ML_Tool.Serialization import parse_json_file
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='代理模型训练参数输入')
|
||||
parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
|
||||
help='配置参数文件绝对路径')
|
||||
args = parser.parse_args()
|
||||
params = parse_json_file(args.param)
|
||||
train_data = torch.tensor()
|
||||
# print(params)
|
||||
#获取训练参数
|
||||
input_Size = params["algorithmParam"]["inputSize"]#输入特征维度
|
||||
output_Size = params["algorithmParam"]["outputSize"]#输出标签维度
|
||||
algorithm = params["algorithmParam"]["algorithm"]#神经网络模型
|
||||
activate_fun = params["algorithmParam"]["activateFun"]#激活函数类型
|
||||
loss_fun = params["algorithmParam"]["lossFun"]#损失函数
|
||||
optimize_fun = params["algorithmParam"]["optimizeFun"]#优化函数
|
||||
export_format = params["algorithmParam"]["exportFormat"]#导出训练模型格式
|
||||
training_ratio = params["algorithmParam"]["trainingRatio"]#训练数据占比
|
||||
load_size = params["algorithmParam"]["loadSize"]#同时最大加载训练数据宽度
|
||||
study_percent = params["algorithmParam"]["studyPercent"]#学习率
|
||||
round_print = params["algorithmParam"]["roundPrint"]#训练多少轮进行打印
|
||||
round_count = params["algorithmParam"]["round"]#总训练轮次
|
||||
pre_dispose_data = params["algorithmParam"]["preDisposeData"]#是否进行数据预处理
|
||||
dispose_method = params["algorithmParam"]["disposeMethod"] # 数据预处理方法
|
||||
data_no_order = params["algorithmParam"]["dataNoOrder"] # 训练数据是否乱序处理
|
||||
#加载所有训练数据
|
||||
source_dir = params["path"] + "/"
|
||||
for data_file in params["files"]:
|
||||
data_file_path = source_dir + data_file
|
||||
ori_data,normalize = get_train_data_from_csv(data_file_path,pre_dispose_data,dispose_method)
|
||||
torch.cat((train_data,ori_data),dim=0)#按行拼接
|
||||
#拆分测试集和训练集
|
||||
split = int(training_ratio / 100 * len(train_data))
|
||||
train_dataset = TensorDataset(train_data[:split,0:input_Size], train_data[:split,input_Size:])
|
||||
test_dataset = TensorDataset(train_data[split:,0:input_Size], train_data[split:,input_Size:])
|
||||
print(train_dataset,test_dataset)
|
||||
train_loader = DataLoader(
|
||||
train_dataset,
|
||||
batch_size = load_size, #批量加载数据
|
||||
shuffle = data_no_order #数据打乱
|
||||
)
|
||||
|
||||
test_loader = DataLoader(
|
||||
test_dataset,
|
||||
batch_size= load_size, #批量加载数据
|
||||
shuffle=False #验证集默认不打乱
|
||||
)
|
||||
|
||||
#初始化模型
|
||||
DEVICE = torch.device("cpu")#默认CPU训练
|
||||
model = PolyModel(input_size = input_Size,output_size=output_Size).to(DEVICE)
|
||||
#初始化损失函数
|
||||
loss_selector = LossFunctionSelector()
|
||||
criterion = loss_selector.get_loss(loss_fun)
|
||||
#初始化优化器
|
||||
optim_selector = OptimizerSelector()
|
||||
|
||||
optimizer = optim_selector.get_optimizer(model.parameters(),optimize_fun, lr=study_percent)
|
||||
# 在训练循环前初始化损失记录列表
|
||||
train_losses = []
|
||||
test_losses = []
|
||||
for epoch in range(round_count):
|
||||
#模型启用训练模式
|
||||
epoch_train_loss = 0
|
||||
model.train()
|
||||
for features,labels in train_loader:
|
||||
features,labels= features.to(DEVICE),labels.to(DEVICE)
|
||||
pred = model(features)
|
||||
loss = criterion(pred.squeeze(), labels)
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
#计算损失
|
||||
epoch_train_loss += loss.item() * features.size(0)
|
||||
|
||||
# 计算平均训练损失并记录
|
||||
avg_train_loss = epoch_train_loss / len(train_loader.dataset)
|
||||
train_losses.append(avg_train_loss)
|
||||
|
||||
#模型启用评估模式
|
||||
model.eval()
|
||||
test_loss = 0
|
||||
with torch.no_grad():#关闭梯度下降
|
||||
for features, labels in test_loader:
|
||||
features, labels = features.to(DEVICE), labels.to(DEVICE)
|
||||
preds = model(features)
|
||||
test_loss += criterion(preds.squeeze(), labels).item() * features.size(0)
|
||||
|
||||
avg_test_loss = test_loss / len(test_loader.dataset)
|
||||
test_losses.append(avg_test_loss)
|
||||
#每100次迭代输出一次损失数值
|
||||
if epoch % round_print == 0:
|
||||
print(
|
||||
f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1")
|
||||
with open(source_dir + "training.log", "a") as f:
|
||||
f.write(f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1\n") # 自动换行追加
|
||||
#导出训练后的模型
|
||||
export_model(model,source_dir,"model",export_format)
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user