切换git用户重新进行项目首次归档

FC_ML_Baseline/FC_ML_Baseline_Data_Handler/Data_Load.py

@@ -0,0 +1,148 @@
+'''
+/**
+ * 导入数据文件脚本
+ * 传参：python xxx.py /xx/xx/param.json
+ * param.json 包含：
+ *          {
+ *            files:[xxx.excel,xxx.excel,xxx.excel],
+ *            path: '/xx/xx/xx',//源文件路径
+ *          }
+ * 返回：source.json source.json文件放到  源文件路径下
+ * source.json文件包含：
+ *          表格标题：sourceTitle
+ *          表格数据：sourceData
+ *          稳态必传：averageData
+ *          动态必传：sourceCurveData
+ */
+
+ //源数据表头
+export const sourceTitle = [
+  { value: 'param1', label: '整备质量' },
+  { value: 'param2', label: '迎风面积' },
+  { value: 'param3', label: '风阻系数' },
+  { value: 'param4', label: '转动惯量' },
+  { value: 'param5', label: '最大转速时拖拽扭矩' },
+  { value: 'param6', label: '速比' },
+];
+//源数据
+export const sourceData = [{ param1: 1, param2: 32, param3: 432, param4: 643, param5: 34, param6: 22 }];
+//最大平均最小值
+export const averageData = [
+  { property: '最大值', param1: 1, param2: 32, param3: 432, param4: 643, param5: 34, param6: 22 },
+  { property: '平均值', param1: 1, param2: 32, param3: 432, param4: 643, param5: 34, param6: 22 },
+  { property: '最小值', param1: 1, param2: 32, param3: 432, param4: 643, param5: 34, param6: 22 },
+];
+
+//源数据曲线
+export const sourceCurveData = {
+  legend: ['整备质量', '迎风面积', '风阻系数', '转动惯量', '最大转速时拖拽扭矩', '速比'],
+  xAxis: ['0', '200', '400', '600', '800', '1000', '1200'],
+  series: [
+    { name: '整备质量', data: [120, 132, 101, 134, 90, 230, 210] },
+    { name: '迎风面积', data: [220, 182, 191, 234, 290, 330, 310] },
+    { name: '风阻系数', data: [150, 232, 201, 154, 190, 330, 410] },
+    { name: '滑行阻力曲线', data: [320, 332, 301, 334, 390, 330, 320] },
+    { name: '转动惯量', data: [820, 932, 901, 934, 1290, 1330, 1320] },
+  ],
+};
+
+'''
+import argparse
+
+from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_data_from_csv_feature
+from FC_ML_Model.Model_Train_Data import TrainData
+from FC_ML_Tool.Serialization import parse_json_file
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='导入数据文件参数')
+    parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\param.json',
+                        help='配置参数文件绝对路径')
+    parser.add_argument('--export', default='source.json',
+                        help='导出JSON文件名')
+    args = parser.parse_args()
+    params = parse_json_file(args.param)
+    train_data = TrainData()
+    print(params)
+    source_dir = params["path"] + "/"
+    #需要保证同批数据特征一致，否则无法抛出异常
+    for data_file in params["files"]:
+        data_file_path = source_dir + data_file
+        label_name,source_data,data_min,data_max,data_mean,sampled_indices,data_sample = get_data_from_csv_feature(data_file_path)
+        #拾取标签名
+        if len(train_data.source_title) == 0:
+            param_label = 1
+            for label_name in label_name:
+                dic_name = {"param" + str(param_label): label_name}
+                param_label += 1
+                train_data.source_title.append(dic_name)
+        #拾取源数据
+        if len(train_data.source_data) == 0:
+            for rows in source_data:
+                cols = rows.tolist()
+                param_label = 1
+                dic_source_data = {}
+                for col in cols:
+                    dic_source_data["param" + str(param_label)] = col
+                    param_label += 1
+                train_data.source_data.append(dic_source_data)
+        #拾取特征数据
+        if len(train_data.average_data) == 0:
+            #首次直接记录
+            property_max = {}
+            property_max["property"] = "最大值"
+            property_min = {}
+            property_min["property"] = "最小值"
+            property_mean = {}
+            property_mean["property"] = "平均值"
+            param_label = 1
+            for value in data_max.tolist():
+                property_max["param" + str(param_label)] = value
+                param_label += 1
+            param_label = 1
+            for value in data_mean.tolist():
+                property_mean["param" + str(param_label)] = value
+                param_label += 1
+            param_label = 1
+            for value in data_min.tolist():
+                property_min["param" + str(param_label)] = value
+                param_label += 1
+            train_data.average_data.append(property_max)
+            train_data.average_data.append(property_mean)
+            train_data.average_data.append(property_min)
+        else:
+            #非首次对数据进行合并操作
+            param_label = 1
+            for value in data_max.tolist():
+                if train_data.average_data[0]["param" + str(param_label)] < value:
+                    train_data.average_data[0]["param" + str(param_label)] = value
+                param_label += 1
+            param_label = 1
+            for value in data_mean.tolist():
+                train_data.average_data[0]["param" + str(param_label)] = (train_data.average_data[0]["param" + str(param_label)] + value) / 2
+                param_label += 1
+            param_label = 1
+            for value in data_min.tolist():
+                if train_data.average_data[0]["param" + str(param_label)] > value:
+                    train_data.average_data[0]["param" + str(param_label)] = value
+                param_label += 1
+        #拾取曲线数据
+        if len(train_data.source_curve_data["legend"]) == 0:
+            train_data.source_curve_data["legend"] = train_data.source_title
+            train_data.source_curve_data["xAxis"] = sampled_indices.tolist()
+            param_label = 1
+            for col_idx in range(data_sample.size(1)):  # 获取列数
+                serie = {}
+                serie["name"] = "param1" + str(param_label)
+                param_label += 1
+                serie["data"] = []
+                column = data_sample[:, col_idx]  # 获取第col_idx列
+                serie["data"].append(column.tolist())
+                train_data.source_curve_data["series"].append(serie)
+        else: #补充曲线数据
+            param_label = 0
+            for col_idx in range(data_sample.size(1)):  # 获取列数
+                column = data_sample[:, col_idx]  # 获取第col_idx列
+                train_data.source_curve_data["series"][param_label].append(column.tolist())
+    print(train_data)
+    with open(params["path"] + "/" + args.export, 'w') as f:
+        f.write(train_data.to_json())  # 写入后会覆盖原有内容

FC_ML_Baseline/FC_ML_Baseline_Data_Handler/__init__.py

@@ -0,0 +1 @@
+#代理模型数据预处独立功能包

FC_ML_Baseline/FC_ML_Baseline_Predict/Model_Pred.py

@@ -0,0 +1,56 @@
+import argparse
+import json
+
+import torch
+from openpyxl.styles.builtins import output
+
+from FC_ML_Data.FC_ML_Data_Process.Data_Process_Normalization import Normalizer
+from FC_ML_NN_Model.Poly_Model import PolyModel
+from FC_ML_Tool.Serialization import parse_json_file
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='代理模型训练参数输入')
+    parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
+                        help='配置参数文件绝对路径')
+    args = parser.parse_args()
+    params = parse_json_file(args.param)
+    source_dir = params["path"] + "/"
+    model_file = source_dir + params["modelFile"]
+    inputs = []
+    names = []
+    #获取输入特征
+    for input_value in params["input"]:
+        inputs.append(input_value["value"])
+        names.append(input_value["name"])
+    #记载模型进行预测
+    input_size = params["modelParams"]["inputSize"]
+    output_size = params["modelParams"]["outputSize"]
+    model_path = params["path"] + "/" + params["modelFile"]
+    device = torch.device('cpu')
+    model = PolyModel(input_size,output_size).to(device)
+    model.load_state_dict(torch.load(model_file))
+    model.eval()
+
+    #加载数据处理器
+    normalization_type = params["modelParams"]["normalizerType"]
+    normalization_max = params["modelParams"]["normalizerMax"]
+    normalization_min = params["modelParams"]["normalizerMin"]
+    normalizer = Normalizer(method=normalization_type)
+    normalizer.load_params(normalization_type,normalization_min,normalization_max)
+    input_data = normalizer.transform(torch.tensor(inputs))
+
+    #执行模型预测
+    with torch.no_grad():
+        output_data = model(input_data)
+        print(f"Prediction result: {output_data.item():.4f}")
+        output_data_ori = normalizer.inverse_transform(output_data)
+        print(f"Prediction real result: {output_data_ori.item():.4f}")
+        #输出预测结果到文件中
+        output_datas = output_data_ori.tolist()
+        json_str = {}
+        if len(output_datas) == len(names):
+            for i in range(len(names)):
+                json_str[names[i]] = output_datas[i]
+
+        with open(source_dir + "forecast.json", ) as f:
+            f.write(json.dumps(json_str, indent=None, ensure_ascii=False))

FC_ML_Baseline/FC_ML_Baseline_Test/Data_Handle/param.json

@@ -0,0 +1,4 @@
+{
+  "files": ["sample1.CSV"],
+  "path": "D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle"
+}

FC_ML_Baseline/FC_ML_Baseline_Test/Train/param.json

@@ -0,0 +1,20 @@
+{
+  "path": ["D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Train"],
+  "algorithmParam": {
+    "algorithm": "多项式拟合",
+    "activateFun": "sigmod",
+    "lossFun": "l1",
+    "optimizeFun": "sgd",
+    "exportFormat": ".onnx",
+    "trainingRatio": 80,
+    "loadSize": 32,
+    "studyPercent": 0.001,
+    "stepCounts": 3,
+    "roundPrint": 11,
+    "round": 1001,
+    "preDisposeData": false,
+    "disposeMethod": "minmax",
+    "dataNoOrder": false
+  },
+  "algorithm": "基础神经网络NN"
+}

FC_ML_Baseline/FC_ML_Baseline_Test/pred/param.json

@@ -0,0 +1,20 @@
+{
+  "modelFile": "model.onnx",
+  "path": "D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\pred",
+  "modelParams": {
+    "inputSize": 3,
+    "outputSize": 3,
+    "normalizerType": "minmax",
+    "normalizerMax": 100,
+    "normalizerMin": 10
+  },
+  "input": [
+    {
+      "name": "质量",
+      "value": 1
+    }, {
+      "name": "系数",
+      "value": 2
+    }
+  ]
+}

FC_ML_Baseline/FC_ML_Baseline_Train/Train_Proxy_Model.py

@@ -0,0 +1,151 @@
+'''
+/**
+ * 开始训练脚本
+ * 传参：python xxx.py 源文件路径 文件名  /xx/xx/param.json
+ * param.json 包含：
+ *          {
+ *
+ *            path: '/xx/xx/xx',//源文件路径
+ *            algorithmParam:{
+                algorithm: '多项式拟合',
+                activateFun: 'sigmod',
+                lossFun: 'l1',
+                optimizeFun: 'sgd',
+                exportFormat: '.onnx',
+                trainingRatio: 80,
+                loadSize: 32,
+                studyPercent: 0.001,
+                stepCounts: 3,
+                roundPrint: 11,
+                round: 1001,
+                preDisposeData: false,
+                disposeMethod: 'minmax',
+                dataNoOrder: false,
+              },
+ *            algorithm: '基础神经网络NN',//线性拟合 多项式拟合 基础神经网络NN 循环神经网络RNN 长短记忆神经网络LSTM
+ *          }
+ * 生成training.json文件放到  源文件路径下
+ * training.json文件包含 ：
+ *          日志曲线：logCurveData
+ *          表格数据：sourceData
+ *          稳态必传：averageData
+ *          动态必传：sourceCurveData
+ *
+ * 生成 training.log日志文件放到  源文件路径下
+ */
+'''
+import argparse
+from locale import normalize
+
+import torch
+from torch.utils.data import TensorDataset, DataLoader
+
+from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_data_from_csv_feature, get_train_data_from_csv
+from FC_ML_Data.FC_ML_Data_Output.Data_Output_Pytorch import export_model
+from FC_ML_Loss_Function.Loss_Function_Selector import LossFunctionSelector
+from FC_ML_Model.Model_Train_Data import TrainData
+from FC_ML_NN_Model.Poly_Model import PolyModel
+from FC_ML_Optim_Function.Optimizer_Selector import OptimizerSelector
+from FC_ML_Tool.Serialization import parse_json_file
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='代理模型训练参数输入')
+    parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
+                        help='配置参数文件绝对路径')
+    args = parser.parse_args()
+    params = parse_json_file(args.param)
+    train_data = torch.tensor()
+    # print(params)
+    #获取训练参数
+    input_Size = params["algorithmParam"]["inputSize"]#输入特征维度
+    output_Size = params["algorithmParam"]["outputSize"]#输出标签维度
+    algorithm = params["algorithmParam"]["algorithm"]#神经网络模型
+    activate_fun = params["algorithmParam"]["activateFun"]#激活函数类型
+    loss_fun = params["algorithmParam"]["lossFun"]#损失函数
+    optimize_fun = params["algorithmParam"]["optimizeFun"]#优化函数
+    export_format = params["algorithmParam"]["exportFormat"]#导出训练模型格式
+    training_ratio = params["algorithmParam"]["trainingRatio"]#训练数据占比
+    load_size = params["algorithmParam"]["loadSize"]#同时最大加载训练数据宽度
+    study_percent = params["algorithmParam"]["studyPercent"]#学习率
+    round_print = params["algorithmParam"]["roundPrint"]#训练多少轮进行打印
+    round_count = params["algorithmParam"]["round"]#总训练轮次
+    pre_dispose_data = params["algorithmParam"]["preDisposeData"]#是否进行数据预处理
+    dispose_method = params["algorithmParam"]["disposeMethod"]  # 数据预处理方法
+    data_no_order = params["algorithmParam"]["dataNoOrder"]  # 训练数据是否乱序处理
+    #加载所有训练数据
+    source_dir = params["path"] + "/"
+    for data_file in params["files"]:
+        data_file_path = source_dir + data_file
+        ori_data,normalize = get_train_data_from_csv(data_file_path,pre_dispose_data,dispose_method)
+        torch.cat((train_data,ori_data),dim=0)#按行拼接
+    #拆分测试集和训练集
+    split = int(training_ratio / 100 * len(train_data))
+    train_dataset = TensorDataset(train_data[:split,0:input_Size], train_data[:split,input_Size:])
+    test_dataset = TensorDataset(train_data[split:,0:input_Size], train_data[split:,input_Size:])
+    print(train_dataset,test_dataset)
+    train_loader = DataLoader(
+        train_dataset,
+        batch_size = load_size, #批量加载数据
+        shuffle = data_no_order #数据打乱
+    )
+
+    test_loader = DataLoader(
+        test_dataset,
+        batch_size= load_size, #批量加载数据
+        shuffle=False #验证集默认不打乱
+    )
+
+    #初始化模型
+    DEVICE = torch.device("cpu")#默认CPU训练
+    model = PolyModel(input_size = input_Size,output_size=output_Size).to(DEVICE)
+    #初始化损失函数
+    loss_selector = LossFunctionSelector()
+    criterion = loss_selector.get_loss(loss_fun)
+    #初始化优化器
+    optim_selector = OptimizerSelector()
+
+    optimizer = optim_selector.get_optimizer(model.parameters(),optimize_fun, lr=study_percent)
+    # 在训练循环前初始化损失记录列表
+    train_losses = []
+    test_losses = []
+    for epoch in range(round_count):
+        #模型启用训练模式
+        epoch_train_loss = 0
+        model.train()
+        for features,labels in train_loader:
+            features,labels= features.to(DEVICE),labels.to(DEVICE)
+            pred = model(features)
+            loss = criterion(pred.squeeze(), labels)
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            #计算损失
+            epoch_train_loss += loss.item() * features.size(0)
+
+        # 计算平均训练损失并记录
+        avg_train_loss = epoch_train_loss / len(train_loader.dataset)
+        train_losses.append(avg_train_loss)
+
+        #模型启用评估模式
+        model.eval()
+        test_loss = 0
+        with torch.no_grad():#关闭梯度下降
+            for features, labels in test_loader:
+                features, labels = features.to(DEVICE), labels.to(DEVICE)
+                preds = model(features)
+                test_loss += criterion(preds.squeeze(), labels).item() * features.size(0)
+
+        avg_test_loss = test_loss / len(test_loader.dataset)
+        test_losses.append(avg_test_loss)
+        #每100次迭代输出一次损失数值
+        if epoch % round_print == 0:
+            print(
+                f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1")
+            with open(source_dir + "training.log", "a") as f:
+                f.write(f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1\n")  # 自动换行追加
+    #导出训练后的模型
+    export_model(model,source_dir,"model",export_format)
+
+
+
+