update bq

Signed-off-by: cwj <talkingwallace@sohu.com>

examples/benchmark_quality/hetero_sbt/hetero_sbt_benchmark.yaml

@@ -101,39 +101,32 @@ data:
     table_name: student_hetero_host
     namespace: experiment
     role: host_0  
-hetero_sbt-binary-0-breast:
+# hetero_sbt-binary-0-breast:
+#   local:
+#     script: "./xgb.py"
+#     conf: "./xgb_breast_config.yaml"
+#   FATE-hetero-sbt:
+#     script: "./pipeline_hetero_sbt.py"
+#     conf: "./sbt_breast_config.yaml"
+#   compare_setting:
+#     relative_tol: 0.01
+hetero_sbt-regression:
   local:
     script: "./xgb.py"
-    conf: "./breast_lr_sklearn_config.yaml"
+    conf: "./xgb_student_config.yaml"
   FATE-hetero-sbt:
-    script: "./pipeline-hetero-sbt.py"
-    conf: "./breast_config.yaml"
+    script: "./pipeline_hetero_sbt.py"
+    conf: "./sbt_student_config.yaml"
   compare_setting:
-    relative_tol: 0.01
-hetero_sbt-binary-1-default-credit:
-  local:
-    script: "./xgb.py"
-    conf: "./default_credit_lr_sklearn_config.yaml"
-  FATE-hetero-sbt:
-    script: "./pipeline-hetero-sbt.py"
-    conf: "./default_credit_config.yaml"
-  compare_setting:
-    relative_tol: 0.01
-hetero_sbt-binary-2-epsilon-5k:
-  local:
-    script: "./xgb.py"
-    conf: "./epsilon_5k_lr_sklearn_config.yaml"
-  FATE-hetero-sbt:
-    script: "./pipeline-hetero-sbt.py"
-    conf: "./epsilon_5k_config.yaml"
-  compare_setting:
-    relative_tol: 0.01
-hetero_sbt-binary-regression:
-  local:
-    script: "./xgb.py"
-    conf: "./give_credit_lr_sklearn_config.yaml"
-  FATE-hetero-sbt:
-    script: "./pipeline-hetero-sbt.py"
-    conf: "./give_credit_config.yaml"
-  compare_setting:
-    relative_tol: 0.01
+    relative_tol: 0.05
+# hetero_sbt-binary-1-default-credit:
+#   local:
+#     script: "./xgb.py"
+#     conf: "./xgb_default_credit_config.yaml"
+#   FATE-hetero-sbt:
+#     script: "./pipeline_hetero_sbt.py"
+#     conf: "./sbt_default_credit_config.yaml"
+#   compare_setting:
+#     relative_tol: 0.01
+
+

examples/benchmark_quality/hetero_sbt/xgb.py

@@ -56,14 +56,13 @@ def main(config="../../config.yaml", param="./xgb_breast_config.yaml"):
         mse = mean_squared_error(y_test, y_pred)
         rmse = math.sqrt(mse)
         mae = mean_absolute_error(y_test, y_pred)
-        print("mse: {}, rmse: {}, mae: {}".format(mse, rmse, mae))
+        return {}, {"rmse": rmse}
     else:
         model = xgb.XGBClassifier(**config_param)
         model.fit(x_train, y_train)
         y_pred = model.predict(x_test)
         y_prob = model.predict_proba(x_test)[:, 1]
 
-
         try:
             auc_score = roc_auc_score(y_test, y_prob)
         except BaseException:

examples/benchmark_quality/hetero_sbt/xgb_student_config.yaml

@@ -5,5 +5,5 @@ label_name: "y"
 max_depth: 3
 max_bin: 32
 learning_rate: 0.3
-n_estimators: 20
+n_estimators: 50
 objective: "reg:squarederror"

examples/benchmark_quality/homo_nn/fed_nn_student_config.yaml

@@ -6,5 +6,5 @@ label_name: "y"
 in_feat: 13
 out_feat: 8
 lr: 0.01
-batch_size: 32
-epochs: 50
+batch_size: 1024 
+epochs: 60

examples/benchmark_quality/homo_nn/fed_nn_vehicle_config.yaml

@@ -5,7 +5,7 @@ idx: "id"
 label_name: "y"
 in_feat: 18
 out_feat: 8
-lr: 0.1
+lr: 0.01
 batch_size: 32
-epochs: 100
+epochs: 110
 class_num: 4

examples/benchmark_quality/homo_nn/homo_nn_benchmark.yaml

@@ -178,24 +178,24 @@ homo_nn-binary-0-breast:
     conf: "./fed_nn_breast_config.yaml"
   compare_setting:
     relative_tol: 0.01
-# homo_nn-binary-1-default-credit:
-#   local:
-#     script: "./local_nn_binary.py"
-#     conf: "./local_nn_default_credit_config.yaml"
-#   FATE-homo-nn:
-#     script: "./pipeline_nn_binary.py"
-#     conf: "./fed_nn_default_credit_config.yaml"
-#   compare_setting:
-#     relative_tol: 0.01
-# homo_nn-binary-2-give-credit:
-#   local:
-#     script: "./local_nn_binary.py"
-#     conf: "./local_nn_give_credit_config.yaml"
-#   FATE-homo-nn:
-#     script: "./pipeline_nn_binary.py"
-#     conf: "./fed_nn_give_credit_config.yaml"
-#   compare_setting:
-#     relative_tol: 0.01
+homo_nn-binary-1-default-credit:
+  local:
+    script: "./local_nn_binary.py"
+    conf: "./local_nn_default_credit_config.yaml"
+  FATE-homo-nn:
+    script: "./pipeline_nn_binary.py"
+    conf: "./fed_nn_default_credit_config.yaml"
+  compare_setting:
+    relative_tol: 0.01
+homo_nn-binary-2-give-credit:
+  local:
+    script: "./local_nn_binary.py"
+    conf: "./local_nn_give_credit_config.yaml"
+  FATE-homo-nn:
+    script: "./pipeline_nn_binary.py"
+    conf: "./fed_nn_give_credit_config.yaml"
+  compare_setting:
+    relative_tol: 0.01
 homo_nn-regression-student:
   local:
     script: "./local_nn_regression.py"
@@ -213,4 +213,4 @@ homo_nn-multi-vehicle:
     script: "./pipeline_nn_multi.py"
     conf: "./fed_nn_vehicle_config.yaml"
   compare_setting:
-    relative_tol: 0.01
+    relative_tol: 0.05

examples/benchmark_quality/homo_nn/local_nn_binary.py

@@ -26,6 +26,10 @@
 import tqdm
 
 
+seed = 114514
+torch.manual_seed(seed)
+torch.cuda.manual_seed_all(seed)
+
 def main(config="../../config.yaml", param="./local_nn_breast_config.yaml", namespace=""):
     # obtain config
     if isinstance(config, str):

examples/benchmark_quality/homo_nn/local_nn_multi.py

@@ -25,6 +25,9 @@
 from fate.ml.evaluation.classification import MultiAccuracy, MultiPrecision, MultiRecall
 import tqdm
 
+seed = 114514
+torch.manual_seed(seed)
+torch.cuda.manual_seed_all(seed)
 
 def main(config="../../config.yaml", param="", namespace=""):
     # obtain config
@@ -80,18 +83,13 @@ def main(config="../../config.yaml", param="", namespace=""):
         y_train_pred = model(X).numpy()
 
     # compute accuracy
-    acc = MultiAccuracy()(y_train_pred, y)
-    print(f"accuracy: {acc}")
+    acc = MultiAccuracy()(y_train_pred, y).get_raw_data()
     # compute precision
-    precision = MultiPrecision()(y_train_pred, y)
-    print(f"precision: {precision}")
+    precision = MultiPrecision()(y_train_pred, y).get_raw_data()
     # compute recall
-    recall = MultiRecall()(y_train_pred, y)
-    print(f"recall: {recall}")
+    recall = MultiRecall()(y_train_pred, y).get_raw_data()
 
-    result = {"multi_accuracy": acc,
-            "multi_precision": precision,
-            "multi_recall": recall}
+    result = {"multi_accuracy": float(acc)}
 
     return {}, result
 

examples/benchmark_quality/homo_nn/local_nn_regression.py

@@ -25,6 +25,10 @@
 from sklearn.metrics import mean_squared_error, mean_absolute_error
 import tqdm
 
+seed = 114514
+torch.manual_seed(seed)
+torch.cuda.manual_seed_all(seed)
+
 
 def main(config="../../config.yaml", param="", namespace=""):
     # obtain config

examples/benchmark_quality/homo_nn/local_nn_student_config.yaml

@@ -6,5 +6,5 @@ label_name: "y"
 in_feat: 13
 out_feat: 8
 lr: 0.01
-batch_size: 32
-epochs: 50
+batch_size: 1024
+epochs: 100

examples/benchmark_quality/homo_nn/local_nn_vehicle_config.yaml

@@ -5,7 +5,7 @@ idx: "id"
 label_name: "y"
 in_feat: 18
 out_feat: 8
-lr: 0.1
+lr: 0.01
 batch_size: 32
 epochs: 100
 class_num: 4

examples/benchmark_quality/homo_nn/pipeline_nn_binary.py

@@ -68,7 +68,7 @@ def main(config="../../config.yaml", param="./fed_nn_breast_config.yaml", namesp
         ), 
         loss=nn.BCELoss(),
         optimizer=optim.Adam(lr=lr),
-        training_args=TrainingArguments(num_train_epochs=epochs, per_device_train_batch_size=batch_size),
+        training_args=TrainingArguments(num_train_epochs=epochs, per_device_train_batch_size=batch_size, seed=114514),
         fed_args=FedAVGArguments(),
         task_type='binary'
         )

examples/benchmark_quality/homo_nn/pipeline_nn_multi.py

@@ -64,7 +64,7 @@ def main(config="../../config.yaml", param="", namespace=""):
         model=ModelLoader('multi_model', 'Multi', feat=in_feat, class_num=class_num), 
         loss=nn.CrossEntropyLoss(),
         optimizer=optim.Adam(lr=lr),
-        training_args=TrainingArguments(num_train_epochs=epochs, per_device_train_batch_size=batch_size),
+        training_args=TrainingArguments(num_train_epochs=epochs, per_device_train_batch_size=batch_size, seed=114514),
         fed_args=FedAVGArguments(),
         task_type='multi'
         )
@@ -103,7 +103,6 @@ def main(config="../../config.yaml", param="", namespace=""):
     pipeline.compile()
     pipeline.fit()
 
-    print(pipeline.get_task_info("eval_0").get_output_metric())
     result_summary = parse_summary_result(pipeline.get_task_info("eval_0").get_output_metric()[0]["data"])
     data_summary = {"train": {"guest": guest_train_data["name"], "host": host_train_data["name"]},
                     "test": {"guest": guest_train_data["name"], "host": host_train_data["name"]}

examples/benchmark_quality/homo_nn/pipeline_nn_regression.py

@@ -67,7 +67,7 @@ def main(config="../../config.yaml", param="", namespace=""):
         ), 
         loss=nn.MSELoss(),
         optimizer=optim.Adam(lr=lr),
-        training_args=TrainingArguments(num_train_epochs=epochs, per_device_train_batch_size=batch_size),
+        training_args=TrainingArguments(num_train_epochs=epochs, per_device_train_batch_size=batch_size, seed=114514),
         fed_args=FedAVGArguments(),
         task_type='regression'
         )