Merge branch 'dev-2.0.0-beta-debugging' into dev-2.0.0-homo-lr

examples/pipeline/test_lr_sid.py

@@ -0,0 +1,73 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from fate_client.pipeline import FateFlowPipeline
+from fate_client.pipeline.components.fate import CoordinatedLR, Intersection
+from fate_client.pipeline.components.fate import Evaluation
+from fate_client.pipeline.interface import DataWarehouseChannel
+
+pipeline = FateFlowPipeline().set_roles(guest="9999", host="9998", arbiter="9998")
+
+intersect_0 = Intersection("intersect_0", method="raw")
+intersect_0.guest.component_setting(input_data=DataWarehouseChannel(name="breast_hetero_guest_sid",
+                                                                    namespace="experiment"))
+intersect_0.hosts[0].component_setting(input_data=DataWarehouseChannel(name="breast_hetero_host_sid",
+                                                                       namespace="experiment"))
+lr_0 = CoordinatedLR("lr_0",
+                     epochs=10,
+                     batch_size=100,
+                     optimizer={"method": "sgd", "optimizer_params": {"lr": 0.01}, "alpha": 0.5},
+                     init_param={"fit_intercept": True},
+                     train_data=intersect_0.outputs["output_data"])
+
+"""lr_0.guest.component_setting(train_data=DataWarehouseChannel(name="breast_hetero_guest_sid",
+                                                             namespace="experiment"))
+lr_0.hosts[0].component_setting(train_data=DataWarehouseChannel(name="breast_hetero_host_sid",
+                                                                namespace="experiment"))"""
+
+evaluation_0 = Evaluation("evaluation_0",
+                          runtime_roles=["guest"],
+                          input_data=lr_0.outputs["train_output_data"])
+
+# pipeline.add_task(feature_scale_0)
+# pipeline.add_task(feature_scale_1)
+pipeline.add_task(intersect_0)
+pipeline.add_task(lr_0)
+pipeline.add_task(evaluation_0)
+# pipeline.add_task(hetero_feature_binning_0)
+pipeline.compile()
+print(pipeline.get_dag())
+pipeline.fit()
+
+# print(pipeline.get_task_info("statistics_0").get_output_model())
+print(pipeline.get_task_info("lr_0").get_output_model())
+print(pipeline.get_task_info("lr_0").get_output_metrics())
+print(f"evaluation metrics: ")
+print(pipeline.get_task_info("evaluation_0").get_output_metrics())
+
+pipeline.deploy([lr_0])
+
+predict_pipeline = FateFlowPipeline()
+
+deployed_pipeline = pipeline.get_deployed_pipeline()
+lr_0.guest.component_setting(test_data=DataWarehouseChannel(name="breast_hetero_guest_sid",
+                                                            namespace="experiment"))
+lr_0.hosts[0].component_setting(test_data=DataWarehouseChannel(name="breast_hetero_host_sid",
+                                                               namespace="experiment"))
+
+predict_pipeline.add_task(deployed_pipeline)
+predict_pipeline.compile()
+# print("\n\n\n")
+# print(predict_pipeline.compile().get_dag())
+predict_pipeline.predict()

examples/pipeline/test_single_linr.py

@@ -0,0 +1,72 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from fate_client.pipeline import FateFlowPipeline
+from fate_client.pipeline.components.fate import CoordinatedLinR
+from fate_client.pipeline.components.fate import Evaluation
+from fate_client.pipeline.interface import DataWarehouseChannel
+
+pipeline = FateFlowPipeline().set_roles(guest="9999", host="9998", arbiter="9998")
+
+"""feature_scale_0 = FeatureScale(name="feature_scale_0",
+                               method="min_max",
+                               train_data=intersection_0.outputs["output_data"])
+
+feature_scale_1 = FeatureScale(name="feature_scale_1",
+                               test_data=intersection_1.outputs["output_data"],
+                               input_model=feature_scale_0.outputs["output_model"])"""
+
+linr_0 = CoordinatedLinR("linr_0",
+                         max_iter=10,
+                         batch_size=-1,
+                         init_param={"fit_intercept": False})
+
+linr_0.guest.component_setting(train_data=DataWarehouseChannel(name="motor_hetero_guest",
+                                                               namespace="experiment"))
+linr_0.hosts[0].component_setting(train_data=DataWarehouseChannel(name="motor_hetero_host",
+                                                                  namespace="experiment"))
+
+evaluation_0 = Evaluation("evaluation_0",
+                          runtime_roles="guest",
+                          input_data=linr_0.outputs["train_output_data"])
+
+# pipeline.add_task(feature_scale_0)
+# pipeline.add_task(feature_scale_1)
+pipeline.add_task(linr_0)
+pipeline.add_task(evaluation_0)
+# pipeline.add_task(hetero_feature_binning_0)
+pipeline.compile()
+print(pipeline.get_dag())
+pipeline.fit()
+
+# print(pipeline.get_task_info("statistics_0").get_output_model())
+print(pipeline.get_task_info("linr_0").get_output_model())
+print(pipeline.get_task_info("linr_0").get_output_data())
+print(pipeline.get_task_info("evaluation_0").get_output_metrics())
+
+pipeline.deploy([linr_0])
+
+predict_pipeline = FateFlowPipeline()
+
+deployed_pipeline = pipeline.get_deployed_pipeline()
+deployed_pipeline.linr_0.guest.component_setting(test_data=DataWarehouseChannel(name="breast_hetero_guest",
+                                                                                namespace="experiment"))
+deployed_pipeline.linr_0.hosts[0].component_setting(test_data=DataWarehouseChannel(name="breast_hetero_host",
+                                                                                   namespace="experiment"))
+
+predict_pipeline.add_task(deployed_pipeline)
+predict_pipeline.compile()
+# print("\n\n\n")
+# print(predict_pipeline.compile().get_dag())
+predict_pipeline.predict()

examples/pipeline/test_single_lr.py

@@ -19,31 +19,29 @@
 
 pipeline = FateFlowPipeline().set_roles(guest="9999", host="9998", arbiter="9998")
 
-"""feature_scale_0 = FeatureScale(name="feature_scale_0",
-                               method="min_max",
-                               train_data=intersection_0.outputs["output_data"])
-
-feature_scale_1 = FeatureScale(name="feature_scale_1",
-                               test_data=intersection_1.outputs["output_data"],
-                               input_model=feature_scale_0.outputs["output_model"])"""
-
 lr_0 = CoordinatedLR("lr_0",
-                     max_iter=10,
-                     batch_size=-1,
-                     init_param={"fit_intercept": False})
+                     epochs=10,
+                     batch_size=100,
+                     optimizer={"method": "sgd", "optimizer_params": {"lr": 0.1}, "alpha": 0.5},
+                     init_param={"fit_intercept": True})
+lr_1 = CoordinatedLR("lr_1", input_model=lr_0.outputs["output_model"],
+                     test_data=DataWarehouseChannel(name="breast_hetero_guest",
+                                                    namespace="experiment_64")
+                     )
 
 lr_0.guest.component_setting(train_data=DataWarehouseChannel(name="breast_hetero_guest",
-                                                             namespace="experiment"))
-lr_0.hosts[0].component_setting(train_data=DataWarehouseChannel(name="breast_hetero_host",
-                                                                namespace="experiment"))
+                                                             namespace="experiment_64"))
+lr_0.hosts[0].component_setting(train_data=DataWarehouseChannel(name="breast_hetero_guest",
+                                                                namespace="experiment_64"))
 
 evaluation_0 = Evaluation("evaluation_0",
-                          runtime_roles="guest",
+                          runtime_roles=["guest"],
                           input_data=lr_0.outputs["train_output_data"])
 
 # pipeline.add_task(feature_scale_0)
 # pipeline.add_task(feature_scale_1)
 pipeline.add_task(lr_0)
+pipeline.add_task(lr_1)
 pipeline.add_task(evaluation_0)
 # pipeline.add_task(hetero_feature_binning_0)
 pipeline.compile()
@@ -52,18 +50,19 @@
 
 # print(pipeline.get_task_info("statistics_0").get_output_model())
 print(pipeline.get_task_info("lr_0").get_output_model())
-print(pipeline.get_task_info("lr_0").get_output_data())
-print(pipeline.get_task_info("evaluation_0").get_output_metrics())
+print(pipeline.get_task_info("lr_0").get_output_metric())
+print(f"evaluation metric: ")
+print(pipeline.get_task_info("evaluation_0").get_output_metric())
 
 pipeline.deploy([lr_0])
 
 predict_pipeline = FateFlowPipeline()
 
 deployed_pipeline = pipeline.get_deployed_pipeline()
-lr_0.guest.component_setting(test_data=DataWarehouseChannel(name="breast_hetero_guest",
-                                                            namespace="experiment"))
-lr_0.hosts[0].component_setting(test_data=DataWarehouseChannel(name="breast_hetero_host",
-                                                               namespace="experiment"))
+deployed_pipeline.lr_0.guest.component_setting(test_data=DataWarehouseChannel(name="breast_hetero_guest_data",
+                                                                              namespace="experiment"))
+deployed_pipeline.lr_0.hosts[0].component_setting(test_data=DataWarehouseChannel(name="breast_hetero_guest_data",
+                                                                                 namespace="experiment"))
 
 predict_pipeline.add_task(deployed_pipeline)
 predict_pipeline.compile()

examples/pipeline/test_single_lr_multi.py

@@ -0,0 +1,73 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from fate_client.pipeline import FateFlowPipeline
+from fate_client.pipeline.components.fate import CoordinatedLR
+from fate_client.pipeline.components.fate import Evaluation
+from fate_client.pipeline.interface import DataWarehouseChannel
+
+pipeline = FateFlowPipeline().set_roles(guest="9999", host="9998", arbiter="9998")
+
+"""feature_scale_0 = FeatureScale(name="feature_scale_0",
+                               method="min_max",
+                               train_data=intersection_0.outputs["output_data"])
+
+feature_scale_1 = FeatureScale(name="feature_scale_1",
+                               test_data=intersection_1.outputs["output_data"],
+                               input_model=feature_scale_0.outputs["output_model"])"""
+
+lr_0 = CoordinatedLR("lr_0",
+                     epochs=10,
+                     batch_size=-1,
+                     init_param={"fit_intercept": False})
+
+lr_0.guest.component_setting(train_data=DataWarehouseChannel(name="vehicle_scale_hetero_guest",
+                                                             namespace="experiment_64"))
+lr_0.hosts[0].component_setting(train_data=DataWarehouseChannel(name="vehicle_scale_hetero_guest",
+                                                                namespace="experiment_64"))
+
+evaluation_0 = Evaluation("evaluation_0",
+                          default_eval_metrics="multi",
+                          runtime_roles=["guest"],
+                          input_data=lr_0.outputs["train_output_data"])
+
+# pipeline.add_task(feature_scale_0)
+# pipeline.add_task(feature_scale_1)
+pipeline.add_task(lr_0)
+pipeline.add_task(evaluation_0)
+# pipeline.add_task(hetero_feature_binning_0)
+pipeline.compile()
+print(pipeline.get_dag())
+pipeline.fit()
+
+# print(pipeline.get_task_info("statistics_0").get_output_model())
+print(pipeline.get_task_info("lr_0").get_output_model())
+print(pipeline.get_task_info("lr_0").get_output_data())
+print(pipeline.get_task_info("evaluation_0").get_output_metrics())
+
+pipeline.deploy([lr_0])
+
+predict_pipeline = FateFlowPipeline()
+
+deployed_pipeline = pipeline.get_deployed_pipeline()
+deployed_pipeline.lr_0.guest.component_setting(test_data=DataWarehouseChannel(name="breast_hetero_guest",
+                                                                              namespace="experiment_64"))
+deployed_pipeline.lr_0.hosts[0].component_setting(test_data=DataWarehouseChannel(name="breast_hetero_host",
+                                                                                 namespace="experiment_64"))
+
+predict_pipeline.add_task(deployed_pipeline)
+predict_pipeline.compile()
+# print("\n\n\n")
+# print(predict_pipeline.compile().get_dag())
+predict_pipeline.predict()

examples/pipeline/test_upload_sid.py

@@ -0,0 +1,53 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from fate_client.pipeline import FateFlowPipeline
+
+pipeline = FateFlowPipeline().set_roles(
+    local="0")
+pipeline.set_site_role("local")
+pipeline.set_site_party_id("0")
+
+meta = {'delimiter': ',',
+        'dtype': 'float64',
+        'input_format': 'dense',
+        'label_type': 'int64',
+        'label_name': 'y',
+        'match_id_name': 'id',
+        'match_id_range': 0,
+        'sample_id_name': 'sid',
+        'tag_value_delimiter': ':',
+        'tag_with_value': False,
+        'weight_type': 'float64'}
+
+pipeline.transform_local_file_to_dataframe(  # file="${abs_path_of_data_guest}",
+    meta=meta, head=True, extend_sid=False,
+    namespace="experiment",
+    name="breast_hetero_guest_sid")
+
+meta = {'delimiter': ',',
+        'dtype': 'float64',
+        'input_format': 'dense',
+        'label_type': 'int64',
+        'sample_id_name': 'sid',
+        'match_id_range': 0,
+        'match_id_name': 'id',
+        'tag_value_delimiter': ':',
+        'tag_with_value': False,
+        'weight_type': 'float64'}
+
+pipeline.transform_local_file_to_dataframe(  # file="${abs_path_of_data_guest}",
+    meta=meta, head=True, extend_sid=False,
+    namespace="experiment",
+    name="breast_hetero_host_sid")

python/fate/arch/__init__.py

@@ -14,7 +14,7 @@
 #  limitations under the License.
 #
 
-from .context import Context
+from .context import CipherKit, Context
 from .unify import URI, Backend, device
 
-__all__ = ["Backend", "device", "Context", "URI"]
+__all__ = ["Backend", "device", "Context", "URI", "CipherKit"]