Merge pull request #5305 from FederatedAI/dev-2.0.0-rc-sp3

dev 2.0.0 rc sp3

configs/default.yaml

@@ -0,0 +1,47 @@
+communicator:
+  verbose: False
+debug:
+  debug_mode: False
+  validation_mode: False
+encoder:
+  precision_bits: 16
+functions:
+  max_method: "log_reduction"
+
+  # exponential function
+  exp_iterations: 8
+
+  # reciprocal configuration
+  reciprocal_method: "NR"
+  reciprocal_nr_iters: 10
+  reciprocal_log_iters: 1
+  reciprocal_all_pos: False
+  reciprocal_initial: null
+
+  # sqrt configuration
+  sqrt_nr_iters: 3
+  sqrt_nr_initial: null
+
+  # sigmoid / tanh configuration
+  sigmoid_tanh_method: "reciprocal"
+  sigmoid_tanh_terms: 32
+
+  # log configuration
+  log_iterations: 2
+  log_exp_iterations: 8
+  log_order: 8
+
+  # trigonometry configuration
+  trig_iterations: 10
+
+  # error function configuration:
+  erf_iterations: 8
+mpc:
+  active_security: False
+  provider: "TFP"
+  protocol: "beaver"
+nn:
+  dpsmpc:
+    protocol: "layer_estimation"
+    skip_loss_forward: True
+    cache_pred_size: True

examples/pipeline/feature_correlation/feature_correlation_testsuite.yaml

@@ -0,0 +1,38 @@
+data:
+  - file: examples/data/breast_hetero_guest.csv
+    meta:
+      delimiter: ","
+      dtype: float64
+      input_format: dense
+      label_type: int64
+      label_name: y
+      match_id_name: id
+      match_id_range: 0
+      tag_value_delimiter: ":"
+      tag_with_value: false
+      weight_type: float64
+    partitions: 4
+    head: true
+    extend_sid: true
+    table_name: breast_hetero_guest
+    namespace: experiment
+    role: guest_0
+  - file: examples/data/breast_hetero_host.csv
+    meta:
+      delimiter: ","
+      dtype: float64
+      input_format: dense
+      match_id_name: id
+      match_id_range: 0
+      tag_value_delimiter: ":"
+      tag_with_value: false
+      weight_type: float64
+    partitions: 4
+    head: true
+    extend_sid: true
+    table_name: breast_hetero_host
+    namespace: experiment
+    role: host_0
+tasks:
+  feature-correlation:
+    script: test_feature_correlation.py

examples/pipeline/feature_correlation/test_feature_correlation.py

@@ -0,0 +1,62 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import argparse
+
+from fate_client.pipeline import FateFlowPipeline
+from fate_client.pipeline.components.fate import PSI, FeatureCorrelation
+from fate_client.pipeline.interface import DataWarehouseChannel
+from fate_client.pipeline.utils import test_utils
+
+
+def main(config="../config.yaml", namespace=""):
+    if isinstance(config, str):
+        config = test_utils.load_job_config(config)
+    parties = config.parties
+    guest = parties.guest[0]
+    host = parties.host[0]
+
+    pipeline = FateFlowPipeline().set_parties(guest=guest, host=host)
+    if config.task_cores:
+        pipeline.conf.set("task_cores", config.task_cores)
+    if config.timeout:
+        pipeline.conf.set("timeout", config.timeout)
+
+    psi_0 = PSI("psi_0")
+    psi_0.guest.task_setting(input_data=DataWarehouseChannel(name="breast_hetero_guest",
+                                                             namespace=f"experiment{namespace}"))
+    psi_0.hosts[0].task_setting(input_data=DataWarehouseChannel(name="breast_hetero_host",
+                                                                namespace=f"experiment{namespace}"))
+
+    feature_corr_0 = FeatureCorrelation("feature_corr_0",
+                                        input_data=psi_0.outputs["output_data"])
+
+    pipeline.add_task(psi_0)
+    pipeline.add_task(feature_corr_0)
+
+    pipeline.compile()
+    print(pipeline.get_dag())
+    pipeline.fit()
+
+    print(pipeline.get_task_info("feature_corr_0").get_output_model())
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("PIPELINE DEMO")
+    parser.add_argument("--config", type=str, default="../config.yaml",
+                        help="config file")
+    parser.add_argument("--namespace", type=str, default="",
+                        help="namespace for data stored in FATE")
+    args = parser.parse_args()
+    main(config=args.config, namespace=args.namespace)

java/osx/osx-broker/src/main/java/org/fedai/osx/broker/eggroll/ErJob.java

@@ -27,13 +27,13 @@ public class ErJob extends BaseProto<Meta.Job> {
 
     String id;
     String name;
-    List<ErStore> inputs;
-    List<ErStore> outputs;
+    List<ErJobIO> inputs;
+    List<ErJobIO> outputs;
     List<ErFunctor> functors;
     Map<String, String> options;
 
-    public ErJob(String id, String name, List<ErStore> inputs,
-                 List<ErStore> outputs,
+    public ErJob(String id, String name, List<ErJobIO> inputs,
+                 List<ErJobIO> outputs,
                  List<ErFunctor> functors,
                  Map<String, String> options) {
         this.id = id;
@@ -51,15 +51,15 @@ public static ErJob parseFromPb(Meta.Job job) {
         String id = job.getId();
         String name = job.getName();
         Map<String, String> options = job.getOptionsMap();
-        List<Meta.Store> inputMeta = job.getInputsList();
-        List<ErStore> input = Lists.newArrayList();
+        List<Meta.JobIO> inputMeta = job.getInputsList();
+        List<ErJobIO> input = Lists.newArrayList();
         if (inputMeta != null) {
-            input = inputMeta.stream().map(ErStore::parseFromPb).collect(Collectors.toList());
+            input = inputMeta.stream().map(ErJobIO::parseFromPb).collect(Collectors.toList());
         }
-        List<Meta.Store> outputMeta = job.getOutputsList();
-        List<ErStore> output = Lists.newArrayList();
+        List<Meta.JobIO> outputMeta = job.getOutputsList();
+        List<ErJobIO> output = Lists.newArrayList();
         if (output != null) {
-            output = outputMeta.stream().map(ErStore::parseFromPb).collect(Collectors.toList());
+            output = outputMeta.stream().map(ErJobIO::parseFromPb).collect(Collectors.toList());
         }
         List<ErFunctor> functors = Lists.newArrayList();
         List<Meta.Functor> functorMeta = job.getFunctorsList();
@@ -90,19 +90,19 @@ public void setName(String name) {
         this.name = name;
     }
 
-    public List<ErStore> getInputs() {
+    public List<ErJobIO> getInputs() {
         return inputs;
     }
 
-    public void setInputs(List<ErStore> inputs) {
+    public void setInputs(List<ErJobIO> inputs) {
         this.inputs = inputs;
     }
 
-    public List<ErStore> getOutputs() {
+    public List<ErJobIO> getOutputs() {
         return outputs;
     }
 
-    public void setOutputs(List<ErStore> outputs) {
+    public void setOutputs(List<ErJobIO> outputs) {
         this.outputs = outputs;
     }
 
@@ -124,18 +124,12 @@ public void setOptions(Map<String, String> options) {
 
     @Override
     Meta.Job toProto() {
-
-        return Meta.Job.newBuilder().setId(id).setName(name)
+        return Meta.Job.newBuilder()
+                .setId(id)
+                .setName(name)
                 .addAllFunctors(this.functors.stream().map(ErFunctor::toProto).collect(Collectors.toList())).
-                addAllInputs(inputs.stream().map(ErStore::toProto).collect(Collectors.toList())).putAllOptions(options).build();
-
+                addAllInputs(inputs.stream().map(ErJobIO::toProto).collect(Collectors.toList()))
+                .addAllOutputs(outputs.stream().map(ErJobIO::toProto).collect(Collectors.toList()))
+                .putAllOptions(options).build();
     }
-}
-
-
-//case class ErJob(id: String,
-//                 name: String = StringConstants.EMPTY,
-//                 inputs: Array[ErStore],
-//                 outputs: Array[ErStore] = Array(),
-//        functors: Array[ErFunctor],
-//        options: Map[String, String] = Map[String, String]())
+}

java/osx/osx-broker/src/main/java/org/fedai/osx/broker/eggroll/ErJobIO.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 The FATE Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.fedai.osx.broker.eggroll;
+
+import com.webank.eggroll.core.meta.Meta;
+
+public class ErJobIO extends BaseProto<Meta.JobIO> {
+
+    ErStore erStore;
+    ErSerdes keySerdes;
+    ErSerdes valueSerdes;
+    ErPartitioner partitioner;
+
+    public ErJobIO(ErStore erStore, ErSerdes keySerdes, ErSerdes valueSerdes, ErPartitioner partitioner) {
+        this.erStore = erStore;
+        this.keySerdes = keySerdes;
+        this.valueSerdes = valueSerdes;
+        this.partitioner = partitioner;
+    }
+
+    public static ErJobIO parseFromPb(Meta.JobIO jobIO) {
+        ErStore erStore = ErStore.parseFromPb(jobIO.getStore());
+        ErSerdes key_serdes = ErSerdes.parseFromPb(jobIO.getKeySerdes());
+        ErSerdes value_serdes = ErSerdes.parseFromPb(jobIO.getValueSerdes());
+        ErPartitioner partitioner = ErPartitioner.parseFromPb(jobIO.getPartitioner());
+        ErJobIO erJobIO = new ErJobIO(erStore, key_serdes, value_serdes, partitioner);
+        return erJobIO;
+    }
+
+    public ErSerdes getKeySerdes() {
+        return keySerdes;
+    }
+
+    public void setKeySerdes(ErSerdes keySerdes) {
+        this.keySerdes = keySerdes;
+    }
+
+    public ErSerdes getValueSerdes() {
+        return valueSerdes;
+    }
+
+    public void setValueSerdes(ErSerdes valueSerdes) {
+        this.valueSerdes = valueSerdes;
+    }
+
+    public ErPartitioner getPartitioner() {
+        return partitioner;
+    }
+
+    public void setPartitioner(ErPartitioner partitioner) {
+        this.partitioner = partitioner;
+    }
+
+    public ErStore getErStore() {
+        return erStore;
+    }
+
+    public void setErStore(ErStore erStore) {
+        this.erStore = erStore;
+    }
+
+    @Override
+    Meta.JobIO toProto() {
+        Meta.JobIO.Builder builder = Meta.JobIO.newBuilder();
+        builder.setStore(erStore.toProto())
+                .setKeySerdes(keySerdes.toProto())
+                .setValueSerdes(valueSerdes.toProto())
+                .setPartitioner(partitioner.toProto());
+        return builder.build();
+    }
+}

java/osx/osx-broker/src/main/java/org/fedai/osx/broker/eggroll/ErPartitioner.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 The FATE Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.fedai.osx.broker.eggroll;
+
+import com.google.protobuf.ByteString;
+import com.webank.eggroll.core.meta.Meta;
+
+public class ErPartitioner extends BaseProto<Meta.Partitioner> {
+
+    Integer t;
+    ByteString body;
+
+    public ErPartitioner(int t, ByteString body) {
+        this.t = t;
+        this.body = body;
+    }
+
+    public ErPartitioner(int t) {
+        this(t, ByteString.EMPTY);
+    }
+
+    public static ErPartitioner parseFromPb(Meta.Partitioner partitioner) {
+        int t = partitioner.getType();
+        ByteString body = partitioner.getBody();
+        return new ErPartitioner(t, body);
+    }
+
+    public int getT() {
+        return t;
+    }
+
+    public void setT(Integer t) {
+        this.t = t;
+    }
+
+    public ByteString getBody() {
+        return body;
+    }
+
+    public void setBody(ByteString body) {
+        this.body = body;
+    }
+
+
+    @Override
+    Meta.Partitioner toProto() {
+        Meta.Partitioner.Builder builder = Meta.Partitioner.newBuilder()
+                .setType(this.t)
+                .setBody(this.body);
+        return builder.build();
+    }
+}