Merge pull request #152 from FederatedAI/dev-2.1.0

Update 2.1.0
FederatedAI · Mar 6, 2024 · 54dbeef · 54dbeef
2 parents 5383f59 + ed44037
commit 54dbeef
Show file tree

Hide file tree

Showing 37 changed files with 339 additions and 166 deletions.
diff --git a/RELEASE.md b/RELEASE.md
@@ -1,3 +1,10 @@
+## Release 2.1.0
+### Major improvements
+* Pipeline: add supports for fate-llm 2.0
+  * newly added LLMModelLoader, LLMDatasetLoader, LLMDataFuncLoader
+  * newly added configuration parsing of seq2seq_runner and ot_runner
+* Pipeline: unified input interface of components
+
 ## Release 2.0.0
 ### Feature Highlights
 > FATE-Client 2.0: Building Scalable Federated DSL for Application Layer Interconnection

diff --git a/doc/pipeline.md b/doc/pipeline.md
@@ -100,11 +100,11 @@ Below lists data input and output of all components:
 | Coordinated-LR           | CoordinatedLR          | train_data, validate_data, test_data, cv_data | train_output_data, validate_output_data, test_output_data, cv_output_datas |
 | Coordinated-LinR         | CoordinatedLinR        | train_data, validate_data, test_data, cv_data | train_output_data, validate_output_data, test_output_data, cv_output_datas |
 | Homo-LR                  | HomoLR                 | train_data, validate_data, test_data, cv_data | train_output_data, validate_output_data, test_output_data, cv_output_datas |
-| Homo-NN                  | HomoNN                 | train_data, validate_data, test_data | train_data_output, predict_data_output|
-| Hetero-NN   | HeteroNN    | train_data, validate_data, test_data | train_data_output, predict_data_output|
-| Hetero Secure Boosting   | HeteroSecureBoost      | train_data, validate_data, test_data, cv_data | train_data_output, test_output_data, cv_output_datas |
-| Evaluation               | Evaluation             | input_data                                    |                                                                            |
-| Union                    | Union                  | input_data_list                               | output_data                                                                |
+| Homo-NN                  | HomoNN                 | train_data, validate_data, test_data          | train_output_data, test_output_data                                        |
+| Hetero-NN                | HeteroNN               | train_data, validate_data, test_data          | train_output_data, test_output_data                                        |
+| Hetero Secure Boosting   | HeteroSecureBoost      | train_data, validate_data, test_data, cv_data | train_output_data, test_output_data, cv_output_datas                       |
+| Evaluation               | Evaluation             | input_datas                                   |                                                                            |
+| Union                    | Union                  | input_datas                                   | output_data                                                                |
 
 ### Model
 
@@ -114,23 +114,23 @@ Model training components also may take `warm_start_model`, but note that only o
 
 Below lists model input and output of all components:
 
-| Algorithm                | Component Name         | Model Input                   | Model Output |
-|--------------------------|------------------------|-------------------------------|--------------|
-| PSI                      | PSI                    |                               |              |
-| Sampling                 | Sample                 |                               |              |
-| Data Split               | DataSplit              |                               |              |
-| Feature Scale            | FeatureScale           | input_model                   | output_model |
-| Data Statistics          | Statistics             |                               | output_model |
-| Hetero Feature Binning   | HeteroFeatureBinning   | input_model                   | output_model |
-| Hetero Feature Selection | HeteroFeatureSelection | input_models, input_model     | output_model |
-| Coordinated-LR           | CoordinatedLR          | input_model, warm_start_model | output_model |
-| Coordinated-LinR         | CoordinatedLinR        | input_model, warm_start_model | output_model |
-| Homo-LR                  | HomoLR                 | input_model, warm_start_model | output_model |
-| Homo-NN                  | HomoNN                 | train_model_input, predict_model_input, train_model_output | train_model_output |
-| Hetero-NN|HeteroNN|train_model_input, predict_model_input, train_model_output| train_model_output|
-| Hetero Secure Boosting   | HeteroSecureBoost      | train_model_input, predict_model_input, train_model_output | train_model_output |
-| Evaluation               | Evaluation             |                               |              |
-| Union                    | Union                  |                               |              |
+| Algorithm                | Component Name         | Model Input                    | Model Output |
+|--------------------------|------------------------|--------------------------------|--------------|
+| PSI                      | PSI                    |                                |              |
+| Sampling                 | Sample                 |                                |              |
+| Data Split               | DataSplit              |                                |              |
+| Feature Scale            | FeatureScale           | input_model                    | output_model |
+| Data Statistics          | Statistics             |                                | output_model |
+| Hetero Feature Binning   | HeteroFeatureBinning   | input_model                    | output_model |
+| Hetero Feature Selection | HeteroFeatureSelection | input_models, input_model      | output_model |
+| Coordinated-LR           | CoordinatedLR          | input_model, warm_start_model  | output_model |
+| Coordinated-LinR         | CoordinatedLinR        | input_model, warm_start_model  | output_model |
+| Homo-LR                  | HomoLR                 | input_model, warm_start_model  | output_model |
+| Homo-NN                  | HomoNN                 | input_model, warm_start_model  | output_model |
+| Hetero-NN                | HeteroNN               | input_model, warm_start_model  | output_model |
+| Hetero Secure Boosting   | HeteroSecureBoost      | input_model, warm_start_model  | output_model |
+| Evaluation               | Evaluation             |                                |              |
+| Union                    | Union                  |                                |              |
 
 ## Build A Pipeline
 

diff --git a/python/MANIFEST.in b/python/MANIFEST.in
@@ -1,3 +1,4 @@
 include fate_client/pipeline/component_define/fate/*.yaml
 include fate_client/pipeline/*.yaml
 include fate_client/*.yaml
+include fate_client/pipeline/components/fate/nn/*.yaml
diff --git a/python/fate_client/__init__.py b/python/fate_client/__init__.py
@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "2.0.0"
+__version__ = "2.1.0"
diff --git a/python/fate_client/flow_sdk/api/data.py b/python/fate_client/flow_sdk/api/data.py
@@ -41,8 +41,8 @@ def upload(self, file: str, head: bool, partitions: int, meta: dict, namespace:
             {'code': 0, 'message': 'success','data':{...}]}
         """
         kwargs = locals()
-        if not os.path.exists(file):
-            raise Exception(f"{file} is not exist, please check the file path")
+        # if not os.path.exists(file):
+        #     raise Exception(f"{file} is not exist, please check the file path")
         params = filter_invalid_params(**kwargs)
         return self._post(url='/data/component/upload', json=params)
 

diff --git a/python/fate_client/pipeline/component_define/fate/coordinated_linr.yaml b/python/fate_client/pipeline/component_define/fate/coordinated_linr.yaml
@@ -2,8 +2,8 @@ component:
   name: coordinated_linr
   description: ''
   provider: fate
-  version: 2.0.0
-  labels: [ ]
+  version: 2.1.0
+  labels: []
   roles:
     - guest
     - host
@@ -373,8 +373,9 @@ component:
         types:
           - json_metric
         optional: false
-        stages: [ ]
-        roles: [ ]
+        stages: []
+        roles: []
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+
diff --git a/python/fate_client/pipeline/component_define/fate/coordinated_lr.yaml b/python/fate_client/pipeline/component_define/fate/coordinated_lr.yaml
@@ -2,8 +2,8 @@ component:
   name: coordinated_lr
   description: ''
   provider: fate
-  version: 2.0.0
-  labels: [ ]
+  version: 2.1.0
+  labels: []
   roles:
     - guest
     - host
@@ -383,8 +383,9 @@ component:
         types:
           - json_metric
         optional: false
-        stages: [ ]
-        roles: [ ]
+        stages: []
+        roles: []
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+
diff --git a/python/fate_client/pipeline/component_define/fate/data_split.yaml b/python/fate_client/pipeline/component_define/fate/data_split.yaml
@@ -2,8 +2,8 @@ component:
   name: data_split
   description: ''
   provider: fate
-  version: 2.0.0
-  labels: [ ]
+  version: 2.1.0
+  labels: []
   roles:
     - guest
     - host
@@ -111,7 +111,7 @@ component:
           - host
         description: ''
         is_multi: false
-    model: { }
+    model: {}
   output_artifacts:
     data:
       train_output_data:
@@ -147,7 +147,7 @@ component:
           - host
         description: ''
         is_multi: false
-    model: { }
+    model: {}
     metric:
       metric:
         types:
@@ -161,3 +161,4 @@ component:
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+
diff --git a/python/fate_client/pipeline/component_define/fate/dataframe_transformer.yaml b/python/fate_client/pipeline/component_define/fate/dataframe_transformer.yaml
@@ -2,7 +2,7 @@ component:
   name: dataframe_transformer
   description: ''
   provider: fate
-  version: 2.0.0
+  version: 2.1.0
   labels: []
   roles:
     - local

diff --git a/python/fate_client/pipeline/component_define/fate/evaluation.yaml b/python/fate_client/pipeline/component_define/fate/evaluation.yaml
@@ -2,7 +2,7 @@ component:
   name: evaluation
   description: ''
   provider: fate
-  version: 2.0.0
+  version: 2.1.0
   labels: []
   roles:
     - guest
@@ -54,7 +54,7 @@ component:
           use 'label' in the input dataframe
   input_artifacts:
     data:
-      input_data:
+      input_datas:
         types:
           - dataframe
         optional: false
@@ -82,3 +82,4 @@ component:
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+
diff --git a/python/fate_client/pipeline/component_define/fate/feature_correlation.yaml b/python/fate_client/pipeline/component_define/fate/feature_correlation.yaml
@@ -2,8 +2,8 @@ component:
   name: feature_correlation
   description: ''
   provider: fate
-  version: 2.0.0
-  labels: [ ]
+  version: 2.1.0
+  labels: []
   roles:
     - guest
     - host
@@ -65,9 +65,9 @@ component:
           - host
         description: ''
         is_multi: false
-    model: { }
+    model: {}
   output_artifacts:
-    data: { }
+    data: {}
     model:
       output_model:
         types:
@@ -93,3 +93,4 @@ component:
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+
diff --git a/python/fate_client/pipeline/component_define/fate/feature_scale.yaml b/python/fate_client/pipeline/component_define/fate/feature_scale.yaml
@@ -2,7 +2,7 @@ component:
   name: feature_scale
   description: ''
   provider: fate
-  version: 2.0.0
+  version: 2.1.0
   labels: []
   roles:
     - guest
@@ -29,7 +29,7 @@ component:
         title: typing.Union[list, dict]
         anyOf:
           - type: array
-            items: { }
+            items: {}
           - type: object
         default:
           - 0
@@ -164,8 +164,9 @@ component:
         types:
           - json_metric
         optional: false
-        stages: [ ]
-        roles: [ ]
+        stages: []
+        roles: []
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+
diff --git a/python/fate_client/pipeline/component_define/fate/hetero_feature_binning.yaml b/python/fate_client/pipeline/component_define/fate/hetero_feature_binning.yaml
@@ -2,8 +2,8 @@ component:
   name: hetero_feature_binning
   description: ''
   provider: fate
-  version: 2.0.0
-  labels: [ ]
+  version: 2.1.0
+  labels: []
   roles:
     - guest
     - host
@@ -266,8 +266,9 @@ component:
         types:
           - json_metric
         optional: false
-        stages: [ ]
-        roles: [ ]
+        stages: []
+        roles: []
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+
diff --git a/python/fate_client/pipeline/component_define/fate/hetero_feature_selection.yaml b/python/fate_client/pipeline/component_define/fate/hetero_feature_selection.yaml
@@ -2,8 +2,8 @@ component:
   name: hetero_feature_selection
   description: ''
   provider: fate
-  version: 2.0.0
-  labels: [ ]
+  version: 2.1.0
+  labels: []
   roles:
     - guest
     - host
@@ -178,8 +178,8 @@ component:
     manual_param:
       type: ModelMetaclass
       default:
-        keep_col: [ ]
-        filter_out_col: [ ]
+        keep_col: []
+        filter_out_col: []
       optional: true
       description: manual filter param
       type_meta:
@@ -188,13 +188,13 @@ component:
         properties:
           keep_col:
             title: Keep Col
-            default: [ ]
+            default: []
             type: array
             items:
               type: string
           filter_out_col:
             title: Filter Out Col
-            default: [ ]
+            default: []
             type: array
             items:
               type: string
@@ -309,8 +309,9 @@ component:
         types:
           - json_metric
         optional: false
-        stages: [ ]
-        roles: [ ]
+        stages: []
+        roles: []
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+
diff --git a/python/fate_client/pipeline/component_define/fate/hetero_nn.yaml b/python/fate_client/pipeline/component_define/fate/hetero_nn.yaml
@@ -2,7 +2,7 @@ component:
   name: hetero_nn
   description: ''
   provider: fate
-  version: 2.0.0
+  version: 2.1.0
   labels: []
   roles:
     - guest
@@ -87,7 +87,7 @@ component:
         description: ''
         is_multi: false
     model:
-      train_model_input:
+      warm_start_model:
         types:
           - model_directory
         optional: true
@@ -98,7 +98,7 @@ component:
           - host
         description: ''
         is_multi: false
-      predict_model_input:
+      input_model:
         types:
           - model_directory
         optional: false
@@ -111,7 +111,7 @@ component:
         is_multi: false
   output_artifacts:
     data:
-      train_data_output:
+      train_output_data:
         types:
           - dataframe
         optional: true
@@ -122,7 +122,7 @@ component:
           - host
         description: ''
         is_multi: false
-      predict_data_output:
+      test_output_data:
         types:
           - dataframe
         optional: true
@@ -134,7 +134,7 @@ component:
         description: ''
         is_multi: false
     model:
-      train_model_output:
+      output_model:
         types:
           - model_directory
         optional: true
@@ -155,3 +155,4 @@ component:
         description: metric, invisible for user
         is_multi: false
 schema_version: v1
+