feat: Added Project object to Feast Objects (#4475)

* feat: Added Project object to Feast Objects

Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com>

Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com>

* fix: Extend FeastError and fixed integration tests

Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com>

* fix: Small optimization to test_modify_feature_views_success test

Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com>

* fix: Added Project object to template and quick start

Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com>

---------

Signed-off-by: Bhargav Dodla <bdodla@expediagroup.com>
Co-authored-by: Bhargav Dodla <bdodla@expediagroup.com>

Author: EXPEbdodla

docs/getting-started/quickstart.md

@@ -103,12 +103,17 @@ from feast import (
     FeatureView,
     Field,
     FileSource,
+    Project,
     PushSource,
     RequestSource,
 )
 from feast.on_demand_feature_view import on_demand_feature_view
 from feast.types import Float32, Float64, Int64
 
+# Define a project for the feature repo
+project = Project(name="my_project", description="A project for driver statistics")
+
+
 # Define an entity for the driver. You can think of an entity as a primary key used to
 # fetch features.
 driver = Entity(name="driver", join_keys=["driver_id"])

protos/feast/core/Permission.proto

@@ -45,6 +45,7 @@ message PermissionSpec {
     VALIDATION_REFERENCE = 7;
     SAVED_DATASET = 8;
     PERMISSION = 9;
+    PROJECT = 10;
   }
 
   repeated Type types = 3;

protos/feast/core/Project.proto

@@ -0,0 +1,52 @@
+//
+// * Copyright 2020 The Feast Authors
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// *     https://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+//
+
+syntax = "proto3";
+
+package feast.core;
+option java_package = "feast.proto.core";
+option java_outer_classname = "ProjectProto";
+option go_package = "github.com/feast-dev/feast/go/protos/feast/core";
+
+import "google/protobuf/timestamp.proto";
+
+message Project {
+    // User-specified specifications of this entity.
+    ProjectSpec spec = 1;
+    // System-populated metadata for this entity.
+    ProjectMeta meta = 2;
+}
+
+message ProjectSpec {
+    // Name of the Project
+    string name = 1;
+
+    // Description of the Project
+    string description = 2;
+
+    // User defined metadata
+    map<string,string> tags = 3;
+
+    // Owner of the Project
+    string owner = 4;
+}
+
+message ProjectMeta {
+    // Time when the Project is created
+    google.protobuf.Timestamp created_timestamp = 1;
+    // Time when the Project is last updated with registry changes (Apply stage)
+    google.protobuf.Timestamp last_updated_timestamp = 2;
+}

protos/feast/core/Registry.proto

@@ -33,8 +33,9 @@ import "feast/core/SavedDataset.proto";
 import "feast/core/ValidationProfile.proto";
 import "google/protobuf/timestamp.proto";
 import "feast/core/Permission.proto";
+import "feast/core/Project.proto";
 
-// Next id: 17
+// Next id: 18
 message Registry {
     repeated Entity entities = 1;
     repeated FeatureTable feature_tables = 2;
@@ -47,12 +48,13 @@ message Registry {
     repeated ValidationReference validation_references = 13;
     Infra infra = 10;
     // Tracking metadata of Feast by project
-    repeated ProjectMetadata project_metadata = 15;
+    repeated ProjectMetadata project_metadata = 15 [deprecated = true];
 
     string registry_schema_version = 3; // to support migrations; incremented when schema is changed
     string version_id = 4; // version id, random string generated on each update of the data; now used only for debugging purposes
     google.protobuf.Timestamp last_updated = 5;
     repeated Permission permissions = 16;
+    repeated Project projects = 17;
 }
 
 message ProjectMetadata {

protos/feast/registry/RegistryServer.proto

@@ -15,6 +15,7 @@ import "feast/core/SavedDataset.proto";
 import "feast/core/ValidationProfile.proto";
 import "feast/core/InfraObject.proto";
 import "feast/core/Permission.proto";
+import "feast/core/Project.proto";
 
 service RegistryServer{
   // Entity RPCs
@@ -67,6 +68,12 @@ service RegistryServer{
   rpc ListPermissions (ListPermissionsRequest) returns (ListPermissionsResponse) {}
   rpc DeletePermission (DeletePermissionRequest) returns (google.protobuf.Empty) {}
 
+  // Project RPCs
+  rpc ApplyProject (ApplyProjectRequest) returns (google.protobuf.Empty) {}
+  rpc GetProject (GetProjectRequest) returns (feast.core.Project) {}
+  rpc ListProjects (ListProjectsRequest) returns (ListProjectsResponse) {}
+  rpc DeleteProject (DeleteProjectRequest) returns (google.protobuf.Empty) {}
+
   rpc ApplyMaterialization (ApplyMaterializationRequest) returns (google.protobuf.Empty) {}
   rpc ListProjectMetadata (ListProjectMetadataRequest) returns (ListProjectMetadataResponse) {}
   rpc UpdateInfra (UpdateInfraRequest) returns (google.protobuf.Empty) {}
@@ -356,3 +363,29 @@ message DeletePermissionRequest {
   string project = 2;
   bool commit = 3;
 }
+
+// Projects
+
+message ApplyProjectRequest {
+  feast.core.Project project = 1;
+  bool commit = 2;
+}
+
+message GetProjectRequest {
+  string name = 1;
+  bool allow_cache = 2;
+}
+
+message ListProjectsRequest {
+  bool allow_cache = 1;
+  map<string,string> tags = 2;
+}
+
+message ListProjectsResponse {
+  repeated feast.core.Project projects = 1;
+}
+
+message DeleteProjectRequest {
+  string name = 1;
+  bool commit = 2;
+}

sdk/python/feast/__init__.py

@@ -18,6 +18,7 @@
 from .feature_view import FeatureView
 from .field import Field
 from .on_demand_feature_view import OnDemandFeatureView
+from .project import Project
 from .repo_config import RepoConfig
 from .stream_feature_view import StreamFeatureView
 from .value_type import ValueType
@@ -49,4 +50,5 @@
     "PushSource",
     "RequestSource",
     "AthenaSource",
+    "Project",
 ]

sdk/python/feast/cli.py

@@ -254,6 +254,79 @@ def data_source_list(ctx: click.Context, tags: list[str]):
     print(tabulate(table, headers=["NAME", "CLASS"], tablefmt="plain"))
 
 
+@cli.group(name="projects")
+def projects_cmd():
+    """
+    Access projects
+    """
+    pass
+
+
+@projects_cmd.command("describe")
+@click.argument("name", type=click.STRING)
+@click.pass_context
+def project_describe(ctx: click.Context, name: str):
+    """
+    Describe a project
+    """
+    store = create_feature_store(ctx)
+
+    try:
+        project = store.get_project(name)
+    except FeastObjectNotFoundException as e:
+        print(e)
+        exit(1)
+
+    print(
+        yaml.dump(
+            yaml.safe_load(str(project)), default_flow_style=False, sort_keys=False
+        )
+    )
+
+
+@projects_cmd.command("current_project")
+@click.pass_context
+def project_current(ctx: click.Context):
+    """
+    Returns the current project configured with FeatureStore object
+    """
+    store = create_feature_store(ctx)
+
+    try:
+        project = store.get_project(name=None)
+    except FeastObjectNotFoundException as e:
+        print(e)
+        exit(1)
+
+    print(
+        yaml.dump(
+            yaml.safe_load(str(project)), default_flow_style=False, sort_keys=False
+        )
+    )
+
+
+@projects_cmd.command(name="list")
+@tagsOption
+@click.pass_context
+def project_list(ctx: click.Context, tags: list[str]):
+    """
+    List all projects
+    """
+    store = create_feature_store(ctx)
+    table = []
+    tags_filter = utils.tags_list_to_dict(tags)
+    for project in store.list_projects(tags=tags_filter):
+        table.append([project.name, project.description, project.tags, project.owner])
+
+    from tabulate import tabulate
+
+    print(
+        tabulate(
+            table, headers=["NAME", "DESCRIPTION", "TAGS", "OWNER"], tablefmt="plain"
+        )
+    )
+
+
 @cli.group(name="entities")
 def entities_cmd():
     """

sdk/python/feast/diff/registry_diff.py

@@ -11,6 +11,7 @@
 from feast.infra.registry.base_registry import BaseRegistry
 from feast.infra.registry.registry import FEAST_OBJECT_TYPES, FeastObjectType
 from feast.permissions.permission import Permission
+from feast.project import Project
 from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
 from feast.protos.feast.core.Entity_pb2 import Entity as EntityProto
 from feast.protos.feast.core.FeatureService_pb2 import (
@@ -371,6 +372,11 @@ def apply_diff_to_registry(
             TransitionType.CREATE,
             TransitionType.UPDATE,
         ]:
+            if feast_object_diff.feast_object_type == FeastObjectType.PROJECT:
+                registry.apply_project(
+                    cast(Project, feast_object_diff.new_feast_object),
+                    commit=False,
+                )
             if feast_object_diff.feast_object_type == FeastObjectType.DATA_SOURCE:
                 registry.apply_data_source(
                     cast(DataSource, feast_object_diff.new_feast_object),

sdk/python/feast/errors.py

@@ -480,6 +480,16 @@ def __init__(self, name, project=None):
             super().__init__(f"Permission {name} does not exist")
 
 
+class ProjectNotFoundException(FeastError):
+    def __init__(self, project):
+        super().__init__(f"Project {project} does not exist in registry")
+
+
+class ProjectObjectNotFoundException(FeastObjectNotFoundException):
+    def __init__(self, name, project=None):
+        super().__init__(f"Project {name} does not exist")
+
+
 class ZeroRowsQueryResult(FeastError):
     def __init__(self, query: str):
         super().__init__(f"This query returned zero rows:\n{query}")

sdk/python/feast/feast_object.py

@@ -1,5 +1,8 @@
 from typing import Union, get_args
 
+from feast.project import Project
+from feast.protos.feast.core.Project_pb2 import ProjectSpec
+
 from .batch_feature_view import BatchFeatureView
 from .data_source import DataSource
 from .entity import Entity
@@ -23,6 +26,7 @@
 
 # Convenience type representing all Feast objects
 FeastObject = Union[
+    Project,
     FeatureView,
     OnDemandFeatureView,
     BatchFeatureView,
@@ -36,6 +40,7 @@
 ]
 
 FeastObjectSpecProto = Union[
+    ProjectSpec,
     FeatureViewSpec,
     OnDemandFeatureViewSpec,
     StreamFeatureViewSpec,