run only cloud registries in cloud it tests

Signed-off-by: tokoko <togurg14@freeuni.edu.ge>

Author: tokoko

Makefile

@@ -86,10 +86,8 @@ test-python-integration-local:
 	@(docker info > /dev/null 2>&1 && \
 		FEAST_IS_LOCAL_TEST=True \
 		FEAST_LOCAL_ONLINE_CONTAINER=True \
-		python -m pytest -n 8 --color=yes --integration \
-			-k "not gcs_registry and \
- 				not s3_registry and \
- 				not test_lambda_materialization and \
+		python -m pytest -n 8 --durations=5 --color=yes --integration \
+			-k "not test_lambda_materialization and \
  				not test_snowflake_materialization" \
 		sdk/python/tests \
 	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";

sdk/python/tests/integration/registration/test_feature_store.py

@@ -33,46 +33,6 @@
 from tests.utils.data_source_test_creator import prep_file_source
 
 
-@pytest.mark.integration
-@pytest.mark.parametrize(
-    "test_feature_store",
-    [
-        lazy_fixture("feature_store_with_gcs_registry"),
-        lazy_fixture("feature_store_with_s3_registry"),
-    ],
-)
-def test_apply_entity_integration(test_feature_store):
-    entity = Entity(
-        name="driver_car_id",
-        description="Car driver id",
-        tags={"team": "matchmaking"},
-    )
-
-    # Register Entity
-    test_feature_store.apply([entity])
-
-    entities = test_feature_store.list_entities()
-
-    entity = entities[0]
-    assert (
-        len(entities) == 1
-        and entity.name == "driver_car_id"
-        and entity.description == "Car driver id"
-        and "team" in entity.tags
-        and entity.tags["team"] == "matchmaking"
-    )
-
-    entity = test_feature_store.get_entity("driver_car_id")
-    assert (
-        entity.name == "driver_car_id"
-        and entity.description == "Car driver id"
-        and "team" in entity.tags
-        and entity.tags["team"] == "matchmaking"
-    )
-
-    test_feature_store.teardown()
-
-
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_feature_store",
@@ -108,82 +68,6 @@ def test_feature_view_inference_success(test_feature_store, dataframe_source):
 
         test_feature_store.teardown()
 
-
-@pytest.mark.integration
-@pytest.mark.parametrize(
-    "test_feature_store",
-    [
-        lazy_fixture("feature_store_with_gcs_registry"),
-        lazy_fixture("feature_store_with_s3_registry"),
-    ],
-)
-def test_apply_feature_view_integration(test_feature_store):
-    # Create Feature Views
-    batch_source = FileSource(
-        file_format=ParquetFormat(),
-        path="file://feast/*",
-        timestamp_field="ts_col",
-        created_timestamp_column="timestamp",
-    )
-
-    entity = Entity(name="fs1_my_entity_1", join_keys=["test"])
-
-    fv1 = FeatureView(
-        name="my_feature_view_1",
-        schema=[
-            Field(name="fs1_my_feature_1", dtype=Int64),
-            Field(name="fs1_my_feature_2", dtype=String),
-            Field(name="fs1_my_feature_3", dtype=Array(String)),
-            Field(name="fs1_my_feature_4", dtype=Array(Bytes)),
-            Field(name="test", dtype=Int64),
-        ],
-        entities=[entity],
-        tags={"team": "matchmaking"},
-        source=batch_source,
-        ttl=timedelta(minutes=5),
-    )
-
-    # Register Feature View
-    test_feature_store.apply([fv1, entity])
-
-    feature_views = test_feature_store.list_feature_views()
-
-    # List Feature Views
-    assert (
-        len(feature_views) == 1
-        and feature_views[0].name == "my_feature_view_1"
-        and feature_views[0].features[0].name == "fs1_my_feature_1"
-        and feature_views[0].features[0].dtype == Int64
-        and feature_views[0].features[1].name == "fs1_my_feature_2"
-        and feature_views[0].features[1].dtype == String
-        and feature_views[0].features[2].name == "fs1_my_feature_3"
-        and feature_views[0].features[2].dtype == Array(String)
-        and feature_views[0].features[3].name == "fs1_my_feature_4"
-        and feature_views[0].features[3].dtype == Array(Bytes)
-        and feature_views[0].entities[0] == "fs1_my_entity_1"
-    )
-
-    feature_view = test_feature_store.get_feature_view("my_feature_view_1")
-    assert (
-        feature_view.name == "my_feature_view_1"
-        and feature_view.features[0].name == "fs1_my_feature_1"
-        and feature_view.features[0].dtype == Int64
-        and feature_view.features[1].name == "fs1_my_feature_2"
-        and feature_view.features[1].dtype == String
-        and feature_view.features[2].name == "fs1_my_feature_3"
-        and feature_view.features[2].dtype == Array(String)
-        and feature_view.features[3].name == "fs1_my_feature_4"
-        and feature_view.features[3].dtype == Array(Bytes)
-        and feature_view.entities[0] == "fs1_my_entity_1"
-    )
-
-    test_feature_store.delete_feature_view("my_feature_view_1")
-    feature_views = test_feature_store.list_feature_views()
-    assert len(feature_views) == 0
-
-    test_feature_store.teardown()
-
-
 @pytest.fixture
 def feature_store_with_local_registry():
     fd, registry_path = mkstemp()
@@ -199,44 +83,3 @@ def feature_store_with_local_registry():
     )
 
 
-@pytest.fixture
-def feature_store_with_gcs_registry():
-    from google.cloud import storage
-
-    storage_client = storage.Client()
-    bucket_name = f"feast-registry-test-{int(time.time() * 1000)}"
-    bucket = storage_client.bucket(bucket_name)
-    bucket = storage_client.create_bucket(bucket)
-    bucket.add_lifecycle_delete_rule(
-        age=14
-    )  # delete buckets automatically after 14 days
-    bucket.patch()
-    bucket.blob("registry.db")
-
-    return FeatureStore(
-        config=RepoConfig(
-            registry=f"gs://{bucket_name}/registry.db",
-            project="default",
-            provider="gcp",
-            entity_key_serialization_version=2,
-        )
-    )
-
-
-@pytest.fixture
-def feature_store_with_s3_registry():
-    aws_registry_path = os.getenv(
-        "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries"
-    )
-    return FeatureStore(
-        config=RepoConfig(
-            registry=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db",
-            project="default",
-            provider="aws",
-            online_store=DynamoDBOnlineStoreConfig(
-                region=os.getenv("AWS_REGION", "us-west-2")
-            ),
-            offline_store=FileOfflineStoreConfig(),
-            entity_key_serialization_version=2,
-        )
-    )

sdk/python/tests/integration/registration/test_universal_registry.py

@@ -232,20 +232,32 @@ def mock_remote_registry():
     registry.stub = RegistryServer_pb2_grpc.RegistryServerStub(mock_channel)
     yield registry
 
+if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "False":
+    all_fixtures=[
+        "s3_registry", 
+        "gcs_registry"
+    ]
+else:
+    all_fixtures = [
+        "local_registry", 
+        "minio_registry", 
+        "pg_registry", 
+        "mysql_registry", 
+        "sqlite_registry", 
+        "mock_remote_registry"
+    ]
+
+
+# sql_fixtures = [
+#     "pg_registry", 
+#     "mysql_registry", 
+#     "sqlite_registry", 
+# ]
 
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_registry",
-    [
-        lazy_fixture("local_registry"),
-        lazy_fixture("gcs_registry"),
-        lazy_fixture("s3_registry"),
-        lazy_fixture("minio_registry"),
-        lazy_fixture("pg_registry"),
-        lazy_fixture("mysql_registry"),
-        lazy_fixture("sqlite_registry"),
-        lazy_fixture("mock_remote_registry"),
-    ],
+    [lazy_fixture(f) for f in all_fixtures]
 )
 def test_apply_entity_success(test_registry):
     entity = Entity(
@@ -305,16 +317,7 @@ def assert_project_uuid(project, project_uuid, test_registry):
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_registry",
-    [
-        lazy_fixture("local_registry"),
-        lazy_fixture("gcs_registry"),
-        lazy_fixture("s3_registry"),
-        lazy_fixture("minio_registry"),
-        lazy_fixture("pg_registry"),
-        lazy_fixture("mysql_registry"),
-        lazy_fixture("sqlite_registry"),
-        lazy_fixture("mock_remote_registry"),
-    ],
+    [lazy_fixture(f) for f in all_fixtures],
 )
 def test_apply_feature_view_success(test_registry):
     # Create Feature Views
@@ -492,16 +495,7 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame:
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_registry",
-    [
-        lazy_fixture("local_registry"),
-        lazy_fixture("gcs_registry"),
-        lazy_fixture("s3_registry"),
-        lazy_fixture("minio_registry"),
-        lazy_fixture("pg_registry"),
-        lazy_fixture("mysql_registry"),
-        lazy_fixture("sqlite_registry"),
-        lazy_fixture("mock_remote_registry"),
-    ],
+    [lazy_fixture(f) for f in all_fixtures],
 )
 def test_apply_data_source(test_registry):
     # Create Feature Views
@@ -564,16 +558,7 @@ def test_apply_data_source(test_registry):
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_registry",
-    [
-        lazy_fixture("local_registry"),
-        lazy_fixture("gcs_registry"),
-        lazy_fixture("s3_registry"),
-        lazy_fixture("minio_registry"),
-        lazy_fixture("pg_registry"),
-        lazy_fixture("mysql_registry"),
-        lazy_fixture("sqlite_registry"),
-        lazy_fixture("mock_remote_registry"),
-    ],
+    [lazy_fixture(f) for f in all_fixtures],
 )
 def test_modify_feature_views_success(test_registry):
     # Create Feature Views
@@ -809,16 +794,7 @@ def test_registry_cache(test_registry):
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_registry",
-    [
-        lazy_fixture("local_registry"),
-        lazy_fixture("gcs_registry"),
-        lazy_fixture("s3_registry"),
-        lazy_fixture("minio_registry"),
-        lazy_fixture("pg_registry"),
-        lazy_fixture("mysql_registry"),
-        lazy_fixture("sqlite_registry"),
-        lazy_fixture("mock_remote_registry"),
-    ],
+    [lazy_fixture(f) for f in all_fixtures],
 )
 def test_apply_stream_feature_view_success(test_registry):
     # Create Feature Views