diff --git a/protos/feast/core/OnDemandFeatureView.proto b/protos/feast/core/OnDemandFeatureView.proto index 741d46e39e..c43b33c1d2 100644 --- a/protos/feast/core/OnDemandFeatureView.proto +++ b/protos/feast/core/OnDemandFeatureView.proto @@ -50,6 +50,7 @@ message OnDemandFeatureViewSpec { oneof transformation { UserDefinedFunction user_defined_function = 5; + OnDemandSubstraitTransformation on_demand_substrait_transformation = 9; } // Description of the on demand feature view. @@ -89,3 +90,7 @@ message UserDefinedFunction { // The string representation of the udf string body_text = 3; } + +message OnDemandSubstraitTransformation { + bytes substrait_plan = 1; +} \ No newline at end of file diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 706f2ec4e4..586286a3d4 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,5 +1,6 @@ import copy import functools +import inspect import warnings from datetime import datetime from types import FunctionType @@ -17,6 +18,7 @@ from feast.feature_view_projection import FeatureViewProjection from feast.field import Field, from_value_type from feast.on_demand_pandas_transformation import OnDemandPandasTransformation +from feast.on_demand_substrait_transformation import OnDemandSubstraitTransformation from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( OnDemandFeatureView as OnDemandFeatureViewProto, ) @@ -210,6 +212,9 @@ def to_proto(self) -> OnDemandFeatureViewProto: user_defined_function=self.transformation.to_proto() if type(self.transformation) == OnDemandPandasTransformation else None, + on_demand_substrait_transformation=self.transformation.to_proto() # type: ignore + if type(self.transformation) == OnDemandSubstraitTransformation + else None, description=self.description, tags=self.tags, owner=self.owner, @@ -255,6 +260,13 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): transformation = OnDemandPandasTransformation.from_proto( on_demand_feature_view_proto.spec.user_defined_function ) + elif ( + on_demand_feature_view_proto.spec.WhichOneof("transformation") + == "on_demand_substrait_transformation" + ): + transformation = OnDemandSubstraitTransformation.from_proto( + on_demand_feature_view_proto.spec.on_demand_substrait_transformation + ) else: raise Exception("At least one transformation type needs to be provided") @@ -460,10 +472,47 @@ def mainify(obj) -> None: obj.__module__ = "__main__" def decorator(user_function): - udf_string = dill.source.getsource(user_function) - mainify(user_function) + return_annotation = inspect.signature(user_function).return_annotation + if ( + return_annotation + and return_annotation.__module__ == "ibis.expr.types.relations" + and return_annotation.__name__ == "Table" + ): + import ibis + import ibis.expr.datatypes as dt + from ibis_substrait.compiler.core import SubstraitCompiler + + compiler = SubstraitCompiler() + + input_fields: Field = [] + + for s in sources: + if type(s) == FeatureView: + fields = s.projection.features + else: + fields = s.features + + input_fields.extend( + [ + ( + f.name, + dt.dtype( + feast_value_type_to_pandas_type(f.dtype.to_value_type()) + ), + ) + for f in fields + ] + ) + + expr = user_function(ibis.table(input_fields, "t")) - transformation = OnDemandPandasTransformation(user_function, udf_string) + transformation = OnDemandSubstraitTransformation( + substrait_plan=compiler.compile(expr).SerializeToString() + ) + else: + udf_string = dill.source.getsource(user_function) + mainify(user_function) + transformation = OnDemandPandasTransformation(user_function, udf_string) on_demand_feature_view_obj = OnDemandFeatureView( name=user_function.__name__, diff --git a/sdk/python/feast/on_demand_substrait_transformation.py b/sdk/python/feast/on_demand_substrait_transformation.py new file mode 100644 index 0000000000..4e92e77dc8 --- /dev/null +++ b/sdk/python/feast/on_demand_substrait_transformation.py @@ -0,0 +1,50 @@ +import pandas as pd +import pyarrow +import pyarrow.substrait as substrait # type: ignore # noqa + +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + OnDemandSubstraitTransformation as OnDemandSubstraitTransformationProto, +) + + +class OnDemandSubstraitTransformation: + def __init__(self, substrait_plan: bytes): + """ + Creates an OnDemandSubstraitTransformation object. + + Args: + substrait_plan: The user-provided substrait plan. + """ + self.substrait_plan = substrait_plan + + def transform(self, df: pd.DataFrame) -> pd.DataFrame: + def table_provider(names, schema: pyarrow.Schema): + return pyarrow.Table.from_pandas(df[schema.names]) + + table: pyarrow.Table = pyarrow.substrait.run_query( + self.substrait_plan, table_provider=table_provider + ).read_all() + return table.to_pandas() + + def __eq__(self, other): + if not isinstance(other, OnDemandSubstraitTransformation): + raise TypeError( + "Comparisons should only involve OnDemandSubstraitTransformation class objects." + ) + + if not super().__eq__(other): + return False + + return self.substrait_plan == other.substrait_plan + + def to_proto(self) -> OnDemandSubstraitTransformationProto: + return OnDemandSubstraitTransformationProto(substrait_plan=self.substrait_plan) + + @classmethod + def from_proto( + cls, + on_demand_substrait_transformation_proto: OnDemandSubstraitTransformationProto, + ): + return OnDemandSubstraitTransformation( + substrait_plan=on_demand_substrait_transformation_proto.substrait_plan + ) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index f20bc05df9..45c4eb765d 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -10,7 +10,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -34,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==4.0 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -55,15 +57,17 @@ babel==2.14.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert +bidict==0.23.1 + # via ibis-framework black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -122,7 +126,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -137,7 +141,7 @@ cryptography==41.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) db-dtypes==1.2.0 # via google-cloud-bigquery @@ -213,9 +217,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -226,10 +230,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -244,7 +244,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -264,13 +264,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -282,15 +282,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -304,7 +304,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -312,11 +312,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -337,7 +343,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.2 # via jupyterlab -ipython==8.21.0 +ipython==8.22.1 # via # great-expectations # ipykernel @@ -367,7 +373,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -412,7 +418,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -428,6 +434,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -441,6 +449,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -455,7 +465,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -463,6 +473,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -477,7 +489,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -491,7 +503,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -501,6 +513,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -519,6 +532,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -536,6 +550,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -544,6 +559,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -552,7 +569,7 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) platformdirs==3.11.0 # via @@ -580,7 +597,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -598,6 +615,7 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus + # substrait psutil==5.9.0 # via # feast (setup.py) @@ -621,7 +639,10 @@ pyarrow==15.0.0 # db-dtypes # feast (setup.py) # google-cloud-bigquery + # ibis-framework # snowflake-connector-python +pyarrow-hotfix==0.6 + # via ibis-framework pyasn1==0.5.1 # via # pyasn1-modules @@ -634,12 +655,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -648,6 +669,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -659,14 +681,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -699,6 +723,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -712,6 +737,7 @@ python-json-logger==2.0.7 pytz==2024.1 # via # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -719,6 +745,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -769,7 +796,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -805,7 +834,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -826,13 +855,19 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 # via fastapi +substrait==0.12.1 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 @@ -865,6 +900,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -893,7 +929,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -913,11 +949,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -931,11 +967,13 @@ typing-extensions==4.9.0 # azure-storage-blob # fastapi # great-expectations + # ibis-framework # mypy # pydantic # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 3943662d01..7141cd0f25 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -38,7 +38,7 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -131,7 +131,7 @@ partd==1.4.1 # via dask proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -141,11 +141,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -212,13 +212,12 @@ typing-extensions==4.9.0 # pydantic # pydantic-core # sqlalchemy2-stubs + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 - # via - # feast (setup.py) - # uvicorn + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt deleted file mode 100644 index bc9ddbe8e7..0000000000 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ /dev/null @@ -1,1000 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt -# -alabaster==0.7.16 - # via sphinx -altair==4.2.2 - # via great-expectations -annotated-types==0.6.0 - # via pydantic -anyio==4.2.0 - # via - # httpx - # jupyter-server - # starlette - # watchfiles -appdirs==1.4.4 - # via fissix -argon2-cffi==23.1.0 - # via jupyter-server -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -arrow==1.3.0 - # via isoduration -asn1crypto==1.5.1 - # via snowflake-connector-python -assertpy==1.1 - # via feast (setup.py) -asttokens==2.4.1 - # via stack-data -async-lru==2.0.4 - # via jupyterlab -async-timeout==4.0.3 - # via redis -attrs==23.2.0 - # via - # bowler - # jsonschema - # referencing -avro==1.10.0 - # via feast (setup.py) -azure-core==1.30.0 - # via - # azure-identity - # azure-storage-blob -azure-identity==1.15.0 - # via feast (setup.py) -azure-storage-blob==12.19.0 - # via feast (setup.py) -babel==2.14.0 - # via - # jupyterlab-server - # sphinx -beautifulsoup4==4.12.3 - # via nbconvert -black==22.12.0 - # via feast (setup.py) -bleach==6.1.0 - # via nbconvert -boto3==1.34.42 - # via - # feast (setup.py) - # moto -botocore==1.34.42 - # via - # boto3 - # moto - # s3transfer -bowler==0.9.0 - # via feast (setup.py) -build==1.0.3 - # via - # feast (setup.py) - # pip-tools -bytewax==0.18.2 - # via feast (setup.py) -cachecontrol==0.14.0 - # via firebase-admin -cachetools==5.3.2 - # via google-auth -cassandra-driver==3.29.0 - # via feast (setup.py) -certifi==2024.2.2 - # via - # httpcore - # httpx - # kubernetes - # minio - # requests - # snowflake-connector-python -cffi==1.16.0 - # via - # argon2-cffi-bindings - # cryptography - # snowflake-connector-python -cfgv==3.4.0 - # via pre-commit -charset-normalizer==3.3.2 - # via - # requests - # snowflake-connector-python -click==8.1.7 - # via - # black - # bowler - # dask - # feast (setup.py) - # geomet - # great-expectations - # moreorless - # pip-tools - # uvicorn -cloudpickle==3.0.0 - # via dask -colorama==0.4.6 - # via - # feast (setup.py) - # great-expectations -comm==0.2.1 - # via - # ipykernel - # ipywidgets -coverage[toml]==7.4.1 - # via pytest-cov -cryptography==41.0.7 - # via - # azure-identity - # azure-storage-blob - # feast (setup.py) - # great-expectations - # moto - # msal - # pyjwt - # pyopenssl - # snowflake-connector-python - # types-pyopenssl - # types-redis -dask==2024.2.0 - # via feast (setup.py) -db-dtypes==1.2.0 - # via google-cloud-bigquery -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -deprecation==2.1.0 - # via testcontainers -dill==0.3.8 - # via - # bytewax - # feast (setup.py) - # multiprocess -distlib==0.3.8 - # via virtualenv -docker==7.0.0 - # via - # feast (setup.py) - # testcontainers -docutils==0.19 - # via sphinx -entrypoints==0.4 - # via altair -exceptiongroup==1.2.0 - # via - # anyio - # ipython - # pytest -execnet==2.0.2 - # via pytest-xdist -executing==2.0.1 - # via stack-data -fastapi==0.109.2 - # via feast (setup.py) -fastavro==1.9.4 - # via - # feast (setup.py) - # pandavro -fastjsonschema==2.19.1 - # via nbformat -filelock==3.13.1 - # via - # snowflake-connector-python - # virtualenv -firebase-admin==5.4.0 - # via feast (setup.py) -fissix==21.11.13 - # via bowler -flake8==6.0.0 - # via feast (setup.py) -fqdn==1.5.1 - # via jsonschema -fsspec==2023.12.2 - # via - # dask - # feast (setup.py) -geojson==2.5.0 - # via rockset -geomet==0.2.1.post1 - # via cassandra-driver -google-api-core[grpc]==2.17.1 - # via - # feast (setup.py) - # firebase-admin - # google-api-python-client - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-core - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-api-python-client==2.118.0 - # via firebase-admin -google-auth==2.27.0 - # via - # google-api-core - # google-api-python-client - # google-auth-httplib2 - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-httplib2==0.2.0 - # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 - # via feast (setup.py) -google-cloud-bigquery-storage==2.24.0 - # via feast (setup.py) -google-cloud-bigtable==2.23.0 - # via feast (setup.py) -google-cloud-core==2.4.1 - # via - # google-cloud-bigquery - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-cloud-datastore==2.19.0 - # via feast (setup.py) -google-cloud-firestore==2.14.0 - # via firebase-admin -google-cloud-storage==2.14.0 - # via - # feast (setup.py) - # firebase-admin -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via - # google-cloud-bigquery - # google-cloud-storage -googleapis-common-protos[grpc]==1.62.0 - # via - # feast (setup.py) - # google-api-core - # grpc-google-iam-v1 - # grpcio-status -great-expectations==0.18.8 - # via feast (setup.py) -greenlet==3.0.3 - # via sqlalchemy -grpc-google-iam-v1==0.13.0 - # via google-cloud-bigtable -grpcio==1.60.1 - # via - # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools -grpcio-health-checking==1.60.1 - # via feast (setup.py) -grpcio-reflection==1.60.1 - # via feast (setup.py) -grpcio-status==1.60.1 - # via google-api-core -grpcio-testing==1.60.1 - # via feast (setup.py) -grpcio-tools==1.60.1 - # via feast (setup.py) -gunicorn==21.2.0 - # via feast (setup.py) -h11==0.14.0 - # via - # httpcore - # uvicorn -happybase==1.2.0 - # via feast (setup.py) -hazelcast-python-client==5.3.0 - # via feast (setup.py) -hiredis==2.3.2 - # via feast (setup.py) -httpcore==1.0.3 - # via httpx -httplib2==0.22.0 - # via - # google-api-python-client - # google-auth-httplib2 -httptools==0.6.1 - # via uvicorn -httpx==0.26.0 - # via - # feast (setup.py) - # jupyterlab -identify==2.5.34 - # via pre-commit -idna==3.6 - # via - # anyio - # httpx - # jsonschema - # requests - # snowflake-connector-python -imagesize==1.4.1 - # via sphinx -importlib-metadata==6.11.0 - # via - # dask - # feast (setup.py) -importlib-resources==6.1.1 - # via feast (setup.py) -iniconfig==2.0.0 - # via pytest -ipykernel==6.29.2 - # via jupyterlab -ipython==8.21.0 - # via - # great-expectations - # ipykernel - # ipywidgets -ipywidgets==8.1.2 - # via great-expectations -isodate==0.6.1 - # via azure-storage-blob -isoduration==20.11.0 - # via jsonschema -isort==5.13.2 - # via feast (setup.py) -jedi==0.19.1 - # via ipython -jinja2==3.1.3 - # via - # altair - # feast (setup.py) - # great-expectations - # jupyter-server - # jupyterlab - # jupyterlab-server - # moto - # nbconvert - # sphinx -jmespath==1.0.1 - # via - # boto3 - # botocore -json5==0.9.14 - # via jupyterlab-server -jsonpatch==1.33 - # via great-expectations -jsonpointer==2.4 - # via - # jsonpatch - # jsonschema -jsonschema[format-nongpl]==4.21.1 - # via - # altair - # feast (setup.py) - # great-expectations - # jupyter-events - # jupyterlab-server - # nbformat -jsonschema-specifications==2023.12.1 - # via jsonschema -jupyter-client==8.6.0 - # via - # ipykernel - # jupyter-server - # nbclient -jupyter-core==5.7.1 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # nbclient - # nbconvert - # nbformat -jupyter-events==0.9.0 - # via jupyter-server -jupyter-lsp==2.2.2 - # via jupyterlab -jupyter-server==2.12.5 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook - # notebook-shim -jupyter-server-terminals==0.5.2 - # via jupyter-server -jupyterlab==4.1.1 - # via notebook -jupyterlab-pygments==0.3.0 - # via nbconvert -jupyterlab-server==2.25.3 - # via - # jupyterlab - # notebook -jupyterlab-widgets==3.0.10 - # via ipywidgets -kubernetes==20.13.0 - # via feast (setup.py) -locket==1.0.0 - # via partd -makefun==1.15.2 - # via great-expectations -markupsafe==2.1.5 - # via - # jinja2 - # nbconvert - # werkzeug -marshmallow==3.20.2 - # via great-expectations -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -mccabe==0.7.0 - # via flake8 -minio==7.1.0 - # via feast (setup.py) -mistune==3.0.2 - # via - # great-expectations - # nbconvert -mmh3==4.1.0 - # via feast (setup.py) -mock==2.0.0 - # via feast (setup.py) -moreorless==0.4.0 - # via bowler -moto==4.2.14 - # via feast (setup.py) -msal==1.26.0 - # via - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -msgpack==1.0.7 - # via cachecontrol -multiprocess==0.70.16 - # via bytewax -mypy==1.8.0 - # via - # feast (setup.py) - # sqlalchemy -mypy-extensions==1.0.0 - # via - # black - # mypy -mypy-protobuf==3.1.0 - # via feast (setup.py) -nbclient==0.9.0 - # via nbconvert -nbconvert==7.16.0 - # via jupyter-server -nbformat==5.9.2 - # via - # great-expectations - # jupyter-server - # nbclient - # nbconvert -nest-asyncio==1.6.0 - # via ipykernel -nodeenv==1.8.0 - # via pre-commit -notebook==7.1.0 - # via great-expectations -notebook-shim==0.2.3 - # via - # jupyterlab - # notebook -numpy==1.24.4 - # via - # altair - # db-dtypes - # feast (setup.py) - # great-expectations - # pandas - # pandavro - # pyarrow - # scipy -oauthlib==3.2.2 - # via requests-oauthlib -overrides==7.7.0 - # via jupyter-server -packaging==23.2 - # via - # build - # dask - # db-dtypes - # deprecation - # docker - # google-cloud-bigquery - # great-expectations - # gunicorn - # ipykernel - # jupyter-server - # jupyterlab - # jupyterlab-server - # marshmallow - # msal-extensions - # nbconvert - # pytest - # snowflake-connector-python - # sphinx -pandas==1.5.3 - # via - # altair - # db-dtypes - # feast (setup.py) - # google-cloud-bigquery - # great-expectations - # pandavro - # snowflake-connector-python -pandavro==1.5.2 - # via feast (setup.py) -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.3 - # via jedi -partd==1.4.1 - # via dask -pathspec==0.12.1 - # via black -pbr==6.0.0 - # via mock -pexpect==4.9.0 - # via ipython -pip-tools==7.3.0 - # via feast (setup.py) -platformdirs==3.11.0 - # via - # black - # jupyter-core - # snowflake-connector-python - # virtualenv -pluggy==1.4.0 - # via pytest -ply==3.11 - # via thriftpy2 -portalocker==2.8.2 - # via msal-extensions -pre-commit==3.3.1 - # via feast (setup.py) -prometheus-client==0.20.0 - # via jupyter-server -prompt-toolkit==3.0.43 - # via ipython -proto-plus==1.23.0 - # via - # feast (setup.py) - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore -protobuf==4.23.3 - # via - # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools - # mypy-protobuf - # proto-plus -psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel -psycopg2-binary==2.9.9 - # via feast (setup.py) -ptyprocess==0.7.0 - # via - # pexpect - # terminado -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via feast (setup.py) -py-cpuinfo==9.0.0 - # via pytest-benchmark -py4j==0.10.9.7 - # via pyspark -pyarrow==15.0.0 - # via - # db-dtypes - # feast (setup.py) - # google-cloud-bigquery - # snowflake-connector-python -pyasn1==0.5.1 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pybindgen==0.22.1 - # via feast (setup.py) -pycodestyle==2.10.0 - # via flake8 -pycparser==2.21 - # via cffi -pydantic==2.6.1 - # via - # fastapi - # feast (setup.py) - # great-expectations -pydantic-core==2.16.2 - # via pydantic -pyflakes==3.0.1 - # via flake8 -pygments==2.17.2 - # via - # feast (setup.py) - # ipython - # nbconvert - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.2.11 - # via feast (setup.py) -pymysql==1.1.0 - # via feast (setup.py) -pyodbc==5.1.0 - # via feast (setup.py) -pyopenssl==23.3.0 - # via snowflake-connector-python -pyparsing==3.1.1 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.0.0 - # via build -pyspark==3.5.0 - # via feast (setup.py) -pytest==7.4.4 - # via - # feast (setup.py) - # pytest-benchmark - # pytest-cov - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 - # via feast (setup.py) -pytest-cov==4.1.0 - # via feast (setup.py) -pytest-lazy-fixture==0.6.3 - # via feast (setup.py) -pytest-mock==1.10.4 - # via feast (setup.py) -pytest-ordering==0.6 - # via feast (setup.py) -pytest-timeout==1.4.2 - # via feast (setup.py) -pytest-xdist==3.5.0 - # via feast (setup.py) -python-dateutil==2.8.2 - # via - # arrow - # botocore - # google-cloud-bigquery - # great-expectations - # jupyter-client - # kubernetes - # moto - # pandas - # rockset - # trino -python-dotenv==1.0.1 - # via uvicorn -python-json-logger==2.0.7 - # via jupyter-events -pytz==2024.1 - # via - # great-expectations - # pandas - # snowflake-connector-python - # trino -pyyaml==6.0.1 - # via - # dask - # feast (setup.py) - # jupyter-events - # kubernetes - # pre-commit - # responses - # uvicorn -pyzmq==25.1.2 - # via - # ipykernel - # jupyter-client - # jupyter-server -redis==4.6.0 - # via feast (setup.py) -referencing==0.33.0 - # via - # jsonschema - # jsonschema-specifications - # jupyter-events -regex==2023.12.25 - # via feast (setup.py) -requests==2.31.0 - # via - # azure-core - # cachecontrol - # docker - # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # google-cloud-storage - # great-expectations - # jupyterlab-server - # kubernetes - # moto - # msal - # requests-oauthlib - # responses - # snowflake-connector-python - # sphinx - # trino -requests-oauthlib==1.3.1 - # via kubernetes -responses==0.25.0 - # via moto -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events -rockset==2.1.0 - # via feast (setup.py) -rpds-py==0.18.0 - # via - # jsonschema - # referencing -rsa==4.9 - # via google-auth -ruamel-yaml==0.17.17 - # via great-expectations -s3transfer==0.10.0 - # via boto3 -scipy==1.12.0 - # via great-expectations -send2trash==1.8.2 - # via jupyter-server -six==1.16.0 - # via - # asttokens - # azure-core - # bleach - # geomet - # happybase - # isodate - # kubernetes - # mock - # pandavro - # python-dateutil - # rfc3339-validator - # thriftpy2 -sniffio==1.3.0 - # via - # anyio - # httpx -snowballstemmer==2.2.0 - # via sphinx -snowflake-connector-python[pandas]==3.7.0 - # via feast (setup.py) -sortedcontainers==2.4.0 - # via snowflake-connector-python -soupsieve==2.5 - # via beautifulsoup4 -sphinx==6.2.1 - # via feast (setup.py) -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx -sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy -stack-data==0.6.3 - # via ipython -starlette==0.36.3 - # via fastapi -tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) -terminado==0.18.0 - # via - # jupyter-server - # jupyter-server-terminals -testcontainers==3.7.1 - # via feast (setup.py) -thriftpy2==0.4.17 - # via happybase -tinycss2==1.2.1 - # via nbconvert -toml==0.10.2 - # via feast (setup.py) -tomli==2.0.1 - # via - # black - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pyproject-hooks - # pytest -tomlkit==0.12.3 - # via snowflake-connector-python -toolz==0.12.1 - # via - # altair - # dask - # partd -tornado==6.4 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado -tqdm==4.66.2 - # via - # feast (setup.py) - # great-expectations -traitlets==5.14.1 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # jupyter-events - # jupyter-server - # jupyterlab - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -trino==0.327.0 - # via feast (setup.py) -typeguard==4.1.5 - # via feast (setup.py) -types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf -types-pymysql==1.1.0.1 - # via feast (setup.py) -types-pyopenssl==24.0.0.20240130 - # via types-redis -types-python-dateutil==2.8.19.20240106 - # via - # arrow - # feast (setup.py) -types-pytz==2024.1.0.20240203 - # via feast (setup.py) -types-pyyaml==6.0.12.12 - # via feast (setup.py) -types-redis==4.6.0.20240106 - # via feast (setup.py) -types-requests==2.30.0.0 - # via feast (setup.py) -types-setuptools==69.0.0.20240125 - # via feast (setup.py) -types-tabulate==0.9.0.20240106 - # via feast (setup.py) -types-urllib3==1.26.25.14 - # via types-requests -typing-extensions==4.9.0 - # via - # anyio - # async-lru - # azure-core - # azure-storage-blob - # fastapi - # great-expectations - # mypy - # pydantic - # pydantic-core - # snowflake-connector-python - # sqlalchemy2-stubs - # uvicorn -tzlocal==5.2 - # via - # great-expectations - # trino -uri-template==1.3.0 - # via jsonschema -uritemplate==4.1.1 - # via google-api-python-client -urllib3==1.26.18 - # via - # botocore - # docker - # feast (setup.py) - # great-expectations - # kubernetes - # minio - # requests - # responses - # rockset -uvicorn[standard]==0.27.1 - # via feast (setup.py) -uvloop==0.19.0 - # via uvicorn -virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit -volatile==2.1.0 - # via bowler -watchfiles==0.21.0 - # via uvicorn -wcwidth==0.2.13 - # via prompt-toolkit -webcolors==1.13 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.7.0 - # via - # jupyter-server - # kubernetes -websockets==12.0 - # via uvicorn -werkzeug==3.0.1 - # via moto -wheel==0.42.0 - # via pip-tools -widgetsnbextension==4.0.10 - # via ipywidgets -wrapt==1.16.0 - # via testcontainers -xmltodict==0.13.0 - # via moto -zipp==3.17.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt deleted file mode 100644 index 3943662d01..0000000000 --- a/sdk/python/requirements/py3.11-requirements.txt +++ /dev/null @@ -1,234 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt -# -annotated-types==0.6.0 - # via pydantic -anyio==4.2.0 - # via - # httpx - # starlette - # watchfiles -appdirs==1.4.4 - # via fissix -attrs==23.2.0 - # via - # bowler - # jsonschema - # referencing -bowler==0.9.0 - # via feast (setup.py) -certifi==2024.2.2 - # via - # httpcore - # httpx - # requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # bowler - # dask - # feast (setup.py) - # moreorless - # uvicorn -cloudpickle==3.0.0 - # via dask -colorama==0.4.6 - # via feast (setup.py) -dask==2024.2.0 - # via feast (setup.py) -dill==0.3.8 - # via feast (setup.py) -exceptiongroup==1.2.0 - # via anyio -fastapi==0.109.2 - # via feast (setup.py) -fastavro==1.9.4 - # via - # feast (setup.py) - # pandavro -fissix==21.11.13 - # via bowler -fsspec==2024.2.0 - # via dask -greenlet==3.0.3 - # via sqlalchemy -grpcio==1.60.1 - # via - # feast (setup.py) - # grpcio-health-checking - # grpcio-reflection - # grpcio-tools -grpcio-health-checking==1.60.1 - # via feast (setup.py) -grpcio-reflection==1.60.1 - # via feast (setup.py) -grpcio-tools==1.60.1 - # via feast (setup.py) -gunicorn==21.2.0 - # via feast (setup.py) -h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.3 - # via httpx -httptools==0.6.1 - # via uvicorn -httpx==0.26.0 - # via feast (setup.py) -idna==3.6 - # via - # anyio - # httpx - # requests -importlib-metadata==6.11.0 - # via - # dask - # feast (setup.py) -importlib-resources==6.1.1 - # via feast (setup.py) -jinja2==3.1.3 - # via feast (setup.py) -jsonschema==4.21.1 - # via feast (setup.py) -jsonschema-specifications==2023.12.1 - # via jsonschema -locket==1.0.0 - # via partd -markupsafe==2.1.5 - # via jinja2 -mmh3==4.1.0 - # via feast (setup.py) -moreorless==0.4.0 - # via bowler -mypy==1.8.0 - # via sqlalchemy -mypy-extensions==1.0.0 - # via mypy -mypy-protobuf==3.1.0 - # via feast (setup.py) -numpy==1.24.4 - # via - # feast (setup.py) - # pandas - # pandavro - # pyarrow -packaging==23.2 - # via - # dask - # gunicorn -pandas==1.5.3 - # via - # feast (setup.py) - # pandavro -pandavro==1.5.2 - # via feast (setup.py) -partd==1.4.1 - # via dask -proto-plus==1.23.0 - # via feast (setup.py) -protobuf==4.23.3 - # via - # feast (setup.py) - # grpcio-health-checking - # grpcio-reflection - # grpcio-tools - # mypy-protobuf - # proto-plus -pyarrow==15.0.0 - # via feast (setup.py) -pydantic==2.6.1 - # via - # fastapi - # feast (setup.py) -pydantic-core==2.16.2 - # via pydantic -pygments==2.17.2 - # via feast (setup.py) -python-dateutil==2.8.2 - # via pandas -python-dotenv==1.0.1 - # via uvicorn -pytz==2024.1 - # via pandas -pyyaml==6.0.1 - # via - # dask - # feast (setup.py) - # uvicorn -referencing==0.33.0 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via feast (setup.py) -rpds-py==0.18.0 - # via - # jsonschema - # referencing -six==1.16.0 - # via - # pandavro - # python-dateutil -sniffio==1.3.0 - # via - # anyio - # httpx -sqlalchemy[mypy]==1.4.51 - # via - # feast (setup.py) - # sqlalchemy -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy -starlette==0.36.3 - # via fastapi -tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) -toml==0.10.2 - # via feast (setup.py) -tomli==2.0.1 - # via mypy -toolz==0.12.1 - # via - # dask - # partd -tqdm==4.66.2 - # via feast (setup.py) -typeguard==4.1.5 - # via feast (setup.py) -types-protobuf==4.24.0.20240129 - # via mypy-protobuf -typing-extensions==4.9.0 - # via - # anyio - # fastapi - # mypy - # pydantic - # pydantic-core - # sqlalchemy2-stubs - # uvicorn -urllib3==2.2.0 - # via requests -uvicorn[standard]==0.27.1 - # via - # feast (setup.py) - # uvicorn -uvloop==0.19.0 - # via uvicorn -volatile==2.1.0 - # via bowler -watchfiles==0.21.0 - # via uvicorn -websockets==12.0 - # via uvicorn -zipp==3.17.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index afa43ec2a2..b006c1c621 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,14 +4,13 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt # - alabaster==0.7.13 # via sphinx altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -35,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==3.1.2 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -66,11 +67,11 @@ black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -129,7 +130,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -219,9 +220,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -232,10 +233,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -250,7 +247,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -270,13 +267,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -288,15 +285,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -310,7 +307,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -318,11 +315,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==4.1.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==2.29.1 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -344,6 +347,7 @@ importlib-metadata==6.11.0 # jupyterlab-server # nbconvert # sphinx + # typeguard importlib-resources==6.1.1 # via # feast (setup.py) @@ -384,7 +388,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -429,7 +433,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -445,6 +449,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -458,6 +464,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -472,7 +480,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -480,6 +488,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==0.6.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -494,7 +504,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -508,7 +518,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -518,6 +528,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -536,6 +547,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -553,6 +565,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -561,6 +574,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -571,7 +586,7 @@ pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) pkgutil-resolve-name==1.3.10 # via jsonschema @@ -601,7 +616,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -617,6 +632,7 @@ protobuf==4.23.3 # grpcio-status # grpcio-testing # grpcio-tools + # ibis-substrait # mypy-protobuf # proto-plus psutil==5.9.0 @@ -655,12 +671,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -669,6 +685,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -680,14 +697,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -720,6 +739,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -734,6 +754,7 @@ pytz==2024.1 # via # babel # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -741,6 +762,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -791,7 +813,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -819,6 +843,7 @@ six==1.16.0 # isodate # kubernetes # mock + # multipledispatch # pandavro # python-dateutil # rfc3339-validator @@ -829,7 +854,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -850,9 +875,13 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==10.6.4 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 @@ -889,6 +918,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -917,7 +947,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -937,11 +967,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -949,6 +979,7 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.9.0 # via + # annotated-types # anyio # async-lru # azure-core @@ -956,13 +987,16 @@ typing-extensions==4.9.0 # black # fastapi # great-expectations + # ibis-framework # ipython # mypy # pydantic # pydantic-core + # rich # snowflake-connector-python # sqlalchemy2-stubs # starlette + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 079064a9ec..541beecf0d 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -89,6 +89,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) + # typeguard importlib-resources==6.1.1 # via # feast (setup.py) @@ -136,7 +137,7 @@ pkgutil-resolve-name==1.3.10 # via jsonschema proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -146,11 +147,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -184,7 +185,9 @@ sniffio==1.3.0 # anyio # httpx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -209,6 +212,7 @@ types-protobuf==4.24.0.20240129 # via mypy-protobuf typing-extensions==4.9.0 # via + # annotated-types # anyio # fastapi # mypy @@ -216,13 +220,12 @@ typing-extensions==4.9.0 # pydantic-core # sqlalchemy2-stubs # starlette + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 - # via - # feast (setup.py) - # uvicorn + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 6c26f889e2..e17d545d38 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -10,7 +10,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -34,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==4.0 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -55,15 +57,17 @@ babel==2.14.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert +bidict==0.23.1 + # via ibis-framework black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -122,7 +126,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -137,7 +141,7 @@ cryptography==41.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) db-dtypes==1.2.0 # via google-cloud-bigquery @@ -213,9 +217,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -226,10 +230,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -244,7 +244,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -264,13 +264,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -282,15 +282,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -304,7 +304,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -312,11 +312,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -338,6 +344,7 @@ importlib-metadata==6.11.0 # jupyterlab-server # nbconvert # sphinx + # typeguard importlib-resources==6.1.1 # via feast (setup.py) iniconfig==2.0.0 @@ -374,7 +381,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -419,7 +426,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -435,6 +442,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -448,6 +457,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -462,7 +473,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -470,6 +481,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -484,7 +497,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -498,7 +511,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -508,6 +521,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -526,6 +540,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -543,6 +558,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -551,6 +567,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -559,7 +577,7 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) platformdirs==3.11.0 # via @@ -587,7 +605,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -605,6 +623,7 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus + # substrait psutil==5.9.0 # via # feast (setup.py) @@ -628,7 +647,10 @@ pyarrow==15.0.0 # db-dtypes # feast (setup.py) # google-cloud-bigquery + # ibis-framework # snowflake-connector-python +pyarrow-hotfix==0.6 + # via ibis-framework pyasn1==0.5.1 # via # pyasn1-modules @@ -641,12 +663,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -655,6 +677,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -666,14 +689,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -706,6 +731,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -719,6 +745,7 @@ python-json-logger==2.0.7 pytz==2024.1 # via # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -726,6 +753,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -776,7 +804,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -814,7 +844,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -835,13 +865,19 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 # via fastapi +substrait==0.12.1 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 @@ -874,6 +910,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -902,7 +939,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -922,11 +959,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -941,6 +978,7 @@ typing-extensions==4.9.0 # black # fastapi # great-expectations + # ibis-framework # ipython # mypy # pydantic @@ -948,6 +986,7 @@ typing-extensions==4.9.0 # snowflake-connector-python # sqlalchemy2-stubs # starlette + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 182cb7ad07..12c14a03ae 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -38,7 +38,7 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -89,6 +89,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) + # typeguard importlib-resources==6.1.1 # via feast (setup.py) jinja2==3.1.3 @@ -131,7 +132,7 @@ partd==1.4.1 # via dask proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -141,11 +142,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -179,7 +180,9 @@ sniffio==1.3.0 # anyio # httpx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -211,8 +214,9 @@ typing-extensions==4.9.0 # pydantic-core # sqlalchemy2-stubs # starlette + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 # via feast (setup.py) diff --git a/sdk/python/tests/unit/test_on_demand_substrait_transformation.py b/sdk/python/tests/unit/test_on_demand_substrait_transformation.py new file mode 100644 index 0000000000..c9d30c5b7a --- /dev/null +++ b/sdk/python/tests/unit/test_on_demand_substrait_transformation.py @@ -0,0 +1,112 @@ +import os +import tempfile +from datetime import datetime, timedelta + +import pandas as pd + +from feast import Entity, FeatureStore, FeatureView, FileSource, RepoConfig +from feast.driver_test_data import create_driver_hourly_stats_df +from feast.field import Field +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Float32, Float64, Int64 + + +def test_ibis_pandas_parity(): + with tempfile.TemporaryDirectory() as data_dir: + store = FeatureStore( + config=RepoConfig( + project="test_on_demand_substrait_transformation", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + # Generate test data. + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) + driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") + driver_df.to_parquet(path=driver_stats_path, allow_truncated_timestamps=True) + + driver = Entity(name="driver", join_keys=["driver_id"]) + + driver_stats_source = FileSource( + name="driver_hourly_stats_source", + path=driver_stats_path, + timestamp_field="event_timestamp", + created_timestamp_column="created", + ) + + driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + ) + + @on_demand_feature_view( + sources=[driver_stats_fv], + schema=[Field(name="conv_rate_plus_acc", dtype=Float64)], + ) + def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_acc"] = inputs["conv_rate"] + inputs["acc_rate"] + return df + + from ibis.expr.types import Table + + @on_demand_feature_view( + sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], + schema=[Field(name="conv_rate_plus_acc_substrait", dtype=Float64)], + ) + def substrait_view(inputs: Table) -> Table: + return inputs.select( + (inputs["conv_rate"] + inputs["acc_rate"]).name( + "conv_rate_plus_acc_substrait" + ) + ) + + store.apply( + [driver, driver_stats_source, driver_stats_fv, substrait_view, pandas_view] + ) + + entity_df = pd.DataFrame.from_dict( + { + # entity's join key -> entity values + "driver_id": [1001, 1002, 1003], + # "event_timestamp" (reserved key) -> timestamps + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + ], + } + ) + + training_df = store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "substrait_view:conv_rate_plus_acc_substrait", + "pandas_view:conv_rate_plus_acc", + ], + ).to_df() + + assert training_df["conv_rate_plus_acc"].equals( + training_df["conv_rate_plus_acc_substrait"] + ) diff --git a/setup.py b/setup.py index 818b9d66b1..3334d2e939 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<=2024.1.0"] -BYTEWAX_REQUIRED = ["bytewax==0.18.2", "docker>=5.0.2", "kubernetes<=20.13.0"] +BYTEWAX_REQUIRED = ["bytewax==0.15.1", "docker>=5.0.2", "kubernetes<=20.13.0"] SNOWFLAKE_REQUIRED = [ "snowflake-connector-python[pandas]>=3,<4", @@ -144,6 +144,11 @@ "hazelcast-python-client>=5.1", ] +IBIS_REQUIRED = [ + "ibis-framework", + "ibis-substrait" +] + CI_REQUIRED = ( [ "build", @@ -201,6 +206,7 @@ + AZURE_REQUIRED + ROCKSET_REQUIRED + HAZELCAST_REQUIRED + + IBIS_REQUIRED ) @@ -368,6 +374,7 @@ def run(self): "cassandra": CASSANDRA_REQUIRED, "hazelcast": HAZELCAST_REQUIRED, "rockset": ROCKSET_REQUIRED, + "ibis": IBIS_REQUIRED }, include_package_data=True, license="Apache",