chore: Change dev workflow to use uv (and pixi) both for lock and ins…

…tall (#4166) * change dev workflow to use uv (and pixi) Signed-off-by: tokoko <togurg14@freeuni.edu.ge> * pin ibis versions Signed-off-by: tokoko <togurg14@freeuni.edu.ge> * remove redundant make command Signed-off-by: tokoko <togurg14@freeuni.edu.ge> --------- Signed-off-by: tokoko <togurg14@freeuni.edu.ge>
feast-dev · May 8, 2024 · 34d3635 · 34d3635
1 parent e88f1e3
commit 34d3635
Show file tree

Hide file tree

Showing 11 changed files with 657 additions and 1,082 deletions.
diff --git a/Makefile b/Makefile
@@ -47,7 +47,7 @@ install-python-ci-dependencies-uv:
 	python setup.py build_python_protos --inplace
 
 lock-python-ci-dependencies:
-	python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
+	uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
 
 package-protos:
 	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
@@ -60,13 +60,15 @@ install-python:
 	python setup.py develop
 
 lock-python-dependencies:
-	python -m piptools compile -U --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt
+	uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt 
 
 lock-python-dependencies-all:
-	pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.9-requirements.txt"
-	pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt"
-	pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.10-requirements.txt"
-	pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt"
+	pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt"
+	pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt"
+	pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt"
+	pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt"
+	pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt"
+	pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt"
 
 benchmark-python:
 	FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests
@@ -359,9 +361,6 @@ kill-trino-locally:
 install-protoc-dependencies:
 	pip install --ignore-installed protobuf==4.24.0 "grpcio-tools>=1.56.2,<2" mypy-protobuf==3.1.0
 
-install-feast-ci-locally:
-	pip install -e ".[ci]"
-
 # Docker
 
 build-docker: build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker

diff --git a/docs/project/development-guide.md b/docs/project/development-guide.md
@@ -123,43 +123,43 @@ Note that this means if you are midway through working through a PR and rebase,
 Setting up your development environment for Feast Python SDK / CLI:
 1. Ensure that you have Docker installed in your environment. Docker is used to provision service dependencies during testing, and build images for feature servers and other components.
    - Please note that we use [Docker with BuiltKit](https://docs.docker.com/develop/develop-images/build_enhancements/).
-2. Ensure that you have `make`, Python (3.8 and above) with `pip`, installed.
+2. Ensure that you have `make` and Python (3.9 or above) installed.
 3. _Recommended:_ Create a virtual environment to isolate development dependencies to be installed
   ```sh
   # create & activate a virtual environment
   python -m venv venv/
   source venv/bin/activate
   ```
-4. Upgrade `pip` if outdated
-  ```sh
-  pip install --upgrade pip
-  ```
-5. (M1 Mac only): Follow the [dev guide](https://github.com/feast-dev/feast/issues/2105)
-6. Install pip-tools
-  ```sh
-  pip install pip-tools
-  ```
-7. (Optional): Install Node & Yarn. Then run the following to build Feast UI artifacts for use in `feast ui`
+4. (M1 Mac only): Follow the [dev guide](https://github.com/feast-dev/feast/issues/2105)
+5. Install uv
+It is recommended to use uv for managing python dependencies.
+```sh
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+or
+```ssh
+pip install uv
+```
+6. (Optional): Install Node & Yarn. Then run the following to build Feast UI artifacts for use in `feast ui`
 ```
 make build-ui
 ```
-8. Install mysql (needed for ci dependencies)
+7. (Optional) install pixi
+pixi is necessary to run step 8 for all python versions at once.
 ```sh
-brew install mysql
+curl -fsSL https://pixi.sh/install.sh | bash
 ```
-9. Install development dependencies for Feast Python SDK / CLI
+8. (Optional): Recompile python lock files
+If you make changes to requirements or simply want to update python lock files to reflect latest versioons.
 ```sh
-pip install -e ".[dev]"
-```
-
-This will allow the installed feast version to automatically reflect changes to your local development version of Feast without needing to reinstall everytime you make code changes.
-
-10. Compile the protubufs
+make lock-python-dependencies-all
+``` 
+9. Install development dependencies for Feast Python SDK / CLI
+This will install package versions from the lock file, install editable version of feast and compile protobufs.
 ```sh
-make compile-protos-python
+make install-python-ci-dependencies-uv
 ```
-
-11. Spin up Docker Image
+10. Spin up Docker Image
 ```sh
 docker build -t docker-whale -f ./sdk/python/feast/infra/feature_servers/multicloud/Dockerfile .
 ```