Use pip-tools to handle Python deps

.github/workflows/linting-and-tests.yml

@@ -127,7 +127,8 @@ jobs:
         # makemigrations --check = Exit with a non-zero status if model changes are missing migrations
         # and don't actually write them.
         run: |
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install pip-tools
+          pip-sync requirements.txt requirements-dev.txt
           python manage.py makemigrations --check
           python manage.py lintmigrations
 
@@ -184,7 +185,8 @@ jobs:
         working-directory: engine
         run: |
           apt-get update && apt-get install -y netcat-traditional
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install pip-tools
+          pip-sync requirements.txt requirements-dev.txt
           ./wait_for_test_mysql_start.sh && pytest -x
 
   unit-test-backend-postgresql-rabbitmq:
@@ -233,7 +235,8 @@ jobs:
       - name: Unit Test Backend
         working-directory: engine
         run: |
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install pip-tools
+          pip-sync requirements.txt requirements-dev.txt
           pytest -x
 
   unit-test-backend-sqlite-redis:
@@ -272,7 +275,8 @@ jobs:
         working-directory: engine
         run: |
           apt-get update && apt-get install -y netcat-traditional
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install pip-tools
+          pip-sync requirements.txt requirements-dev.txt
           pytest -x
 
   unit-test-pd-migrator:
@@ -288,7 +292,8 @@ jobs:
       - name: Unit Test PD Migrator
         working-directory: tools/pagerduty-migrator
         run: |
-          pip install -r requirements.txt
+          pip install pip-tools
+          pip-sync requirements.txt
           pytest -x
 
   mypy:
@@ -306,7 +311,8 @@ jobs:
       - name: mypy Static Type Checking
         working-directory: engine
         run: |
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install pip-tools
+          pip-sync requirements.txt requirements-dev.txt
           mypy .
 
   end-to-end-tests:

Makefile

@@ -28,6 +28,10 @@ DEV_HELM_FILE = $(DEV_ENV_DIR)/helm-local.yml
 DEV_HELM_USER_SPECIFIC_FILE = $(DEV_ENV_DIR)/helm-local.dev.yml
 
 ENGINE_DIR = ./engine
+VENV_DIR = ./venv
+REQUIREMENTS_DEV_IN = $(ENGINE_DIR)/requirements-dev.in
+REQUIREMENTS_DEV_TXT = $(ENGINE_DIR)/requirements-dev.txt
+REQUIREMENTS_IN = $(ENGINE_DIR)/requirements.in
 REQUIREMENTS_TXT = $(ENGINE_DIR)/requirements.txt
 REQUIREMENTS_ENTERPRISE_TXT = $(ENGINE_DIR)/requirements-enterprise.txt
 SQLITE_DB_FILE = $(ENGINE_DIR)/oncall.db
@@ -237,20 +241,29 @@ backend-debug-disable: _backend-debug-disable stop start
 define backend_command
 	export `grep -v '^#' $(DEV_ENV_FILE) | xargs -0` && \
 	export BROKER_TYPE=$(BROKER_TYPE) && \
+	. ./venv/bin/activate && \
 	cd engine && \
 	$(1)
 endef
 
 backend-bootstrap:
-	pip install -U pip wheel
-	pip install -r $(REQUIREMENTS_TXT)
+	python3.11 -m venv $(VENV_DIR)
+	$(VENV_DIR)/bin/pip install -U pip wheel pip-tools
+	$(VENV_DIR)/bin/pip-sync $(REQUIREMENTS_TXT) $(REQUIREMENTS_DEV_TXT)
 	@if [ -f $(REQUIREMENTS_ENTERPRISE_TXT) ]; then \
-		pip install -r $(REQUIREMENTS_ENTERPRISE_TXT); \
+		$(VENV_DIR)/bin/pip install -r $(REQUIREMENTS_ENTERPRISE_TXT); \
 	fi
 
 backend-migrate:
 	$(call backend_command,python manage.py migrate)
 
+backend-compile-deps:
+	pip-compile --strip-extras $(REQUIREMENTS_IN)
+	pip-compile --strip-extras $(REQUIREMENTS_DEV_IN)
+
+backend-upgrade-deps:
+	pip-compile --strip-extras --upgrade $(REQUIREMENTS_IN)
+
 run-backend-server:
 	$(call backend_command,python manage.py runserver 0.0.0.0:8080)
 

Tiltfile

@@ -44,7 +44,7 @@ docker_build_sub(
     live_update=[
         sync("./engine/", "/etc/app"),
         run(
-            "cd /etc/app && pip install -r requirements.txt",
+            "cd /etc/app && pip install pip-tools && pip-sync",
             trigger="./engine/requirements.txt",
         ),
     ],

dev/README.md

@@ -189,13 +189,11 @@ See the `django-silk` documentation [here](https://github.com/jazzband/django-si
 By default everything runs inside Docker. If you would like to run the backend services outside of Docker
 (for integrating w/ PyCharm for example), follow these instructions:
 
-1. Create a Python 3.11 virtual environment using a method of your choosing (ex.
-   [venv](https://docs.python.org/3.11/library/venv.html) or [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
-   Make sure the virtualenv is "activated".
+1. Make sure you have Python 3.11 installed.
 2. `postgres` is a dependency on some of our Python dependencies (notably `psycopg2`
    ([docs](https://www.psycopg.org/docs/install.html#prerequisites))). Please visit
    [here](https://www.postgresql.org/download/) for installation instructions.
-3. `make backend-bootstrap` - installs all backend dependencies
+3. `make backend-bootstrap` - will create the virtual env and install all backend dependencies
 4. Modify your `.env.dev` by copying the contents of one of `.env.mysql.dev`, `.env.postgres.dev`,
    or `.env.sqlite.dev` into `.env.dev` (you should exclude the `GF_` prefixed environment variables).
 
@@ -209,6 +207,22 @@ By default everything runs inside Docker. If you would like to run the backend s
 - `make run-backend-server` - runs the HTTP server
 - `make run-backend-celery` - runs Celery workers
 
+### Adding or updating Python dependencies
+
+We are using [pip-tools](https://github.com/jazzband/pip-tools) to manage our dependencies. It helps
+making builds deterministic, controlling deps (and indirect deps) upgrades (and versions consistency)
+avoiding unexpected (and potentially breaking) changes.
+
+We keep our direct deps in `requirements.in` from which we generate (through `pip-compile`) the
+`requirements.txt` (where all deps are pinned). We also constrain dev (and enterprise) deps based
+on our base requirements. Check [how to update deps](https://github.com/jazzband/pip-tools?tab=readme-ov-file#updating-requirements).
+
+`pip install -r requirements.txt` will keep working (the difference is that this should never
+bring additional dependencies or different versions not listed there), and when starting an env
+from scratch, it would be the same as running `pip-sync`. `pip-sync` on the other hand will also
+ensure to clean up any deps not listed in the requirements, keeping the env exactly as described
+in `requirements.txt`.
+
 ## UI E2E Tests
 
 We've developed a suite of "end-to-end" integration tests using [Playwright](https://playwright.dev/). These tests

engine/requirements-dev.in

@@ -0,0 +1,16 @@
+-c requirements.txt
+celery-types==0.18.0
+django-filter-stubs==0.1.3
+django-stubs[compatible-mypy]==4.2.2
+djangorestframework-stubs[compatible-mypy]==3.14.2
+httpretty==1.1.4
+mypy==1.4.1
+pre-commit==2.15.0
+pytest==7.3.1
+pytest-django==4.5.2
+pytest-xdist[psutil]==3.5.0
+pytest_factoryboy==2.5.1
+types-beautifulsoup4==4.12.0.5
+types-PyMySQL==1.0.19.7
+types-python-dateutil==2.8.19.13
+types-requests==2.31.0.1

engine/requirements-dev.txt

@@ -1,15 +1,169 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --strip-extras ./engine/requirements-dev.in
+#
+asgiref==3.7.2
+    # via
+    #   -c ./engine/requirements.txt
+    #   django
 celery-types==0.18.0
+    # via -r ./engine/requirements-dev.in
+certifi==2024.2.2
+    # via
+    #   -c ./engine/requirements.txt
+    #   requests
+cfgv==3.4.0
+    # via pre-commit
+charset-normalizer==3.3.2
+    # via
+    #   -c ./engine/requirements.txt
+    #   requests
+distlib==0.3.8
+    # via virtualenv
+django==4.2.10
+    # via
+    #   -c ./engine/requirements.txt
+    #   django-stubs
+    #   django-stubs-ext
 django-filter-stubs==0.1.3
-django-stubs[compatible-mypy]==4.2.2
-djangorestframework-stubs[compatible-mypy]==3.14.2
+    # via -r ./engine/requirements-dev.in
+django-stubs==4.2.2
+    # via
+    #   -r ./engine/requirements-dev.in
+    #   django-filter-stubs
+    #   django-stubs
+    #   djangorestframework-stubs
+django-stubs-ext==4.2.7
+    # via django-stubs
+djangorestframework-stubs==3.14.2
+    # via
+    #   -r ./engine/requirements-dev.in
+    #   django-filter-stubs
+    #   djangorestframework-stubs
+execnet==2.0.2
+    # via pytest-xdist
+factory-boy==2.12.0
+    # via
+    #   -c ./engine/requirements.txt
+    #   pytest-factoryboy
+faker==23.1.0
+    # via
+    #   -c ./engine/requirements.txt
+    #   factory-boy
+filelock==3.13.1
+    # via virtualenv
+httpretty==1.1.4
+    # via -r ./engine/requirements-dev.in
+identify==2.5.34
+    # via pre-commit
+idna==3.6
+    # via
+    #   -c ./engine/requirements.txt
+    #   requests
+inflection==0.5.1
+    # via
+    #   -c ./engine/requirements.txt
+    #   pytest-factoryboy
+iniconfig==2.0.0
+    # via pytest
 mypy==1.4.1
+    # via
+    #   -r ./engine/requirements-dev.in
+    #   django-filter-stubs
+    #   django-stubs
+    #   djangorestframework-stubs
+mypy-extensions==1.0.0
+    # via mypy
+nodeenv==1.8.0
+    # via pre-commit
+packaging==23.2
+    # via pytest
+platformdirs==4.2.0
+    # via virtualenv
+pluggy==1.4.0
+    # via pytest
 pre-commit==2.15.0
+    # via -r ./engine/requirements-dev.in
+psutil==5.9.4
+    # via
+    #   -c ./engine/requirements.txt
+    #   pytest-xdist
 pytest==7.3.1
+    # via
+    #   -r ./engine/requirements-dev.in
+    #   pytest-django
+    #   pytest-factoryboy
+    #   pytest-xdist
 pytest-django==4.5.2
-pytest_factoryboy==2.5.1
+    # via -r ./engine/requirements-dev.in
+pytest-factoryboy==2.5.1
+    # via -r ./engine/requirements-dev.in
+pytest-xdist==3.5.0
+    # via
+    #   -r ./engine/requirements-dev.in
+    #   pytest-xdist
+python-dateutil==2.8.2
+    # via
+    #   -c ./engine/requirements.txt
+    #   faker
+pyyaml==6.0.1
+    # via
+    #   -c ./engine/requirements.txt
+    #   pre-commit
+requests==2.31.0
+    # via
+    #   -c ./engine/requirements.txt
+    #   djangorestframework-stubs
+six==1.16.0
+    # via
+    #   -c ./engine/requirements.txt
+    #   python-dateutil
+sqlparse==0.4.4
+    # via
+    #   -c ./engine/requirements.txt
+    #   django
+toml==0.10.2
+    # via
+    #   -c ./engine/requirements.txt
+    #   pre-commit
 types-beautifulsoup4==4.12.0.5
-types-PyMySQL==1.0.19.7
+    # via -r ./engine/requirements-dev.in
+types-html5lib==1.1.11.20240106
+    # via types-beautifulsoup4
+types-pymysql==1.0.19.7
+    # via -r ./engine/requirements-dev.in
 types-python-dateutil==2.8.19.13
+    # via -r ./engine/requirements-dev.in
+types-pytz==2024.1.0.20240203
+    # via django-stubs
+types-pyyaml==6.0.12.12
+    # via
+    #   django-stubs
+    #   djangorestframework-stubs
 types-requests==2.31.0.1
-httpretty==1.1.4
-pytest-xdist[psutil]==3.5.0
+    # via
+    #   -r ./engine/requirements-dev.in
+    #   djangorestframework-stubs
+types-urllib3==1.26.25.14
+    # via types-requests
+typing-extensions==4.9.0
+    # via
+    #   -c ./engine/requirements.txt
+    #   celery-types
+    #   django-filter-stubs
+    #   django-stubs
+    #   django-stubs-ext
+    #   djangorestframework-stubs
+    #   mypy
+    #   pytest-factoryboy
+urllib3==1.26.18
+    # via
+    #   -c ./engine/requirements.txt
+    #   requests
+virtualenv==20.25.0
+    # via pre-commit
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools