Fix Install API Client Dependencies Only (#1572)

* Add `requirements/base.txt` and `requirements/openapi.txt` again
* Add GitHub Action to verify that package can be installed with API dependencies only
* Add tests to verify that `requirements/base.txt` and `requirements/openapi.txt` are in sync with `pyproject.toml`
* Add `toml` and `types-toml` dev dependencies

.github/workflows/test_api_deps_only.yml

@@ -0,0 +1,49 @@
+
+name: Install API Dependencies Only
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  detect-code-changes:
+    name: Detect Code Changes
+    runs-on: ubuntu-latest
+    outputs:
+      run-tests: ${{ steps.filter.outputs.run-tests }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          list-files: shell
+          filters: |
+            run-tests:
+              - '!docs/**'
+              - '!examples/**'
+              - '!benchmarks/**'
+
+  test:
+    name: Test
+    needs: detect-code-changes
+    if: needs.detect-code-changes.outputs.run-tests == 'true'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install API Dependencies
+        run: |
+          make install-api-only
+      - name: Test API Client
+        run: |
+          # Test if ApiWorkflowClient can be created.
+          export LIGHTLY_SERVER_LOCATION="localhost:-1"
+          python -c "from lightly.api import ApiWorkflowClient; ApiWorkflowClient(token='abc')"

Makefile

@@ -82,6 +82,12 @@ dist: clean
 install: clean 
 	pip install .
 
+# Should be same command as in the docs:
+# https://docs.lightly.ai/docs/install-lightly#install-the-lightly-python-client
+install-api-only: clean
+	pip install -r requirements/base.txt
+	pip install . --no-deps
+
 # uninstall package from active site
 uninstall: clean
 	pip uninstall lightly

pyproject.toml

@@ -79,12 +79,14 @@ dev = [
   "opencv-python",
   "scikit-learn",
   "pandas",
+  "toml",
   "torchmetrics",
   "lightning-bolts", # for LARS optimizer
   "black==23.1.0", # frozen version to avoid differences between CI and local dev machines
   "isort==5.11.5", # frozen version to avoid differences between CI and local dev machines
   "mypy==1.4.1", # frozen version to avoid differences between CI and local dev machines
-  "types-python-dateutil"
+  "types-python-dateutil",
+  "types-toml"
 ]
 minimal = [
   "certifi==2017.4.17",
@@ -121,6 +123,7 @@ minimal = [
   "opencv-python==4.4.0.46",
   "scikit-learn==0.24.2",
   "pandas",
+  "toml",
   "matplotlib",
   "av==8.0.3"
 ]

requirements/README.md

@@ -0,0 +1,26 @@
+All dependencies are tracked in `pyproject.toml` and no new files should be added to
+the `requirements` directory.
+
+We maintain `base.txt` to allow installing the package only with the dependencies
+necessary to use the API part of the package. The package can be installed with API
+only dependencies by running:
+```
+pip install -r requirements/base.txt
+pip install lightly --no-deps
+```
+This is also documented in our Lightly Worker docs:
+https://docs.lightly.ai/docs/install-lightly#install-the-lightly-python-client
+
+It is currently not possible to move these dependencies to an optional dependency
+group in `pyproject.toml` because pip does not support installing only optional
+dependencies. See https://github.com/pypa/pip/issues/11440
+
+`openapi.txt` is automatically created by the API generator and should not be modified
+manually.
+
+There are tests in [`tests/test_requirements.py`](../tests/test_requirements.py) that
+check that the dependencies in `base.txt` are in sync with the dependencies in
+`pyproject.toml` and `openapi.txt`.
+
+There is also a [GitHub Action](../.github/workflows/test_api_deps_only.yml) that
+verifies that installing only the API part of the package works correctly. 

requirements/base.txt

@@ -0,0 +1,13 @@
+# Minimal dependencies to use the package with the API client.
+# See requirements/README.md for more information.
+certifi>=14.05.14
+hydra-core>=1.0.0
+lightly_utils~=0.0.0
+numpy>=1.18.1,<2
+python_dateutil>=2.5.3
+requests>=2.23.0
+six>=1.10
+tqdm>=4.44
+urllib3 >= 1.25.3
+pydantic >= 1.10.5, < 2
+aenum >= 3.1.11

requirements/openapi.txt

@@ -0,0 +1,5 @@
+python_dateutil >= 2.5.3
+setuptools >= 21.0.0
+urllib3 >= 1.25.3
+pydantic >= 1.10.5, < 2
+aenum >= 3.1.11

tests/test_requirements.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+from typing import List, Set
+
+import toml
+
+
+def test_requirements_base__pyproject() -> None:
+    """Check that all dependencies in requirements/base.txt are also in
+    pyproject.toml."""
+    missing_deps = _requirements("base") - _pyproject("dependencies")
+    assert not missing_deps
+
+
+def test_requirements_base__openapi() -> None:
+    """Check that all dependencies in requirements/openapi.txt are also in
+    requirements/base.txt."""
+    openapi = {
+        dep for dep in _requirements("openapi") if not dep.startswith("setuptools")
+    }
+    missing_deps = openapi - _requirements("base")
+    assert not missing_deps
+
+
+def _pyproject(name: str) -> Set[str]:
+    """Returns dependencies from pyproject.toml."""
+    return _normalize_dependencies(toml.load("pyproject.toml")["project"][name])
+
+
+def _requirements(name: str) -> Set[str]:
+    """Returns dependencies from requirements files."""
+    path = Path(__file__).parent.parent / "requirements" / f"{name}.txt"
+    return _normalize_dependencies(path.read_text().splitlines())
+
+
+def _normalize_dependencies(deps: List[str]) -> Set[str]:
+    """Remove spaces, empty lines, and comments."""
+    return {dep.replace(" ", "") for dep in deps if dep and not dep.startswith("#")}