refactor: 💡 set libcommon as an "editable" dependency (#699)

* refactor: 💡 set libcommon as an "editable" dependency All the jobs, services and workers share the same version of libs/libcommon, which is the current version. Before, we had to update the libcommon version, run poetry build, update the version in the other projects' pyproject.toml and poetry update it. The workflow when updating libcommon will be a lot simpler now. Also: vscode will be able to follow the reference to the source of the libcommon code, instead of showing the packaged dependency in .venv/. Caveat: this also means that now, modifying something in libcommon will affect all the jobs, services and workers, so we need to have to be more careful. This also means that more CI jobs will be run on every libcommon update, and that when deploying on prod, all the images will be updated and redeployed. * refactor: 💡 fix libcommon dependency in the Dockerfiles * feat: 🎸 udpate docker images
huggingface · Jan 25, 2023 · 02ca898 · 02ca898
1 parent f320f88
commit 02ca898
Show file tree

Hide file tree

Showing 100 changed files with 64 additions and 72 deletions.
diff --git a/chart/docker-images.yaml b/chart/docker-images.yaml
@@ -2,14 +2,14 @@
   "dockerImage": {
     "reverseProxy": "docker.io/nginx:1.20",
     "jobs": {
-      "mongodbMigration": "huggingface/datasets-server-jobs-mongodb_migration:sha-0abd269"
+      "mongodbMigration": "huggingface/datasets-server-jobs-mongodb_migration:sha-05a1740"
     },
     "services": {
-      "admin": "huggingface/datasets-server-services-admin:sha-0abd269",
-      "api": "huggingface/datasets-server-services-api:sha-0abd269"
+      "admin": "huggingface/datasets-server-services-admin:sha-05a1740",
+      "api": "huggingface/datasets-server-services-api:sha-05a1740"
     },
     "workers": {
-      "datasets_based": "huggingface/datasets-server-workers-datasets_based:sha-0abd269"
+      "datasets_based": "huggingface/datasets-server-workers-datasets_based:sha-05a1740"
     }
   }
 }
diff --git a/jobs/mongodb_migration/Dockerfile b/jobs/mongodb_migration/Dockerfile
@@ -23,7 +23,7 @@ RUN pip install --no-cache-dir "poetry==$POETRY_VERSION"
 WORKDIR /src
 COPY jobs/mongodb_migration/poetry.lock ./jobs/mongodb_migration/poetry.lock
 COPY jobs/mongodb_migration/pyproject.toml ./jobs/mongodb_migration/pyproject.toml
-COPY libs/libcommon/dist ./libs/libcommon/dist
+COPY libs/libcommon ./libs/libcommon
 WORKDIR /src/jobs/mongodb_migration/
 RUN poetry install --no-cache
 COPY jobs/mongodb_migration/src ./src

diff --git a/jobs/mongodb_migration/poetry.lock b/jobs/mongodb_migration/poetry.lock
diff --git a/jobs/mongodb_migration/pyproject.toml b/jobs/mongodb_migration/pyproject.toml
@@ -7,7 +7,7 @@ license = "Apache-2.0"
 
 [tool.poetry.dependencies]
 environs = "^9.5.0"
-libcommon = { path = "../../libs/libcommon/dist/libcommon-0.6.8-py3-none-any.whl", develop = false }
+libcommon = {path = "../../libs/libcommon", develop = true}
 python = "3.9.15"
 
 [tool.poetry.group.dev.dependencies]

diff --git a/libs/libcommon/dist/libcommon-0.3.0-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.3.0-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.3.0.tar.gz b/libs/libcommon/dist/libcommon-0.3.0.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.3.1-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.3.1-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.3.1.tar.gz b/libs/libcommon/dist/libcommon-0.3.1.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.3.2-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.3.2-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.3.2.tar.gz b/libs/libcommon/dist/libcommon-0.3.2.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.3.3-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.3.3-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.3.3.tar.gz b/libs/libcommon/dist/libcommon-0.3.3.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.3.4-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.3.4-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.3.4.tar.gz b/libs/libcommon/dist/libcommon-0.3.4.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.4.0-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.4.0-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.4.0.tar.gz b/libs/libcommon/dist/libcommon-0.4.0.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.4.1-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.4.1-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.4.1.tar.gz b/libs/libcommon/dist/libcommon-0.4.1.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.4.2-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.4.2-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.4.2.tar.gz b/libs/libcommon/dist/libcommon-0.4.2.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.4.3-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.4.3-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.4.3.tar.gz b/libs/libcommon/dist/libcommon-0.4.3.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.0-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.0-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.0.tar.gz b/libs/libcommon/dist/libcommon-0.5.0.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.1-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.1-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.1.tar.gz b/libs/libcommon/dist/libcommon-0.5.1.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.10-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.10-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.10.tar.gz b/libs/libcommon/dist/libcommon-0.5.10.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.11-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.11-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.11.tar.gz b/libs/libcommon/dist/libcommon-0.5.11.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.2-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.2-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.2.tar.gz b/libs/libcommon/dist/libcommon-0.5.2.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.3-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.3-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.3.tar.gz b/libs/libcommon/dist/libcommon-0.5.3.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.4-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.4-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.4.tar.gz b/libs/libcommon/dist/libcommon-0.5.4.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.5-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.5-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.5.tar.gz b/libs/libcommon/dist/libcommon-0.5.5.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.6-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.6-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.6.tar.gz b/libs/libcommon/dist/libcommon-0.5.6.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.7-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.7-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.7.tar.gz b/libs/libcommon/dist/libcommon-0.5.7.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.8-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.8-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.8.tar.gz b/libs/libcommon/dist/libcommon-0.5.8.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.5.9-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.5.9-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.5.9.tar.gz b/libs/libcommon/dist/libcommon-0.5.9.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.0-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.0-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.0.tar.gz b/libs/libcommon/dist/libcommon-0.6.0.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.1-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.1-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.1.tar.gz b/libs/libcommon/dist/libcommon-0.6.1.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.2-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.2-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.2.tar.gz b/libs/libcommon/dist/libcommon-0.6.2.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.3-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.3-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.3.tar.gz b/libs/libcommon/dist/libcommon-0.6.3.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.4-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.4-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.4.tar.gz b/libs/libcommon/dist/libcommon-0.6.4.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.5-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.5-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.5.tar.gz b/libs/libcommon/dist/libcommon-0.6.5.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.6-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.6-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.6.tar.gz b/libs/libcommon/dist/libcommon-0.6.6.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.7-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.7-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.7.tar.gz b/libs/libcommon/dist/libcommon-0.6.7.tar.gz
diff --git a/libs/libcommon/dist/libcommon-0.6.8-py3-none-any.whl b/libs/libcommon/dist/libcommon-0.6.8-py3-none-any.whl
diff --git a/libs/libcommon/dist/libcommon-0.6.8.tar.gz b/libs/libcommon/dist/libcommon-0.6.8.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.0-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.0-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.0.tar.gz b/libs/libcommon/dist/libutils-0.1.0.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.1-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.1-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.1.tar.gz b/libs/libcommon/dist/libutils-0.1.1.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.10-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.10-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.10.tar.gz b/libs/libcommon/dist/libutils-0.1.10.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.11-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.11-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.11.tar.gz b/libs/libcommon/dist/libutils-0.1.11.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.2-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.2-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.2.tar.gz b/libs/libcommon/dist/libutils-0.1.2.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.3-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.3-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.3.tar.gz b/libs/libcommon/dist/libutils-0.1.3.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.4-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.4-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.4.tar.gz b/libs/libcommon/dist/libutils-0.1.4.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.5-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.5-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.5.tar.gz b/libs/libcommon/dist/libutils-0.1.5.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.6-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.6-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.6.tar.gz b/libs/libcommon/dist/libutils-0.1.6.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.7-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.7-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.7.tar.gz b/libs/libcommon/dist/libutils-0.1.7.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.8-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.8-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.8.tar.gz b/libs/libcommon/dist/libutils-0.1.8.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.1.9-py3-none-any.whl b/libs/libcommon/dist/libutils-0.1.9-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.1.9.tar.gz b/libs/libcommon/dist/libutils-0.1.9.tar.gz
diff --git a/libs/libcommon/dist/libutils-0.2.0-py3-none-any.whl b/libs/libcommon/dist/libutils-0.2.0-py3-none-any.whl
diff --git a/libs/libcommon/dist/libutils-0.2.0.tar.gz b/libs/libcommon/dist/libutils-0.2.0.tar.gz
diff --git a/services/admin/Dockerfile b/services/admin/Dockerfile
@@ -23,7 +23,7 @@ RUN pip install --no-cache-dir "poetry==$POETRY_VERSION"
 WORKDIR /src
 COPY services/admin/poetry.lock ./services/admin/poetry.lock
 COPY services/admin/pyproject.toml ./services/admin/pyproject.toml
-COPY libs/libcommon/dist ./libs/libcommon/dist
+COPY libs/libcommon ./libs/libcommon
 WORKDIR /src/services/admin/
 RUN poetry install --no-cache
 COPY services/admin/src ./src

diff --git a/services/admin/poetry.lock b/services/admin/poetry.lock
diff --git a/services/admin/pyproject.toml b/services/admin/pyproject.toml
@@ -7,7 +7,7 @@ license = "Apache-2.0"
 
 [tool.poetry.dependencies]
 environs = "^9.5.0"
-libcommon = { path = "../../libs/libcommon/dist/libcommon-0.6.8-py3-none-any.whl", develop = false }
+libcommon = {path = "../../libs/libcommon", develop = true}
 python = "3.9.15"
 starlette = "^0.16.0"
 starlette-prometheus = "^0.9.0"

diff --git a/services/api/Dockerfile b/services/api/Dockerfile
@@ -23,7 +23,7 @@ RUN pip install --no-cache-dir "poetry==$POETRY_VERSION"
 WORKDIR /src
 COPY services/api/poetry.lock ./services/api/poetry.lock
 COPY services/api/pyproject.toml ./services/api/pyproject.toml
-COPY libs/libcommon/dist ./libs/libcommon/dist
+COPY libs/libcommon ./libs/libcommon
 WORKDIR /src/services/api/
 RUN poetry install --no-cache
 COPY services/api/src ./src

diff --git a/services/api/poetry.lock b/services/api/poetry.lock
diff --git a/services/api/pyproject.toml b/services/api/pyproject.toml
@@ -9,7 +9,7 @@ license = "Apache-2.0"
 environs = "^9.5.0"
 huggingface-hub = "^0.11.0"
 jsonschema = "^4.16.0"
-libcommon = { path = "../../libs/libcommon/dist/libcommon-0.6.8-py3-none-any.whl", develop = false }
+libcommon = {path = "../../libs/libcommon", develop = true}
 python = "3.9.15"
 starlette = "^0.16.0"
 starlette-prometheus = "^0.9.0"

diff --git a/tools/Python.mk b/tools/Python.mk
@@ -10,10 +10,6 @@ lock:
 	poetry lock
 	poetry install
 
-.PHONY: build
-build:
-	poetry build
-
 # Check that source code meets quality standards + security
 .PHONY: quality
 quality:

diff --git a/workers/datasets_based/Dockerfile b/workers/datasets_based/Dockerfile
@@ -27,7 +27,7 @@ WORKDIR /src
 COPY workers/datasets_based/vendors ./workers/datasets_based/vendors/
 COPY workers/datasets_based/poetry.lock ./workers/datasets_based/poetry.lock
 COPY workers/datasets_based/pyproject.toml ./workers/datasets_based/pyproject.toml
-COPY libs/libcommon/dist ./libs/libcommon/dist
+COPY libs/libcommon ./libs/libcommon
 WORKDIR /src/workers/datasets_based/
 RUN poetry install --no-cache
 COPY workers/datasets_based/src ./src

diff --git a/workers/datasets_based/poetry.lock b/workers/datasets_based/poetry.lock
diff --git a/workers/datasets_based/pyproject.toml b/workers/datasets_based/pyproject.toml
@@ -17,7 +17,7 @@ gdown = "^4.2.0"
 huggingface-hub = "^0.11.0"
 kenlm = { url = "https://github.com/kpu/kenlm/archive/master.zip" }
 kss = "^2.6.0"
-libcommon = { path = "../../libs/libcommon/dist/libcommon-0.6.8-py3-none-any.whl", develop = false }
+libcommon = {path = "../../libs/libcommon", develop = true}
 lm-dataformat = "^0.0.20"
 lxml = "^4.9.1"
 nlp = "^0.4.0"