Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: 🐛 install missing dependency #647

Merged
merged 4 commits into from
Nov 24, 2022
Merged

fix: 🐛 install missing dependency #647

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f0c1e0b
fix: 🐛 install missing dependency
severo Nov 24, 2022
c1804c2
fix: 🐛 downgrade pymongo for workers
severo Nov 24, 2022
2d81b2f
feat: 🎸 update libcache and libqueue
severo Nov 24, 2022
8ba01bb
feat: 🎸 upgrade docker images
severo Nov 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/_quality-python.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
run: poetry run bandit -r src
- name: Run pip-audit (datasets worker)
if: ${{ inputs.is-datasets-worker == true }}
run: bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d')"
run: bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^pymongo==/,+110 d' | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d')"
- name: Run pip-audit
if: ${{ inputs.is-datasets-worker == false }}
run: bash -c 'poetry run pip-audit -r <(poetry export -f requirements.txt --with dev)'
if: ${{ inputs.is-datasets-worker == false && inputs.is-library-with-pymongo == false }}
run: bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^pymongo==/,+110 d')"
10 changes: 5 additions & 5 deletions chart/docker-images.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
"dockerImage": {
"reverseProxy": "docker.io/nginx:1.20",
"jobs": {
"mongodbMigration": "huggingface/datasets-server-jobs-mongodb_migration:sha-b6d4c8a"
"mongodbMigration": "huggingface/datasets-server-jobs-mongodb_migration:sha-2d81b2f"
},
"services": {
"admin": "huggingface/datasets-server-services-admin:sha-b6d4c8a",
"api": "huggingface/datasets-server-services-api:sha-b6d4c8a"
"admin": "huggingface/datasets-server-services-admin:sha-2d81b2f",
"api": "huggingface/datasets-server-services-api:sha-2d81b2f"
},
"workers": {
"splits": "huggingface/datasets-server-workers-splits:sha-2bb27cb",
"firstRows": "huggingface/datasets-server-workers-first_rows:sha-2bb27cb"
"splits": "huggingface/datasets-server-workers-splits:sha-2d81b2f",
"firstRows": "huggingface/datasets-server-workers-first_rows:sha-2d81b2f"
}
}
}
225 changes: 130 additions & 95 deletions jobs/mongodb_migration/poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions jobs/mongodb_migration/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ license = "Apache-2.0"

[tool.poetry.dependencies]
environs = "^9.5.0"
libcache = { path = "../../libs/libcache/dist/libcache-0.4.1-py3-none-any.whl", develop = false }
libcache = { path = "../../libs/libcache/dist/libcache-0.4.3-py3-none-any.whl", develop = false }
libcommon = { path = "../../libs/libcommon/dist/libcommon-0.3.3-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.11-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.13-py3-none-any.whl", develop = false }
python = "3.9.6"

[tool.poetry.group.dev.dependencies]
Expand Down
Binary file added libs/libcache/dist/libcache-0.4.2-py3-none-any.whl
View file
Open in desktop
Binary file not shown.
Binary file added libs/libcache/dist/libcache-0.4.2.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added libs/libcache/dist/libcache-0.4.3-py3-none-any.whl
View file
Open in desktop
Binary file not shown.
Binary file added libs/libcache/dist/libcache-0.4.3.tar.gz
View file
Open in desktop
Binary file not shown.
225 changes: 129 additions & 96 deletions libs/libcache/poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion libs/libcache/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@
authors = ["Sylvain Lesage <sylvain.lesage@huggingface.co>"]
description = "Library for the cache in mongodb"
name = "libcache"
version = "0.4.1"
version = "0.4.3"
license = "Apache-2.0"

[tool.poetry.dependencies]
appdirs = "^1.4.4"
environs = "^9.5.0"
mongo-types = "0.15.1"
mongoengine = "^0.24.1"
pymongo = { extras = ["srv"], version = "^3.13.0" }
python = "3.9.6"

[tool.poetry.group.dev.dependencies]
Expand Down
Binary file added libs/libqueue/dist/libqueue-0.4.12-py3-none-any.whl
View file
Open in desktop
Binary file not shown.
Binary file added libs/libqueue/dist/libqueue-0.4.12.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added libs/libqueue/dist/libqueue-0.4.13-py3-none-any.whl
View file
Open in desktop
Binary file not shown.
Binary file added libs/libqueue/dist/libqueue-0.4.13.tar.gz
View file
Open in desktop
Binary file not shown.
225 changes: 129 additions & 96 deletions libs/libqueue/poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion libs/libqueue/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
authors = ["Sylvain Lesage <sylvain.lesage@huggingface.co>"]
description = "Library for the jobs queue in mongodb"
name = "libqueue"
version = "0.4.11"
version = "0.4.13"
license = "Apache-2.0"

[tool.poetry.dependencies]
Expand All @@ -11,6 +11,7 @@ mongo-types = "0.15.1"
mongoengine = "^0.24.1"
packaging = "^21.3"
psutil = "^5.9.2"
pymongo = { extras = ["srv"], version = "^3.13.0" }
python = "3.9.6"

[tool.poetry.group.dev.dependencies]
Expand Down
225 changes: 130 additions & 95 deletions services/admin/poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions services/admin/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ license = "Apache-2.0"
[tool.poetry.dependencies]
environs = "^9.5.0"
huggingface-hub = "^0.11.0"
libcache = { path = "../../libs/libcache/dist/libcache-0.4.1-py3-none-any.whl", develop = false }
libcache = { path = "../../libs/libcache/dist/libcache-0.4.3-py3-none-any.whl", develop = false }
libcommon = { path = "../../libs/libcommon/dist/libcommon-0.3.3-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.11-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.13-py3-none-any.whl", develop = false }
python = "3.9.6"
starlette = "^0.16.0"
starlette-prometheus = "^0.9.0"
Expand Down
225 changes: 130 additions & 95 deletions services/api/poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions services/api/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ license = "Apache-2.0"
environs = "^9.5.0"
huggingface-hub = "^0.11.0"
jsonschema = "^4.16.0"
libcache = { path = "../../libs/libcache/dist/libcache-0.4.1-py3-none-any.whl", develop = false }
libcache = { path = "../../libs/libcache/dist/libcache-0.4.3-py3-none-any.whl", develop = false }
libcommon = { path = "../../libs/libcommon/dist/libcommon-0.3.3-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.11-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.13-py3-none-any.whl", develop = false }
python = "3.9.6"
starlette = "^0.16.0"
starlette-prometheus = "^0.9.0"
Expand Down
3 changes: 2 additions & 1 deletion tools/PythonAudit.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
.PHONY: pip-audit
pip-audit:
bash -c 'poetry run pip-audit -r <(poetry export -f requirements.txt --with dev)'
bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^pymongo==/,+110 d')"
# ^ we remove problematic lines to have a working pip-audit. See https://github.com/pypa/pip-audit/issues/84#issuecomment-1326203111 for "requests"
2 changes: 1 addition & 1 deletion workers/first_rows/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ run:

.PHONY: pip-audit
pip-audit:
bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d' | sed '/^libcache @/,+1 d' | sed '/^libcommon @/,+1 d' | sed '/^libqueue @/,+1 d')"
bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^pymongo==/,+110 d' | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d' | sed '/^libcache @/,+1 d' | sed '/^libcommon @/,+1 d' | sed '/^libqueue @/,+1 d')"
# ^ we remove problematic lines to have a working pip-audit. See https://github.com/pypa/pip-audit/issues/84#issuecomment-1326203111 for "requests"
35 changes: 28 additions & 7 deletions workers/first_rows/poetry.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions workers/first_rows/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ gdown = "^4.2.0"
huggingface-hub = "^0.11.0"
kenlm = { url = "https://github.com/kpu/kenlm/archive/master.zip" }
kss = "^2.6.0"
libcache = { path = "../../libs/libcache/dist/libcache-0.4.1-py3-none-any.whl", develop = false }
libcache = { path = "../../libs/libcache/dist/libcache-0.4.3-py3-none-any.whl", develop = false }
libcommon = { path = "../../libs/libcommon/dist/libcommon-0.3.3-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.11-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.13-py3-none-any.whl", develop = false }
lm-dataformat = "^0.0.20"
lxml = "^4.9.1"
nlp = "^0.4.0"
Expand Down
2 changes: 1 addition & 1 deletion workers/splits/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ run:

.PHONY: pip-audit
pip-audit:
bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d')"
bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^pymongo==/,+110 d' | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d')"
# ^ we remove problematic lines to have a working pip-audit. See https://github.com/pypa/pip-audit/issues/84#issuecomment-1326203111 for "requests"
35 changes: 28 additions & 7 deletions workers/splits/poetry.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions workers/splits/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ gdown = "^4.2.0"
huggingface-hub = "^0.11.0"
kenlm = { url = "https://github.com/kpu/kenlm/archive/master.zip" }
kss = "^2.6.0"
libcache = { path = "../../libs/libcache/dist/libcache-0.4.1-py3-none-any.whl", develop = false }
libcache = { path = "../../libs/libcache/dist/libcache-0.4.3-py3-none-any.whl", develop = false }
libcommon = { path = "../../libs/libcommon/dist/libcommon-0.3.3-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.11-py3-none-any.whl", develop = false }
libqueue = { path = "../../libs/libqueue/dist/libqueue-0.4.13-py3-none-any.whl", develop = false }
lm-dataformat = "^0.0.20"
lxml = "^4.9.1"
nlp = "^0.4.0"
Expand Down