fix: 🐛 set the modules cache inside /tmp (#418)

by default, it was created in `/.cache` which cannot be written.
huggingface · Jun 23, 2022 · f2f5706 · f2f5706
1 parent ec95bbc
commit f2f5706
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 5 deletions.
diff --git a/infra/charts/datasets-server/templates/datasets-worker/_container.tpl b/infra/charts/datasets-server/templates/datasets-worker/_container.tpl
@@ -7,9 +7,9 @@
     value: {{ .Values.datasetsWorker.datasetsRevision | quote }}
   - name: HF_DATASETS_CACHE
     value: "{{ .Values.datasetsWorker.cacheDirectory }}/datasets"
-  # note: HF_MODULES_CACHE is not set to a shared directory
-  # we let "datasets" find a directory by itself: we want a different cache for the modules for each worker
-  # and the size should remain so small that we don't need to worry about putting it on an external storage
+  - name: HF_MODULES_CACHE
+    value: "/tmp/modules-cache"
+  # the size should remain so small that we don't need to worry about putting it on an external storage
   # see https://github.com/huggingface/datasets-server/issues/248
   - name: HF_TOKEN
     # see https://kubernetes.io/docs/concepts/configuration/secret/#creating-a-secret

diff --git a/infra/charts/datasets-server/templates/splits-worker/_container.tpl b/infra/charts/datasets-server/templates/splits-worker/_container.tpl
@@ -8,8 +8,9 @@
   - name: HF_DATASETS_CACHE
     value: "{{ .Values.splitsWorker.cacheDirectory }}/datasets"
   # note: HF_MODULES_CACHE is not set to a shared directory
-  # we let "datasets" find a directory by itself: we want a different cache for the modules for each worker
-  # and the size should remain so small that we don't need to worry about putting it on an external storage
+  - name: HF_MODULES_CACHE
+    value: "/tmp/modules-cache"
+  # the size should remain so small that we don't need to worry about putting it on an external storage
   # see https://github.com/huggingface/datasets-server/issues/248
   - name: HF_TOKEN
     # see https://kubernetes.io/docs/concepts/configuration/secret/#creating-a-secret