Fix private to public

chart/docker-images.yaml

@@ -1,7 +1,7 @@
 {
   "dockerImage": {
     "admin": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-admin:sha-49a60c5",
-    "api": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-api:sha-fe75069",
+    "api": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-api:sha-8d9e37d",
     "reverseProxy": "docker.io/nginx:1.20",
     "worker": {
       "splits": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-0dff3bf",

chart/templates/api/_container.tpl

@@ -14,6 +14,14 @@
     value: {{ .Values.api.assetsDirectory | quote }}
   - name: HF_ENDPOINT
     value: {{ .Values.hfEndpoint | quote }}
+  - name: HF_TOKEN
+    # see https://kubernetes.io/docs/concepts/configuration/secret/#creating-a-secret
+    # and https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables
+    valueFrom:
+      secretKeyRef:
+        name: {{ .Values.secrets.hfToken | quote }}
+        key: HF_TOKEN
+        optional: false
   - name: LOG_LEVEL
     value: {{ .Values.api.logLevel | quote }}
   - name: MAX_AGE_LONG_SECONDS

chart/templates/worker/first-rows/_container.tpl

@@ -13,7 +13,7 @@
   - name: HF_DATASETS_CACHE
     value: "{{ .Values.worker.firstRows.cacheDirectory }}/datasets"
   - name: HF_ENDPOINT
-    value: "{{ .Values.hfEndpoint }}"
+    value: {{ .Values.hfEndpoint | quote }}
   # note: HF_MODULES_CACHE is not set to a shared directory
   - name: HF_MODULES_CACHE
     value: "/tmp/modules-cache"

chart/templates/worker/splits/_container.tpl

@@ -13,7 +13,7 @@
   - name: HF_DATASETS_CACHE
     value: "{{ .Values.worker.splits.cacheDirectory }}/datasets"
   - name: HF_ENDPOINT
-    value: "{{ .Values.hfEndpoint }}"
+    value: {{ .Values.hfEndpoint | quote }}
   - name: HF_MODULES_CACHE
     value: "/tmp/modules-cache"
   # the size should remain so small that we don't need to worry about putting it on an external storage

services/api/README.md

@@ -12,6 +12,7 @@ Set environment variables to configure the following aspects:
 - `ASSETS_DIRECTORY`: directory where the asset files are stored. Defaults to empty, in which case the assets are located in the `datasets_server_assets` subdirectory inside the OS default cache directory.
 - `HF_AUTH_PATH`: the path of the external authentication service, on the hub (see `HF_ENDPOINT`). The string must contain `%s` which will be replaced with the dataset name. The external authentication service must return 200, 401, 403 or 404. If empty, the authentication is disabled. Defaults to "/api/datasets/%s/auth-check".
 - `HF_ENDPOINT`: URL of the HuggingFace Hub. Defaults to `https://huggingface.co`.
+- `HF_TOKEN`: App Access Token (ask moonlanding administrators to get one, only the `read` role is required), to access the gated datasets. Defaults to empty.
 - `LOG_LEVEL`: log level, among `DEBUG`, `INFO`, `WARNING`, `ERROR` and `CRITICAL`. Defaults to `INFO`.
 - `MAX_AGE_LONG_SECONDS`: number of seconds to set in the `max-age` header on data endpoints. Defaults to `120` (2 minutes).
 - `MAX_AGE_SHORT_SECONDS`: number of seconds to set in the `max-age` header on technical endpoints. Defaults to `10` (10 seconds).

services/api/pyproject.toml

@@ -6,6 +6,7 @@ version = "0.1.3"
 license = "Apache-2.0"
 
 [tool.poetry.dependencies]
+huggingface-hub = "^0.9.1"
 libcache = { path = "../../libs/libcache/dist/libcache-0.2.1-py3-none-any.whl", develop = false }
 libqueue = { path = "../../libs/libqueue/dist/libqueue-0.2.0-py3-none-any.whl", develop = false }
 libutils = { path = "../../libs/libutils/dist/libutils-0.2.0-py3-none-any.whl", develop = false }
@@ -24,15 +25,20 @@ mypy = "0.812"
 poetryup = "^0.3.8"
 pytest = "^6.2.5"
 pytest-cov = "^2.12.1"
-responses = "^0.21.0"
+pytest-httpserver = "^1.0.6"
 safety = "^2.1.1"
 
 [build-system]
 build-backend = "poetry.core.masonry.api"
 requires = ["poetry-core>=1.0.0"]
 
 [tool.pytest.ini_options]
+addopts = "-k 'not deprecated'"
 filterwarnings = ["ignore::DeprecationWarning"]
+markers = [
+    "deprecated: tests on deprecated code (deselect with '-m \"not deprecated\"')",
+    "wip: tests being developed"
+]
 
 [tool.coverage.run]
 source = ["api"]

services/api/src/api/app.py

@@ -22,6 +22,8 @@
     APP_PORT,
     ASSETS_DIRECTORY,
     EXTERNAL_AUTH_URL,
+    HF_ENDPOINT,
+    HF_TOKEN,
     LOG_LEVEL,
     MONGO_CACHE_DATABASE,
     MONGO_QUEUE_DATABASE,
@@ -32,7 +34,7 @@
 from api.routes.healthcheck import healthcheck_endpoint
 from api.routes.splits import create_splits_endpoint
 from api.routes.valid import create_is_valid_endpoint, valid_endpoint
-from api.routes.webhook import webhook_endpoint
+from api.routes.webhook import create_webhook_endpoint
 
 
 def create_app() -> Starlette:
@@ -53,12 +55,24 @@ def create_app() -> Starlette:
         Route("/valid", endpoint=valid_endpoint),
         Route("/is-valid", endpoint=create_is_valid_endpoint(EXTERNAL_AUTH_URL)),
         # ^ called by https://github.com/huggingface/model-evaluator
-        Route("/first-rows", endpoint=create_first_rows_endpoint(EXTERNAL_AUTH_URL)),
-        Route("/splits", endpoint=create_splits_endpoint(EXTERNAL_AUTH_URL)),
+        Route(
+            "/first-rows",
+            endpoint=create_first_rows_endpoint(
+                external_auth_url=EXTERNAL_AUTH_URL, hf_endpoint=HF_ENDPOINT, hf_token=HF_TOKEN
+            ),
+        ),
+        Route(
+            "/splits",
+            endpoint=create_splits_endpoint(
+                external_auth_url=EXTERNAL_AUTH_URL, hf_endpoint=HF_ENDPOINT, hf_token=HF_TOKEN
+            ),
+        ),
     ]
     to_protect: List[BaseRoute] = [
         # called by the Hub webhooks
-        Route("/webhook", endpoint=webhook_endpoint, methods=["POST"]),
+        Route(
+            "/webhook", endpoint=create_webhook_endpoint(hf_endpoint=HF_ENDPOINT, hf_token=HF_TOKEN), methods=["POST"]
+        ),
     ]
     protected: List[BaseRoute] = [
         Route("/healthcheck", endpoint=healthcheck_endpoint),