refactor: 💡 use pathlib instead of os.path

.github/workflows/s-admin-build.yml

@@ -4,7 +4,7 @@ on:
   push:
     paths:
       - 'services/admin/Dockerfile'
-      - 'services/admin/src'
+      - 'services/admin/src/**'
       - 'services/admin/poetry.lock'
       - 'services/admin/pyproject.toml'
       - '.github/workflows/s-admin-build.yml'

.github/workflows/s-api-build.yml

@@ -4,7 +4,7 @@ on:
   push:
     paths:
       - 'services/api/Dockerfile'
-      - 'services/api/src'
+      - 'services/api/src/**'
       - 'services/api/poetry.lock'
       - 'services/api/pyproject.toml'
       - '.github/workflows/s-api-build.yml'

.github/workflows/s-worker-build.yml

@@ -4,7 +4,7 @@ on:
   push:
     paths:
       - 'services/worker/Dockerfile'
-      - 'services/worker/src'
+      - 'services/worker/src/**'
       - 'services/worker/poetry.lock'
       - 'services/worker/pyproject.toml'
       - '.github/workflows/s-worker-build.yml'

chart/docker-images.yaml

@@ -1,13 +1,13 @@
 {
   "dockerImage": {
-    "admin": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-admin:sha-fff7ce4",
-    "api": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-api:sha-fff7ce4",
+    "admin": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-admin:sha-75a29ae",
+    "api": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-api:sha-75a29ae",
     "reverseProxy": "docker.io/nginx:1.20",
     "worker": {
-      "datasets": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-fff7ce4",
-      "firstRows": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-fff7ce4",
-      "splits": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-fff7ce4",
-      "splitsNext": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-fff7ce4"
+      "datasets": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-75a29ae",
+      "firstRows": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-75a29ae",
+      "splits": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-75a29ae",
+      "splitsNext": "707930574880.dkr.ecr.us-east-1.amazonaws.com/hub-datasets-server-worker:sha-75a29ae"
     }
   }  
 }

e2e/tests/utils.py

@@ -1,7 +1,7 @@
 import json
 import os
 import time
-from os.path import dirname, join
+from pathlib import Path
 from typing import Any, Dict, Optional, Tuple
 
 import requests
@@ -117,8 +117,8 @@ def refresh_poll_splits_next_first_rows(
 
 
 def get_openapi_body_example(path, status, example_name):
-    root = dirname(dirname(dirname(__file__)))
-    openapi_filename = join(root, "chart", "static-files", "openapi.json")
+    root = Path(__file__).resolve().parent.parent.parent
+    openapi_filename = root / "chart" / "static-files" / "openapi.json"
     with open(openapi_filename) as json_file:
         openapi = json.load(json_file)
     return openapi["paths"][path]["get"]["responses"][str(status)]["content"]["application/json"]["examples"][

services/worker/src/worker/asset.py

@@ -1,5 +1,6 @@
 import logging
-import os
+from os import makedirs
+from pathlib import Path
 from typing import List, Tuple, TypedDict
 
 import soundfile  # type:ignore
@@ -16,11 +17,11 @@
 ASSET_DIR_MODE = 0o755
 
 
-def create_asset_dir(dataset: str, config: str, split: str, row_idx: int, column: str) -> Tuple[str, str]:
+def create_asset_dir(dataset: str, config: str, split: str, row_idx: int, column: str) -> Tuple[Path, str]:
     assets_dir = init_assets_dir(ASSETS_DIRECTORY)
-    dir_path = os.path.join(assets_dir, dataset, DATASET_SEPARATOR, config, split, str(row_idx), column)
+    dir_path = Path(assets_dir).resolve() / dataset / DATASET_SEPARATOR / config / split / str(row_idx) / column
     url_dir_path = f"{dataset}/{DATASET_SEPARATOR}/{config}/{split}/{row_idx}/{column}"
-    os.makedirs(dir_path, ASSET_DIR_MODE, exist_ok=True)
+    makedirs(dir_path, ASSET_DIR_MODE, exist_ok=True)
     return dir_path, url_dir_path
 
 
@@ -35,7 +36,7 @@ def create_image_file(
     assets_base_url: str,
 ) -> str:
     dir_path, url_dir_path = create_asset_dir(dataset, config, split, row_idx, column)
-    file_path = os.path.join(dir_path, filename)
+    file_path = dir_path / filename
     image.save(file_path)
     return f"{assets_base_url}/{url_dir_path}/{filename}"
 
@@ -58,8 +59,8 @@ def create_audio_files(
     wav_filename = "audio.wav"
     mp3_filename = "audio.mp3"
     dir_path, url_dir_path = create_asset_dir(dataset, config, split, row_idx, column)
-    wav_file_path = os.path.join(dir_path, wav_filename)
-    mp3_file_path = os.path.join(dir_path, mp3_filename)
+    wav_file_path = dir_path / wav_filename
+    mp3_file_path = dir_path / mp3_filename
     soundfile.write(wav_file_path, array, sampling_rate)
     segment = AudioSegment.from_wav(wav_file_path)
     segment.export(mp3_file_path, format="mp3")

services/worker/tests/conftest.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 
@@ -7,7 +8,7 @@
 
 @pytest.fixture(scope="session")
 def config():
-    return {"image_file": os.path.join(os.path.dirname(__file__), "data", "test_image_rgb.jpg")}
+    return {"image_file": str(Path(__file__).resolve().parent / "data" / "test_image_rgb.jpg")}
 
 
 os.environ["HF_ENDPOINT"] = HF_ENDPOINT