feat: 🎸 ensure the supported datasets are not private

huggingface · Jan 30, 2023 · c3905d4 · c3905d4
1 parent a8648f2
commit c3905d4
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/libs/libcommon/src/libcommon/dataset.py b/libs/libcommon/src/libcommon/dataset.py
@@ -216,4 +216,8 @@ def check_support(
 
 
 def get_supported_datasets(hf_endpoint: str, hf_token: Optional[str] = None) -> list[str]:
-    return [d.id for d in HfApi(endpoint=hf_endpoint, token=hf_token).list_datasets() if d.id is not None]
+    return [
+        d.id
+        for d in HfApi(endpoint=hf_endpoint, token=hf_token).list_datasets()
+        if d.id is not None and d.private is False
+    ]