ML-2296: Add support for datastore profiles (#455)

* Add support for datastore profiles

* Addressing Gal's comments

* Move utils unit tests to separate file

integration/test_filesystems_integration.py

@@ -38,7 +38,6 @@
     build_flow,
 )
 from storey.dtypes import V3ioError
-from storey.utils import get_remaining_path
 
 
 @pytest.fixture()
@@ -665,10 +664,3 @@ def test_filter_before_after_partitioned_outer_other_partition(setup_teardown_te
     ]
 
     assert read_back_result == expected, f"{read_back_result}\n!=\n{expected}"
-
-
-def test_get_path_utils():
-    url = "wasbs://mycontainer@myaccount.blob.core.windows.net/path/to/object.csv"
-    schema, path = get_remaining_path(url)
-    assert path == "mycontainer/path/to/object.csv"
-    assert schema == "wasbs"

storey/utils.py

@@ -127,7 +127,7 @@ def get_remaining_path(url):
     if "://" in url:
         parsed_url = urlparse(url)
         scheme = parsed_url.scheme.lower()
-        if scheme in ("v3io", "dbfs"):
+        if scheme in ("ds", "v3io", "dbfs"):
             remaining_path = parsed_url.path
         elif scheme in ["wasb", "wasbs"]:
             remaining_path = f"{parsed_url.username}{parsed_url.path}"
@@ -138,6 +138,12 @@ def get_remaining_path(url):
 
 def url_to_file_system(url, storage_options):
     scheme, remaining_path = get_remaining_path(url)
+    if url.startswith("ds://"):
+        parsed_url = urlparse(url)
+        if parsed_url.password:
+            scheme = parsed_url.password
+        else:
+            raise ValueError("Datastore profile URL is expected to have underlying scheme embedded as password")
     if scheme:
         load_fs_dependencies(scheme)
 

tests/test_utils.py

@@ -0,0 +1,31 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from fsspec.implementations.local import LocalFileSystem
+
+from storey.utils import get_remaining_path, url_to_file_system
+
+
+def test_get_path_utils():
+    url = "wasbs://mycontainer@myaccount.blob.core.windows.net/path/to/object.csv"
+    schema, path = get_remaining_path(url)
+    assert path == "mycontainer/path/to/object.csv"
+    assert schema == "wasbs"
+
+
+def test_ds_get_path_utils():
+    url = "ds://:file@profile/path/to/object.csv"
+    fs, path = url_to_file_system(url, "")
+    assert path == "/path/to/object.csv"
+    assert isinstance(fs, LocalFileSystem)