fsspec and upath for cloud/remote paths

 - make use of universal_pathlib to enable cloud/remote paths in pybids
 - remove uri prefix when passing to validator
Co-authored-by: KarlHanEdn <tianming@ualberta.ca>

bids/config.py

@@ -1,7 +1,7 @@
 ''' Utilities for manipulating package-level settings. '''
 
 import json
-from pathlib import Path
+from upath import UPath as Path
 import os
 import warnings
 

bids/conftest.py

@@ -14,7 +14,7 @@
 """
 
 import os
-from pathlib import Path
+from upath import UPath as Path
 from unittest.mock import patch
 
 import pytest

bids/layout/db.py

@@ -2,7 +2,7 @@
 Database-related functionality.
 """
 
-from pathlib import Path
+from upath import UPath as Path
 import re
 import sqlite3
 from functools import lru_cache

bids/layout/index.py

@@ -3,8 +3,9 @@
 import os
 import json
 import re
+import fsspec
 from collections import defaultdict
-from pathlib import Path
+from upath import UPath as Path
 from functools import partial, lru_cache
 
 from bids_validator import BIDSValidator
@@ -173,7 +174,7 @@ def _validate_file(self, f):
 
         # BIDS validator expects absolute paths, but really these are relative
         # to the BIDS project root.
-        to_check = f.relative_to(self._layout._root)
+        to_check = Path(f.path).relative_to(Path(self._layout._root.path)) # use .path then Path() to drop the uri prefix
         # Pretend the path is an absolute path
         to_check = Path('/') / to_check
         # bids-validator works with posix paths only
@@ -202,8 +203,9 @@ def _index_dir(self, path, config, force=None):
         for c in config:
             config_entities.update(c.entities)
 
+
         # Get lists of 1st-level subdirectories and files in the path directory
-        _, dirnames, filenames = next(os.walk(path))
+        _, dirnames, filenames = next(path.fs.walk(path.path))
 
         # Move any directories suffixed with .zarr to filenames
         zarrnames = [entry for entry in dirnames if Path(entry).suffix == '.zarr']
@@ -303,7 +305,7 @@ def _index_metadata(self):
         # if they correspond to data files that are indexed
         @lru_cache(maxsize=None)
         def load_json(path):
-            with open(path, 'r', encoding='utf-8') as handle:
+            with Path(path).fs.open(Path(path).path, 'r', encoding='utf-8') as handle:
                 try:
                     return json.load(handle)
                 except (UnicodeDecodeError, json.JSONDecodeError) as e:
@@ -324,7 +326,7 @@ def load_json(path):
 
                 payload = None
                 if ext == '.json':
-                    payload = partial(load_json, bf.path)
+                    payload = partial(load_json, bf)
                 else:
                     filenames.append(bf)
 
@@ -496,3 +498,4 @@ def create_association_pair(src, dst, kind, kind2=None):
         self.session.bulk_save_objects(all_objs)
         self.session.bulk_insert_mappings(Tag, all_tag_dicts)
         self.session.commit()
+

bids/layout/layout.py

@@ -7,7 +7,7 @@
 import copy
 import enum
 import difflib
-from pathlib import Path
+from upath import UPath as Path
 import warnings
 from typing import Hashable
 

bids/layout/models.py

@@ -3,6 +3,7 @@
 import re
 import os
 from pathlib import Path
+from upath import UPath
 import warnings
 import json
 from copy import deepcopy
@@ -63,19 +64,19 @@ def _init_on_load(self):
     def _sanitize_init_args(self, kwargs):
         """ Prepare initialization arguments for serialization """
         if 'root' in kwargs:
-            kwargs['root'] = str(Path(kwargs['root']).absolute())
+            kwargs['root'] = str(UPath(kwargs['root']).absolute())
 
         if 'config' in kwargs and isinstance(kwargs['config'], list):
             kwargs['config'] = [
-                str(Path(config).absolute())
+                str(UPath(config).absolute())
                 if isinstance(config, os.PathLike) else config
                 for config in kwargs['config']
             ]
 
         # Get abspaths
         if kwargs.get('derivatives') not in (None, True, False):
             kwargs['derivatives'] = [
-                str(Path(der).absolute())
+                str(UPath(der).absolute())
                 for der in listify(kwargs['derivatives'])
                 ]
 
@@ -157,11 +158,11 @@ def load(self, config, session=None):
         A Config instance.
         """
 
-        if isinstance(config, (str, Path)):
+        if isinstance(config, (str, Path, UPath)):
             config_paths = get_option('config_paths')
             if config in config_paths:
                 config = config_paths[config]
-            if not Path(config).exists():
+            if not UPath(config).exists():
                 raise ValueError("{} is not a valid path.".format(config))
             else:
                 with open(config, 'r') as f:
@@ -213,11 +214,11 @@ def __init__(self, filename):
 
     @property
     def _path(self):
-        return Path(self.path)
+        return UPath(self.path)
 
     @property
     def _dirname(self):
-        return Path(self.dirname)
+        return UPath(self.dirname)
 
     def __getattr__(self, attr):
         # Ensures backwards compatibility with old File_ namedtuple, which is
@@ -245,7 +246,7 @@ def __fspath__(self):
     def relpath(self):
         """Return path relative to layout root"""
         root = object_session(self).query(LayoutInfo).first().root
-        return str(Path(self.path).relative_to(root))
+        return str(UPath(self.path).relative_to(root))
 
     def get_associations(self, kind=None, include_parents=False):
         """Get associated files, optionally limiting by association kind.
@@ -370,7 +371,7 @@ def copy(self, path_patterns, symbolic_link=False, root=None,
         if self._path.is_absolute() or root is None:
             path = self._path
         else:
-            path = Path(root) / self._path
+            path = UPath(root) / self._path
 
         if not path.exists():
             raise ValueError("Target filename to copy/symlink (%s) doesn't "

bids/layout/utils.py

@@ -1,5 +1,5 @@
 """Miscellaneous layout-related utilities."""
-from pathlib import Path
+from upath import UPath as Path
 
 from .. import config as cf
 from ..utils import make_bidsfile, listify

bids/layout/validation.py

@@ -1,6 +1,6 @@
 """Functionality related to validation of BIDSLayouts and BIDS projects."""
 
-from pathlib import Path
+from upath import UPath as Path
 import json
 import re
 import warnings
@@ -65,7 +65,6 @@ def validate_root(root, validate):
                         "containing the BIDS dataset.")
 
     root = root.absolute()
-
     if not root.exists():
         raise ValueError("BIDS root does not exist: %s" % root)
 
@@ -83,7 +82,7 @@ def validate_root(root, validate):
     else:
         err = None
         try:
-            with open(target, 'r', encoding='utf-8') as desc_fd:
+            with target.fs.open(target.path, 'r', encoding='utf-8') as desc_fd:
                 description = json.load(desc_fd)
         except (UnicodeDecodeError, json.JSONDecodeError) as e:
             description = None

bids/layout/writing.py

@@ -9,7 +9,7 @@
 from string import Formatter
 from itertools import product
 from ..utils import listify
-from pathlib import Path
+from upath import UPath as Path
 
 __all__ = ['build_path', 'write_to_file']
 

bids/utils.py

@@ -2,7 +2,7 @@
 
 import re
 import os
-from pathlib import Path
+from upath import UPath as Path
 
 
 def listify(obj):

pyproject.toml

@@ -32,6 +32,9 @@ dependencies = [
   "bids-validator>=1.11",  # Keep up-to-date to ensure support for recent modalities
   "num2words >=0.5.5",
   "click >=8.0",
+  "fsspec==2024.3.1",
+  "universal_pathlib==0.2.2",
+  "gcsfs==2024.3.1",
 ]
 dynamic = ["version"]
 

setup.py

@@ -3,6 +3,6 @@
 import versioneer
 
 setup(
-    version=versioneer.get_version(),
+    version='0.16.5-dev',
     cmdclass=versioneer.get_cmdclass(),
 )