Merge pull request #1094 from akhanf/master

FIX: cloud paths checking against patterns

bids/layout/index.py

@@ -41,12 +41,17 @@ def _extract_entities(bidsfile, entities):
 
 def _check_path_matches_patterns(path, patterns, root=None):
     """Check if the path matches at least one of the provided patterns. """
+
     if not patterns:
         return False
 
     path = path.absolute()
     if root is not None:
-        path = Path("/") / path.relative_to(root)
+
+        if isinstance(path,Path):
+            path = Path("/") / Path(path.path).relative_to(Path(root.path))
+        else:
+            path = Path("/") / path.relative_to(root)
 
     # Path now can be downcast to str
     path = str(path)
@@ -182,11 +187,11 @@ def _validate_file(self, f):
         return self.validator.is_bids(to_check)
 
     def _index_dir(self, path, config, force=None):
-
-        abs_path = self._layout._root / path
+        root_path = Path(self._layout._root.path) # drops the uri prefix if it is there
+        abs_path = root_path / Path(path.path).relative_to(root_path)
 
         # Derivative directories must always be added separately
-        if self._layout._root.joinpath('derivatives') in abs_path.parents:
+        if root_path.joinpath('derivatives') in abs_path.parents:
             return [], []
 
         config = list(config)  # Shallow copy
@@ -207,6 +212,7 @@ def _index_dir(self, path, config, force=None):
         # Get lists of 1st-level subdirectories and files in the path directory
         _, dirnames, filenames = next(path.fs.walk(path.path))
 
+
         # Symbolic links are returned as filenames even if they point to directories
         # Temporary list to store symbolic links that need to be removed from filenames
         links_to_dirs = []

bids/layout/tests/test_remote_bids.py

@@ -0,0 +1,26 @@
+"""Tests runs layout on bids examples and make sure all files are caught"""
+
+# TODO
+# - add more 'vanilla' datasets
+# - missing files in micr?
+
+import pytest
+from botocore.exceptions import NoCredentialsError
+from upath import UPath
+
+from bids.layout import BIDSLayout
+
+# Values for the number of files by downloading dataset first
+
+
+@pytest.mark.parametrize(
+    "dataset, nb_files",
+    [
+        (UPath("s3://openneuro.org/ds000102", anon=True), 136),
+    ],
+)
+@pytest.mark.xfail(raises=NoCredentialsError)
+def test_layout_on_s3_datasets_no_derivatives(dataset, nb_files):
+    layout = BIDSLayout(dataset)
+    files = layout.get()
+    assert len(files) == nb_files

pyproject.toml

@@ -52,6 +52,7 @@ test = [
   "coverage[toml]",
   "altair",
   "pytest-xdist",
+  "s3fs" #for testing remote uri
 ]
 model_reports = [
   "jinja2",