diff --git a/bids/layout/index.py b/bids/layout/index.py index 8f163228..048c11d2 100644 --- a/bids/layout/index.py +++ b/bids/layout/index.py @@ -41,12 +41,17 @@ def _extract_entities(bidsfile, entities): def _check_path_matches_patterns(path, patterns, root=None): """Check if the path matches at least one of the provided patterns. """ + if not patterns: return False path = path.absolute() if root is not None: - path = Path("/") / path.relative_to(root) + + if isinstance(path,Path): + path = Path("/") / Path(path.path).relative_to(Path(root.path)) + else: + path = Path("/") / path.relative_to(root) # Path now can be downcast to str path = str(path) @@ -182,11 +187,11 @@ def _validate_file(self, f): return self.validator.is_bids(to_check) def _index_dir(self, path, config, force=None): - - abs_path = self._layout._root / path + root_path = Path(self._layout._root.path) # drops the uri prefix if it is there + abs_path = root_path / Path(path.path).relative_to(root_path) # Derivative directories must always be added separately - if self._layout._root.joinpath('derivatives') in abs_path.parents: + if root_path.joinpath('derivatives') in abs_path.parents: return [], [] config = list(config) # Shallow copy @@ -207,6 +212,7 @@ def _index_dir(self, path, config, force=None): # Get lists of 1st-level subdirectories and files in the path directory _, dirnames, filenames = next(path.fs.walk(path.path)) + # Symbolic links are returned as filenames even if they point to directories # Temporary list to store symbolic links that need to be removed from filenames links_to_dirs = [] diff --git a/bids/layout/tests/test_remote_bids.py b/bids/layout/tests/test_remote_bids.py new file mode 100644 index 00000000..cff21672 --- /dev/null +++ b/bids/layout/tests/test_remote_bids.py @@ -0,0 +1,26 @@ +"""Tests runs layout on bids examples and make sure all files are caught""" + +# TODO +# - add more 'vanilla' datasets +# - missing files in micr? + +import pytest +from botocore.exceptions import NoCredentialsError +from upath import UPath + +from bids.layout import BIDSLayout + +# Values for the number of files by downloading dataset first + + +@pytest.mark.parametrize( + "dataset, nb_files", + [ + (UPath("s3://openneuro.org/ds000102", anon=True), 136), + ], +) +@pytest.mark.xfail(raises=NoCredentialsError) +def test_layout_on_s3_datasets_no_derivatives(dataset, nb_files): + layout = BIDSLayout(dataset) + files = layout.get() + assert len(files) == nb_files diff --git a/pyproject.toml b/pyproject.toml index 5602b721..d8711ff6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ test = [ "coverage[toml]", "altair", "pytest-xdist", + "s3fs" #for testing remote uri ] model_reports = [ "jinja2",