diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m index 86715331..dd818ae9 100644 --- a/+bids/+internal/append_to_layout.m +++ b/+bids/+internal/append_to_layout.m @@ -1,4 +1,4 @@ -function subject = append_to_layout(file, subject, modality, schema) +function [subject, p] = append_to_layout(file, subject, modality, schema) % % appends a file to the BIDS layout by parsing it according to the provided schema % @@ -40,7 +40,7 @@ p = bids.internal.parse_filename(file, entities); % do not index json files when using the schema - if ~isempty(p) && strcmp(p.ext, '.json') + if isempty(p) || (~isempty(p) && strcmp(p.ext, '.json')) return end diff --git a/+bids/+internal/ends_with.m b/+bids/+internal/ends_with.m new file mode 100644 index 00000000..5233a3c9 --- /dev/null +++ b/+bids/+internal/ends_with.m @@ -0,0 +1,23 @@ +function res = ends_with(str, pattern) + % + % Checks id character array 'str' ends with 'pat' + % + % USAGE res = bids.internal.endsWith(str, pat) + % + % str - character array + % pat - character array + % + % __________________________________________________________________________ + % + % Based on spm_file.m and spm_select.m from SPM12. + % __________________________________________________________________________ + + % Copyright (C) 2011-2018 Guillaume Flandin, Wellcome Centre for Human Neuroimaging + res = false; + l_pat = length(pattern); + if l_pat > length(str) + return + end + res = strcmp(str(end - l_pat + 1:end), pattern); + +end diff --git a/+bids/+internal/get_meta_list.m b/+bids/+internal/get_meta_list.m new file mode 100644 index 00000000..f4bc581e --- /dev/null +++ b/+bids/+internal/get_meta_list.m @@ -0,0 +1,92 @@ +function metalist = get_meta_list(filename, pattern) + % + % Read a BIDS's file metadata according to the inheritance principle + % + % USAGE:: + % + % meta = bids.internal.get_metadata(filename, pattern = '^.*%s\\.json$') + % + % :param filename: fullpath name of file following BIDS standard + % :type filename: string + % :param pattern: Regular expression matching the metadata file (default is ``'^.*%s\\.json$'``) + % If provided, it must at least be ``'%s'``. + % :type pattern: string + % + % + % metalist - list of paths to metafiles + % __________________________________________________________________________ + + % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging + % Copyright (C) 2018--, BIDS-MATLAB developers + + if nargin == 1 + pattern = '^.*%s\\.json$'; + end + + pth = fileparts(filename); + p = bids.internal.parse_filename(filename); + metalist = {}; + + % Default assumes we are dealing with a file in the root directory + % like "participants.tsv" + % If the file has underscore separated entities ("sub-01_T1w.nii") + % then we look through the hierarchy for potential metadata file associated + % with queried file. + N = 1; + if isfield(p, 'entities') + N = 3; + % -There is a session level in the hierarchy + if isfield(p.entities, 'ses') && ~isempty(p.entities.ses) + N = N + 1; + end + end + + for n = 1:N + + % List the potential metadata files associated with this file suffix type + % Default is to assume it is a JSON file + metafile = bids.internal.file_utils('FPList', pth, sprintf(pattern, p.suffix)); + + if isempty(metafile) + metafile = {}; + else + metafile = cellstr(metafile); + end + + % For all those files we find which one is potentially associated with + % the file of interest + for i = 1:numel(metafile) + + p2 = bids.internal.parse_filename(metafile{i}); + entities = {}; + if isfield(p2, 'entities') + entities = fieldnames(p2.entities); + end + + % Check if this metadata file contains + % - the same entity-label pairs + % - same suffix + % as its data file counterpart + ismeta = true; + if ~strcmp(p.suffix, p2.suffix) + ismeta = false; + end + for j = 1:numel(entities) + if ~isfield(p.entities, entities{j}) || ... + ~strcmp(p.entities.(entities{j}), p2.entities.(entities{j})) + ismeta = false; + break + end + end + + % append path to list + if ismeta + metalist{end + 1, 1} = metafile{i}; %#ok + end + + end + + % Go up to the parent folder + pth = fullfile(pth, '..'); + end +end diff --git a/+bids/+internal/get_metadata.m b/+bids/+internal/get_metadata.m index 1c0759f1..b732cafc 100644 --- a/+bids/+internal/get_metadata.m +++ b/+bids/+internal/get_metadata.m @@ -1,101 +1,38 @@ -function meta = get_metadata(filename, pattern) +function meta = get_metadata(metafile) % % Read a BIDS's file metadata according to the inheritance principle % % USAGE:: % - % meta = bids.internal.get_metadata(filename, pattern = '^.*%s\\.json$') + % meta = bids.internal.get_metadata(metafile) % - % :param filename: fullpath name of file following BIDS standard - % :type filename: string - % :param pattern: Regular expression matching the metadata file (default is ``'^.*%s\\.json$'``) - % If provided, it must at least be ``'%s'``. - % :type pattern: string + % :param metafile: list of fullpath names of metadata files. + % :type metafile: string or array of strings % + % :returns: - :meta: metadata structure % - % meta - metadata structure + % .. todo + % + % add explanation on how the inheritance principle is implemented. % __________________________________________________________________________ % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging % Copyright (C) 2018--, BIDS-MATLAB developers - % assume most files are of the form *_suffix.json - % add an exception for files with no suffix, like participants.tsv - if nargin == 1 - pattern = '^.*_?%s\\.json$'; - end - - pth = fileparts(filename); - p = bids.internal.parse_filename(filename); - meta = struct(); + metafile = cellstr(metafile); - N = 3; - - % -There is a session level in the hierarchy - if isfield(p, 'entities') && isfield(p.entities, 'ses') && ~isempty(p.entities.ses) - N = N + 1; - end - - % -Loop from the directory where the file of interest is back to the - % top level of the BIDS hierarchy - for n = 1:N - - % -List the potential metadata files associated with this file suffix type - % Default is to assume it is a JSON file - metafile = bids.internal.file_utils('FPList', pth, sprintf(pattern, p.suffix)); - - if isempty(metafile) - metafile = {}; + for i = 1:numel(metafile) + if bids.internal.ends_with(metafile{i}, '.json') + meta = update_metadata(meta, bids.util.jsondecode(metafile{i}), metafile{i}); else - metafile = cellstr(metafile); - end - - % -For all those files we find which one is potentially associated with - % the file of interest - for i = 1:numel(metafile) - - p2 = bids.internal.parse_filename(metafile{i}); - entities = {}; - if isfield(p2, 'entities') - entities = fieldnames(p2.entities); - end - - % Check if this metadata file contains - % - the same entity-label pairs - % - same prefix - % as its data file counterpart - ismeta = true; - if ~strcmp(p.suffix, p2.suffix) - ismeta = false; - end - for j = 1:numel(entities) - if ~isfield(p.entities, entities{j}) || ... - ~strcmp(p.entities.(entities{j}), p2.entities.(entities{j})) - ismeta = false; - break - end - end - - % -Read the content of the metadata file if it is a JSON file and update - % the metadata concerning the file of interest otherwise store the filename - if ismeta - if strcmp(p2.ext, '.json') - meta = update_metadata(meta, bids.util.jsondecode(metafile{i}), metafile{i}); - else - meta.filename = metafile{i}; - end - end - + meta.filename = metafile{i}; end - % -Go up to the parent folder - pth = fullfile(pth, '..'); - end if isempty(meta) - warning('No metadata for %s', filename); + warning('No metadata for %s', metafile); end end diff --git a/+bids/+internal/parse_filename.m b/+bids/+internal/parse_filename.m index 29927797..cea3ad97 100644 --- a/+bids/+internal/parse_filename.m +++ b/+bids/+internal/parse_filename.m @@ -53,8 +53,16 @@ end % identidy an eventual prefix to the file - tmp = regexp(parts{1}, '(sub)', 'split'); - p.prefix = tmp{1}; + % and amends the sub entity accordingly + p.prefix = ''; + if strfind(parts{1}, 'sub-') + tmp = regexp(parts{1}, '(sub-)', 'split'); + p.prefix = tmp{1}; + if ~isempty(p.prefix) + p.entities.sub = p.entities.([p.prefix 'sub']); + p.entities = rmfield(p.entities, [p.prefix 'sub']); + end + end % -Extra fields can be added to the structure and ordered specifically. if nargin == 2 diff --git a/+bids/+internal/starts_with.m b/+bids/+internal/starts_with.m new file mode 100644 index 00000000..272be3d8 --- /dev/null +++ b/+bids/+internal/starts_with.m @@ -0,0 +1,23 @@ +function res = starts_with(str, pattern) + % + % Checks id character array 'str' starts with 'pat' + % + % USAGE res = bids.internal.startsWith(str, pat) + % + % str - character array + % pat - character array + % + % __________________________________________________________________________ + % + % Based on spm_file.m and spm_select.m from SPM12. + % __________________________________________________________________________ + + % Copyright (C) 2011-2018 Guillaume Flandin, Wellcome Centre for Human Neuroimaging + res = false; + l_pat = length(pattern); + if l_pat > length(str) + return + end + res = strcmp(str(1:l_pat), pattern); + +end diff --git a/+bids/copy_to_derivative.m b/+bids/copy_to_derivative.m new file mode 100644 index 00000000..76c1f2a9 --- /dev/null +++ b/+bids/copy_to_derivative.m @@ -0,0 +1,254 @@ +function derivatives = copy_to_derivative(BIDS, out_path, pipeline_name, filter, unzip, force, skip_dep, use_schema, verbose) + % + % Copy selected data from BIDS layout to given derivatives folder, + % returning layout of new derivatives folder + % + % USAGE:: + % + % derivatives = copy_to_derivative(BIDS, out_path, query) + % + % :param BIDS: BIDS directory name or BIDS structure (from bids.layout) + % :type BIDS: structure or string + % :param out_path: path to directory containing the derivatives + % :type out_path: string + % :param pipeline_name: name of pipeline to use + % :type pipeline_name: string + % :param query: list of filters to choose what files to copy (see bids.query) + % :type query: structure or cell + % :param unzip: If ``true`` (default) then all ``.gz`` files will be unzipped + % after being copied. + % :type unzip: boolean + % + % All the metadata of each file is read through the whole hierarchy + % and dumped into one side-car json file for each file copied. + % In practice this "unravels" the inheritance principle. + % The presence of this metadata file is also used to prevent the file from + % being copied again on a successive run. + % + % + % __________________________________________________________________________ + % + % BIDS (Brain Imaging Data Structure): https://bids.neuroimaging.io/ + % The brain imaging data structure, a format for organizing and + % describing outputs of neuroimaging experiments. + % K. J. Gorgolewski et al, Scientific Data, 2016. + % __________________________________________________________________________ + + % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging + % Copyright (C) 2021--, BIDS-MATLAB developers + + narginchk(2, Inf); + + if nargin < 3 + pipeline_name = 'bids-matlab'; + end + + if nargin < 4 + filter = []; + end + + if nargin < 5 || isempty(unzip) + unzip = true; + end + + if nargin < 6 || isempty(force) + force = false; + end + + if nargin < 7 || isempty(skip_dep) + skip_dep = false; + end + + if nargin < 8 || isempty(use_schema) + use_schema = true; + end + + if nargin < 9 || isempty(verbose) + verbose = false; + end + + derivatives = []; + + BIDS = bids.layout(BIDS, use_schema); + + % Check that we actually have to copy something + data_list = bids.query(BIDS, 'data', filter); + subjects_list = bids.query(BIDS, 'subjects', filter); + + if isempty(data_list) + warning('No data found for this query'); + return + else + fprintf('Found %d files in %d subjects\n', length(data_list), length(subjects_list)); + end + + % Determine and create output directory + if nargin < 2 || isempty(out_path) + out_path = fullfile(BIDS.dir, '..', 'derivatives'); + end + if ~exist(out_path, 'dir') + mkdir(out_path); + end + derivatives_folder = fullfile(out_path, pipeline_name); + if ~exist(derivatives_folder, 'dir') + mkdir(derivatives_folder); + end + + % Creating / loading description + descr_file = fullfile(derivatives_folder, 'dataset_description.json'); + if exist(descr_file, 'file') + description = bids.util.jsondecode(descr_file); + else + description = BIDS.description; + end + + % Create / update GeneratedBy + pipeline.Name = mfilename; + pipeline.Version = ''; + pipeline.Container = ''; + if isfield(description, 'GeneratedBy') + description.GeneratedBy = [description.GeneratedBy; pipeline]; + else + description.GeneratedBy = pipeline; + end + + bids.util.jsonencode(descr_file, description, struct('Indent', ' ')); + + % extracting participants.tsv file? + + % looping over selected files + for iFile = 1:numel(data_list) + copy_file(BIDS, derivatives_folder, data_list{iFile}, unzip, force, skip_dep, verbose); + end + + %% + derivatives = bids.layout(derivatives_folder, use_schema); +end + +function copy_file(BIDS, derivatives_folder, data_file, unzip, force, skip_dep, verbose) + + info = bids.internal.return_file_info(BIDS, data_file); + file = BIDS.subjects(info.sub_idx).(info.modality)(info.file_idx); + + out_dir = fullfile(derivatives_folder, ... + BIDS.subjects(info.sub_idx).name, ... + BIDS.subjects(info.sub_idx).session, ... + info.modality); + + output_metadata_file = fullfile(out_dir, ... + strrep(file.filename, file.ext, '.json')); + + %% ignore already existing files + % avoid circular references + if ~force && exist(output_metadata_file, 'file') + if verbose + fprintf(1, '\n skipping: %s', file.filename); + end + return + else + if verbose + fprintf(1, '\n copying: %s', file.filename); + end + file.meta = bids.internal.get_metadata(file.metafile); + end + + if ~exist(out_dir, 'dir') + mkdir(out_dir); + end + + %% copy data file + % we follow any eventual symlink + % and then unzip the data if necessary + copy_with_symlink(data_file, fullfile(out_dir, file.filename)); + unzip_data(file, out_dir, unzip); + + %% export metadata + % All the metadata of each file is read through the whole hierarchy + % and dumped into one side-car json file for each file copied + % In practice this "unravels" the inheritance principle + if ~strcmpi(file.ext, '.json') % skip if data file is json + bids.util.jsonencode(output_metadata_file, file.meta); + end + % checking that json is created + if ~exist(output_metadata_file, 'file') + error('Failed to create sidecar json file: %s', output_metadata_file); + end + + %% dealing with dependencies + if ~skip_dep + + dependencies = fieldnames(file.dependencies); + + for dep = 1:numel(dependencies) + for ifile = 1:numel(file.dependencies.(dependencies{dep})) + + dep_file = file.dependencies.(dependencies{dep}){ifile}; + if exist(dep_file, 'file') + copy_file(BIDS, derivatives_folder, dep_file, unzip, force, skip_dep, verbose); + else + warning(['Dependency file ' dep_file ' not found']); + end + + end + end + + end + +end + +function copy_with_symlink(src, target) + % + % Follows symbolic link to copy data: + % Might be necessary for datasets curated with datalad + % + % Comment from Guillaume: + % I think we should make a system() call only out of necessity. + % We could test for symlinks within a isunix condition and only use cp -L for these? + % + % Though datalad should run on windows too + % + + command = 'cp -R -L -f'; + + try + status = system( ... + sprintf('%s %s %s', ... + command, ... + src, ... + target)); + + if status > 0 + message = [ ... + 'Copying data with system command failed: ' ... + 'Are you running Windows?\n', ... + 'Will use matlab/octave copyfile command instead.\n', ... + 'May be an issue if your data set contains symbolic links' ... + '(e.g. if you use datalad or git-annex.)']; + error(message); + end + + catch + + fprintf(1, 'Using octave/matlab to copy files.'); + [status, message, messageId] = copyfile(src, target); + if ~status + warning([messageId ': ' message]); + return + end + + end + +end + +function unzip_data(file, out_dir, unzip) + if ~unzip + return + end + % to ensure a consistent behavior with matlab and octave + if bids.internal.ends_with(file.ext, '.gz') + gunzip(fullfile(out_dir, file.filename)); + if exist(fullfile(out_dir, file.filename), 'file') + delete(fullfile(out_dir, file.filename)); + end + end +end diff --git a/+bids/layout.m b/+bids/layout.m index 920ba205..87267a38 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -125,7 +125,7 @@ %% Dependencies % ========================================================================== - BIDS = manage_intended_for(BIDS); + BIDS = manage_dependencies(BIDS); end @@ -152,8 +152,8 @@ % NOTE: *_scans.json files can stored at the root level % and this should implemented when querying scans.tsv content + metadata subject.scans = bids.internal.file_utils('FPList', ... - subject.path, ... - ['^' subjname, '.*_scans.tsv' '$']); + subject.path, ... + ['^' subjname, '.*_scans.tsv' '$']); modality_groups = bids.schema.return_modality_groups(schema); @@ -165,14 +165,8 @@ % so the parsing is unconstrained for iModality = 1:numel(modalities) switch modalities{iModality} - case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg', 'pet'} + case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg', 'pet', 'fmap', 'dwi', 'perf'} subject = parse_using_schema(subject, modalities{iModality}, schema); - case 'dwi' - subject = parse_dwi(subject, schema); - case 'fmap' - subject = parse_fmap(subject, schema); - case 'perf' - subject = parse_perf(subject, schema); otherwise % in case we are going schemaless % and the modality is not one of the usual suspect @@ -195,159 +189,41 @@ file_list = return_file_list(modality, subject, schema); - for i = 1:numel(file_list) - - subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); - - end - - end - -end - -function subject = parse_dwi(subject, schema) - - % -------------------------------------------------------------------------- - % Diffusion imaging data - % -------------------------------------------------------------------------- + for iFile = 1:size(file_list, 1) - modality = 'dwi'; - pth = fullfile(subject.path, modality); + [subject, parsing] = bids.internal.append_to_layout(file_list{iFile}, ... + subject, ... + modality, ... + schema); - if exist(pth, 'dir') + if ~isempty(parsing) - subject = bids.internal.add_missing_field(subject, modality); + subject = index_dependencies(subject, modality, file_list{iFile}); - file_list = return_file_list(modality, subject, schema); + switch subject.(modality)(end).suffix - for i = 1:numel(file_list) + case 'asl' - subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); + subject.(modality)(end).meta = []; - % if this file is a nifti image we add the bval and bvec as dependencies - if ~isempty(subject.(modality)) && ... - any(strcmp(subject.(modality)(end).ext, {'.nii', '.nii.gz'})) + subject.(modality)(end).meta = bids.internal.get_metadata(subject.(modality)(iFile).metafile); - fullpath_filename = fullfile(subject.path, modality, file_list{i}); + aslcontext_file = strrep(subject.perf(end).filename, ... + ['_asl' subject.perf(end).ext], ... + '_aslcontext.tsv'); + subject.(modality)(end).dependencies.context = manage_tsv( ... + struct('content', [], ... + 'meta', []), ... + pth, ... + aslcontext_file); - % bval & bvec file - % ------------------------------------------------------------------ - % TODO: they can also be stored at higher levels (inheritance principle) - % TODO: refactor and deal with all this after file indexing - bvalfile = bids.internal.get_metadata(fullpath_filename, '^.*%s\\.bval$'); - if isfield(bvalfile, 'filename') - subject.dwi(end).dependencies.bval = bids.util.tsvread(bvalfile.filename); - end + subject.(modality)(end) = manage_M0(subject.perf(end), pth); - bvecfile = bids.internal.get_metadata(fullpath_filename, '^.*%s\\.bvec$'); - if isfield(bvalfile, 'filename') - subject.dwi(end).dependencies.bvec = bids.util.tsvread(bvecfile.filename); end end end - end -end - -function subject = parse_perf(subject, schema) - - % -------------------------------------------------------------------------- - % ASL perfusion imaging data - % -------------------------------------------------------------------------- - - modality = 'perf'; - pth = fullfile(subject.path, 'perf'); - - if exist(pth, 'dir') - - subject = bids.internal.add_missing_field(subject, modality); - - file_list = return_file_list(modality, subject, schema); - - for i = 1:numel(file_list) - - subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); - - switch subject.perf(i).suffix - - case 'asl' - - subject.perf(i).meta = []; - subject.perf(i).dependencies = []; - - subject.perf(i).meta = bids.internal.get_metadata( ... - fullfile( ... - subject.path, ... - modality, ... - file_list{i})); - - aslcontext_file = strrep(subject.perf(i).filename, ... - ['_asl' subject.perf(i).ext], ... - '_aslcontext.tsv'); - subject.perf(i).dependencies.context = manage_tsv( ... - struct('content', [], 'meta', []), ... - pth, ... - aslcontext_file); - - subject.perf(i) = manage_asllabeling(subject.perf(i), pth); - - subject.perf(i) = manage_M0(subject.perf(i), pth); - - end - - end - - end - -end - -function subject = parse_fmap(subject, schema) - - modality = 'fmap'; - pth = fullfile(subject.path, modality); - - if exist(pth, 'dir') - - subject = bids.internal.add_missing_field(subject, modality); - - file_list = return_file_list(modality, subject, schema); - - for i = 1:numel(file_list) - - subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); - - switch subject.fmap(i).suffix - - % -A single, real fieldmap image - case {'fieldmap', 'magnitude'} - subject.fmap(i).dependencies.magnitude = strrep(file_list{idx(i)}, ... - '_fieldmap.nii', ... - '_magnitude.nii'); - - % Phase difference image and at least one magnitude image - case {'phasediff'} - subject.fmap(i).dependencies.magnitude = { ... - strrep(file_list{i}, ... - '_phasediff.nii', ... - '_magnitude1.nii'), ... - strrep(file_list{i}, ... - '_phasediff.nii', ... - '_magnitude2.nii')}; % optional - - % Two phase images and two magnitude images - case {'phase1', 'phase2'} - subject.fmap(i).dependencies.magnitude = { ... - strrep(file_list{i}, ... - '_phase1.nii', ... - '_magnitude1.nii'), ... - strrep(file_list{i}, ... - '_phase1.nii', ... - '_magnitude2.nii')}; - - end - - end end @@ -407,6 +283,15 @@ function tolerant_message(use_schema, msg) end end +function subject_path = return_subject_path(subject) + % get "subject path" without the session folder (if it exists) + subject_path = subject.path; + tmp = bids.internal.file_utils(subject_path, 'filename'); + if strcmp(tmp(1:3), 'ses') + subject_path = bids.internal.file_utils(subject_path, 'path'); + end +end + function file_list = return_file_list(modality, subject, schema) % We list anything but json files @@ -417,8 +302,8 @@ function tolerant_message(use_schema, msg) % TODO % this does not cover coordsystem.json - % jn to omit json but not .pos file for headshape.pos + % jn to omit json but not .pos file for headshape.pos pattern = '_([a-zA-Z0-9]+){1}\\..*[^jn]'; prefix = ''; if isempty(schema) @@ -437,19 +322,71 @@ function tolerant_message(use_schema, msg) if strcmp(modality, 'meg') && ~isempty(d) for i = 1:size(d, 1) - file_list{end + 1, 1} = d(i, :); + file_list{end + 1, 1} = d(i, :); %#ok<*AGROW> end end end +function subject = index_dependencies(subject, modality, file) + % + % Each file structure contains dependencies sub-structure with guaranteed fields: + % + % - explicit: list of data files containing "IntendedFor" referencing current file. + % see the manage_dependencies function + % + % - data: list of files with same name but different extension. + % This combines files that are split in header and data + % (like in Brainvision), also takes care of bval/bvec files + % + % - group: list of files that have same name except extension and suffix. + % This groups file that logically need each other, + % like functional mri and events tabular file. + % It also takes care of fmap magnitude1/2 and phasediff. + + pth = fullfile(subject.path, modality); + fullpath_filename = fullfile(pth, file); + + subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); + subject.(modality)(end).dependencies.explicit = {}; + subject.(modality)(end).dependencies.data = {}; + subject.(modality)(end).dependencies.group = {}; + + ext = subject.(modality)(end).ext; + suffix = subject.(modality)(end).suffix; + pattern = strrep(file, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); + candidates = bids.internal.file_utils('List', pth, ['^' pattern '$']); + candidates = cellstr(candidates); + + for ii = 1:numel(candidates) + + if strcmp(candidates{ii}, file) + continue + end + + if bids.internal.ends_with(candidates{ii}, '.json') + continue + end + + match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); + % different suffix + if isempty(match) + subject.(modality)(end).dependencies.group{end + 1, 1} = fullfile(pth, candidates{ii}); + % same suffix + else + subject.(modality)(end).dependencies.data{end + 1, 1} = fullfile(pth, candidates{ii}); + end + + end + +end + function structure = manage_tsv(structure, pth, filename) - % Retunrs the content and metadata of a TSV file (if they exist) + % Returns the content and metadata of a TSV file (if they exist) % % NOTE: inheritance principle not implemented. % Does NOT look for the metadata of a file at higher levels % - % ext = bids.internal.file_utils(filename, 'ext'); tsv_file = bids.internal.file_utils('FPList', ... @@ -473,132 +410,47 @@ function tolerant_message(use_schema, msg) end -function BIDS = manage_intended_for(BIDS) - - % Loops through all the files with potential ``intentedFor`` metadata - % and creates an ``intended_for`` field with the fullpath list of all the target files - % it is intended for that exist. +function BIDS = manage_dependencies(BIDS) + % + % Loops over all files and retrieve all files that current file depends on % - % Also update the structure of each target file with an ``informed_by`` field - - suffix_with_intended_for = { ... - 'phasediff'; ... - 'phase1'; ... - 'phase2'; ... - 'fieldmap'; ... - 'epi'; ... - 'm0scan'; ... - 'coordsystem'}; - - for iSuffix = 1:numel(suffix_with_intended_for) - file_list = bids.query(BIDS, 'data', 'suffix', suffix_with_intended_for(iSuffix)); - - for iFile = 1:size(file_list, 1) - - info_src = bids.internal.return_file_info(BIDS, file_list{iFile}); - - [metadata, intended_for] = check_intended_for_exist(BIDS.subjects(info_src.sub_idx), ... - fullfile( ... - info_src.path, ... - info_src.modality, ... - info_src.filename)); - - BIDS.subjects(info_src.sub_idx).(info_src.modality)(info_src.file_idx).meta = metadata; - BIDS.subjects(info_src.sub_idx).(info_src.modality)(info_src.file_idx).intended_for = ... - intended_for; - % update "dependency" field of target file - informed_by = []; - for iTargetFile = 1:size(intended_for, 1) + file_list = bids.query(BIDS, 'data'); - info_tgt = bids.internal.return_file_info(BIDS, intended_for{iTargetFile}); + for iFile = 1:size(file_list, 1) - % TODO: this should probably check that the informed_by field does not - % already exist in the structure - % TODO: This assumes that a file will only be informed by a single file from - % this moodality - informed_by.(info_src.modality) = file_list{iFile}; - BIDS.subjects(info_tgt.sub_idx).(info_tgt.modality)(info_tgt.file_idx).informed_by = ... - informed_by; - end + info_src = bids.internal.return_file_info(BIDS, file_list{iFile}); + file = BIDS.subjects(info_src.sub_idx).(info_src.modality)(info_src.file_idx); + metadata = bids.internal.get_metadata(file.metafile); + % If the file A is intended for file B + % then we update the dependencies.explicit field structrure of file B + % so it contains the fullpath to file A + % + % This way when one queries info about B, then it is easy to know what + % other is present to help with analysis. + intended = {}; + if isfield(metadata, 'IntendedFor') + intended = cellstr(metadata.IntendedFor); end - end - -end - -function [metadata, intended_for] = check_intended_for_exist(subject, filename) - - metadata = bids.internal.get_metadata(filename); - - intended_for = {}; - - if isempty(metadata) || ~isfield(metadata, 'IntendedFor') - warning('Missing field IntendedFor for %s', filename); - return - - else - - path_intended_for = {}; - - if ischar(metadata.IntendedFor) - path_intended_for{1, 1} = metadata.IntendedFor; - - elseif isstruct(metadata.IntendedFor) - for iPath = 1:length(metadata.IntendedFor) - path_intended_for{iPath, 1} = metadata.IntendedFor(iPath); %#ok<*AGROW> + for iIntended = 1:numel(intended) + dest = fullfile(BIDS.dir, BIDS.subjects(info_src.sub_idx).name, ... + intended{iIntended}); + if ~exist(dest, 'file') + warning(['IntendedFor file ' dest ' from ' file.filename ' not found']); + continue end - - end - end - - subject_path = return_subject_path(subject); - - for iPath = 1:size(path_intended_for, 1) - - % create a fullname path for current operating system - fullpath_filename = fullfile(subject_path, ... - strrep(path_intended_for{iPath}, '/', filesep)); - - % check if this file is missing - if ~exist(fullpath_filename, 'file') - warning(['Missing: ' fullpath_filename]); - - else - intended_for{end + 1, 1} = fullpath_filename; - + info_dest = bids.internal.return_file_info(BIDS, dest); + BIDS.subjects(info_dest.sub_idx).(info_dest.modality)(info_dest.file_idx) ... + .dependencies.explicit{end + 1, 1} = file_list{iFile}; end - end - -end - -function subject_path = return_subject_path(subject) - % get "subject path" without the session folder (if it exists) - subject_path = subject.path; - tmp = bids.internal.file_utils(subject_path, 'filename'); - if strcmp(tmp(1:3), 'ses') - subject_path = bids.internal.file_utils(subject_path, 'path'); - end -end - -function perf = manage_asllabeling(perf, pth) - % labeling image metadata (OPTIONAL) - % --------------------------- - metafile = fullfile(pth, strrep(perf.filename, ... - ['_asl' perf.ext], ... - '_asllabeling.jpg')); - - if exist(metafile, 'file') - [~, Ffile] = fileparts(metafile); - perf.dependencies.labeling_image.filename = [Ffile '.jpg']; end end function perf = manage_M0(perf, pth) - % M0 field is flexible: if ~isfield(perf.meta, 'M0Type') diff --git a/+bids/query.m b/+bids/query.m index 9cc7eb96..5bad63eb 100644 --- a/+bids/query.m +++ b/+bids/query.m @@ -19,8 +19,27 @@ % - 'modalities' % :type query: string % + % Queries can "filtered" by passing more arguments key-value pairs as a list of + % strings or as a cell or a structure + % + % Example 1:: + % + % data = bids.query(BIDS, 'data', ... + % 'sub', '01', ... + % 'task', 'stopsignalwithpseudowordnaming', ... + % 'extension', '.nii.gz', ... + % 'suffix', 'bold'); + % + % + % Example 2:: + % + % filters = struct('sub', '01', ... + % 'task', 'stopsignalwithpseudowordnaming', ... + % 'extension', '.nii.gz', ... + % 'suffix', 'bold'); + % + % data = bids.query(BIDS, 'data', filters); % - % __________________________________________________________________________ % % BIDS (Brain Imaging Data Structure): https://bids.neuroimaging.io/ @@ -44,6 +63,7 @@ 'suffixes', ... 'data', ... 'metadata', ... + 'metafiles', ... 'dependencies', ... 'extensions', ... 'prefixes'}; @@ -80,12 +100,14 @@ result = regexprep(result, '^[a-zA-Z0-9]+-', ''); result(cellfun('isempty', result)) = []; - case {'modalities', 'data'} + case {'modalities', 'data', 'metafiles'} result = result'; case {'metadata', 'dependencies'} if numel(result) == 1 result = result{1}; + else + result = result'; end case {'tasks', 'runs', 'suffixes', 'extensions', 'prefixes'} @@ -97,8 +119,18 @@ function options = parse_query(options) - if numel(options) == 1 && isstruct(options{1}) - options = [fieldnames(options{1}), struct2cell(options{1})]; + if numel(options) == 1 + + if isstruct(options{1}) + options = [fieldnames(options{1}), struct2cell(options{1})]; + + elseif iscell(options{1}) + options = options{1}; + + elseif isempty(options{1}) + options = cell(0, 2); + return + end else if mod(numel(options), 2) @@ -213,25 +245,24 @@ result = union(result, allmods(hasmod)); case 'data' - if isfield(d(k), 'filename') - result{end + 1} = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename); - end + result{end + 1} = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename); + + case 'metafiles' + fmeta = BIDS.subjects(i).(modalities{j})(k).metafile; + result = [result; fmeta]; case 'metadata' - if isfield(d(k), 'filename') - - f = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename); - result{end + 1, 1} = bids.internal.get_metadata(f); - if ~isempty(target) - try - result{end} = subsref(result{end}, target); - catch - warning('Non-existent field for metadata.'); - result{end} = []; - end + fmeta = BIDS.subjects(i).(modalities{j})(k).metafile; + result{end + 1, 1} = bids.internal.get_metadata(fmeta); + if ~isempty(target) + try + result{end} = subsref(result{end}, target); + catch + warning('Non-existent field for metadata.'); + result{end} = []; end - end + % if status && isfield(d(k),'meta') % result{end+1} = d(k).meta; % end @@ -247,9 +278,7 @@ end case 'suffixes' - if isfield(d(k), 'suffix') - result{end + 1} = d(k).suffix; - end + result{end + 1} = d(k).suffix; case 'prefixes' if isfield(d(k), 'prefix') @@ -257,9 +286,7 @@ end case 'extensions' - if isfield(d(k), 'ext') - result{end + 1} = d(k).ext; - end + result{end + 1} = d(k).ext; case 'dependencies' if isfield(d(k), 'dependencies') diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 88e50fee..d5f740e3 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -23,7 +23,6 @@ jobs: run: | sudo apt-get -y -qq update sudo apt-get -y install octave liboctave-dev - sudo apt-get -y install nodejs npm - name: Clone bids-matlab uses: actions/checkout@v2 @@ -31,10 +30,6 @@ jobs: submodules: true fetch-depth: 2 - - name: Install BIDS validator - run: | - npm install `cat npm-requirements.txt` - - name: Install JSONio run: | git clone git://github.com/gllmflndn/JSONio.git --depth 1 @@ -42,17 +37,31 @@ jobs: mkoctfile --mex jsonread.c jsmn.c -DJSMN_PARENT_LINKS cd .. - - name: Install bids example + - name: Install bids-example and data run: | cd tests git clone git://github.com/bids-standard/bids-examples.git --depth 1 - cd .. + cd utils + octave $OCTFLAGS --eval "download_moae_ds" + cd ../.. + + # ------------------------------ + # if tests on real data are needed + # + # sudo apt-get -y install datalad + # mkdir data; cd data + # datalad clone https://github.com/OpenNeuroDatasets/ds000001.git + # datalad get ds000001/sub-01/func + # datalad clone https://github.com/OpenNeuroDatasets/ds000117.git + # datalad get ds000117/sub-01/ses-mri/func/*run-0[13]* + # datalad get ds000117/sub-01/ses-mri/fmap/ + # cd .. - name: MOxUnit Action uses: joergbrech/moxunit-action@master with: tests: tests # files or directories containing the MOxUnit test cases - src: +bids # directories to be added to the Octave search path before running the tests. + src: +bids # directories to be added to the Octave search path before running the tests. ext: JSONio tests/utils # External resources to add to the search put (excluded from coverage) # data: # Directory for test data with_coverage: true diff --git a/.gitignore b/.gitignore index 6a705523..6a956d4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,15 @@ *.asv *.*~ - -/tests/bids-examples -/tests/*.tsv +*.swp # ignore virtual env env/* -# ignore bids-specification repo that gets cloned during CI +# ignore bids-specification repo (necessary for convert_schema.py) bids-specification/* -.prettierrc \ No newline at end of file +# ignore input and output data involved in tests +data/* +derivatives/* + +.prettierrc diff --git a/miss_hit.cfg b/miss_hit.cfg index e418050c..58b9d5ce 100644 --- a/miss_hit.cfg +++ b/miss_hit.cfg @@ -29,5 +29,5 @@ tab_width: 2 # metrics limit for the code quality (https://florianschanda.github.io/miss_hit/metrics.html) metric "cnest": limit 6 metric "file_length": limit 600 -metric "cyc": limit 20 -metric "parameters": limit 8 \ No newline at end of file +metric "cyc": limit 18 +metric "parameters": limit 6 \ No newline at end of file diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 00000000..f303df46 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,8 @@ + +*.tsv + +bids-examples + +data/MoAEpilot/* +data/ds*/* +data/derivatives*/* diff --git a/tests/README.md b/tests/README.md index 02844abc..3f3e95c9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -52,6 +52,25 @@ cd tests git clone git://github.com/bids-standard/bids-examples.git --depth 1 ``` +#### Datasets with content + +The function `download_moae_ds.m` will download a lightweight dataset from the +SPM website. + +To get more complex data sets, to test things you can use datalad. + +```bash +mkdir data +datalad clone https://github.com/OpenNeuroDatasets/ds000001.git +datalad get ds000001/sub-01/ + +datalad clone https://github.com/OpenNeuroDatasets/ds000117.git +datalad get ds000117/sub-01/ses-mri/func +datalad get ds000117/sub-01/ses-mri/fmap + + +``` + ## Add helper functions to the path There are a some help functions you need to add to the Matlab / Octave path to diff --git a/tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_midthickness.json b/tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_midthickness.json similarity index 100% rename from tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_midthickness.json rename to tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_midthickness.json diff --git a/tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii b/tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii similarity index 100% rename from tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii rename to tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii diff --git a/tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_thickness.json b/tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_thickness.json similarity index 100% rename from tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_thickness.json rename to tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_thickness.json diff --git a/tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii b/tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii similarity index 100% rename from tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii rename to tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii diff --git a/tests/data/SurfaceData/sub-06_space-individual_den-native_thickness.dscalar.nii b/tests/data/surface_data/sub-06_space-individual_den-native_thickness.dscalar.nii similarity index 100% rename from tests/data/SurfaceData/sub-06_space-individual_den-native_thickness.dscalar.nii rename to tests/data/surface_data/sub-06_space-individual_den-native_thickness.dscalar.nii diff --git a/tests/data/SurfaceData/sub-06_space-individual_den-native_thickness.json b/tests/data/surface_data/sub-06_space-individual_den-native_thickness.json similarity index 100% rename from tests/data/SurfaceData/sub-06_space-individual_den-native_thickness.json rename to tests/data/surface_data/sub-06_space-individual_den-native_thickness.json diff --git a/tests/data/synthetic/CHANGES b/tests/data/synthetic/CHANGES new file mode 100644 index 00000000..cc28676a --- /dev/null +++ b/tests/data/synthetic/CHANGES @@ -0,0 +1,4 @@ +1.0.1 2019-03-20 + - BIDS version. +1.0.0 1999-05-13 + - Initial release. diff --git a/tests/data/synthetic/README b/tests/data/synthetic/README new file mode 100644 index 00000000..959ca6ba --- /dev/null +++ b/tests/data/synthetic/README @@ -0,0 +1,26 @@ + ___ ____ __ __ +/ __)( _ \( \/ ) Statistical Parametric Mapping +\__ \ )___/ ) ( Wellcome Centre for Human Neuroimaging +(___/(__) (_/\/\_) https://www.fil.ion.ucl.ac.uk/spm/ + + MoAEpilot example epoch (block) fMRI dataset +________________________________________________________________________ + +This experiment was conducted by Geraint Rees under the direction of +Karl Friston and the FIL methods group. The purpose was to explore new +equipment and techniques. As such it has not been formally written up, +and is freely available for personal education and evaluation purposes. + +These whole brain BOLD/EPI images were acquired on a modified 2T +Siemens MAGNETOM Vision system. Each acquisition consisted of 64 +contiguous slices (64x64x64 3mm x 3mm x 3mm voxels). Acquisition took +6.05s, with the scan to scan repeat time (RT) set arbitrarily to 7s. + +96 acquisitions were made (RT=7s), in blocks of 6, giving 16 42s +blocks. The condition for successive blocks alternated between rest and +auditory stimulation, starting with rest. Auditory stimulation was +bi-syllabic words presented binaurally at a rate of 60 per minute. Due +to T1 effects it is advisable to discard the first few scans (there were +no "dummy" lead-in scans). + +A structural image was also acquired. diff --git a/tests/data/MoAEpilot/T1w.json b/tests/data/synthetic/T1w.json similarity index 100% rename from tests/data/MoAEpilot/T1w.json rename to tests/data/synthetic/T1w.json diff --git a/tests/data/synthetic/dataset_description.json b/tests/data/synthetic/dataset_description.json new file mode 100644 index 00000000..7948d385 --- /dev/null +++ b/tests/data/synthetic/dataset_description.json @@ -0,0 +1,9 @@ +{ + "BIDSVersion": "1.2.0", + "Name": "Mother of All Experiments", + "Authors": [ + "Geraint Rees", + "Karl Friston" + ], + "ReferencesAndLinks": ["https://www.fil.ion.ucl.ac.uk/spm/data/auditory/"] +} \ No newline at end of file diff --git a/tests/data/MoAEpilot/sub-01/anat/sub-01_T1w.json b/tests/data/synthetic/sub-01/anat/sub-01_T1w.json similarity index 100% rename from tests/data/MoAEpilot/sub-01/anat/sub-01_T1w.json rename to tests/data/synthetic/sub-01/anat/sub-01_T1w.json diff --git a/tests/data/MoAEpilot/sub-01/anat/sub-01_T1w.nii.gz b/tests/data/synthetic/sub-01/anat/sub-01_T1w.nii.gz similarity index 100% rename from tests/data/MoAEpilot/sub-01/anat/sub-01_T1w.nii.gz rename to tests/data/synthetic/sub-01/anat/sub-01_T1w.nii.gz diff --git a/tests/data/MoAEpilot/sub-01/func/sub-01_task-auditory_bold.json b/tests/data/synthetic/sub-01/func/sub-01_task-auditory_bold.json similarity index 100% rename from tests/data/MoAEpilot/sub-01/func/sub-01_task-auditory_bold.json rename to tests/data/synthetic/sub-01/func/sub-01_task-auditory_bold.json diff --git a/tests/data/MoAEpilot/sub-01/func/sub-01_task-auditory_bold.nii.gz b/tests/data/synthetic/sub-01/func/sub-01_task-auditory_bold.nii.gz similarity index 100% rename from tests/data/MoAEpilot/sub-01/func/sub-01_task-auditory_bold.nii.gz rename to tests/data/synthetic/sub-01/func/sub-01_task-auditory_bold.nii.gz diff --git a/tests/data/synthetic/task-auditory_bold.json b/tests/data/synthetic/task-auditory_bold.json new file mode 100644 index 00000000..68017a5b --- /dev/null +++ b/tests/data/synthetic/task-auditory_bold.json @@ -0,0 +1,10 @@ +{ + "RepetitionTime": 7, + "NumberOfVolumesDiscardedByScanner": 0, + "NumberOfVolumesDiscardedByUser": 12, + "TaskName": "Auditory", + "TaskDescription": "The condition for successive blocks alternated between rest and auditory stimulation, starting with rest. Auditory stimulation was with bi-syllabic words presented binaurally at a rate of 60 per minute.", + "Manufacturer": "Siemens", + "ManufacturersModelName": "MAGNETOM Vision", + "MagneticFieldStrength": 2 +} \ No newline at end of file diff --git a/tests/test_bids_query.m b/tests/test_bids_query.m index f46dbff5..912b197d 100644 --- a/tests/test_bids_query.m +++ b/tests/test_bids_query.m @@ -10,6 +10,47 @@ end +function test_query_basic() + + pth_bids_example = get_test_data_dir(); + + BIDS = bids.layout(fullfile(pth_bids_example, 'ds007')); + + tasks = { ... + 'stopsignalwithletternaming', ... + 'stopsignalwithmanualresponse', ... + 'stopsignalwithpseudowordnaming'}; + assertEqual(bids.query(BIDS, 'tasks'), tasks); + + assert(isempty(bids.query(BIDS, 'runs', 'suffix', 'T1w'))); + + runs = {'01', '02'}; + assertEqual(bids.query(BIDS, 'runs'), runs); + assertEqual(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs); + + % make sure that query can work with filter + filters = {'sub', {'01', '03'}; ... + 'task', {'stopsignalwithletternaming', ... + 'stopsignalwithmanualresponse'}; ... + 'run', '02'; ... + 'suffix', 'bold'}; + + files_cell_filter = bids.query(BIDS, 'data', filters); + assertEqual(size(files_cell_filter, 1), 4); + + filters = struct('run', '02', ... + 'suffix', 'bold'); + filters.sub = {'01', '03'}; + filters.task = {'stopsignalwithletternaming', ... + 'stopsignalwithmanualresponse'}; + + files_struct_filter = bids.query(BIDS, 'data', filters); + assertEqual(size(files_struct_filter, 1), 4); + + assertEqual(files_cell_filter, files_struct_filter); + +end + function test_query_extension() pth_bids_example = get_test_data_dir(); @@ -108,26 +149,6 @@ function test_query_modalities() end -function test_query_basic() - - pth_bids_example = get_test_data_dir(); - - BIDS = bids.layout(fullfile(pth_bids_example, 'ds007')); - - tasks = { ... - 'stopsignalwithletternaming', ... - 'stopsignalwithmanualresponse', ... - 'stopsignalwithpseudowordnaming'}; - assertEqual(bids.query(BIDS, 'tasks'), tasks); - - assert(isempty(bids.query(BIDS, 'runs', 'suffix', 'T1w'))); - - runs = {'01', '02'}; - assertEqual(bids.query(BIDS, 'runs'), runs); - assertEqual(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs); - -end - function test_query_subjects() pth_bids_example = get_test_data_dir(); diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m index 4ed55f6e..8fff5e4f 100644 --- a/tests/test_bids_query_asl.m +++ b/tests/test_bids_query_asl.m @@ -6,14 +6,35 @@ initTestSuite; end -function test_bids_query_asl_basic() - % - % asl queries - % +function test_bids_query_asl_basic_asl002() + + pth_bids_example = get_test_data_dir(); + + BIDS = bids.layout(fullfile(pth_bids_example, 'asl002')); + + modalities = {'anat', 'perf'}; + assertEqual(bids.query(BIDS, 'modalities'), modalities); + + suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'}; + assertEqual(bids.query(BIDS, 'suffixes'), suffixes); + + filename = bids.query(BIDS, 'data', 'sub', 'Sub103', 'suffix', 'm0scan'); + basename = bids.internal.file_utils(filename, 'basename'); + assertEqual(basename, {'sub-Sub103_m0scan.nii'}); + + assert(~isempty(BIDS.subjects.perf(1).dependencies.explicit)); + + dependencies = bids.query(BIDS, 'dependencies', 'sub', 'Sub103', 'suffix', 'asl'); + assert(any(ismember( ... + bids.internal.file_utils(dependencies.group, 'filename'), ... + 'sub-Sub103_aslcontext.tsv'))); + +end + +function test_bids_query_asl_basic_asl001() pth_bids_example = get_test_data_dir(); - %% 'asl001' BIDS = bids.layout(fullfile(pth_bids_example, 'asl001')); modalities = {'anat', 'perf'}; @@ -31,26 +52,18 @@ function test_bids_query_asl_basic() meta = bids.query(BIDS, 'metadata', 'sub', 'Sub103', 'suffix', 'asl'); dependencies = bids.query(BIDS, 'dependencies', 'sub', 'Sub103', 'suffix', 'asl'); - assertEqual(dependencies.labeling_image.filename, 'sub-Sub103_asllabeling.jpg'); + assert(any(ismember( ... + bids.internal.file_utils(dependencies.group, 'filename'), ... + 'sub-Sub103_asllabeling.jpg'))); dependencies.context; dependencies.m0; - %% 'asl002' - BIDS = bids.layout(fullfile(pth_bids_example, 'asl002')); - - modalities = {'anat', 'perf'}; - assertEqual(bids.query(BIDS, 'modalities'), modalities); - - suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'}; - assertEqual(bids.query(BIDS, 'suffixes'), suffixes); +end - filename = bids.query(BIDS, 'data', 'sub', 'Sub103', 'suffix', 'm0scan'); - basename = bids.internal.file_utils(filename, 'basename'); - assertEqual(basename, {'sub-Sub103_m0scan.nii'}); +function test_bids_query_asl_basic_asl003() - assert(isfield(BIDS.subjects.perf(4), 'intended_for')); + pth_bids_example = get_test_data_dir(); - %% 'asl003' BIDS = bids.layout(fullfile(pth_bids_example, 'asl003')); modalities = {'anat', 'perf'}; @@ -59,7 +72,12 @@ function test_bids_query_asl_basic() suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'}; assertEqual(bids.query(BIDS, 'suffixes'), suffixes); - %% 'asl004' +end + +function test_bids_query_asl_basic_asl004() + + pth_bids_example = get_test_data_dir(); + BIDS = bids.layout(fullfile(pth_bids_example, 'asl004')); modalities = {'anat', 'fmap', 'perf'}; @@ -72,7 +90,7 @@ function test_bids_query_asl_basic() basename = bids.internal.file_utils(filename, 'basename'); assertEqual(basename, {'sub-Sub1_dir-pa_m0scan.nii'}); - assert(isfield(BIDS.subjects.fmap, 'intended_for')); - assert(isfield(BIDS.subjects.perf(4), 'intended_for')); + assert(~isempty(BIDS.subjects.perf(1).dependencies.explicit)); + assert(~isempty(BIDS.subjects.perf(4).dependencies)); end diff --git a/tests/test_bids_query_dwi.m b/tests/test_bids_query_dwi.m index 4807f2a0..29e5d4b4 100644 --- a/tests/test_bids_query_dwi.m +++ b/tests/test_bids_query_dwi.m @@ -28,6 +28,7 @@ function test_bids_query_dwi_basic() 'suffix', 'dwi', ... 'extension', '.nii.gz'); - assertEqual(dependencies.bval(1:11), [0 repmat(2400, 1, 10)]); + bval = bids.util.tsvread(dependencies.data{1}); + assertEqual(bval(1:11), [0 repmat(2400, 1, 10)]); end diff --git a/tests/test_bids_query_fmap.m b/tests/test_bids_query_fmap.m index dc49636e..0b48f700 100644 --- a/tests/test_bids_query_fmap.m +++ b/tests/test_bids_query_fmap.m @@ -13,11 +13,10 @@ function test_query_extension() BIDS = bids.layout(fullfile(pth_bids_example, '7t_trt')); - BIDS.subjects(1).fmap(3).intended_for; - BIDS.subjects(1).func(1).informed_by; + BIDS.subjects(1).func(1).dependencies.explicit; BIDS = bids.layout(fullfile(pth_bids_example, 'hcp_example_bids')); % sub-100307 - BIDS.subjects(1).fmap(3).intended_for{1}; + BIDS.subjects(1).anat(1).dependencies.explicit; end diff --git a/tests/test_bids_query_sessions_scans_tsv.m b/tests/test_bids_query_sessions_scans_tsv.m index 3154ce24..3f9194a9 100644 --- a/tests/test_bids_query_sessions_scans_tsv.m +++ b/tests/test_bids_query_sessions_scans_tsv.m @@ -19,7 +19,6 @@ function test_query_sessions_tsv() assert(~isempty(BIDS.subjects(1).sess)); assert(~isempty(BIDS.subjects(1).scans)); - end function test_query_scans_tsv() diff --git a/tests/test_copy_to_derivative.m b/tests/test_copy_to_derivative.m new file mode 100644 index 00000000..b5bf7a62 --- /dev/null +++ b/tests/test_copy_to_derivative.m @@ -0,0 +1,87 @@ +function test_suite = test_copy_to_derivative %#ok<*STOUT> + try % assignment of 'localfunctions' is necessary in Matlab >= 2016 + test_functions = localfunctions(); %#ok<*NASGU> + catch % no problem; early Matlab versions can use initTestSuite fine + end + initTestSuite; +end + +function test_copy_to_derivative_MoAE() + + input_dir = download_moae_ds(true()); + out_path = []; + + BIDS = fullfile(input_dir, 'MoAEpilot'); + + pipeline_name = 'bids-matlab'; + + derivatives = bids.copy_to_derivative(BIDS, out_path, pipeline_name); + +end + +function test_copy_to_derivative_ds000117() + + pth_bids_example = get_test_data_dir(); + input_dir = fullfile(pth_bids_example, 'ds000117'); + + % to test on real data uncomment the following line + % see tests/README.md to see how to install the data + % + % input_dir = fullfile('..', 'data', 'ds000117'); + + out_path = fullfile(pwd, 'data', 'derivatives'); + + if exist(out_path, 'dir') + rmdir(out_path, 's'); + end + + BIDS = fullfile(input_dir); + + filters = struct('sub', '01', ... + 'modality', 'func', ... + 'suffix', 'bold'); + filters.run = {'01'; '03'}; + + %% + pipeline_name = []; + unzip = false; + force = false; + skip_dependencies = false; + use_schema = false; + verbose = true; + + derivatives = bids.copy_to_derivative(BIDS, ... + out_path, ... + pipeline_name, ... + filters, ... + unzip, ... + force, ... + skip_dependencies, ... + use_schema, ... + verbose); + + copied_files = bids.query(derivatives, 'data'); + assertEqual(size(copied_files, 1), 13); + + %% same but we skip dependencies + if exist(out_path, 'dir') + rmdir(out_path, 's'); + end + skip_dependencies = true; + + derivatives = bids.copy_to_derivative(BIDS, ... + out_path, ... + pipeline_name, ... + filters, ... + unzip, ... + force, ... + skip_dependencies, ... + use_schema, ... + verbose); + + copied_files = bids.query(derivatives, 'data'); + assertEqual(size(copied_files, 1), 4); + + %% add test to check that only files that conform to schema are copied + +end diff --git a/tests/test_get_metadata.m b/tests/test_get_metadata.m index e88278a2..186bb9b3 100644 --- a/tests/test_get_metadata.m +++ b/tests/test_get_metadata.m @@ -23,15 +23,13 @@ function test_get_metadata_basic() % also tests inheritance principle: metadata are passed on to lower levels % unless they are overriden by metadate already present at lower levels - pth = fullfile(fileparts(mfilename('fullpath')), 'data', 'MoAEpilot'); + pth = fullfile(fileparts(mfilename('fullpath')), 'data', 'synthetic'); % define the expected output from bids query metadata func.RepetitionTime = 7; - - func_sub_01.RepetitionTime = 10; - anat.FlipAngle = 5; + func_sub_01.RepetitionTime = 10; anat_sub_01.FlipAngle = 10; anat_sub_01.Manufacturer = 'Siemens'; @@ -40,7 +38,7 @@ function test_get_metadata_basic() %% test func metadata base directory metadata = bids.query(BIDS, 'metadata', 'suffix', 'bold'); -% assert(metadata.RepetitionTime == func.RepetitionTime); + % assert(metadata.RepetitionTime == func.RepetitionTime); %% test func metadata subject 01 metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'suffix', 'bold'); @@ -48,7 +46,7 @@ function test_get_metadata_basic() %% test anat metadata base directory metadata = bids.query(BIDS, 'metadata', 'suffix', 'T1w'); -% assert(metadata.FlipAngle == anat.FlipAngle); + % assert(metadata.FlipAngle == anat.FlipAngle); %% test anat metadata subject 01 metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'suffix', 'T1w'); @@ -57,30 +55,28 @@ function test_get_metadata_basic() end -function test_get_metadata_participants() - % test files with no underscore in name. +function test_get_metadata_internal() pth_bids_example = get_test_data_dir(); - file = fullfile(pth_bids_example, 'ds001', 'participants.tsv'); - side_car = fullfile(pth_bids_example, 'ds001', 'participants.json'); - metadata = bids.internal.get_metadata(file); - expected_metadata = bids.util.jsondecode(side_car); - assertEqual(metadata, expected_metadata); + BIDS = bids.layout(fullfile(pth_bids_example, 'ds000117')); + + bids.internal.get_metadata(BIDS(1).subjects(2).anat(1).metafile); end -function test_get_metadata_internal() +function test_get_metadata_participants() + % test files with no underscore in name. pth_bids_example = get_test_data_dir(); - BIDS = bids.layout(fullfile(pth_bids_example, 'ds000117')); + file = fullfile(pth_bids_example, 'ds001', 'participants.tsv'); + side_car = fullfile(pth_bids_example, 'ds001', 'participants.json'); + + metalist = bids.internal.get_meta_list(file); + metadata = bids.internal.get_metadata(metalist); - bids.internal.get_metadata( ... - fullfile( ... - BIDS(1).subjects(2).path, ... - 'anat', ... - BIDS(1).subjects(2).anat(1).filename), ... - '%s'); + expected_metadata = bids.util.jsondecode(side_car); + assertEqual(metadata, expected_metadata); end diff --git a/tests/test_get_metadata_suffixes.m b/tests/test_get_metadata_suffixes.m index 9890bdb9..9ee503f4 100644 --- a/tests/test_get_metadata_suffixes.m +++ b/tests/test_get_metadata_suffixes.m @@ -9,24 +9,36 @@ function test_get_metadata_suffixes_basic() % ensures that "similar" suffixes are distinguished - data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'SurfaceData'); + data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'surface_data'); file = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_thickness.shape.gii'); side_car = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_thickness.json'); - metadata = bids.internal.get_metadata(file); + + metalist = bids.internal.get_meta_list(file); + metadata = bids.internal.get_metadata(metalist); + expected_metadata = bids.util.jsondecode(side_car); + assertEqual(metadata, expected_metadata); file = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii'); side_car = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_midthickness.json'); - metadata = bids.internal.get_metadata(file); + + metalist = bids.internal.get_meta_list(file); + metadata = bids.internal.get_metadata(metalist); + expected_metadata = bids.util.jsondecode(side_car); + assertEqual(metadata, expected_metadata); file = fullfile(data_dir, 'sub-06_space-individual_den-native_thickness.dscalar.nii'); side_car = fullfile(data_dir, 'sub-06_space-individual_den-native_thickness.json'); - metadata = bids.internal.get_metadata(file); + + metalist = bids.internal.get_meta_list(file); + metadata = bids.internal.get_metadata(metalist); + expected_metadata = bids.util.jsondecode(side_car); + assertEqual(metadata, expected_metadata); end diff --git a/tests/test_layout_derivatives.m b/tests/test_layout_derivatives.m index 092bddbd..a1b2bdda 100644 --- a/tests/test_layout_derivatives.m +++ b/tests/test_layout_derivatives.m @@ -25,7 +25,7 @@ function test_layout_prefix() 'prefix', 'swua'); basename = bids.internal.file_utils(data, 'basename'); assertEqual(basename, {'swuasub-01_task-balloonanalogrisktask_run-01_bold'}); - + assertEqual(bids.query(BIDS, 'prefixes'), {'swua'}); end diff --git a/tests/test_parse_filename.m b/tests/test_parse_filename.m index 6b658d1d..18ed1ea2 100644 --- a/tests/test_parse_filename.m +++ b/tests/test_parse_filename.m @@ -16,7 +16,7 @@ function test_parse_filename_prefix() 'suffix', 'bold', ... 'prefix', 'swua', ... 'ext', '.nii', ... - 'entities', struct('swuasub', '16', ... + 'entities', struct('sub', '16', ... 'run', '1', ... 'task', 'rest')); diff --git a/tests/test_return_modality_regular_expression.m b/tests/test_return_modality_regular_expression.m index b3984f5b..6b8b8edd 100644 --- a/tests/test_return_modality_regular_expression.m +++ b/tests/test_return_modality_regular_expression.m @@ -20,7 +20,7 @@ assertEqual(regular_expression, expected_expression); - data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'MoAEpilot', 'sub-01', 'anat'); + data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'synthetic', 'sub-01', 'anat'); subject_name = 'sub-01'; file = bids.internal.file_utils('List', data_dir, sprintf(expected_expression, subject_name)); diff --git a/tests/test_tsvwrite.m b/tests/test_tsvwrite.m index b557df8d..fb7b7295 100644 --- a/tests/test_tsvwrite.m +++ b/tests/test_tsvwrite.m @@ -99,44 +99,44 @@ function test_tsvwrite_basic() % as a possible data shape for the input opf TSV write % function test_tsvwrite_row_wise_structure() -% +% % pth = fileparts(mfilename('fullpath')); -% +% % tsv_file = fullfile(pth, 'sub-01_task-STRUCTURE_events.tsv'); -% +% % logFile(1, 1).onset = 2; % logFile(1, 1).trial_type = 'motion_up'; % logFile(1, 1).duration = 1; % logFile(1, 1).speed = []; % logFile(1, 1).is_fixation = true; -% +% % logFile(2, 1).onset = NaN; % logFile(2, 1).trial_type = 'static'; % logFile(2, 1).duration = 4; % logFile(2, 1).speed = 4; % logFile(2, 1).is_fixation = 3; -% +% % bids.util.tsvwrite(tsv_file, logFile); -% +% % % read the file & % % check the extra columns of the header and some of the content -% +% % FID = fopen(tsv_file, 'r'); % C = textscan(FID, '%s%s%s%s%s', 'Delimiter', '\t', 'EndOfLine', '\n'); -% +% % % check header % assertEqual(C{4}{1}, 'speed'); -% +% % % check that empty values are entered as NaN: logFile.speed(1) % assertEqual(C{4}{2}, 'n/a'); -% +% % % check that missing fields are entered as NaN: logFile.speed(2) % assertEqual(C{4}{3}, '4'); -% +% % % check that NaN are written as : logFile.onset(2) % assertEqual(C{1}{3}, 'n/a'); % -% +% % % check values entered properly: logFile.is_fixation(2) % assertEqual(C{5}{3}, '3'); -% -% end \ No newline at end of file +% +% end diff --git a/tests/utils/download_moae_ds.m b/tests/utils/download_moae_ds.m new file mode 100644 index 00000000..fd3caa35 --- /dev/null +++ b/tests/utils/download_moae_ds.m @@ -0,0 +1,37 @@ +function output_dir = download_moae_ds(downloadData) + % + % Will download the lightweight "Mother of all experiment" dataset from the + % SPM website. + % + % Copyright (C) 2021--, BIDS-MATLAB developers + + if nargin < 1 + downloadData = true(); + end + + output_dir = fullfile(fileparts(mfilename('fullpath')), '..', 'data'); + + if downloadData + + % URL of the data set to download + URL = 'http://www.fil.ion.ucl.ac.uk/spm/download/data/MoAEpilot/MoAEpilot.bids.zip'; + + % clean previous runs + if exist(fullfile(output_dir, 'MoAEpilot'), 'dir') + rmdir(fullfile(output_dir, 'MoAEpilot'), 's'); + end + + %% Get data + fprintf('%-10s:', 'Downloading dataset...'); + urlwrite(URL, 'MoAEpilot.zip'); + fprintf(1, ' Done\n\n'); + + fprintf('%-10s:', 'Unzipping dataset...'); + unzip('MoAEpilot.zip'); + delete('MoAEpilot.zip'); + movefile('MoAEpilot', fullfile(output_dir)); + fprintf(1, ' Done\n\n'); + + end + +end