Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FIX] fix force false skipped when unzipping files #584

Merged
merged 1 commit into from
Jun 18, 2023
Merged

[FIX] fix force false skipped when unzipping files #584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 20 additions & 4 deletions +bids/copy_to_derivative.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,19 +268,19 @@ function copy_file(BIDS, derivatives_folder, data_file, unzip_files, force, skip

%% ignore already existing files
% avoid circular references
if ~force && exist(fullfile(out_dir, file.filename), 'file')
if ~force && output_file_exists(out_dir, file, unzip_files)
if verbose
fprintf(1, '\n skipping: %s', bids.internal.format_path(file.filename));
end
return

else
file.meta = bids.internal.get_metadata(file.metafile);
end

if ~exist(out_dir, 'dir')
mkdir(out_dir);
end

bids.util.mkdir(out_dir);

%% copy data file
% we follow any eventual symlink and gunzip the data
copy_with_symlink(data_file, fullfile(out_dir, file.filename), unzip_files, verbose);
Expand Down Expand Up @@ -407,3 +407,19 @@ function copy_dependencies(file, BIDS, derivatives_folder, unzip, force, skip_de
function status = is_gunzipped(file)
status = bids.internal.ends_with(file, '.gz');
end

function status = output_file_exists(out_dir, file, unzip_files)

status = false;

if exist(fullfile(out_dir, file.filename), 'file')
status = true;
end

if unzip_files && ...
is_gunzipped(file.filename) && ...
exist(fullfile(out_dir, file.filename(1:end - 3)), 'file')
status = true;
end

end
119 changes: 89 additions & 30 deletions tests/test_copy_to_derivative.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,90 @@
initTestSuite;
end

function test_copy_to_derivative_unzip_force_false_572
% unzip should not occur if output file already exist and force is false

[pth, out_path, filter, cfg] = fixture('MoAEpilot');

pipeline_name = 'bids-matlab';
unzip = true;
force = false;
use_schema = true;
verbose = cfg.verbose;
skip_dependencies = true;

bids.copy_to_derivative(pth, ...
'pipeline_name', pipeline_name, ...
'out_path', out_path, ...
'filter', filter, ...
'use_schema', use_schema, ...
'unzip', unzip, ...
'force', force, ...
'skip_dep', skip_dependencies, ...
'verbose', verbose);

derivatives = bids.layout(fullfile(out_path, pipeline_name), ...
'use_schema', false, 'verbose', verbose);

t1w = bids.query(derivatives, 'data', 'modality', 'anat');
folder = fileparts(t1w{1});
files = dir(fullfile(folder, '*.nii'));
timestamp = files.date;

pause(1);

bids.copy_to_derivative(pth, ...
'pipeline_name', pipeline_name, ...
'out_path', out_path, ...
'filter', filter, ...
'use_schema', use_schema, ...
'unzip', unzip, ...
'force', force, ...
'skip_dep', skip_dependencies, ...
'verbose', verbose);

t1w = bids.query(derivatives, 'data', 'modality', 'anat');
folder = fileparts(t1w{1});
files = dir(fullfile(folder, '*.nii'));
assertEqual(files.date, timestamp);

derivatives = bids.layout(fullfile(out_path, pipeline_name), ...
'use_schema', false, 'verbose', verbose);

zipped_files = bids.query(derivatives, 'data', 'extension', '.nii.gz');
assertEqual(numel(zipped_files), 0);

end

function test_copy_to_derivative_unzip_force_false

[pth, out_path, filter, cfg] = fixture('MoAEpilot');

pipeline_name = 'bids-matlab';
unzip = true;
force = false;
use_schema = true;
verbose = cfg.verbose;
skip_dependencies = true;

bids.copy_to_derivative(pth, ...
'pipeline_name', pipeline_name, ...
'out_path', out_path, ...
'filter', filter, ...
'use_schema', use_schema, ...
'unzip', unzip, ...
'force', force, ...
'skip_dep', skip_dependencies, ...
'verbose', verbose);

derivatives = bids.layout(fullfile(out_path, pipeline_name), ...
'use_schema', false, 'verbose', verbose);

zipped_files = bids.query(derivatives, 'data', 'extension', '.nii.gz');
assertEqual(numel(zipped_files), 0);

end

function test_copy_to_derivative_exclude_with_regex()

[BIDS, out_path, filter, cfg] = fixture('ds002');
Expand Down Expand Up @@ -39,8 +123,6 @@ function test_copy_to_derivative_exclude_with_regex()
'.*_events.json');
assert(isempty(no_file_expected));

teardown(out_path);

end

function test_copy_to_derivative_GeneratedBy()
Expand All @@ -63,8 +145,6 @@ function test_copy_to_derivative_GeneratedBy()

assertEqual(BIDS.description.GeneratedBy.Name, 'SPM12');

teardown(out_path);

end

function test_copy_to_derivative_basic()
Expand Down Expand Up @@ -100,13 +180,11 @@ function test_copy_to_derivative_basic()
'skip_dep', skip_dependencies, ...
'verbose', verbose);

teardown(out_path);

end

function test_copy_to_derivative_unzip

[pth, out_path, filter, cfg, bu_folder] = fixture('MoAEpilot');
[pth, out_path, filter, cfg] = fixture('MoAEpilot');

pipeline_name = 'bids-matlab';
unzip = true;
Expand All @@ -131,8 +209,6 @@ function test_copy_to_derivative_basic()
zipped_files = bids.query(derivatives, 'data', 'extension', '.nii.gz');
assertEqual(numel(zipped_files), 0);

teardown_moae(bu_folder);

end

function test_copy_to_derivative_dependencies()
Expand Down Expand Up @@ -163,10 +239,9 @@ function test_copy_to_derivative_dependencies()
copied_files = bids.query(derivatives, 'data');
assertEqual(size(copied_files, 1), 10);

teardown(out_path);
bids.util.mkdir(out_path);

%%
[BIDS, out_path, filter] = fixture('qmri_mp2rageme');

skip_dependencies = false;

bids.copy_to_derivative(BIDS, ...
Expand All @@ -185,8 +260,6 @@ function test_copy_to_derivative_dependencies()
copied_files = bids.query(derivatives, 'data');
assertEqual(size(copied_files, 1), 11);

teardown(out_path);

end

function test_copy_to_derivative_sessions_scans_tsv
Expand Down Expand Up @@ -216,13 +289,9 @@ function test_copy_to_derivative_dependencies()
assert(~isempty(derivatives.subjects(1).scans));
assertEqual(derivatives.subjects(1).sess, derivatives.subjects(2).sess);

teardown(out_path);

end

function [BIDS, out_path, filter, cfg, bu_folder] = fixture(dataset)

bu_folder = '';
function [BIDS, out_path, filter, cfg] = fixture(dataset)

cfg = set_test_cfg();

Expand All @@ -234,11 +303,7 @@ function test_copy_to_derivative_dependencies()
%
% input_dir = fullfile('..', 'data', 'ds000117');

out_path = fullfile(pwd, 'data', dataset, 'derivatives');

if exist(out_path, 'dir')
teardown(out_path);
end
out_path = fullfile(temp_dir, 'data', dataset, 'derivatives');

bids.util.mkdir(out_path);

Expand Down Expand Up @@ -269,8 +334,6 @@ function test_copy_to_derivative_dependencies()

if ~isdir(fullfile(BIDS, 'sub-01'))

bu_folder = fixture_moae();

bids.util.download_ds('source', 'spm', ...
'demo', 'moae', ...
'force', true, ...
Expand All @@ -295,7 +358,3 @@ function test_copy_to_derivative_dependencies()
end

end

function teardown(out_path)
rmdir(out_path, 's');
end