From 6749a517aca68889ccf3ab92e0a3b37f3823ad0b Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Thu, 18 Apr 2024 13:31:27 +0200 Subject: [PATCH 1/9] cellranger_mapping integration test added --- .../cellranger_mapping/integration_test.sh | 2 + .../ingestion/cellranger_mapping/test.nf | 10 +++++ .../cellranger_mapping/config.vsh.yaml | 37 +++++++++++++++++++ .../ingestion/cellranger_mapping/script.py | 18 +++++++++ 4 files changed, 67 insertions(+) create mode 100644 src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml create mode 100644 src/workflows/test_workflows/ingestion/cellranger_mapping/script.py diff --git a/src/workflows/ingestion/cellranger_mapping/integration_test.sh b/src/workflows/ingestion/cellranger_mapping/integration_test.sh index 1d039ec7d0e..b09ce44e83d 100755 --- a/src/workflows/ingestion/cellranger_mapping/integration_test.sh +++ b/src/workflows/ingestion/cellranger_mapping/integration_test.sh @@ -8,6 +8,8 @@ REPO_ROOT=$(git rev-parse --show-toplevel) # ensure that the command below is run from the root of the repository cd "$REPO_ROOT" +viash ns build -q ingestion/cellranger_mapping --setup cb --platform nextflow + export NXF_VER=21.10.6 nextflow \ diff --git a/src/workflows/ingestion/cellranger_mapping/test.nf b/src/workflows/ingestion/cellranger_mapping/test.nf index 5752785aa5d..fc1fb3eb829 100644 --- a/src/workflows/ingestion/cellranger_mapping/test.nf +++ b/src/workflows/ingestion/cellranger_mapping/test.nf @@ -1,6 +1,7 @@ nextflow.enable.dsl=2 include { cellranger_mapping } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_mapping/main.nf" +include { cellranger_mapping_test } from params.rootDir + "/target/nextflow/test_workflows/cellranger_mapping/cellranger_mapping_test/main.nf" workflow test_wf { // allow changing the resources_test dir @@ -22,6 +23,15 @@ workflow test_wf { // todo: check whether output dir contains fastq files "Output: $output" } + + | cellranger_mapping_test.run( + fromState: { output -> + [ + input: output[1].output_h5mu + ] + } + ) + | toSortedList() | map { output_list -> assert output_list.size() == 1 : "output channel should contain one event" diff --git a/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml new file mode 100644 index 00000000000..c21e5da01c9 --- /dev/null +++ b/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml @@ -0,0 +1,37 @@ +functionality: + name: "cellranger_mapping_test" + namespace: "test_workflows/cellranger_mapping" + description: "This component test the output of the integration test of the cellranger mapping workflow." + authors: + - __merge__: /src/authors/jakub_majercik.yaml + argument_groups: + - name: Inputs + arguments: + - name: "--input" + type: file + required: true + description: Path to h5mu output. + example: foo.final.h5mu + resources: + - type: python_script + path: script.py + - path: /src/utils/setup_logger.py + - path: /src/base/openpipelinetestutils + dest: openpipelinetestutils +platforms: + - type: docker + image: python:3.12-slim + setup: + - type: docker + copy: ["openpipelinetestutils /opt/openpipelinetestutils"] + - type: apt + packages: + - procps + - type: python + packages: /opt/openpipelinetestutils + - type: python + pypi: + - mudata + # - pytest + __merge__: /src/base/requirements/viashpy.yaml + - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_mapping/script.py b/src/workflows/test_workflows/ingestion/cellranger_mapping/script.py new file mode 100644 index 00000000000..72cd659c65f --- /dev/null +++ b/src/workflows/test_workflows/ingestion/cellranger_mapping/script.py @@ -0,0 +1,18 @@ +from mudata import read_h5mu + +##VIASH START +par = { + "input": "input.h5mu" +} + +meta = { + "resources_dir": "resources_test" +} +##VIASH END + +input_mudata = read_h5mu(par["input"]) +expected_colnames = ['gene_symbol', 'feature_types', 'genome'] + +assert list(input_mudata.mod.keys()) == ["rna"], "Input should contain rna modality." +assert list(input_mudata.var.columns) == expected_colnames, f"Input var columns should be: {expected_colnames}." +assert list(input_mudata.mod["rna"].var.columns) == expected_colnames, f"Input mod['rna'] var columns should be: {expected_colnames}." From 36273d3d910fd20166e37f187bb0301c9b6d11bc Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Thu, 25 Apr 2024 14:23:11 +0200 Subject: [PATCH 2/9] basic integration test for cellranger_multi added --- .../ingestion/cellranger_multi/test.nf | 10 +++++ .../cellranger_multi/config.vsh.yaml | 37 +++++++++++++++++++ .../ingestion/cellranger_multi/script.py | 18 +++++++++ 3 files changed, 65 insertions(+) create mode 100644 src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml create mode 100644 src/workflows/test_workflows/ingestion/cellranger_multi/script.py diff --git a/src/workflows/ingestion/cellranger_multi/test.nf b/src/workflows/ingestion/cellranger_multi/test.nf index 0df92d88595..856996272ad 100644 --- a/src/workflows/ingestion/cellranger_multi/test.nf +++ b/src/workflows/ingestion/cellranger_multi/test.nf @@ -1,6 +1,7 @@ nextflow.enable.dsl=2 include { cellranger_multi } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_multi/main.nf" +include { cellranger_multi_test } from params.rootDir + "/target/nextflow/test_workflows/cellranger_multi/cellranger_multi_test/main.nf" workflow test_wf { resources_test = file("${params.rootDir}/resources_test") @@ -29,6 +30,15 @@ workflow test_wf { // todo: check whether output dir contains fastq files "Output: $output" } + + | cellranger_multi_test.run( + fromState: { output -> + [ + input: output[1].output_h5mu + ] + } + ) + | toSortedList() | map { output_list -> assert output_list.size() == 1 : "output channel should contain one event" diff --git a/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml new file mode 100644 index 00000000000..8d54687f068 --- /dev/null +++ b/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml @@ -0,0 +1,37 @@ +functionality: + name: "cellranger_multi_test" + namespace: "test_workflows/cellranger_multi" + description: "This component test the output of the integration test of the cellranger multi workflow." + authors: + - __merge__: /src/authors/jakub_majercik.yaml + argument_groups: + - name: Inputs + arguments: + - name: "--input" + type: file + required: true + description: Path to h5mu output. + example: foo.final.h5mu + resources: + - type: python_script + path: script.py + - path: /src/utils/setup_logger.py + - path: /src/base/openpipelinetestutils + dest: openpipelinetestutils +platforms: + - type: docker + image: python:3.12-slim + setup: + - type: docker + copy: ["openpipelinetestutils /opt/openpipelinetestutils"] + - type: apt + packages: + - procps + - type: python + packages: /opt/openpipelinetestutils + - type: python + pypi: + - mudata + # - pytest + __merge__: /src/base/requirements/viashpy.yaml + - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_multi/script.py b/src/workflows/test_workflows/ingestion/cellranger_multi/script.py new file mode 100644 index 00000000000..b2bb0da0da6 --- /dev/null +++ b/src/workflows/test_workflows/ingestion/cellranger_multi/script.py @@ -0,0 +1,18 @@ +from mudata import read_h5mu + +##VIASH START +par = { + "input": "input.h5mu" +} + +meta = { + "resources_dir": "resources_test" +} +##VIASH END + +input_mudata = read_h5mu(par["input"]) + +assert list(input_mudata.mod.keys()) == ['rna', 'prot', 'vdj_t'] +assert list(input_mudata.uns.keys()) == ['metrics_cellranger'] +expected_metrics = ['Category', 'Library Type', 'Grouped By', 'Group Name', 'Metric Name', 'Metric Value'] +assert input_mudata.uns['metrics_cellranger'].columns.to_list() == expected_metrics From 6b7eafc8facb7370a9233e834c54cea2e4fafd08 Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Mon, 29 Apr 2024 10:02:14 +0200 Subject: [PATCH 3/9] cellranger_preprocessing integration test added --- .../cellranger_postprocessing/test.nf | 46 ++++++++++++++++++- .../cellranger_postprocessing/config.vsh.yaml | 46 +++++++++++++++++++ .../cellranger_postprocessing/script.py | 31 +++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 src/workflows/test_workflows/ingestion/cellranger_postprocessing/config.vsh.yaml create mode 100644 src/workflows/test_workflows/ingestion/cellranger_postprocessing/script.py diff --git a/src/workflows/ingestion/cellranger_postprocessing/test.nf b/src/workflows/ingestion/cellranger_postprocessing/test.nf index d0d92f28e6a..18a1a1d6315 100644 --- a/src/workflows/ingestion/cellranger_postprocessing/test.nf +++ b/src/workflows/ingestion/cellranger_postprocessing/test.nf @@ -2,6 +2,7 @@ nextflow.enable.dsl=2 include { cellranger_postprocessing } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_postprocessing/main.nf" include { from_10xh5_to_h5mu } from params.rootDir + "/target/nextflow/convert/from_10xh5_to_h5mu/main.nf" +include { cellranger_postprocessing_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_postprocessing_test/main.nf" workflow test_wf { // allow changing the resources_test dir @@ -11,6 +12,7 @@ workflow test_wf { [ id: "foo", input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5"), + input_og: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5mu"), perform_correction: true, min_genes: 100, min_counts: 1000, @@ -24,13 +26,32 @@ workflow test_wf { toState: ["input": "output"] ) - | cellranger_postprocessing + | cellranger_postprocessing.run( + toState: {id, output, state -> + output + [ + input_og: state.input_og, + perform_correction: state.perform_correction + ] + } + ) + | view { output -> assert output.size() == 2 : "outputs should contain two elements; [id, out]" assert output[1] instanceof Map : "Output should be a Map." // todo: check whether output dir contains fastq files "Output: $output" } + + | cellranger_postprocessing_test.run( + fromState: {id, state -> + [ + input: state.output, + input_og: state.input_og, + is_corrected: state.perform_correction + ] + } + ) + | toSortedList() | map { output_list -> assert output_list.size() == 1 : "output channel should contain one event" @@ -47,6 +68,7 @@ workflow test_wf2 { [ id: "zing", input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5"), + input_og: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5mu"), perform_correction: false, min_genes: 100, min_counts: 1000, @@ -59,13 +81,33 @@ workflow test_wf2 { fromState: ["input"], toState: ["input": "output"] ) - | cellranger_postprocessing + + | cellranger_postprocessing.run( + toState: {id, output, state -> + output + [ + input_og: state.input_og, + perform_correction: state.perform_correction + ] + } + ) + | view { output -> assert output.size() == 2 : "outputs should contain two elements; [id, out]" assert output[1] instanceof Map : "Output should be a Map." // todo: check whether output dir contains fastq files "Output: $output" } + + | cellranger_postprocessing_test.run( + fromState: {id, state -> + [ + input: state.output, + input_og: state.input_og, + is_corrected: state.perform_correction + ] + } + ) + | toSortedList() | map { output_list -> assert output_list.size() == 1 : "output channel should contain one event" diff --git a/src/workflows/test_workflows/ingestion/cellranger_postprocessing/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_postprocessing/config.vsh.yaml new file mode 100644 index 00000000000..436326edd2c --- /dev/null +++ b/src/workflows/test_workflows/ingestion/cellranger_postprocessing/config.vsh.yaml @@ -0,0 +1,46 @@ +functionality: + name: "cellranger_postprocessing_test" + namespace: "test_workflows/ingestion" + description: "This component test the output of the integration test of the cellranger postprocessing workflow." + authors: + - __merge__: /src/authors/jakub_majercik.yaml + argument_groups: + - name: Inputs + arguments: + - name: "--input" + type: file + required: true + description: Path to h5mu output. + example: foo.final.h5mu + - name: "--input_og" + type: file + required: true + description: Path to the original h5mu file. + example: foo.h5mu + - name: "--is_corrected" + type: boolean + required: true + description: Whether the input file has been corrected. + resources: + - type: python_script + path: script.py + - path: /src/utils/setup_logger.py + - path: /src/base/openpipelinetestutils + dest: openpipelinetestutils +platforms: + - type: docker + image: python:3.12-slim + setup: + - type: docker + copy: ["openpipelinetestutils /opt/openpipelinetestutils"] + - type: apt + packages: + - procps + - type: python + packages: /opt/openpipelinetestutils + - type: python + pypi: + - mudata + # - pytest + __merge__: /src/base/requirements/viashpy.yaml + - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_postprocessing/script.py b/src/workflows/test_workflows/ingestion/cellranger_postprocessing/script.py new file mode 100644 index 00000000000..1090842712a --- /dev/null +++ b/src/workflows/test_workflows/ingestion/cellranger_postprocessing/script.py @@ -0,0 +1,31 @@ +from mudata import read_h5mu +from openpipelinetestutils.asserters import assert_annotation_objects_equal + +##VIASH START +par = { + "input": "input.h5mu", + "input": "input_og.h5mu", + "is_corrected": True +} + +meta = { + "resources_dir": "resources_test" +} +##VIASH END + +input_mudata = read_h5mu(par["input_og"]) +output_mudata = read_h5mu(par["input"]) + +assert input_mudata.mod.keys() == output_mudata.mod.keys(), "Input and output should have the same modalities." + +for modality,input_adata,output_adata in zip(input_mudata.mod.keys(), + input_mudata.mod.values(), + output_mudata.mod.values()): + assert input_adata.n_obs >= output_adata.n_obs, "Output should have less or equal number of observations than input." + assert input_adata.n_vars == output_adata.n_vars, "Output should have the same number of variables as input." + if modality != "rna": + assert_annotation_objects_equal(input_adata, + output_adata) + +if par["is_corrected"]: + assert "cellbender_corrected" in output_mudata.mod["rna"].layers \ No newline at end of file From 1d2e8102f6c2ec01b98a4ff85acb2eba49afb764 Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Tue, 30 Apr 2024 11:34:58 +0200 Subject: [PATCH 4/9] conversion integration test added --- src/workflows/ingestion/conversion/test.nf | 18 ++++++++--- .../ingestion/conversion/config.vsh.yaml | 31 +++++++++++++++++++ .../ingestion/conversion/script.py | 19 ++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml create mode 100644 src/workflows/test_workflows/ingestion/conversion/script.py diff --git a/src/workflows/ingestion/conversion/test.nf b/src/workflows/ingestion/conversion/test.nf index 6dd74a9a03c..058d7774f6b 100644 --- a/src/workflows/ingestion/conversion/test.nf +++ b/src/workflows/ingestion/conversion/test.nf @@ -1,6 +1,7 @@ nextflow.enable.dsl=2 include { conversion } from params.rootDir + "/target/nextflow/workflows/ingestion/conversion/main.nf" +include { conversion_test } from params.rootDir + "/target/nextflow/test_workflows/conversion/conversion_test/main.nf" workflow test_wf { // allow changing the resources_test dir @@ -41,9 +42,18 @@ workflow test_wf { assert output.size() == 2 : "outputs should contain two elements; [id, file]" assert output[1].output.toString().endsWith(".h5mu") : "Output file should be a h5mu file. Found: ${output[1]}" "Output: $output" + } + + | conversion_test.run( + fromState: { output -> + [ + input: output[1].output + ] } - | toSortedList() - | map { output_list -> - assert output_list.size() == 4 : "output channel should contain four events" - } + ) + + | toSortedList() + | map { output_list -> + assert output_list.size() == 4 : "output channel should contain four events" + } } \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml b/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml new file mode 100644 index 00000000000..17824ddd086 --- /dev/null +++ b/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml @@ -0,0 +1,31 @@ +functionality: + name: "conversion_test" + namespace: "test_workflows/conversion" + description: "This component test the output of the integration test of the conversion workflow." + authors: + - __merge__: /src/authors/jakub_majercik.yaml + argument_groups: + - name: Inputs + arguments: + - name: "--input" + type: file + required: true + description: Path to h5mu output. + example: foo.final.h5mu + resources: + - type: python_script + path: script.py + - path: /src/utils/setup_logger.py +platforms: + - type: docker + image: python:3.12-slim + setup: + - type: apt + packages: + - procps + - type: python + pypi: + - mudata + # - pytest + __merge__: /src/base/requirements/viashpy.yaml + - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/conversion/script.py b/src/workflows/test_workflows/ingestion/conversion/script.py new file mode 100644 index 00000000000..b334fde8c90 --- /dev/null +++ b/src/workflows/test_workflows/ingestion/conversion/script.py @@ -0,0 +1,19 @@ +from mudata import read_h5mu + +##VIASH START +par = { + "input": "input.h5mu" +} + +meta = { + "resources_dir": "resources_test" +} +##VIASH END + +input_mudata = read_h5mu(par["input"]) + +assert "rna" in input_mudata.mod.keys() +assert input_mudata.n_obs == 713 +assert input_mudata.mod["rna"].var["feature_types"].unique() == [ + "Gene Expression" +], "Output X should only contain Gene Expression vars." \ No newline at end of file From f0e3f90c7cd28f225635e68c122ad07ba63211f9 Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Mon, 6 May 2024 10:49:27 +0200 Subject: [PATCH 5/9] conversion + bd_rhapsody tests added --- src/workflows/ingestion/bd_rhapsody/test.nf | 12 ++++++++ .../ingestion/bd_rhapsody/config.vsh.yaml | 30 +++++++++++++++++++ .../ingestion/bd_rhapsody/script.py | 23 ++++++++++++++ .../ingestion/conversion/config.vsh.yaml | 3 +- .../ingestion/conversion/script.py | 26 ++++++++++++---- 5 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml create mode 100644 src/workflows/test_workflows/ingestion/bd_rhapsody/script.py diff --git a/src/workflows/ingestion/bd_rhapsody/test.nf b/src/workflows/ingestion/bd_rhapsody/test.nf index eae143e7fe2..33b532fecd6 100644 --- a/src/workflows/ingestion/bd_rhapsody/test.nf +++ b/src/workflows/ingestion/bd_rhapsody/test.nf @@ -1,6 +1,7 @@ nextflow.enable.dsl=2 include { bd_rhapsody } from params.rootDir + "/target/nextflow/workflows/ingestion/bd_rhapsody/main.nf" +include { bd_rhapsody_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/bd_rhapsody/bd_rhapsody_test/main.nf" workflow test_wf { // allow changing the resources_test dir @@ -32,9 +33,20 @@ workflow test_wf { assert data.output_h5mu.toString().endsWith(".h5mu") : "Output file should be a h5mu file. Found: ${output[1]}" "Output: $output" } + + | bd_rhapsody_test.run( + fromState: { output -> + [ + input: output[1].output_h5mu + ] + } + ) + | toList() | view { output_list -> assert output_list.size() == 1 : "output channel should contain one event" } + + // | view { output -> output[1]} // | check_format(args: {""}) // todo: check whether output h5mu has the right slots defined } \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml b/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml new file mode 100644 index 00000000000..639e80454ba --- /dev/null +++ b/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml @@ -0,0 +1,30 @@ +functionality: + name: "bd_rhapsody_test" + namespace: "test_workflows/ingestion/bd_rhapsody" + description: "This component test the output of the integration test of the bd_rhapsody workflow." + authors: + - __merge__: /src/authors/jakub_majercik.yaml + argument_groups: + - name: Inputs + arguments: + - name: "--input" + type: file + required: true + description: Path to h5mu output. + example: foo.final.h5mu + resources: + - type: python_script + path: script.py + - path: /src/utils/setup_logger.py +platforms: + - type: docker + image: python:3.12-slim + setup: + - type: apt + packages: + - procps + - type: python + pypi: + - mudata + - pytest + - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/bd_rhapsody/script.py b/src/workflows/test_workflows/ingestion/bd_rhapsody/script.py new file mode 100644 index 00000000000..925064cf50c --- /dev/null +++ b/src/workflows/test_workflows/ingestion/bd_rhapsody/script.py @@ -0,0 +1,23 @@ +from mudata import read_h5mu +import numpy as np + +##VIASH START +par = { + "input": "input.h5mu" +} + +meta = { + "resources_dir": "resources_test" +} +##VIASH END + +input_mudata = read_h5mu(par["input"]) +expected_var = ['gene_name', 'feature_types', 'reference_file'] +expected_obs = ['run_id', 'library_id', 'sample_id'] + +assert list(input_mudata.mod.keys()) == ["rna"], "Input should contain rna modality." +assert list(input_mudata.var.columns) == expected_var, f"Input var columns should be: {expected_var}." +assert list(input_mudata.mod["rna"].var.columns) == expected_var, f"Input mod['rna'] var columns should be: {expected_var}." +assert list(input_mudata.mod["rna"].obs.columns) == expected_obs, f"Input obs columns should be: {expected_obs}." + +assert np.array_equal(input_mudata.var["feature_types"].unique(), ["Gene Expression"]), "Output X should only contain Gene Expression vars." diff --git a/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml b/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml index 17824ddd086..e91a329db6b 100644 --- a/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml @@ -26,6 +26,5 @@ platforms: - type: python pypi: - mudata - # - pytest - __merge__: /src/base/requirements/viashpy.yaml + - pytest - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/conversion/script.py b/src/workflows/test_workflows/ingestion/conversion/script.py index b334fde8c90..6e99fb21dd3 100644 --- a/src/workflows/test_workflows/ingestion/conversion/script.py +++ b/src/workflows/test_workflows/ingestion/conversion/script.py @@ -1,4 +1,7 @@ from mudata import read_h5mu +# import pytest +import sys +import pathlib ##VIASH START par = { @@ -12,8 +15,21 @@ input_mudata = read_h5mu(par["input"]) -assert "rna" in input_mudata.mod.keys() -assert input_mudata.n_obs == 713 -assert input_mudata.mod["rna"].var["feature_types"].unique() == [ - "Gene Expression" -], "Output X should only contain Gene Expression vars." \ No newline at end of file +def test_run(): + assert "rna" in input_mudata.mod.keys() + assert input_mudata.n_obs == 713 + assert input_mudata.mod["rna"].var["feature_types"].unique() == [ + "Gene Expression" + ], "Output X should only contain Gene Expression vars." + +if __name__ == "__main__": + with open("pytest.ini", "w") as ini_file: + ini_file.write("[pytest]\npython_files=*.viash_script.py\nnorecursedirs=") + + import pytest + sys.exit(pytest.main([str(pathlib.Path(__file__).resolve().parent), + # "-o", "python_files=*.viash_script.sh", + "--verbose", + # "--rootdir", str(pathlib.Path(__file__).resolve().parent) + ])) + # sys.exit(pytest.main(["script.py"])) \ No newline at end of file From e4bdb02526c6faa36b0979bc17464b6c29fba3e7 Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Mon, 6 May 2024 11:32:40 +0200 Subject: [PATCH 6/9] reorganized namespaces --- src/workflows/ingestion/bd_rhapsody/test.nf | 2 +- .../ingestion/cellranger_mapping/test.nf | 2 +- .../ingestion/cellranger_multi/test.nf | 2 +- src/workflows/ingestion/conversion/test.nf | 2 +- .../ingestion/bd_rhapsody/config.vsh.yaml | 2 +- .../cellranger_mapping/config.vsh.yaml | 2 +- .../cellranger_multi/config.vsh.yaml | 2 +- .../ingestion/conversion/config.vsh.yaml | 2 +- .../ingestion/conversion/script.py | 28 +++++-------------- 9 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/workflows/ingestion/bd_rhapsody/test.nf b/src/workflows/ingestion/bd_rhapsody/test.nf index 33b532fecd6..694d11c80a2 100644 --- a/src/workflows/ingestion/bd_rhapsody/test.nf +++ b/src/workflows/ingestion/bd_rhapsody/test.nf @@ -1,7 +1,7 @@ nextflow.enable.dsl=2 include { bd_rhapsody } from params.rootDir + "/target/nextflow/workflows/ingestion/bd_rhapsody/main.nf" -include { bd_rhapsody_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/bd_rhapsody/bd_rhapsody_test/main.nf" +include { bd_rhapsody_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/bd_rhapsody_test/main.nf" workflow test_wf { // allow changing the resources_test dir diff --git a/src/workflows/ingestion/cellranger_mapping/test.nf b/src/workflows/ingestion/cellranger_mapping/test.nf index fc1fb3eb829..91647099f02 100644 --- a/src/workflows/ingestion/cellranger_mapping/test.nf +++ b/src/workflows/ingestion/cellranger_mapping/test.nf @@ -1,7 +1,7 @@ nextflow.enable.dsl=2 include { cellranger_mapping } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_mapping/main.nf" -include { cellranger_mapping_test } from params.rootDir + "/target/nextflow/test_workflows/cellranger_mapping/cellranger_mapping_test/main.nf" +include { cellranger_mapping_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_mapping_test/main.nf" workflow test_wf { // allow changing the resources_test dir diff --git a/src/workflows/ingestion/cellranger_multi/test.nf b/src/workflows/ingestion/cellranger_multi/test.nf index 856996272ad..b17ae9da40f 100644 --- a/src/workflows/ingestion/cellranger_multi/test.nf +++ b/src/workflows/ingestion/cellranger_multi/test.nf @@ -1,7 +1,7 @@ nextflow.enable.dsl=2 include { cellranger_multi } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_multi/main.nf" -include { cellranger_multi_test } from params.rootDir + "/target/nextflow/test_workflows/cellranger_multi/cellranger_multi_test/main.nf" +include { cellranger_multi_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_multi_test/main.nf" workflow test_wf { resources_test = file("${params.rootDir}/resources_test") diff --git a/src/workflows/ingestion/conversion/test.nf b/src/workflows/ingestion/conversion/test.nf index 058d7774f6b..336e455e775 100644 --- a/src/workflows/ingestion/conversion/test.nf +++ b/src/workflows/ingestion/conversion/test.nf @@ -1,7 +1,7 @@ nextflow.enable.dsl=2 include { conversion } from params.rootDir + "/target/nextflow/workflows/ingestion/conversion/main.nf" -include { conversion_test } from params.rootDir + "/target/nextflow/test_workflows/conversion/conversion_test/main.nf" +include { conversion_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/conversion_test/main.nf" workflow test_wf { // allow changing the resources_test dir diff --git a/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml b/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml index 639e80454ba..b9d26eb93ea 100644 --- a/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml @@ -1,6 +1,6 @@ functionality: name: "bd_rhapsody_test" - namespace: "test_workflows/ingestion/bd_rhapsody" + namespace: "test_workflows/ingestion" description: "This component test the output of the integration test of the bd_rhapsody workflow." authors: - __merge__: /src/authors/jakub_majercik.yaml diff --git a/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml index c21e5da01c9..fae72fcc9d2 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml @@ -1,6 +1,6 @@ functionality: name: "cellranger_mapping_test" - namespace: "test_workflows/cellranger_mapping" + namespace: "test_workflows/ingestion" description: "This component test the output of the integration test of the cellranger mapping workflow." authors: - __merge__: /src/authors/jakub_majercik.yaml diff --git a/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml index 8d54687f068..b1b8ce739b8 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml @@ -1,6 +1,6 @@ functionality: name: "cellranger_multi_test" - namespace: "test_workflows/cellranger_multi" + namespace: "test_workflows/ingestion" description: "This component test the output of the integration test of the cellranger multi workflow." authors: - __merge__: /src/authors/jakub_majercik.yaml diff --git a/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml b/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml index e91a329db6b..1d48dd8ae67 100644 --- a/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml @@ -1,6 +1,6 @@ functionality: name: "conversion_test" - namespace: "test_workflows/conversion" + namespace: "test_workflows/ingestion" description: "This component test the output of the integration test of the conversion workflow." authors: - __merge__: /src/authors/jakub_majercik.yaml diff --git a/src/workflows/test_workflows/ingestion/conversion/script.py b/src/workflows/test_workflows/ingestion/conversion/script.py index 6e99fb21dd3..d71c088f2fa 100644 --- a/src/workflows/test_workflows/ingestion/conversion/script.py +++ b/src/workflows/test_workflows/ingestion/conversion/script.py @@ -1,7 +1,5 @@ from mudata import read_h5mu -# import pytest -import sys -import pathlib + ##VIASH START par = { @@ -15,21 +13,9 @@ input_mudata = read_h5mu(par["input"]) -def test_run(): - assert "rna" in input_mudata.mod.keys() - assert input_mudata.n_obs == 713 - assert input_mudata.mod["rna"].var["feature_types"].unique() == [ - "Gene Expression" - ], "Output X should only contain Gene Expression vars." - -if __name__ == "__main__": - with open("pytest.ini", "w") as ini_file: - ini_file.write("[pytest]\npython_files=*.viash_script.py\nnorecursedirs=") - - import pytest - sys.exit(pytest.main([str(pathlib.Path(__file__).resolve().parent), - # "-o", "python_files=*.viash_script.sh", - "--verbose", - # "--rootdir", str(pathlib.Path(__file__).resolve().parent) - ])) - # sys.exit(pytest.main(["script.py"])) \ No newline at end of file +assert "rna" in input_mudata.mod.keys() +assert input_mudata.n_obs == 713 +assert input_mudata.mod["rna"].var["feature_types"].unique() == [ + "Gene Expression" +], "Output X should only contain Gene Expression vars." + \ No newline at end of file From 75ae35c92a1db599d9a2974e7875fd1291007818 Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Tue, 11 Jun 2024 22:50:33 +0200 Subject: [PATCH 7/9] tests updated to use pytest --- src/base/openpipelinetestutils/conftest.py | 13 +++++++ .../ingestion/bd_rhapsody/config.vsh.yaml | 7 +++- .../ingestion/bd_rhapsody/script.py | 28 ++++++++++---- .../cellranger_mapping/config.vsh.yaml | 1 - .../ingestion/cellranger_mapping/script.py | 22 ++++++++--- .../cellranger_multi/config.vsh.yaml | 1 - .../ingestion/cellranger_multi/script.py | 22 ++++++++--- .../cellranger_postprocessing/config.vsh.yaml | 1 - .../cellranger_postprocessing/script.py | 38 ++++++++++++------- .../ingestion/conversion/config.vsh.yaml | 7 +++- .../ingestion/conversion/script.py | 28 +++++++++----- 11 files changed, 123 insertions(+), 45 deletions(-) create mode 100644 src/base/openpipelinetestutils/conftest.py diff --git a/src/base/openpipelinetestutils/conftest.py b/src/base/openpipelinetestutils/conftest.py new file mode 100644 index 00000000000..ca3948eae0e --- /dev/null +++ b/src/base/openpipelinetestutils/conftest.py @@ -0,0 +1,13 @@ +import importlib +import pytest +from pathlib import Path + +def pytest_collect_file(file_path: Path, parent): + if (file_path.name == ".viash_script.sh"): + # Allow file ending in .sh to be imported + importlib.machinery.SOURCE_SUFFIXES.append('.viash_script.sh') + return pytest.Module.from_parent(parent, path=file_path) + + +def pytest_collection_finish(session): + importlib.machinery.SOURCE_SUFFIXES.remove('.viash_script.sh') \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml b/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml index b9d26eb93ea..3e37fbdbff0 100644 --- a/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/bd_rhapsody/config.vsh.yaml @@ -16,15 +16,20 @@ functionality: - type: python_script path: script.py - path: /src/utils/setup_logger.py + - path: /src/base/openpipelinetestutils + dest: openpipelinetestutils platforms: - type: docker image: python:3.12-slim setup: + - type: docker + copy: ["openpipelinetestutils /opt/openpipelinetestutils"] - type: apt packages: - procps - type: python + packages: /opt/openpipelinetestutils pypi: - mudata - - pytest + __merge__: /src/base/requirements/viashpy.yaml - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/bd_rhapsody/script.py b/src/workflows/test_workflows/ingestion/bd_rhapsody/script.py index 925064cf50c..b4749956b55 100644 --- a/src/workflows/test_workflows/ingestion/bd_rhapsody/script.py +++ b/src/workflows/test_workflows/ingestion/bd_rhapsody/script.py @@ -1,5 +1,10 @@ from mudata import read_h5mu import numpy as np +import shutil +import os +import sys +from pathlib import Path +import pytest ##VIASH START par = { @@ -11,13 +16,20 @@ } ##VIASH END -input_mudata = read_h5mu(par["input"]) -expected_var = ['gene_name', 'feature_types', 'reference_file'] -expected_obs = ['run_id', 'library_id', 'sample_id'] +def test_run(): + input_mudata = read_h5mu(par["input"]) + expected_var = ['gene_name', 'feature_types', 'reference_file'] + expected_obs = ['run_id', 'library_id', 'sample_id'] -assert list(input_mudata.mod.keys()) == ["rna"], "Input should contain rna modality." -assert list(input_mudata.var.columns) == expected_var, f"Input var columns should be: {expected_var}." -assert list(input_mudata.mod["rna"].var.columns) == expected_var, f"Input mod['rna'] var columns should be: {expected_var}." -assert list(input_mudata.mod["rna"].obs.columns) == expected_obs, f"Input obs columns should be: {expected_obs}." + assert list(input_mudata.mod.keys()) == ["rna"], "Input should contain rna modality." + assert list(input_mudata.var.columns) == expected_var, f"Input var columns should be: {expected_var}." + assert list(input_mudata.mod["rna"].var.columns) == expected_var, f"Input mod['rna'] var columns should be: {expected_var}." + assert list(input_mudata.mod["rna"].obs.columns) == expected_obs, f"Input obs columns should be: {expected_obs}." -assert np.array_equal(input_mudata.var["feature_types"].unique(), ["Gene Expression"]), "Output X should only contain Gene Expression vars." + assert np.array_equal(input_mudata.var["feature_types"].unique(), ["Gene Expression"]), "Output X should only contain Gene Expression vars." + +if __name__ == "__main__": + HERE_DIR = Path(__file__).resolve().parent + shutil.copyfile(os.path.join(meta['resources_dir'], "openpipelinetestutils", "conftest.py"), + os.path.join(HERE_DIR, "conftest.py")) + sys.exit(pytest.main(["--import-mode=importlib"])) \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml index fae72fcc9d2..cfbb3257684 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/cellranger_mapping/config.vsh.yaml @@ -32,6 +32,5 @@ platforms: - type: python pypi: - mudata - # - pytest __merge__: /src/base/requirements/viashpy.yaml - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_mapping/script.py b/src/workflows/test_workflows/ingestion/cellranger_mapping/script.py index 72cd659c65f..e1cecb1a91b 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_mapping/script.py +++ b/src/workflows/test_workflows/ingestion/cellranger_mapping/script.py @@ -1,4 +1,9 @@ from mudata import read_h5mu +from pathlib import Path +import shutil +import os +import sys +import pytest ##VIASH START par = { @@ -10,9 +15,16 @@ } ##VIASH END -input_mudata = read_h5mu(par["input"]) -expected_colnames = ['gene_symbol', 'feature_types', 'genome'] +def test_run(): + input_mudata = read_h5mu(par["input"]) + expected_colnames = ['gene_symbol', 'feature_types', 'genome'] -assert list(input_mudata.mod.keys()) == ["rna"], "Input should contain rna modality." -assert list(input_mudata.var.columns) == expected_colnames, f"Input var columns should be: {expected_colnames}." -assert list(input_mudata.mod["rna"].var.columns) == expected_colnames, f"Input mod['rna'] var columns should be: {expected_colnames}." + assert list(input_mudata.mod.keys()) == ["rna"], "Input should contain rna modality." + assert list(input_mudata.var.columns) == expected_colnames, f"Input var columns should be: {expected_colnames}." + assert list(input_mudata.mod["rna"].var.columns) == expected_colnames, f"Input mod['rna'] var columns should be: {expected_colnames}." + +if __name__ == "__main__": + HERE_DIR = Path(__file__).resolve().parent + shutil.copyfile(os.path.join(meta['resources_dir'], "openpipelinetestutils", "conftest.py"), + os.path.join(HERE_DIR, "conftest.py")) + sys.exit(pytest.main(["--import-mode=importlib"])) \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml index b1b8ce739b8..418b63ca701 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml @@ -32,6 +32,5 @@ platforms: - type: python pypi: - mudata - # - pytest __merge__: /src/base/requirements/viashpy.yaml - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_multi/script.py b/src/workflows/test_workflows/ingestion/cellranger_multi/script.py index b2bb0da0da6..4ad0ee71abf 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_multi/script.py +++ b/src/workflows/test_workflows/ingestion/cellranger_multi/script.py @@ -1,4 +1,9 @@ from mudata import read_h5mu +from pathlib import Path +import shutil +import os +import sys +import pytest ##VIASH START par = { @@ -10,9 +15,16 @@ } ##VIASH END -input_mudata = read_h5mu(par["input"]) +def test_run(): + input_mudata = read_h5mu(par["input"]) -assert list(input_mudata.mod.keys()) == ['rna', 'prot', 'vdj_t'] -assert list(input_mudata.uns.keys()) == ['metrics_cellranger'] -expected_metrics = ['Category', 'Library Type', 'Grouped By', 'Group Name', 'Metric Name', 'Metric Value'] -assert input_mudata.uns['metrics_cellranger'].columns.to_list() == expected_metrics + assert list(input_mudata.mod.keys()) == ['rna', 'prot', 'vdj_t'] + assert list(input_mudata.uns.keys()) == ['metrics_cellranger'] + expected_metrics = ['Category', 'Library Type', 'Grouped By', 'Group Name', 'Metric Name', 'Metric Value'] + assert input_mudata.uns['metrics_cellranger'].columns.to_list() == expected_metrics + +if __name__ == "__main__": + HERE_DIR = Path(__file__).resolve().parent + shutil.copyfile(os.path.join(meta['resources_dir'], "openpipelinetestutils", "conftest.py"), + os.path.join(HERE_DIR, "conftest.py")) + sys.exit(pytest.main(["--import-mode=importlib"])) \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_postprocessing/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_postprocessing/config.vsh.yaml index 436326edd2c..d728e8beda1 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_postprocessing/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/cellranger_postprocessing/config.vsh.yaml @@ -41,6 +41,5 @@ platforms: - type: python pypi: - mudata - # - pytest __merge__: /src/base/requirements/viashpy.yaml - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/cellranger_postprocessing/script.py b/src/workflows/test_workflows/ingestion/cellranger_postprocessing/script.py index 1090842712a..11189b26c25 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_postprocessing/script.py +++ b/src/workflows/test_workflows/ingestion/cellranger_postprocessing/script.py @@ -1,5 +1,10 @@ from mudata import read_h5mu from openpipelinetestutils.asserters import assert_annotation_objects_equal +from pathlib import Path +import shutil +import os +import sys +import pytest ##VIASH START par = { @@ -13,19 +18,26 @@ } ##VIASH END -input_mudata = read_h5mu(par["input_og"]) -output_mudata = read_h5mu(par["input"]) +def test_run(): + input_mudata = read_h5mu(par["input_og"]) + output_mudata = read_h5mu(par["input"]) -assert input_mudata.mod.keys() == output_mudata.mod.keys(), "Input and output should have the same modalities." + assert input_mudata.mod.keys() == output_mudata.mod.keys(), "Input and output should have the same modalities." -for modality,input_adata,output_adata in zip(input_mudata.mod.keys(), - input_mudata.mod.values(), - output_mudata.mod.values()): - assert input_adata.n_obs >= output_adata.n_obs, "Output should have less or equal number of observations than input." - assert input_adata.n_vars == output_adata.n_vars, "Output should have the same number of variables as input." - if modality != "rna": - assert_annotation_objects_equal(input_adata, - output_adata) + for modality,input_adata,output_adata in zip(input_mudata.mod.keys(), + input_mudata.mod.values(), + output_mudata.mod.values()): + assert input_adata.n_obs >= output_adata.n_obs, "Output should have less or equal number of observations than input." + assert input_adata.n_vars == output_adata.n_vars, "Output should have the same number of variables as input." + if modality != "rna": + assert_annotation_objects_equal(input_adata, + output_adata) -if par["is_corrected"]: - assert "cellbender_corrected" in output_mudata.mod["rna"].layers \ No newline at end of file + if par["is_corrected"]: + assert "cellbender_corrected" in output_mudata.mod["rna"].layers + +if __name__ == "__main__": + HERE_DIR = Path(__file__).resolve().parent + shutil.copyfile(os.path.join(meta['resources_dir'], "openpipelinetestutils", "conftest.py"), + os.path.join(HERE_DIR, "conftest.py")) + sys.exit(pytest.main(["--import-mode=importlib"])) \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml b/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml index 1d48dd8ae67..c3c27569850 100644 --- a/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/conversion/config.vsh.yaml @@ -16,15 +16,20 @@ functionality: - type: python_script path: script.py - path: /src/utils/setup_logger.py + - path: /src/base/openpipelinetestutils + dest: openpipelinetestutils platforms: - type: docker image: python:3.12-slim setup: + - type: docker + copy: ["openpipelinetestutils /opt/openpipelinetestutils"] - type: apt packages: - procps - type: python + packages: /opt/openpipelinetestutils pypi: - mudata - - pytest + __merge__: /src/base/requirements/viashpy.yaml - type: nextflow \ No newline at end of file diff --git a/src/workflows/test_workflows/ingestion/conversion/script.py b/src/workflows/test_workflows/ingestion/conversion/script.py index d71c088f2fa..64a0e6b9da5 100644 --- a/src/workflows/test_workflows/ingestion/conversion/script.py +++ b/src/workflows/test_workflows/ingestion/conversion/script.py @@ -1,5 +1,9 @@ from mudata import read_h5mu - +import shutil +import os +import sys +from pathlib import Path +import pytest ##VIASH START par = { @@ -11,11 +15,17 @@ } ##VIASH END -input_mudata = read_h5mu(par["input"]) - -assert "rna" in input_mudata.mod.keys() -assert input_mudata.n_obs == 713 -assert input_mudata.mod["rna"].var["feature_types"].unique() == [ - "Gene Expression" -], "Output X should only contain Gene Expression vars." - \ No newline at end of file +def test_run(): + input_mudata = read_h5mu(par["input"]) + assert "rna" in input_mudata.mod.keys() + assert input_mudata.n_obs == 713 + assert input_mudata.mod["rna"].var["feature_types"].unique() == [ + "Gene Expression" + ], "Output X should only contain Gene Expression vars." + + +if __name__ == "__main__": + HERE_DIR = Path(__file__).resolve().parent + shutil.copyfile(os.path.join(meta['resources_dir'], "openpipelinetestutils", "conftest.py"), + os.path.join(HERE_DIR, "conftest.py")) + sys.exit(pytest.main(["--import-mode=importlib"])) \ No newline at end of file From 955f9114b70eb8583d8e4441c4fe0c459ff056f4 Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Tue, 6 Aug 2024 14:20:34 +0200 Subject: [PATCH 8/9] updated PR suggestions --- src/workflows/ingestion/bd_rhapsody/test.nf | 6 +----- src/workflows/ingestion/cellranger_mapping/test.nf | 6 +----- src/workflows/ingestion/cellranger_multi/test.nf | 6 +----- src/workflows/ingestion/conversion/test.nf | 6 +----- .../ingestion/cellranger_multi/config.vsh.yaml | 1 + .../ingestion/cellranger_multi/script.py | 11 ++++++----- 6 files changed, 11 insertions(+), 25 deletions(-) diff --git a/src/workflows/ingestion/bd_rhapsody/test.nf b/src/workflows/ingestion/bd_rhapsody/test.nf index 694d11c80a2..f3554824bd9 100644 --- a/src/workflows/ingestion/bd_rhapsody/test.nf +++ b/src/workflows/ingestion/bd_rhapsody/test.nf @@ -35,11 +35,7 @@ workflow test_wf { } | bd_rhapsody_test.run( - fromState: { output -> - [ - input: output[1].output_h5mu - ] - } + fromState: ["input": "output_h5mu"] ) | toList() diff --git a/src/workflows/ingestion/cellranger_mapping/test.nf b/src/workflows/ingestion/cellranger_mapping/test.nf index 91647099f02..422541297db 100644 --- a/src/workflows/ingestion/cellranger_mapping/test.nf +++ b/src/workflows/ingestion/cellranger_mapping/test.nf @@ -25,11 +25,7 @@ workflow test_wf { } | cellranger_mapping_test.run( - fromState: { output -> - [ - input: output[1].output_h5mu - ] - } + fromState: ["input": "output_h5mu"] ) | toSortedList() diff --git a/src/workflows/ingestion/cellranger_multi/test.nf b/src/workflows/ingestion/cellranger_multi/test.nf index 20acf273915..dff8beb29ce 100644 --- a/src/workflows/ingestion/cellranger_multi/test.nf +++ b/src/workflows/ingestion/cellranger_multi/test.nf @@ -32,11 +32,7 @@ workflow test_wf { } | cellranger_multi_test.run( - fromState: { output -> - [ - input: output[1].output_h5mu - ] - } + fromState: ["input": "output_h5mu"] ) | toSortedList() diff --git a/src/workflows/ingestion/conversion/test.nf b/src/workflows/ingestion/conversion/test.nf index 336e455e775..b31ee6b4a44 100644 --- a/src/workflows/ingestion/conversion/test.nf +++ b/src/workflows/ingestion/conversion/test.nf @@ -45,11 +45,7 @@ workflow test_wf { } | conversion_test.run( - fromState: { output -> - [ - input: output[1].output - ] - } + fromState: ["input": "output"] ) | toSortedList() diff --git a/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml b/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml index 418b63ca701..b86047363cc 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml +++ b/src/workflows/test_workflows/ingestion/cellranger_multi/config.vsh.yaml @@ -10,6 +10,7 @@ functionality: - name: "--input" type: file required: true + multiple: true description: Path to h5mu output. example: foo.final.h5mu resources: diff --git a/src/workflows/test_workflows/ingestion/cellranger_multi/script.py b/src/workflows/test_workflows/ingestion/cellranger_multi/script.py index 4ad0ee71abf..b4aca775929 100644 --- a/src/workflows/test_workflows/ingestion/cellranger_multi/script.py +++ b/src/workflows/test_workflows/ingestion/cellranger_multi/script.py @@ -16,12 +16,13 @@ ##VIASH END def test_run(): - input_mudata = read_h5mu(par["input"]) + for input_path in par["input"]: + input_mudata = read_h5mu(input_path) - assert list(input_mudata.mod.keys()) == ['rna', 'prot', 'vdj_t'] - assert list(input_mudata.uns.keys()) == ['metrics_cellranger'] - expected_metrics = ['Category', 'Library Type', 'Grouped By', 'Group Name', 'Metric Name', 'Metric Value'] - assert input_mudata.uns['metrics_cellranger'].columns.to_list() == expected_metrics + assert list(input_mudata.mod.keys()) == ['rna', 'prot', 'vdj_t'] + assert list(input_mudata.uns.keys()) == ['metrics_cellranger'] + expected_metrics = ['Category', 'Library Type', 'Grouped By', 'Group Name', 'Metric Name', 'Metric Value'] + assert input_mudata.uns['metrics_cellranger'].columns.to_list() == expected_metrics if __name__ == "__main__": HERE_DIR = Path(__file__).resolve().parent From 4e631cbe076cb17a7f01619e82d46ec77d2ef62d Mon Sep 17 00:00:00 2001 From: jakubmajercik Date: Tue, 6 Aug 2024 14:26:29 +0200 Subject: [PATCH 9/9] chengelog updated --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c9035c5717..f77fc80d562 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ * `transform/clr` component: Added the option to set the `axis` along which to apply CLR. Possible to override on workflow level as well (PR #767). +* `workflows/test_workflows/ingestion` components & `workflows/ingestion`: Added standalone components for integration testing of ingestion workflows (PR #801). + ## MINOR CHANGES * Bump scvelo to `0.3.2` (PR #828).