Update Viash to 0.9.0-RC7

.github/workflows/integration-test.yml

@@ -81,7 +81,7 @@ jobs:
     - name: Remove unused test resources to save space
       shell: bash
       run: |
-        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" -c 'del(.functionality.dependencies)' | jq -r -c '(.info.config | capture("^(?<dir>.*\/)").dir) as $dir | .functionality.test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
+        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" -c 'del(.dependencies)' | jq -r -c '(.build_info.config | capture("^(?<dir>.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
         to_not_remove=()
         for resource in "${resources[@]}"; do
           if [[ $resource == *"resources_test"* ]]; then
@@ -104,9 +104,8 @@ jobs:
       uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: |
-          .functionality.version := 'integration_build'
-          .platforms[.type == 'docker'].target_tag := 'integration_build'
-        platform: 'docker|nextflow'
+          .version := 'integration_build'
+          .engines[.type == 'docker'].target_tag := 'integration_build'
         src: ${{ matrix.component.dir }}
         query: "^test_workflows"
         setup: build

.github/workflows/main-build.yml

@@ -141,14 +141,14 @@ jobs:
     - uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: |
-          .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
-          .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}'
+          .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
+          .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}'
         parallel: true
         query: ^(?!workflows|test_workflows)
 
     - uses: viash-io/viash-actions/ns-build@v6
       with:
-        config_mod: .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
+        config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
         parallel: true
         query: ^workflows
 
@@ -178,39 +178,39 @@ jobs:
       id: ns_list
       uses: viash-io/viash-actions/ns-list@v6
       with:
-        platform: docker
+        engine: docker
         src: src
         format: json
-        query_namespace: ^(?!workflows|test_workflows)
+        query_namespace: ^(?!workflows)|(workflows/test_workflows)
 
     - name: "List workflows"
       id: ns_list_workflows
       uses: viash-io/viash-actions/ns-list@v6
       with:
         src: src
         format: json
-        query_namespace: ^workflows
+        query_namespace: ^workflows/(?!test_workflows)
 
     - name: "Parse JSON output from 'viash ns list' as input for matrix."
       id: set_matrix
       run: |
         echo "components=$(jq -c '[ .[] | 
           { 
-            "name": .functionality.name,
-            "namespace_separator": .platforms | map(select(.type == "docker"))[0].namespace_separator,
-            "namespace": .functionality.namespace,
-            "config": .info.config,
-            "dir": .info.config | capture("^(?<dir>.*\/)").dir
+            "name": .name,
+            "namespace_separator": .engines | map(select(.type == "docker"))[0].namespace_separator,
+            "namespace": .namespace,
+            "config": .build_info.config,
+            "dir": .build_info.config | capture("^(?<dir>.*\/)").dir
           }
         ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT
 
-        echo "workflows=$(jq -c '[ .[] | . as $config | (.functionality.test_resources // [])[] | select(.type == "nextflow_script", .entrypoint) |
+        echo "workflows=$(jq -c '[ .[] | . as $config | (.test_resources // [])[] | select(.type == "nextflow_script", .entrypoint) |
         { 
-          "name": $config.functionality.name,
-          "namespace": $config.functionality.namespace,
-          "main_script": (($config.info.config | capture("^(?<dir>.*\/)").dir) + "/" + .path),
+          "name": $config.name,
+          "namespace": $config.namespace,
+          "main_script": (($config.build_info.config | capture("^(?<dir>.*\/)").dir) + "/" + .path),
           "entry": .entrypoint,
-          "config": $config.info.config
+          "config": $config.build_info.config
         }
         ] | unique' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT
 
@@ -239,9 +239,9 @@ jobs:
       uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: |
-          .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
-          .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}'
-        platform: docker
+          .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
+          .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}'
+        engine: docker
         src: ${{ matrix.component.dir }}
         setup: build
 
@@ -255,7 +255,7 @@ jobs:
     - name: Push container
       uses: viash-io/viash-actions/ns-build@v6
       with:
-        config_mod: .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
-        platform: docker
+        config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
+        engine: docker
         src: ${{ matrix.component.dir }}
         setup: push

.github/workflows/release-build.yml

@@ -74,7 +74,7 @@ jobs:
     - name: Remove unused test resources to save space
       shell: bash
       run: |
-        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" -c 'del(.functionality.dependencies)' | jq -r -c '(.info.config | capture("^(?<dir>.*\/)").dir) as $dir | .functionality.test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
+        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" -c 'del(.dependencies)' | jq -r -c '(.build_info.config | capture("^(?<dir>.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
         to_not_remove=()
         for resource in "${resources[@]}"; do
           if [[ $resource == *"resources_test"* ]]; then
@@ -157,7 +157,7 @@ jobs:
       run: |
         viash test \
           "${{ matrix.component.config }}" \
-          --config_mod ".platforms[.type == 'docker'].image := 'ghcr.io/openpipelines-bio/${{ matrix.component.namespace }}${{matrix.component.namespace_separator}}${{ matrix.component.name }}:${{ github.event.inputs.version_tag }}'" \
-          --config_mod ".platforms[.type == 'docker'].setup := []" \
+          --config_mod ".engines[.type == 'docker'].image := 'ghcr.io/openpipelines-bio/${{ matrix.component.namespace }}${{matrix.component.namespace_separator}}${{ matrix.component.name }}:${{ github.event.inputs.version_tag }}'" \
+          --config_mod ".engines[.type == 'docker'].setup := []" \
           --cpus 4 \
           --memory "12gb"

.github/workflows/viash-test.yml

@@ -33,7 +33,7 @@ jobs:
     - name: Check if all config can be parsed if there is no unicode support
       run: |
         LANG=C viash ns list > /dev/null
-        JAVA_TOOL_OPTIONS=-Dfile.encoding=ascii viash ns build --parallel --setup meh > /dev/null
+        JAVA_TOOL_OPTIONS=-Dfile.encoding=ascii viash ns build --parallel > /dev/null
 
     - uses: viash-io/viash-actions/project/sync-and-cache-s3@v6
       id: cache
@@ -45,7 +45,7 @@ jobs:
     - id: ns_list
       uses: viash-io/viash-actions/ns-list@v6
       with:
-        platform: docker
+        engine: docker
         format: json
         query_namespace: ^(?!workflows|test_workflows)
 
@@ -58,13 +58,13 @@ jobs:
       run: |
         echo "matrix=$(jq -c '[ .[] | 
           { 
-            "name": .functionality.name,
-            "namespace_separator": .platforms | map(select(.type == "docker"))[0].namespace_separator,
-            "namespace": .functionality.namespace,
-            "config": .info.config,
-            "dir": .info.config | capture("^(?<dir>.*\/)").dir
+            "name": .name,
+            "namespace_separator": .engines | map(select(.type == "docker"))[0].namespace_separator,
+            "namespace": .namespace,
+            "config": .build_info.config,
+            "dir": .build_info.config | capture("^(?<dir>.*\/)").dir
           }
-        ]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT
+        ] | unique' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT
 
   # phase 2
   viash_test:
@@ -84,7 +84,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - uses: viash-io/viash-actions/setup@v6
-
+    - uses: viash-io/viash-actions/project/update-docker-engine@v6
     # use cache
     - name: Cache resources data
       id: restore_cache
@@ -99,7 +99,7 @@ jobs:
       shell: bash
       if: steps.restore_cache.outputs.cache-hit != 'true'
       run: |
-        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" | jq -r -c '(.info.config | capture("^(?<dir>.*\/)").dir) as $dir | .functionality.test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
+        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" | jq -r -c '(.build_info.config | capture("^(?<dir>.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
         for resource in "${resources[@]}"; do
           if [[ $resource == *"resources_test"* ]]; then
             relative_path=${resource#*resources_test/}
@@ -123,7 +123,7 @@ jobs:
       if: steps.restore_cache.outputs.cache-hit == 'true'
       shell: bash
       run: |
-        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" | jq -r -c '(.info.config | capture("^(?<dir>.*\/)").dir) as $dir | .functionality.test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
+        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" | jq -r -c '(.build_info.config | capture("^(?<dir>.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
         to_not_remove=()
         for resource in "${resources[@]}"; do
           if [[ $resource == *"resources_test"* ]]; then
@@ -156,5 +156,7 @@ jobs:
         viash test \
           "${{ matrix.component.config }}" \
           --cpus 4 \
-          --memory "14gb"
+          --memory "14gb" \
+          --engine docker \
+          --runner executable
 

CHANGELOG.md

@@ -12,6 +12,12 @@
 
 * `lianapy`: bumped version to `1.3.0` (PR #827 and PR #862). Additionally, `groupby` is now a required argument.
 
+* `concat`: this component was deprecated and has now been removed, use `concatenate_h5mu` instead (PR #796).
+
+* The `workflows` folder in the root of the project no longer contains symbolic links to the build workflows in `target`.
+  Using any workflows that was previously linked in this directory will now result in an error which will indicate
+  the location of the workflow to be used instead (PR #796).
+
 * `XGBoost`: bump version to `2.0.3` (PR #646).
 
 * Several components: update anndata to `0.10.8` and mudata to `0.2.3` (PR #645). 
@@ -32,6 +38,8 @@
 
 * `reference/cellranger_mkgtf`: Rename `reference/mkgtf` to `reference/cellranger_mkgtf` (PR #846).
 
+* `interactive/run_cellxgene` and `interactive/run_cirrocumulus` were removed (PR #796).
+
 * `reference/build_cellranger_arc_reference`: a default value of "output" is now specified for the argument `--genome`, inline with `reference/build_cellranger_reference` component. Additionally, providing a value for `--organism` is no longer required and its default value of `Homo Sapiens` has been removed (PR #864).
 
 ## NEW FUNCTIONALITY

_viash.yaml

@@ -1,19 +1,23 @@
-viash_version: 0.8.6
+# since 0.9.0-RC7 is not released yet, we need to use documentation by code
+# https://github.com/viash-io/viash/blob/develop/src/main/scala/io/viash/packageConfig/PackageConfig.scala
+viash_version: 0.9.0-RC7
+
+version: dev
 
 source: src
 target: target
 
+organization: "openpipelines-bio"
+
+links:
+  repository: https://github.com/openpipelines-bio/openpipeline
+  docker_registry: ghcr.io
+  homepage: https://openpipelines.bio
+  documentation: https://openpipelines.bio/fundamentals
+  issue_tracker: https://github.com/openpipelines-bio/openpipeline/issues
+
 config_mods: |
-  .functionality.version := 'dev'
-  .functionality.requirements.commands := ['ps']
-  .functionality.arguments[.multiple == true].multiple_sep := ";"
-  .functionality.argument_groups[true].arguments[.multiple == true].multiple_sep := ";"
-  .functionality.test_resources += {path: 'src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}
-  .platforms[.type == 'docker'].namespace_separator := '/'
-  .platforms[.type == 'docker'].target_registry := 'ghcr.io'
-  .platforms[.type == 'docker'].target_organization := 'openpipelines-bio'
-  .platforms[.type == 'docker'].target_image_source := 'https://github.com/openpipelines-bio/openpipeline'
-  .platforms[.type == 'nextflow'].directives.tag := '$id'
-  .platforms[.type == 'nextflow'].auto.simplifyOutput := false
-  .platforms[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")'
-  .functionality.resources += {path: './src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}
+  .test_resources += {path: '/src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}
+  .resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}
+  .runners[.type == 'nextflow'].directives.tag := '$id'
+  .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")'