Skip to content

Commit

Permalink
charliecloud: get container environment from file
Browse files Browse the repository at this point in the history 
With charliecloud v0.22, ENV layers of docker containers are saved to
a file located in $IMAGE/ch/environment when the container is pulled.

Hence it is no longer necessary to pass for example PATH variables
explicitly via the env scope, because we can just read it from /ch/environment

Signed-off-by: Patrick Hüther <p.huether@biologie.uni-muenchen.de>
  • Loading branch information
Patrick Hüther committed Feb 19, 2021
1 parent 4fd0f44 commit 8c7f059
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 24 deletions.
14 changes: 1 addition & 13 deletions docs/charliecloud.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ executed on any platform supporting the Charliecloud engine.
Prerequisites
==============

You will need Charliecloud version ``0.21`` or later installed on your execution environment e.g. your computer or a
You will need Charliecloud version ``0.22`` or later installed on your execution environment e.g. your computer or a
distributed cluster, depending on where you want to run your pipeline.

How it works
Expand Down Expand Up @@ -81,18 +81,6 @@ Whereas this would pull from Docker Hub::
process.container = 'nextflow/examples:latest'
charliecloud.enabled = true

Container environment
=====================

A current limitation is that Charliecloud does not inherit environment variables specified in Docker ENV layers.
This means that if a container has software dependencies installed at a location that is not in the host system's
``$PATH``, it needs to be added explicitly using the :ref:`env scope <config-env>`.

For example::

process.container = 'https://quay.io/biocontainers/multiqc:1.3--py35_2'
charliecloud.enabled = true
env.PATH = '/opt/conda/bin:$PATH'

Multiple containers
===================
Expand Down
2 changes: 1 addition & 1 deletion modules/nextflow/src/main/groovy/nextflow/container/CharliecloudBuilder.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
CharliecloudBuilder build(StringBuilder result) {
assert image

result << 'ch-run --no-home --unset-env="*" -w '
result << 'ch-run --no-home --unset-env="*" -w --set-env=' + image + '/ch/environment '

appendEnv(result)

Expand Down
20 changes: 10 additions & 10 deletions modules/nextflow/src/test/groovy/nextflow/container/CharliecloudBuilderTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,59 +38,59 @@ class CharliecloudBuilderTest extends Specification {
expect:
new CharliecloudBuilder('busybox')
.build()
.runCommand == 'ch-run --no-home -w busybox -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w -b "$PWD":"$PWD" -c "$PWD" busybox --'
.runCommand == 'ch-run --no-home -w busybox -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=busybox/ch/environment -b "$PWD":"$PWD" -c "$PWD" busybox --'

new CharliecloudBuilder('busybox')
.params(runOptions: '-j')
.build()
.runCommand == 'ch-run --no-home -w busybox -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w -j -b "$PWD":"$PWD" -c "$PWD" busybox --'
.runCommand == 'ch-run --no-home -w busybox -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=busybox/ch/environment -j -b "$PWD":"$PWD" -c "$PWD" busybox --'

new CharliecloudBuilder('busybox')
.params(temp: '/foo')
.build()
.runCommand == 'ch-run --no-home -w busybox -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w -b /foo:/tmp -b "$PWD":"$PWD" -c "$PWD" busybox --'
.runCommand == 'ch-run --no-home -w busybox -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=busybox/ch/environment -b /foo:/tmp -b "$PWD":"$PWD" -c "$PWD" busybox --'

new CharliecloudBuilder('busybox')
.addEnv('X=1')
.addEnv(ALPHA:'aaa', BETA: 'bbb')
.build()
.runCommand == 'ch-run --no-home -w busybox -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=<( echo "X=1" ) --set-env=<( echo "ALPHA="aaa"" ) --set-env=<( echo "BETA="bbb"" ) -b "$PWD":"$PWD" -c "$PWD" busybox --'
.runCommand == 'ch-run --no-home -w busybox -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=busybox/ch/environment --set-env=<( echo "X=1" ) --set-env=<( echo "ALPHA="aaa"" ) --set-env=<( echo "BETA="bbb"" ) -b "$PWD":"$PWD" -c "$PWD" busybox --'

new CharliecloudBuilder('ubuntu')
.addMount(path1)
.build()
.runCommand == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p /foo/data/file1 "$PWD"";ch-run --no-home --unset-env="*" -w -b /foo/data/file1:/foo/data/file1 -b "$PWD":"$PWD" -c "$PWD" ubuntu --'
.runCommand == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p /foo/data/file1 "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=ubuntu/ch/environment -b /foo/data/file1:/foo/data/file1 -b "$PWD":"$PWD" -c "$PWD" ubuntu --'

new CharliecloudBuilder('ubuntu')
.addMount(path1)
.addMount(path2)
.build()
.runCommand == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p /foo/data/file1 /bar/data/file2 "$PWD"";ch-run --no-home --unset-env="*" -w -b /foo/data/file1:/foo/data/file1 -b /bar/data/file2:/bar/data/file2 -b "$PWD":"$PWD" -c "$PWD" ubuntu --'
.runCommand == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p /foo/data/file1 /bar/data/file2 "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=ubuntu/ch/environment -b /foo/data/file1:/foo/data/file1 -b /bar/data/file2:/bar/data/file2 -b "$PWD":"$PWD" -c "$PWD" ubuntu --'

new CharliecloudBuilder('ubuntu')
.addMount(path1)
.addMount(path2)
.params(readOnlyInputs: true)
.build()
.runCommand == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p /foo/data/file1 /bar/data/file2 "$PWD"";ch-run --no-home --unset-env="*" -w -b /foo/data/file1:/foo/data/file1 -b /bar/data/file2:/bar/data/file2 -b "$PWD":"$PWD" -c "$PWD" ubuntu --'
.runCommand == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p /foo/data/file1 /bar/data/file2 "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=ubuntu/ch/environment -b /foo/data/file1:/foo/data/file1 -b /bar/data/file2:/bar/data/file2 -b "$PWD":"$PWD" -c "$PWD" ubuntu --'
}

def 'should get run command' () {

when:
def cmd = new CharliecloudBuilder('ubuntu').build().getRunCommand()
then:
cmd == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w -b "$PWD":"$PWD" -c "$PWD" ubuntu --'
cmd == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=ubuntu/ch/environment -b "$PWD":"$PWD" -c "$PWD" ubuntu --'

when:
cmd = new CharliecloudBuilder('ubuntu').build().getRunCommand('bwa --this --that file.fastq')
then:
cmd == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w -b "$PWD":"$PWD" -c "$PWD" ubuntu -- bwa --this --that file.fastq'
cmd == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=ubuntu/ch/environment -b "$PWD":"$PWD" -c "$PWD" ubuntu -- bwa --this --that file.fastq'

when:
cmd = new CharliecloudBuilder('ubuntu').params(entry:'/bin/sh').build().getRunCommand('bwa --this --that file.fastq')
then:
cmd == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w -b "$PWD":"$PWD" -c "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'
cmd == 'ch-run --no-home -w ubuntu -- bash -c "mkdir -p "$PWD"";ch-run --no-home --unset-env="*" -w --set-env=ubuntu/ch/environment -b "$PWD":"$PWD" -c "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'
}

@Unroll
Expand Down

0 comments on commit 8c7f059

Please sign in to comment.