Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor Sentieon #5945

Merged
merged 12 commits into from
Jul 19, 2024
23 changes: 4 additions & 19 deletions modules/nf-core/sentieon/applyvarcal/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ process SENTIEON_APPLYVARCAL {
label 'process_low'
label 'sentieon'

secret 'SENTIEON_LICENSE_BASE64'

conda "${moduleDir}/environment.yml"
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'oras://community.wave.seqera.io/library/sentieon:202308.02--ffce1b7074ce9924' :
Expand All @@ -26,24 +24,11 @@ process SENTIEON_APPLYVARCAL {
script:
def args = task.ext.args ?: ''
def prefix = task.ext.prefix ?: "${meta.id}"
def sentieon_auth_mech_base64 = task.ext.sentieon_auth_mech_base64 ?: ''
def sentieon_auth_data_base64 = task.ext.sentieon_auth_data_base64 ?: ''
def sentieonLicense = secrets.SENTIEON_LICENSE_BASE64 ?
"export SENTIEON_LICENSE=\$(mktemp);echo -e \"${secrets.SENTIEON_LICENSE_BASE64}\" | base64 -d > \$SENTIEON_LICENSE; " :
""
"""
if [ "\${#SENTIEON_LICENSE_BASE64}" -lt "1500" ]; then # If the string SENTIEON_LICENSE_BASE64 is short, then it is an encrypted url.
export SENTIEON_LICENSE=\$(echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d)
else # Localhost license file
# The license file is stored as a nextflow variable like, for instance, this:
# nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
export SENTIEON_LICENSE=\$(mktemp)
echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d > \$SENTIEON_LICENSE
fi

if [ ${sentieon_auth_mech_base64} ] && [ ${sentieon_auth_data_base64} ]; then
# If sentieon_auth_mech_base64 and sentieon_auth_data_base64 are non-empty strings, then Sentieon is mostly likely being run with some test-license.
export SENTIEON_AUTH_MECH=\$(echo -n "${sentieon_auth_mech_base64}" | base64 -d)
export SENTIEON_AUTH_DATA=\$(echo -n "${sentieon_auth_data_base64}" | base64 -d)
echo "Decoded and exported Sentieon test-license system environment variables"
fi
$sentieonLicense

sentieon driver -r ${fasta} --algo ApplyVarCal \\
-v $vcf \\
Expand Down
28 changes: 5 additions & 23 deletions modules/nf-core/sentieon/bwamem/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,6 @@ process SENTIEON_BWAMEM {
label 'process_high'
label 'sentieon'

// NOTE this is not required for the process to run, but it is required for the process to be run in GitHub actions or nf-core MegaTests
// The rest of the secrets aren't really "secrets" because they're not sensitive information for users
// secret SENTIEON_AUTH_DATA

conda "${moduleDir}/environment.yml"
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'oras://community.wave.seqera.io/library/sentieon:202308.02--ffce1b7074ce9924' :
Expand All @@ -28,26 +24,12 @@ process SENTIEON_BWAMEM {
script:
def args = task.ext.args ?: ''
def prefix = task.ext.prefix ?: "${meta.id}"
def sentieon_auth_mech_base64 = task.ext.sentieon_auth_mech_base64 ?: ''
def sentieon_auth_data_base64 = task.ext.sentieon_auth_data_base64 ?: ''
"""
if [ "\${SENTIEON_LICSRVR_IP}" ]; then
# NOTE: This is how pipeline users will use Sentieon in real world
echo "Using a Sentieon License Server"
export SENTIEON_LICENSE="\${SENTIEON_LICSRVR_IP}"
else
# NOTE: This is how pipeline users will test out Sentieon
echo "Localhost license file"
# The license file is stored as a nextflow variable like, for instance, this:
# nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
export SENTIEON_LICENSE=\$(mktemp)
echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d > \$SENTIEON_LICENSE
fi
def sentieonLicense = secrets.SENTIEON_LICENSE_BASE64 ?
"export SENTIEON_LICENSE=\$(mktemp);echo -e \"${secrets.SENTIEON_LICENSE_BASE64}\" | base64 -d > \$SENTIEON_LICENSE; " :
""

if [ "\${SENTIEON_AUTH_MECH}" ] && [ "\${SENTIEON_AUTH_DATA}" ]; then
# NOTE: This should only happen in GitHub Actions or nf-core/megatests
echo "If sentieon_auth_mech and sentieon_auth_data are non-empty strings, then Sentieon is mostly likely being run with some test-license."
fi
"""
$sentieonLicense

INDEX=`find -L ./ -name "*.amb" | sed 's/.amb//'`

Expand Down
4 changes: 2 additions & 2 deletions modules/nf-core/sentieon/bwamem/tests/main.nf.test.snap
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
"id": "test",
"single_end": true
},
"test.bam:md5,c95a3243fcd7df16481f67ad0466696b",
"test.bam.bai:md5,6fc1dff58fab0491ecfa48f016041a18"
"test.bam:md5,58bc2194d084cf8462a3a7f6114c303a",
"test.bam.bai:md5,4fd9b026ff2306cc292cd695e11d5628"
]
],
[
Expand Down
6 changes: 5 additions & 1 deletion modules/nf-core/sentieon/bwamem/tests/nextflow.config
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
env {
SENTIEON_LICSRVR_IP = "$SENTIEON_LICSRVR_IP"
// NOTE This is how nf-core/sarek users will use Sentieon in real world use
SENTIEON_LICENSE = "$SENTIEON_LICSRVR_IP"
// NOTE This should only happen in GitHub actions or nf-core MegaTests
SENTIEON_AUTH_MECH = "$SENTIEON_AUTH_MECH"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might throw error if SENTIEON_AUTH_MECH isn't populated?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we can write some tests for this 🤔 ? I guess different configs... Maybe profiles would be better?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's on the user to configure their env, but that's where the docs comes in (and maybe a warning!). Not certain about profiles, would want to see a working version first and test the UX.

SENTIEON_AUTH_DATA = secrets.SENTIEON_AUTH_DATA
// NOTE This is how nf-core/sarek users will test out Sentieon in Sarek with a license file
// nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
}

process {
Expand Down
23 changes: 4 additions & 19 deletions modules/nf-core/sentieon/datametrics/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ process SENTIEON_DATAMETRICS {
label 'process_medium'
label 'sentieon'

secret 'SENTIEON_LICENSE_BASE64'

conda "${moduleDir}/environment.yml"
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'oras://community.wave.seqera.io/library/sentieon:202308.02--ffce1b7074ce9924' :
Expand All @@ -31,24 +29,11 @@ process SENTIEON_DATAMETRICS {
def args = task.ext.args ?: ''
def prefix = task.ext.prefix ?: "${meta.id}"
def input = bam.sort().collect{"-i $it"}.join(' ')
def sentieon_auth_mech_base64 = task.ext.sentieon_auth_mech_base64 ?: ''
def sentieon_auth_data_base64 = task.ext.sentieon_auth_data_base64 ?: ''
def sentieonLicense = secrets.SENTIEON_LICENSE_BASE64 ?
"export SENTIEON_LICENSE=\$(mktemp);echo -e \"${secrets.SENTIEON_LICENSE_BASE64}\" | base64 -d > \$SENTIEON_LICENSE; " :
""
"""
if [ "\${#SENTIEON_LICENSE_BASE64}" -lt "1500" ]; then # If the string SENTIEON_LICENSE_BASE64 is short, then it is an encrypted url.
export SENTIEON_LICENSE=\$(echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d)
else # Localhost license file
# The license file is stored as a nextflow variable like, for instance, this:
# nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
export SENTIEON_LICENSE=\$(mktemp)
echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d > \$SENTIEON_LICENSE
fi

if [ ${sentieon_auth_mech_base64} ] && [ ${sentieon_auth_data_base64} ]; then
# If sentieon_auth_mech_base64 and sentieon_auth_data_base64 are non-empty strings, then Sentieon is mostly likely being run with some test-license.
export SENTIEON_AUTH_MECH=\$(echo -n "${sentieon_auth_mech_base64}" | base64 -d)
export SENTIEON_AUTH_DATA=\$(echo -n "${sentieon_auth_data_base64}" | base64 -d)
echo "Decoded and exported Sentieon test-license system environment variables"
fi
$sentieonLicense

sentieon \\
driver \\
Expand Down
23 changes: 4 additions & 19 deletions modules/nf-core/sentieon/dedup/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ process SENTIEON_DEDUP {
label 'process_medium'
label 'sentieon'

secret 'SENTIEON_LICENSE_BASE64'

conda "${moduleDir}/environment.yml"
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'oras://community.wave.seqera.io/library/sentieon:202308.02--ffce1b7074ce9924' :
Expand Down Expand Up @@ -36,25 +34,12 @@ process SENTIEON_DEDUP {
def prefix = task.ext.prefix ?: "${meta.id}"
def suffix = task.ext.suffix ?: ".cram" // The suffix should be either ".cram" or ".bam".
def metrics = task.ext.metrics ?: "${prefix}${suffix}.metrics"
def sentieon_auth_mech_base64 = task.ext.sentieon_auth_mech_base64 ?: ''
def sentieon_auth_data_base64 = task.ext.sentieon_auth_data_base64 ?: ''
def input_list = bam.collect{"-i $it"}.join(' ')
def sentieonLicense = secrets.SENTIEON_LICENSE_BASE64 ?
"export SENTIEON_LICENSE=\$(mktemp);echo -e \"${secrets.SENTIEON_LICENSE_BASE64}\" | base64 -d > \$SENTIEON_LICENSE; " :
""
"""
if [ "\${#SENTIEON_LICENSE_BASE64}" -lt "1500" ]; then # If the string SENTIEON_LICENSE_BASE64 is short, then it is an encrypted url.
export SENTIEON_LICENSE=\$(echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d)
else # Localhost license file
# The license file is stored as a nextflow variable like, for instance, this:
# nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
export SENTIEON_LICENSE=\$(mktemp)
echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d > \$SENTIEON_LICENSE
fi

if [ ${sentieon_auth_mech_base64} ] && [ ${sentieon_auth_data_base64} ]; then
# If sentieon_auth_mech_base64 and sentieon_auth_data_base64 are non-empty strings, then Sentieon is mostly likely being run with some test-license.
export SENTIEON_AUTH_MECH=\$(echo -n "${sentieon_auth_mech_base64}" | base64 -d)
export SENTIEON_AUTH_DATA=\$(echo -n "${sentieon_auth_data_base64}" | base64 -d)
echo "Decoded and exported Sentieon test-license system environment variables"
fi
$sentieonLicense

sentieon driver $args $input_list -r ${fasta} --algo LocusCollector $args2 --fun score_info ${prefix}.score
sentieon driver $args3 -t $task.cpus $input_list -r ${fasta} --algo Dedup $args4 --score_info ${prefix}.score --metrics ${metrics} ${prefix}${suffix}
Expand Down
23 changes: 4 additions & 19 deletions modules/nf-core/sentieon/dnamodelapply/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ process SENTIEON_DNAMODELAPPLY {
label 'process_high'
label 'sentieon'

secret 'SENTIEON_LICENSE_BASE64'

conda "${moduleDir}/environment.yml"
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'oras://community.wave.seqera.io/library/sentieon:202308.02--ffce1b7074ce9924' :
Expand All @@ -27,24 +25,11 @@ process SENTIEON_DNAMODELAPPLY {
script:
def args = task.ext.args ?: ''
def prefix = task.ext.prefix ?: "${meta.id}"
def sentieon_auth_mech_base64 = task.ext.sentieon_auth_mech_base64 ?: ''
def sentieon_auth_data_base64 = task.ext.sentieon_auth_data_base64 ?: ''
def sentieonLicense = secrets.SENTIEON_LICENSE_BASE64 ?
"export SENTIEON_LICENSE=\$(mktemp);echo -e \"${secrets.SENTIEON_LICENSE_BASE64}\" | base64 -d > \$SENTIEON_LICENSE; " :
""
"""
if [ "\${#SENTIEON_LICENSE_BASE64}" -lt "1500" ]; then # If the string SENTIEON_LICENSE_BASE64 is short, then it is an encrypted url.
export SENTIEON_LICENSE=\$(echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d)
else # Localhost license file
# The license file is stored as a nextflow variable like, for instance, this:
# nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
export SENTIEON_LICENSE=\$(mktemp)
echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d > \$SENTIEON_LICENSE
fi

if [ ${sentieon_auth_mech_base64} ] && [ ${sentieon_auth_data_base64} ]; then
# If sentieon_auth_mech_base64 and sentieon_auth_data_base64 are non-empty strings, then Sentieon is mostly likely being run with some test-license.
export SENTIEON_AUTH_MECH=\$(echo -n "${sentieon_auth_mech_base64}" | base64 -d)
export SENTIEON_AUTH_DATA=\$(echo -n "${sentieon_auth_data_base64}" | base64 -d)
echo "Decoded and exported Sentieon test-license system environment variables"
fi
$sentieonLicense

sentieon driver \\
-t $task.cpus \\
Expand Down
23 changes: 4 additions & 19 deletions modules/nf-core/sentieon/dnascope/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ process SENTIEON_DNASCOPE {
label 'process_high'
label 'sentieon'

secret 'SENTIEON_LICENSE_BASE64'

conda "${moduleDir}/environment.yml"
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'oras://community.wave.seqera.io/library/sentieon:202308.02--ffce1b7074ce9924' :
Expand Down Expand Up @@ -40,8 +38,6 @@ process SENTIEON_DNASCOPE {
def model_cmd = ml_model ? " --model ${ml_model}" : ''
def pcr_indel_model_cmd = pcr_indel_model ? " --pcr_indel_model ${pcr_indel_model}" : ''
def prefix = task.ext.prefix ?: "${meta.id}"
def sentieon_auth_mech_base64 = task.ext.sentieon_auth_mech_base64 ?: ''
def sentieon_auth_data_base64 = task.ext.sentieon_auth_data_base64 ?: ''
def vcf_cmd = ""
def gvcf_cmd = ""
def base_cmd = '--algo DNAscope ' + dbsnp_cmd + ' '
Expand All @@ -54,22 +50,11 @@ process SENTIEON_DNASCOPE {
gvcf_cmd = base_cmd + args3 + ' ' + model_cmd + pcr_indel_model_cmd + ' --emit_mode gvcf ' + prefix + '.g.vcf.gz'
}

def sentieonLicense = secrets.SENTIEON_LICENSE_BASE64 ?
"export SENTIEON_LICENSE=\$(mktemp);echo -e \"${secrets.SENTIEON_LICENSE_BASE64}\" | base64 -d > \$SENTIEON_LICENSE; " :
""
"""
if [ "\${#SENTIEON_LICENSE_BASE64}" -lt "1500" ]; then # If the string SENTIEON_LICENSE_BASE64 is short, then it is an encrypted url.
export SENTIEON_LICENSE=\$(echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d)
else # Localhost license file
# The license file is stored as a nextflow variable like, for instance, this:
# nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
export SENTIEON_LICENSE=\$(mktemp)
echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d > \$SENTIEON_LICENSE
fi

if [ ${sentieon_auth_mech_base64} ] && [ ${sentieon_auth_data_base64} ]; then
# If sentieon_auth_mech_base64 and sentieon_auth_data_base64 are non-empty strings, then Sentieon is mostly likely being run with some test-license.
export SENTIEON_AUTH_MECH=\$(echo -n "${sentieon_auth_mech_base64}" | base64 -d)
export SENTIEON_AUTH_DATA=\$(echo -n "${sentieon_auth_data_base64}" | base64 -d)
echo "Decoded and exported Sentieon test-license system environment variables"
fi
$sentieonLicense

sentieon driver $args -r $fasta -t $task.cpus -i $bam $interval $vcf_cmd $gvcf_cmd

Expand Down
21 changes: 4 additions & 17 deletions modules/nf-core/sentieon/gvcftyper/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -27,26 +27,13 @@ process SENTIEON_GVCFTYPER {

script:
def prefix = task.ext.prefix ?: "${meta.id}"
def sentieon_auth_mech_base64 = task.ext.sentieon_auth_mech_base64 ?: ''
def sentieon_auth_data_base64 = task.ext.sentieon_auth_data_base64 ?: ''
def gvcfs_input = '-v ' + gvcfs.join(' -v ')
def dbsnp_cmd = dbsnp ? "--dbsnp $dbsnp" : ""
def sentieonLicense = secrets.SENTIEON_LICENSE_BASE64 ?
"export SENTIEON_LICENSE=\$(mktemp);echo -e \"${secrets.SENTIEON_LICENSE_BASE64}\" | base64 -d > \$SENTIEON_LICENSE; " :
""
"""
if [ "\${#SENTIEON_LICENSE_BASE64}" -lt "1500" ]; then # If the string SENTIEON_LICENSE_BASE64 is short, then it is an encrypted url.
export SENTIEON_LICENSE=\$(echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d)
else # Localhost license file
# The license file is stored as a nextflow variable like, for instance, this:
# nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
export SENTIEON_LICENSE=\$(mktemp)
echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d > \$SENTIEON_LICENSE
fi

if [ ${sentieon_auth_mech_base64} ] && [ ${sentieon_auth_data_base64} ]; then
# If sentieon_auth_mech_base64 and sentieon_auth_data_base64 are non-empty strings, then Sentieon is mostly likely being run with some test-license.
export SENTIEON_AUTH_MECH=\$(echo -n "${sentieon_auth_mech_base64}" | base64 -d)
export SENTIEON_AUTH_DATA=\$(echo -n "${sentieon_auth_data_base64}" | base64 -d)
echo "Decoded and exported Sentieon test-license system environment variables"
fi
$sentieonLicense

sentieon driver -r ${fasta} --algo GVCFtyper ${gvcfs_input} ${dbsnp_cmd} ${prefix}.vcf.gz

Expand Down
23 changes: 4 additions & 19 deletions modules/nf-core/sentieon/haplotyper/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ process SENTIEON_HAPLOTYPER {
label 'process_medium'
label 'sentieon'

secret 'SENTIEON_LICENSE_BASE64'

conda "${moduleDir}/environment.yml"
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'oras://community.wave.seqera.io/library/sentieon:202308.02--ffce1b7074ce9924' :
Expand Down Expand Up @@ -36,8 +34,6 @@ process SENTIEON_HAPLOTYPER {
def prefix = task.ext.prefix ?: "${meta.id}"
def dbsnp_command = dbsnp ? "-d $dbsnp " : ""
def interval_command = intervals ? "--interval $intervals" : ""
def sentieon_auth_mech_base64 = task.ext.sentieon_auth_mech_base64 ?: ''
def sentieon_auth_data_base64 = task.ext.sentieon_auth_data_base64 ?: ''
def vcf_cmd = ""
def gvcf_cmd = ""
def base_cmd = '--algo Haplotyper ' + dbsnp_command
Expand All @@ -50,22 +46,11 @@ process SENTIEON_HAPLOTYPER {
gvcf_cmd = base_cmd + args3 + ' --emit_mode gvcf ' + prefix + '.g.vcf.gz'
}

def sentieonLicense = secrets.SENTIEON_LICENSE_BASE64 ?
"export SENTIEON_LICENSE=\$(mktemp);echo -e \"${secrets.SENTIEON_LICENSE_BASE64}\" | base64 -d > \$SENTIEON_LICENSE; " :
""
"""
if [ "\${#SENTIEON_LICENSE_BASE64}" -lt "1500" ]; then # If the string SENTIEON_LICENSE_BASE64 is short, then it is an encrypted url.
export SENTIEON_LICENSE=\$(echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d)
else # Localhost license file
# The license file is stored as a nextflow variable like, for instance, this:
# nextflow secrets set SENTIEON_LICENSE_BASE64 \$(cat <sentieon_license_file.lic> | base64 -w 0)
export SENTIEON_LICENSE=\$(mktemp)
echo -e "\$SENTIEON_LICENSE_BASE64" | base64 -d > \$SENTIEON_LICENSE
fi

if [ ${sentieon_auth_mech_base64} ] && [ ${sentieon_auth_data_base64} ]; then
# If sentieon_auth_mech_base64 and sentieon_auth_data_base64 are non-empty strings, then Sentieon is mostly likely being run with some test-license.
export SENTIEON_AUTH_MECH=\$(echo -n "${sentieon_auth_mech_base64}" | base64 -d)
export SENTIEON_AUTH_DATA=\$(echo -n "${sentieon_auth_data_base64}" | base64 -d)
echo "Decoded and exported Sentieon test-license system environment variables"
fi
$sentieonLicense

sentieon driver $args -r $fasta -t $task.cpus -i $input $interval_command $vcf_cmd $gvcf_cmd

Expand Down
Loading
Loading