+ read_utils.py --version + tee VERSION ++ /opt/viral-ngs/source/docker/calc_mem.py mb 90 + mem_in_mb=464108 + cp /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/SARS_CoV_2_Wuhan_ref_genome.fasta assembly.fasta + grep -v '^>' assembly.fasta + wc -c + tr -d '\n' + tee assembly_length ++ cat assembly_length + '[' 29903 '!=' 0 ']' + '[' minimap2 == novoalign ']' + read_utils.py index_fasta_picard assembly.fasta --loglevel=DEBUG 2024-02-20 01:09:07,884 - cmd:197:main_argparse - INFO - software version: v2.1.33, python version: 3.7.9 | packaged by conda-forge | (default, Dec 9 2020, 21:08:20) [GCC 9.3.0] 2024-02-20 01:09:07,884 - cmd:199:main_argparse - INFO - command: /opt/viral-ngs/source/read_utils.py index_fasta_picard inFasta=assembly.fasta JVMmemory=512m picardOptions=[] loglevel=DEBUG tmp_dir=/tmp tmp_dirKeep=False 2024-02-20 01:09:07,885 - cmd:213:main_argparse - DEBUG - using tempDir: /tmp/tmp-read_utils-index_fasta_picard-krj5onsn 2024-02-20 01:09:07,903 - picard:50:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/picard -Xmx512m -Djava.io.tmpdir=/tmp/tmp-read_utils-index_fasta_picard-krj5onsn CreateSequenceDictionary REFERENCE=/mnt/miniwdl_task_container/work/assembly.sanitized_ids.fasta OUTPUT=assembly.dict INFO 2024-02-20 01:09:08 CreateSequenceDictionary ********** NOTE: Picard's command line syntax is changing. ********** ********** For more information, please see: ********** https://github.com/broadinstitute/picard/wiki/Command-Line-Syntax-Transition-For-Users-(Pre-Transition) ********** ********** The command line looks like this in the new syntax: ********** ********** CreateSequenceDictionary -REFERENCE /mnt/miniwdl_task_container/work/assembly.sanitized_ids.fasta -OUTPUT assembly.dict ********** 01:09:08.827 INFO NativeLibraryLoader - Loading libgkl_compression.so from jar:file:/opt/miniconda/envs/viral-ngs-env/share/picard-2.25.6-0/picard.jar!/com/intel/gkl/native/libgkl_compression.so [Tue Feb 20 01:09:08 UTC 2024] CreateSequenceDictionary OUTPUT=assembly.dict REFERENCE=/mnt/miniwdl_task_container/work/assembly.sanitized_ids.fasta TRUNCATE_NAMES_AT_WHITESPACE=true NUM_SEQUENCES=2147483647 VERBOSITY=INFO QUIET=false VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=5 MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false CREATE_MD5_FILE=false GA4GH_CLIENT_SECRETS=client_secrets.json USE_JDK_DEFLATER=false USE_JDK_INFLATER=false [Tue Feb 20 01:09:08 UTC 2024] Executing as root@943fe7ca9562 on Linux 5.4.0-148-generic amd64; OpenJDK 64-Bit Server VM 1.8.0_282-b08; Deflater: Intel; Inflater: Intel; Provider GCS is not available; Picard version: 2.25.6 [Tue Feb 20 01:09:08 UTC 2024] picard.sam.CreateSequenceDictionary done. Elapsed time: 0.00 minutes. Runtime.totalMemory()=514850816 + read_utils.py index_fasta_samtools assembly.fasta --loglevel=DEBUG 2024-02-20 01:09:09,413 - cmd:197:main_argparse - INFO - software version: v2.1.33, python version: 3.7.9 | packaged by conda-forge | (default, Dec 9 2020, 21:08:20) [GCC 9.3.0] 2024-02-20 01:09:09,413 - cmd:199:main_argparse - INFO - command: /opt/viral-ngs/source/read_utils.py index_fasta_samtools inFasta=assembly.fasta loglevel=DEBUG 2024-02-20 01:09:09,413 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools faidx assembly.fasta + read_utils.py align_and_fix /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam assembly.fasta --outBamAll 5x_cov.simulated_215.all.bam --outBamFiltered 5x_cov.simulated_215.mapped.bam --aligner minimap2 --aligner_options '' --JVMmemory 464108m --loglevel=DEBUG 2024-02-20 01:09:10,186 - cmd:197:main_argparse - INFO - software version: v2.1.33, python version: 3.7.9 | packaged by conda-forge | (default, Dec 9 2020, 21:08:20) [GCC 9.3.0] 2024-02-20 01:09:10,187 - cmd:199:main_argparse - INFO - command: /opt/viral-ngs/source/read_utils.py align_and_fix inBam=/mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam refFasta=assembly.fasta outBamAll=5x_cov.simulated_215.all.bam outBamFiltered=5x_cov.simulated_215.mapped.bam aligner_options= aligner=minimap2 bwa_min_score=None novoalign_amplicons_bed=None amplicon_window=4 JVMmemory=464108m threads=None skip_mark_dupes=False gatk_path=None novoalign_license_path=None loglevel=DEBUG tmp_dir=/tmp tmp_dirKeep=False 2024-02-20 01:09:10,187 - cmd:213:main_argparse - DEBUG - using tempDir: /tmp/tmp-read_utils-align_and_fix-tlt2juqc 2024-02-20 01:09:10,204 - picard:50:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/picard -Xmx464108m -Djava.io.tmpdir=/tmp/tmp-read_utils-align_and_fix-tlt2juqc CreateSequenceDictionary REFERENCE=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.sanitized_ids.fasta OUTPUT=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.dict INFO 2024-02-20 01:09:10 CreateSequenceDictionary ********** NOTE: Picard's command line syntax is changing. ********** ********** For more information, please see: ********** https://github.com/broadinstitute/picard/wiki/Command-Line-Syntax-Transition-For-Users-(Pre-Transition) ********** ********** The command line looks like this in the new syntax: ********** ********** CreateSequenceDictionary -REFERENCE /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.sanitized_ids.fasta -OUTPUT /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.dict ********** 01:09:10.982 INFO NativeLibraryLoader - Loading libgkl_compression.so from jar:file:/opt/miniconda/envs/viral-ngs-env/share/picard-2.25.6-0/picard.jar!/com/intel/gkl/native/libgkl_compression.so [Tue Feb 20 01:09:10 UTC 2024] CreateSequenceDictionary OUTPUT=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.dict REFERENCE=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.sanitized_ids.fasta TRUNCATE_NAMES_AT_WHITESPACE=true NUM_SEQUENCES=2147483647 VERBOSITY=INFO QUIET=false VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=5 MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false CREATE_MD5_FILE=false GA4GH_CLIENT_SECRETS=client_secrets.json USE_JDK_DEFLATER=false USE_JDK_INFLATER=false [Tue Feb 20 01:09:10 UTC 2024] Executing as root@943fe7ca9562 on Linux 5.4.0-148-generic amd64; OpenJDK 64-Bit Server VM 1.8.0_282-b08; Deflater: Intel; Inflater: Intel; Provider GCS is not available; Picard version: 2.25.6 [Tue Feb 20 01:09:11 UTC 2024] picard.sam.CreateSequenceDictionary done. Elapsed time: 0.00 minutes. Runtime.totalMemory()=2058354688 2024-02-20 01:09:11,036 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools faidx /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.fasta 2024-02-20 01:09:11,045 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools view -H -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmp1960jn8o.txt /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam 2024-02-20 01:09:11,082 - misc:342:available_cpu_count - DEBUG - cfs_quota 800000.0, cfs_period 100000.0 2024-02-20 01:09:11,084 - misc:360:available_cpu_count - DEBUG - cgroup_cpus 8, proc_cpus 64, multiprocessing cpus 64 2024-02-20 01:09:11,085 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools view -H -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmphioa334e.txt /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam 2024-02-20 01:09:11,099 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools view -H -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpawcl5j40.txt /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam 2024-02-20 01:09:11,109 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools view -H -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmph6ij6823.A.header.txt /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam 2024-02-20 01:09:11,121 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools view -H -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpyl37m9_w.txt /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam 2024-02-20 01:09:11,132 - misc:342:available_cpu_count - DEBUG - cfs_quota 800000.0, cfs_period 100000.0 2024-02-20 01:09:11,133 - misc:360:available_cpu_count - DEBUG - cgroup_cpus 8, proc_cpus 64, multiprocessing cpus 64 2024-02-20 01:09:11,134 - samtools:88:bam2fq_pipe - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools bam2fq -n /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam | 2024-02-20 01:09:11,140 - minimap2:35:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/minimap2 -R @RG\tID:A\tSM:5x_cov.simulated_215\tLB:5x_cov.simulated_215\tPL:ILLUMINA -x sr -t 8 -2 -a /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.fasta - -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmphfjyoju0.aligned.sam [M::mm_idx_gen::0.005*2.32] collected minimizers [M::mm_idx_gen::0.007*3.19] sorted minimizers [M::main::0.007*3.17] loaded/built the index for 1 target sequence(s) [M::mm_mapopt_update::0.007*3.17] mid_occ = 1000 [M::mm_idx_stat] kmer size: 21; skip: 11; is_hpc: 0; #seq: 1 [M::mm_idx_stat::0.007*3.08] distinct minimizers: 4989 (99.98% are singletons); average occurrences: 1.002; average spacing: 5.979; total length: 29903 [M::bam2fq_mainloop] discarded 0 singletons [M::bam2fq_mainloop] processed 692 reads [M::worker_pipeline::0.020*3.29] mapped 692 sequences [M::main] Version: 2.20-r1061 [M::main] CMD: /opt/miniconda/envs/viral-ngs-env/bin/minimap2 -R @RG\tID:A\tSM:5x_cov.simulated_215\tLB:5x_cov.simulated_215\tPL:ILLUMINA -x sr -t 8 -2 -a -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmphfjyoju0.aligned.sam /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.fasta - [M::main] Real time: 0.022 sec; CPU: 0.067 sec; Peak RSS: 0.027 GB 2024-02-20 01:09:11,176 - misc:342:available_cpu_count - DEBUG - cfs_quota 800000.0, cfs_period 100000.0 2024-02-20 01:09:11,177 - misc:360:available_cpu_count - DEBUG - cgroup_cpus 8, proc_cpus 64, multiprocessing cpus 64 2024-02-20 01:09:11,177 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools sort -@ 8 -T /tmp/tmp-read_utils-align_and_fix-tlt2juqc -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpvw_l8p77.aligned.bam /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmphfjyoju0.aligned.sam [bam_sort_core] merging from 0 files and 8 in-memory blocks... 2024-02-20 01:09:11,229 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools index /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpvw_l8p77.aligned.bam 2024-02-20 01:09:11,241 - picard:50:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/picard -Xmx464108m -Djava.io.tmpdir=/tmp/tmp-read_utils-align_and_fix-tlt2juqc MarkDuplicates INPUT=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpvw_l8p77.aligned.bam OUTPUT=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam METRICS_FILE=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmp63mcynox.metrics CREATE_INDEX=true INFO 2024-02-20 01:09:11 MarkDuplicates ********** NOTE: Picard's command line syntax is changing. ********** ********** For more information, please see: ********** https://github.com/broadinstitute/picard/wiki/Command-Line-Syntax-Transition-For-Users-(Pre-Transition) ********** ********** The command line looks like this in the new syntax: ********** ********** MarkDuplicates -INPUT /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpvw_l8p77.aligned.bam -OUTPUT /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam -METRICS_FILE /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmp63mcynox.metrics -CREATE_INDEX true ********** 01:09:12.186 INFO NativeLibraryLoader - Loading libgkl_compression.so from jar:file:/opt/miniconda/envs/viral-ngs-env/share/picard-2.25.6-0/picard.jar!/com/intel/gkl/native/libgkl_compression.so [Tue Feb 20 01:09:12 UTC 2024] MarkDuplicates INPUT=[/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpvw_l8p77.aligned.bam] OUTPUT=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam METRICS_FILE=/tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmp63mcynox.metrics CREATE_INDEX=true MAX_SEQUENCES_FOR_DISK_READ_ENDS_MAP=50000 MAX_FILE_HANDLES_FOR_READ_ENDS_MAP=8000 SORTING_COLLECTION_SIZE_RATIO=0.25 TAG_DUPLICATE_SET_MEMBERS=false REMOVE_SEQUENCING_DUPLICATES=false TAGGING_POLICY=DontTag CLEAR_DT=true DUPLEX_UMI=false ADD_PG_TAG_TO_READS=true REMOVE_DUPLICATES=false ASSUME_SORTED=false DUPLICATE_SCORING_STRATEGY=SUM_OF_BASE_QUALITIES PROGRAM_RECORD_ID=MarkDuplicates PROGRAM_GROUP_NAME=MarkDuplicates READ_NAME_REGEX= OPTICAL_DUPLICATE_PIXEL_DISTANCE=100 MAX_OPTICAL_DUPLICATE_SET_SIZE=300000 VERBOSITY=INFO QUIET=false VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=5 MAX_RECORDS_IN_RAM=500000 CREATE_MD5_FILE=false GA4GH_CLIENT_SECRETS=client_secrets.json USE_JDK_DEFLATER=false USE_JDK_INFLATER=false [Tue Feb 20 01:09:12 UTC 2024] Executing as root@943fe7ca9562 on Linux 5.4.0-148-generic amd64; OpenJDK 64-Bit Server VM 1.8.0_282-b08; Deflater: Intel; Inflater: Intel; Provider GCS is not available; Picard version: 2.25.6 INFO 2024-02-20 01:09:12 MarkDuplicates Start of doWork freeMemory: 2035882256; totalMemory: 2058354688; maxMemory: 432580067328 INFO 2024-02-20 01:09:12 MarkDuplicates Reading input file and constructing read end information. INFO 2024-02-20 01:09:12 MarkDuplicates Will retain up to 1567319084 data points before spilling to disk. WARNING 2024-02-20 01:09:30 AbstractOpticalDuplicateFinderCommandLineProgram A field field parsed out of a read name was expected to contain an integer and did not. Read name: EPI_ISL_18250690|2021-03-09_128_228. Cause: String 'EPI_ISL_18250690|2021-03-09_128_228' did not start with a parsable number. INFO 2024-02-20 01:09:30 MarkDuplicates Read 692 records. 0 pairs never matched. INFO 2024-02-20 01:09:51 MarkDuplicates After buildSortedReadEndLists freeMemory: 2042320608; totalMemory: 27136098304; maxMemory: 432580067328 INFO 2024-02-20 01:09:51 MarkDuplicates Will retain up to 2147483642 duplicate indices before spilling to disk. INFO 2024-02-20 01:10:02 MarkDuplicates Traversing read pair information and detecting duplicates. INFO 2024-02-20 01:10:02 MarkDuplicates Traversing fragment information and detecting duplicates. INFO 2024-02-20 01:10:02 MarkDuplicates Sorting list of duplicate records. INFO 2024-02-20 01:10:04 MarkDuplicates After generateDuplicateIndexes freeMemory: 27120116808; totalMemory: 44315967488; maxMemory: 432580067328 INFO 2024-02-20 01:10:04 MarkDuplicates Marking 6 records as duplicates. INFO 2024-02-20 01:10:04 MarkDuplicates Found 0 optical duplicate clusters. INFO 2024-02-20 01:10:04 MarkDuplicates Reads are assumed to be ordered by: coordinate INFO 2024-02-20 01:10:04 MarkDuplicates Writing complete. Closing input iterator. INFO 2024-02-20 01:10:04 MarkDuplicates Duplicate Index cleanup. INFO 2024-02-20 01:10:04 MarkDuplicates Getting Memory Stats. INFO 2024-02-20 01:10:04 MarkDuplicates Before output close freeMemory: 44299021616; totalMemory: 44315967488; maxMemory: 432580067328 INFO 2024-02-20 01:10:04 MarkDuplicates Closed outputs. Getting more Memory Stats. INFO 2024-02-20 01:10:04 MarkDuplicates After output close freeMemory: 44299162960; totalMemory: 44315967488; maxMemory: 432580067328 [Tue Feb 20 01:10:04 UTC 2024] picard.sam.markduplicates.MarkDuplicates done. Elapsed time: 0.87 minutes. Runtime.totalMemory()=44315967488 2024-02-20 01:10:07,700 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools index /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam 2024-02-20 01:10:07,717 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools view -H -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmp0ql7peyw.txt /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam 2024-02-20 01:10:07,730 - misc:342:available_cpu_count - DEBUG - cfs_quota 800000.0, cfs_period 100000.0 2024-02-20 01:10:07,731 - misc:360:available_cpu_count - DEBUG - cgroup_cpus 8, proc_cpus 64, multiprocessing cpus 64 2024-02-20 01:10:07,731 - gatk:97:local_realign - DEBUG - Running local realign with 8 threads 2024-02-20 01:10:07,732 - gatk:64:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/gatk3 -Xmx464108m -Djava.io.tmpdir=/tmp/tmp-read_utils-align_and_fix-tlt2juqc -T RealignerTargetCreator -I /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam -R /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.fasta -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpsenzj11a.intervals --num_threads 8 INFO 01:10:09,359 HelpFormatter - ------------------------------------------------------------------------------------ INFO 01:10:09,361 HelpFormatter - The Genome Analysis Toolkit (GATK) v3.8-1-0-gf15c1c3ef, Compiled 2018/02/19 05:43:50 INFO 01:10:09,361 HelpFormatter - Copyright (c) 2010-2016 The Broad Institute INFO 01:10:09,361 HelpFormatter - For support and documentation go to https://software.broadinstitute.org/gatk INFO 01:10:09,361 HelpFormatter - [Tue Feb 20 01:10:09 UTC 2024] Executing on Linux 5.4.0-148-generic amd64 INFO 01:10:09,361 HelpFormatter - OpenJDK 64-Bit Server VM 1.8.0_282-b08 INFO 01:10:09,365 HelpFormatter - Program Args: -T RealignerTargetCreator -I /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam -R /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.fasta -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpsenzj11a.intervals --num_threads 8 INFO 01:10:09,372 HelpFormatter - Executing as root@943fe7ca9562 on Linux 5.4.0-148-generic amd64; OpenJDK 64-Bit Server VM 1.8.0_282-b08. INFO 01:10:09,372 HelpFormatter - Date/Time: 2024/02/20 01:10:09 INFO 01:10:09,373 HelpFormatter - ------------------------------------------------------------------------------------ INFO 01:10:09,373 HelpFormatter - ------------------------------------------------------------------------------------ INFO 01:10:09,379 NativeLibraryLoader - Loading libgkl_compression.so from jar:file:/opt/miniconda/envs/viral-ngs-env/opt/gatk-3.8/GenomeAnalysisTK.jar!/com/intel/gkl/native/libgkl_compression.so INFO 01:10:09,401 GenomeAnalysisEngine - Deflater: IntelDeflater INFO 01:10:09,402 GenomeAnalysisEngine - Inflater: IntelInflater INFO 01:10:09,402 GenomeAnalysisEngine - Strictness is SILENT INFO 01:10:09,620 GenomeAnalysisEngine - Downsampling Settings: Method: BY_SAMPLE, Target Coverage: 1000 INFO 01:10:09,626 SAMDataSource$SAMReaders - Initializing SAMRecords in serial INFO 01:10:09,643 SAMDataSource$SAMReaders - Done initializing BAM readers: total time 0.02 INFO 01:10:09,660 MicroScheduler - Running the GATK in parallel mode with 8 total threads, 1 CPU thread(s) for each of 8 data thread(s), of 8 processors available on this machine INFO 01:10:09,724 GenomeAnalysisEngine - Preparing for traversal over 1 BAM files INFO 01:10:09,734 GenomeAnalysisEngine - Done preparing for traversal INFO 01:10:09,734 ProgressMeter - [INITIALIZATION COMPLETE; STARTING PROCESSING] INFO 01:10:09,734 ProgressMeter - | processed | time | per 1M | | total | remaining INFO 01:10:09,734 ProgressMeter - Location | sites | elapsed | sites | completed | runtime | runtime INFO 01:10:09,739 SAMDataSource$SAMReaders - Initializing SAMRecords in serial INFO 01:10:09,742 SAMDataSource$SAMReaders - Done initializing BAM readers: total time 0.00 INFO 01:10:09,960 ProgressMeter - done 29903.0 0.0 s 7.0 s 99.9% 0.0 s 0.0 s INFO 01:10:09,960 ProgressMeter - Total runtime 0.23 secs, 0.00 min, 0.00 hours INFO 01:10:09,963 MicroScheduler - 6 reads were filtered out during the traversal out of approximately 695 total reads (0.86%) INFO 01:10:09,963 MicroScheduler - -> 0 reads (0.00% of total) failing BadCigarFilter INFO 01:10:09,964 MicroScheduler - -> 0 reads (0.00% of total) failing BadMateFilter INFO 01:10:09,964 MicroScheduler - -> 6 reads (0.86% of total) failing DuplicateReadFilter INFO 01:10:09,964 MicroScheduler - -> 0 reads (0.00% of total) failing FailsVendorQualityCheckFilter INFO 01:10:09,964 MicroScheduler - -> 0 reads (0.00% of total) failing MalformedReadFilter INFO 01:10:09,964 MicroScheduler - -> 0 reads (0.00% of total) failing MappingQualityUnavailableFilter INFO 01:10:09,964 MicroScheduler - -> 0 reads (0.00% of total) failing MappingQualityZeroFilter INFO 01:10:09,964 MicroScheduler - -> 0 reads (0.00% of total) failing NotPrimaryAlignmentFilter INFO 01:10:09,965 MicroScheduler - -> 0 reads (0.00% of total) failing Platform454Filter INFO 01:10:09,965 MicroScheduler - -> 0 reads (0.00% of total) failing UnmappedReadFilter ------------------------------------------------------------------------------------------ Done. ------------------------------------------------------------------------------------------ 2024-02-20 01:10:10,030 - gatk:64:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/gatk3 -Xmx464108m -Djava.io.tmpdir=/tmp/tmp-read_utils-align_and_fix-tlt2juqc -T IndelRealigner -I /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam -R /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.fasta -targetIntervals /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpsenzj11a.intervals -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmp48_1umug.realigned.bam INFO 01:10:11,626 HelpFormatter - ------------------------------------------------------------------------------------ INFO 01:10:11,628 HelpFormatter - The Genome Analysis Toolkit (GATK) v3.8-1-0-gf15c1c3ef, Compiled 2018/02/19 05:43:50 INFO 01:10:11,628 HelpFormatter - Copyright (c) 2010-2016 The Broad Institute INFO 01:10:11,629 HelpFormatter - For support and documentation go to https://software.broadinstitute.org/gatk INFO 01:10:11,629 HelpFormatter - [Tue Feb 20 01:10:11 UTC 2024] Executing on Linux 5.4.0-148-generic amd64 INFO 01:10:11,629 HelpFormatter - OpenJDK 64-Bit Server VM 1.8.0_282-b08 INFO 01:10:11,632 HelpFormatter - Program Args: -T IndelRealigner -I /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpty03_o4o.mkdup.bam -R /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpbohi9w15.ref_copy.fasta -targetIntervals /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmpsenzj11a.intervals -o /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmp48_1umug.realigned.bam INFO 01:10:11,639 HelpFormatter - Executing as root@943fe7ca9562 on Linux 5.4.0-148-generic amd64; OpenJDK 64-Bit Server VM 1.8.0_282-b08. INFO 01:10:11,640 HelpFormatter - Date/Time: 2024/02/20 01:10:11 INFO 01:10:11,640 HelpFormatter - ------------------------------------------------------------------------------------ INFO 01:10:11,640 HelpFormatter - ------------------------------------------------------------------------------------ INFO 01:10:11,811 NativeLibraryLoader - Loading libgkl_compression.so from jar:file:/opt/miniconda/envs/viral-ngs-env/opt/gatk-3.8/GenomeAnalysisTK.jar!/com/intel/gkl/native/libgkl_compression.so INFO 01:10:11,833 GenomeAnalysisEngine - Deflater: IntelDeflater INFO 01:10:11,833 GenomeAnalysisEngine - Inflater: IntelInflater INFO 01:10:11,833 GenomeAnalysisEngine - Strictness is SILENT INFO 01:10:11,919 GenomeAnalysisEngine - Downsampling Settings: No downsampling INFO 01:10:11,925 SAMDataSource$SAMReaders - Initializing SAMRecords in serial INFO 01:10:11,942 SAMDataSource$SAMReaders - Done initializing BAM readers: total time 0.02 INFO 01:10:12,021 GenomeAnalysisEngine - Preparing for traversal over 1 BAM files INFO 01:10:12,025 GenomeAnalysisEngine - Done preparing for traversal INFO 01:10:12,025 ProgressMeter - [INITIALIZATION COMPLETE; STARTING PROCESSING] INFO 01:10:12,025 ProgressMeter - | processed | time | per 1M | | total | remaining INFO 01:10:12,025 ProgressMeter - Location | reads | elapsed | reads | completed | runtime | runtime INFO 01:10:12,033 ReadShardBalancer$1 - Loading BAM index data INFO 01:10:12,033 ReadShardBalancer$1 - Done loading BAM index data INFO 01:10:12,164 ProgressMeter - done 692.0 0.0 s 3.3 m 89.8% 0.0 s 0.0 s INFO 01:10:12,165 ProgressMeter - Total runtime 0.14 secs, 0.00 min, 0.00 hours INFO 01:10:12,170 MicroScheduler - 0 reads were filtered out during the traversal out of approximately 692 total reads (0.00%) INFO 01:10:12,170 MicroScheduler - -> 0 reads (0.00% of total) failing BadCigarFilter INFO 01:10:12,171 MicroScheduler - -> 0 reads (0.00% of total) failing MalformedReadFilter ------------------------------------------------------------------------------------------ Done. ------------------------------------------------------------------------------------------ 2024-02-20 01:10:12,246 - picard:50:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/picard -Xmx464108m -Djava.io.tmpdir=/tmp/tmp-read_utils-align_and_fix-tlt2juqc BuildBamIndex INPUT=5x_cov.simulated_215.all.bam INFO 2024-02-20 01:10:12 BuildBamIndex ********** NOTE: Picard's command line syntax is changing. ********** ********** For more information, please see: ********** https://github.com/broadinstitute/picard/wiki/Command-Line-Syntax-Transition-For-Users-(Pre-Transition) ********** ********** The command line looks like this in the new syntax: ********** ********** BuildBamIndex -INPUT 5x_cov.simulated_215.all.bam ********** 01:10:13.053 INFO NativeLibraryLoader - Loading libgkl_compression.so from jar:file:/opt/miniconda/envs/viral-ngs-env/share/picard-2.25.6-0/picard.jar!/com/intel/gkl/native/libgkl_compression.so [Tue Feb 20 01:10:13 UTC 2024] BuildBamIndex INPUT=5x_cov.simulated_215.all.bam VERBOSITY=INFO QUIET=false VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=5 MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false CREATE_MD5_FILE=false GA4GH_CLIENT_SECRETS=client_secrets.json USE_JDK_DEFLATER=false USE_JDK_INFLATER=false [Tue Feb 20 01:10:13 UTC 2024] Executing as root@943fe7ca9562 on Linux 5.4.0-148-generic amd64; OpenJDK 64-Bit Server VM 1.8.0_282-b08; Deflater: Intel; Inflater: Intel; Provider GCS is not available; Picard version: 2.25.6 INFO 2024-02-20 01:10:13 BuildBamIndex Successfully wrote bam index file /mnt/miniwdl_task_container/work/5x_cov.simulated_215.all.bai [Tue Feb 20 01:10:13 UTC 2024] picard.sam.BuildBamIndex done. Elapsed time: 0.00 minutes. Runtime.totalMemory()=2058354688 2024-02-20 01:10:13,195 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools view -b -q 1 -o 5x_cov.simulated_215.mapped.bam /tmp/tmp-read_utils-align_and_fix-tlt2juqc/tmp7sligcct.filtered_any_mapq.bam 2024-02-20 01:10:13,410 - picard:50:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/picard -Xmx464108m -Djava.io.tmpdir=/tmp/tmp-read_utils-align_and_fix-tlt2juqc BuildBamIndex INPUT=5x_cov.simulated_215.mapped.bam INFO 2024-02-20 01:10:13 BuildBamIndex ********** NOTE: Picard's command line syntax is changing. ********** ********** For more information, please see: ********** https://github.com/broadinstitute/picard/wiki/Command-Line-Syntax-Transition-For-Users-(Pre-Transition) ********** ********** The command line looks like this in the new syntax: ********** ********** BuildBamIndex -INPUT 5x_cov.simulated_215.mapped.bam ********** 01:10:14.178 INFO NativeLibraryLoader - Loading libgkl_compression.so from jar:file:/opt/miniconda/envs/viral-ngs-env/share/picard-2.25.6-0/picard.jar!/com/intel/gkl/native/libgkl_compression.so [Tue Feb 20 01:10:14 UTC 2024] BuildBamIndex INPUT=5x_cov.simulated_215.mapped.bam VERBOSITY=INFO QUIET=false VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=5 MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false CREATE_MD5_FILE=false GA4GH_CLIENT_SECRETS=client_secrets.json USE_JDK_DEFLATER=false USE_JDK_INFLATER=false [Tue Feb 20 01:10:14 UTC 2024] Executing as root@943fe7ca9562 on Linux 5.4.0-148-generic amd64; OpenJDK 64-Bit Server VM 1.8.0_282-b08; Deflater: Intel; Inflater: Intel; Provider GCS is not available; Picard version: 2.25.6 INFO 2024-02-20 01:10:14 BuildBamIndex Successfully wrote bam index file /mnt/miniwdl_task_container/work/5x_cov.simulated_215.mapped.bai [Tue Feb 20 01:10:14 UTC 2024] picard.sam.BuildBamIndex done. Elapsed time: 0.00 minutes. Runtime.totalMemory()=2058354688 + cat /proc/loadavg + grep -v '^>' assembly.fasta + tr -d '\nNn' + wc -c + tee assembly_length_unambiguous + samtools view -c /mnt/miniwdl_task_container/work/_miniwdl_inputs/0/5x_cov.simulated_215.bam + tee reads_provided + samtools view -c 5x_cov.simulated_215.mapped.bam + tee reads_aligned + samtools view -h -F 260 5x_cov.simulated_215.all.bam + samtools flagstat - + tee 5x_cov.simulated_215.all.bam.flagstat.txt + grep properly 5x_cov.simulated_215.all.bam.flagstat.txt + cut -f 1 -d ' ' + tee read_pairs_aligned + samtools view 5x_cov.simulated_215.mapped.bam + cut -f10 + tr -d '\n' + wc -c + tee bases_aligned ++ cat bases_aligned ++ cat assembly_length_unambiguous ++ cat assembly_length_unambiguous + python -c 'print (float(0)/29903) if 29903>0 else print(0)' + reports.py fastqc 5x_cov.simulated_215.mapped.bam 5x_cov.simulated_215.mapped_fastqc.html --out_zip 5x_cov.simulated_215.mapped_fastqc.zip 2024-02-20 01:10:15,569 - cmd:197:main_argparse - INFO - software version: v2.1.33, python version: 3.7.9 | packaged by conda-forge | (default, Dec 9 2020, 21:08:20) [GCC 9.3.0] 2024-02-20 01:10:15,569 - cmd:199:main_argparse - INFO - command: /opt/viral-ngs/source/reports.py fastqc inBam=5x_cov.simulated_215.mapped.bam out_html=5x_cov.simulated_215.mapped_fastqc.html out_zip=5x_cov.simulated_215.mapped_fastqc.zip threads=None 2024-02-20 01:10:15,570 - misc:342:available_cpu_count - DEBUG - cfs_quota 800000.0, cfs_period 100000.0 2024-02-20 01:10:15,570 - misc:360:available_cpu_count - DEBUG - cgroup_cpus 8, proc_cpus 64, multiprocessing cpus 64 2024-02-20 01:10:15,571 - samtools:51:execute - DEBUG - /opt/miniconda/envs/viral-ngs-env/bin/samtools view -H -o /tmp/tmp6v49hjzs.txt 5x_cov.simulated_215.mapped.bam + cat /proc/uptime + cut -f 1 -d ' ' + cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes