diff --git a/CHANGELOG.md b/CHANGELOG.md index 35738baf..31eb5f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,30 @@ If MD5 sum is not listed for a certain release then it means that the container * Miscellaneous goes here + +## [1.4.0] - 2023-10-18 + +### Added + +* Added phasing/imputation tools ``beagle``, ``duohmm``, ``eagle``, ``shapeit5``, ``switchError``, to ``gwas.sif`` container + updated tests + +### Fixed + +* Fix issue that shell script wouldn't capture failing statements + +### Updated + +* Updated `gwas.sif` Dockerfile and installed shell scripts (misc. dependencies updates, installing ``gcta`` version 1.93.3beta2) +* Rebuilt `gwas.sif` using Docker `--no-cache` option to fix missing `minimac4` binary, w. md5 checksum: + + ``` + a1dd235221902741bf5773945a584e47 singularity/gwas.sif + ``` + +### Removed + +* Removed unused `install_miniconda.sh` script from `src/scripts` folder + ## [1.3.9] - 2023-10-17 ### Added @@ -131,7 +155,7 @@ If MD5 sum is not listed for a certain release then it means that the container * Updated file and folder layout, fixing minor documentation issues. Moving from ``m2r2`` to ``Myst-parser`` for Sphinx-generated online docs. * Rebuilt the R container -* ```` +* ``` 1d435af6003bbca95ef8cc062bf666fc singularity/r.sif ``` diff --git a/docker/README.md b/docker/README.md index 4cabbfeb..7bc6e68f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -37,41 +37,50 @@ The detailed description of the available container [files](https://github.com/c | hello.sif | ubuntu | 20.04 | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy) | hello.sif | plink | v1.90b6.18 64-bit (16 Jun 2020) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | gwas.sif | ubuntu | 20.04 | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy) - | gwas.sif | plink | v1.90b6.18 64-bit (16 Jun 2020) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | gwas.sif | plink2 | v2.00a3.6LM 64-bit Intel (14 Aug 2022) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | gwas.sif | plink2_avx2 | v2.00a3.6LM AVX2 Intel (24 Jan 2020) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | gwas.sif | PRSice_linux | 2.3.3 (2020-08-05) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | gwas.sif | simu_linux | v0.9.4 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | bcftools | 1.12 (using htslib 1.12) | [MIT/Expat/GPLv3](https://github.com/samtools/bcftools/blob/develop/LICENSE) + | gwas.sif | beagle | 22Jul22.46e | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | bgenix | 1.1.7 | [Boost](https://www.boost.org/LICENSE_1_0.txt) | gwas.sif | bolt | v2.4 July 22, 2022 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | gwas.sif | gcta64 | version 1.93.2 beta Linux | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | cat-bgen | same version as bgenix | [Boost](https://www.boost.org/LICENSE_1_0.txt) + | gwas.sif | duohmm | 95bd395 | [MIT](https://opensource.org/licenses/MIT) + | gwas.sif | eagle | v2.4.1 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | edit-bgen | same version as bgenix | [Boost](https://www.boost.org/LICENSE_1_0.txt) + | gwas.sif | flashpca_x86-64 | 2.0 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | gcta64 | version 1.93.3 beta 2 Linux | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | gwas.sif | gctb | 2.02 | [MIT](https://opensource.org/licenses/MIT) | gwas.sif | qctool | 2.0.6, revision 18b8f17 | [Boost](https://www.boost.org/LICENSE_1_0.txt) + | gwas.sif | GWAMA | 2.2.2 | [BSD-3-Clause](https://opensource.org/licenses/BSD-3-Clause) + | gwas.sif | HTSlib | 1.12 | [MIT/Expat/Modified-BSD](https://github.com/samtools/htslib/blob/develop/LICENSE) | gwas.sif | king | 2.2.9 - (c) | [permissive](https://www.kingrelatedness.com/Download.shtml) | gwas.sif | metal | version released on 2011-03-25 | - - | gwas.sif | vcftools | 0.1.17 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | gwas.sif | bcftools | 1.12 (using htslib 1.12) | [MIT/Expat/GPLv3](https://github.com/samtools/bcftools/blob/develop/LICENSE) - | gwas.sif | flashpca_x86-64 | 2.0 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | gwas.sif | regenie | v3.2.8 | [MIT/Boost](https://github.com/rgcgithub/regenie/blob/master/LICENSE) - | gwas.sif | GWAMA | 2.2.2 | [BSD-3-Clause](https://opensource.org/licenses/BSD-3-Clause) | gwas.sif | minimac4 | v4.1.0 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | gwas.sif | bgenix | 1.1.7 | [Boost](https://www.boost.org/LICENSE_1_0.txt) - | gwas.sif | cat-bgen | same version as bgenix | [Boost](https://www.boost.org/LICENSE_1_0.txt) - | gwas.sif | edit-bgen | same version as bgenix | [Boost](https://www.boost.org/LICENSE_1_0.txt) - | gwas.sif | HTSlib | 1.12 | [MIT/Expat/Modified-BSD](https://github.com/samtools/htslib/blob/develop/LICENSE) + | gwas.sif | plink | v1.90b6.18 64-bit (16 Jun 2020) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | plink2 | v2.00a3.6LM 64-bit Intel (14 Aug 2022) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | plink2_avx2 | v2.00a3.6LM AVX2 Intel (24 Jan 2020) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | PRSice_linux | 2.3.3 (2020-08-05) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | regenie | v3.2.8 | [MIT/Boost](https://github.com/rgcgithub/regenie/blob/master/LICENSE) | gwas.sif | shapeit4.2 | v4.2.2 | [MIT](https://opensource.org/licenses/MIT) + | gwas.sif | shapeit5 phase_rare | v5.1.1 | [MIT](https://opensource.org/licenses/MIT) + | gwas.sif | shapeit5 phase_common | v5.1.1 | [MIT](https://opensource.org/licenses/MIT) + | gwas.sif | shapeit5 ligate | v5.1.1 | [MIT](https://opensource.org/licenses/MIT) + | gwas.sif | shapeit5 switch | v5.1.1 | [MIT](https://opensource.org/licenses/MIT) + | gwas.sif | shapeit5 xcftools | v5.1.1 | [MIT](https://opensource.org/licenses/MIT) + | gwas.sif | simu_linux | v0.9.4 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | switchError | 6e688b1 | [MIT](https://opensource.org/licenses/MIT) + | gwas.sif | vcftools | 0.1.17 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | python3.sif | ubuntu | 20.04 (LTS) | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy) - | python3.sif | python3 | python 3.10.6 + numpy, pandas, etc. | [PSF](https://docs.python.org/3.10/license.html) | python3.sif | LDpred | 1.0.11 | [MIT](https://opensource.org/licenses/MIT) - | python3.sif | python_convert | github commit bcde562 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | python3.sif | plink | v1.90b6.18 64-bit (16 Jun 2020) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | python3.sif | python3 | python 3.10.6 + numpy, pandas, etc. | [PSF](https://docs.python.org/3.10/license.html) + | python3.sif | python_convert | github commit bcde562 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | r.sif | ubuntu | 20.04 | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy) | r.sif | R | 4.0.5 (2021-03-31) + data.table, ggplot, etc. | [misc](https://www.r-project.org/Licenses/) | r.sif | gcta64 | version 1.93.2 beta Linux | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | r.sif | seqminer | [zhanxw/seqminer@142204d](https://github.com/zhanxw/seqminer/commit/142204d1005553ea87e1740ff97f0286291e41f9) | [GPL](https://github.com/zhanxw/seqminer/blob/master/LICENSE) - | r.sif | rareGWAMA | [dajiangliu/rareGWAMA@72e962d](https://github.com/dajiangliu/rareGWAMA/commit/72e962dae19dc07251244f6c33275ada189c2126) | - | r.sif | GenomicSEM | [GenomicSEM/GenomicSEM@bcbbaff](https://github.com/GenomicSEM/GenomicSEM/commit/bcbbaffff5767acfc5c020409a4dc54fbf07876b) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) - | r.sif | TwoSampleMR | [MRCIEU/TwoSampleMR@c174107](https://github.com/MRCIEU/TwoSampleMR/commit/c174107cfd9ba47cf2f780849a263f37ac472a0e) | [unknown/MIT](https://github.com/MRCIEU/TwoSampleMR#:~:text=Unknown%2C%20MIT%20licenses-,found,-Citation) | r.sif | GSMR | v1.0.9 | [GPL>=v2](https://www.gnu.org/licenses/gpl-2.0.html) + | r.sif | rareGWAMA | [dajiangliu/rareGWAMA@72e962d](https://github.com/dajiangliu/rareGWAMA/commit/72e962dae19dc07251244f6c33275ada189c2126) | - + | r.sif | seqminer | [zhanxw/seqminer@142204d](https://github.com/zhanxw/seqminer/commit/142204d1005553ea87e1740ff97f0286291e41f9) | [GPL](https://github.com/zhanxw/seqminer/blob/master/LICENSE) + | r.sif | TwoSampleMR | [MRCIEU/TwoSampleMR@c174107](https://github.com/MRCIEU/TwoSampleMR/commit/c174107cfd9ba47cf2f780849a263f37ac472a0e) | [unknown/MIT](https://github.com/MRCIEU/TwoSampleMR#:~:text=Unknown%2C%20MIT%20licenses-,found,-Citation) | r.sif | snpStats | v1.40.0 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | saige.sif | ubuntu | 16.04 | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy) | saige.sif | SAIGE | version 0.43 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) diff --git a/docker/dockerfiles/gwas/Dockerfile b/docker/dockerfiles/gwas/Dockerfile index 1bd6eed2..3274a69a 100644 --- a/docker/dockerfiles/gwas/Dockerfile +++ b/docker/dockerfiles/gwas/Dockerfile @@ -1,5 +1,5 @@ # gwas container -FROM 'ubuntu:20.04' +FROM ubuntu:20.04 ENV TZ=Europe ENV DEBIAN_FRONTEND noninteractive @@ -83,7 +83,7 @@ COPY /scripts/install_vcftools.sh /tmp/vcftools/ RUN bash /tmp/vcftools/install_vcftools.sh && \ rm -rf /tmp/vcftools -# HTSlib (dependency for bcftools & shapeit4) +# HTSlib (dependency for bcftools & shapeit4++) WORKDIR /tmp/htslib COPY /scripts/install_htslib.sh /tmp/htslib/ RUN bash /tmp/htslib/install_htslib.sh && \ @@ -134,4 +134,40 @@ COPY /scripts/patches/shapeit4.makefile.diff /tmp/shapeit4/ RUN bash /tmp/shapeit4/install_shapeit4.sh && \ rm -rf /tmp/shapeit4 +# shapeit5 +WORKDIR /tmp/shapeit5 +COPY /scripts/install_shapeit5.sh /tmp/shapeit5/ +COPY /scripts/patches/shapeit5.diff /tmp/shapeit5/ +COPY /scripts/patches/xcftools.diff /tmp/shapeit5/ +RUN bash /tmp/shapeit5/install_shapeit5.sh && \ + rm -rf /tmp/shapeit5 + +# eagle +WORKDIR /tmp/eagle +COPY /scripts/patches/eagle.diff /tmp/eagle/ +COPY /scripts/install_eagle.sh /tmp/eagle/ +RUN bash /tmp/eagle/install_eagle.sh && \ + rm -rf /tmp/eagle + +# switchError +WORKDIR /tmp/switchError +COPY /scripts/patches/switchError.diff /tmp/switchError// +COPY /scripts/install_switchError.sh /tmp/switchError/ +RUN bash /tmp/switchError/install_switchError.sh && \ + rm -rf /tmp/switchError + +# beagle +WORKDIR /tmp/beagle +COPY /scripts/install_beagle.sh /tmp/beagle/ +COPY /scripts/stub.sh /tmp/beagle/ +RUN bash /tmp/beagle/install_beagle.sh && \ + rm -rf /tmp/beagle + + +# duohmm +WORKDIR /tmp/duohmm +COPY /scripts/install_duohmm.sh /tmp/duohmm/ +RUN bash /tmp/duohmm/install_duohmm.sh && \ + rm -rf /tmp/duohmm + WORKDIR /tools diff --git a/docker/scripts/apt_get_essential.sh b/docker/scripts/apt_get_essential.sh index 682f4981..fc085ad9 100644 --- a/docker/scripts/apt_get_essential.sh +++ b/docker/scripts/apt_get_essential.sh @@ -1,6 +1,8 @@ #!/bin/sh +set -euo pipefail + apt-get update && apt-get install -y --no-install-recommends apt-utils=2.0.9 -apt-get update && apt-get install -y --no-install-recommends ca-certificates=20211016~20.04.1 && \ +apt-get update && apt-get install -y --no-install-recommends ca-certificates=20230311ubuntu0.20.04.1 && \ update-ca-certificates # (!) Keep the list below sorted (!) @@ -10,27 +12,27 @@ apt-get update && apt-get install -y --no-install-recommends \ build-essential=12.8ubuntu1 \ bzip2=1.0.8-2 \ cmake=3.16.3-1ubuntu1.20.04.1 \ - curl=7.68.0-1ubuntu2.14 \ + curl=7.68.0-1ubuntu2.19 \ dos2unix=7.4.0-2 \ gfortran=4:9.3.0-1ubuntu2 \ - git=1:2.25.1-1ubuntu3.6 \ + git=1:2.25.1-1ubuntu3.11 \ less=551-1ubuntu0.1 \ libatlas-base-dev=3.10.3-8ubuntu7 \ - libcurl4-openssl-dev=7.68.0-1ubuntu2.14 \ - libgomp1=10.3.0-1ubuntu1~20.04 \ + libcurl4-openssl-dev=7.68.0-1ubuntu2.19 \ + libgomp1=10.5.0-1ubuntu1~20.04 \ libgsl-dev=2.5+dfsg-6build1 \ - libnss3=2:3.49.1-1ubuntu1.8 \ + libnss3=2:3.49.1-1ubuntu1.9 \ libpcre2-dev=10.34-7ubuntu0.1 \ libxt-dev=1:1.1.5-1 \ pandoc=2.5-3build2 \ pandoc-citeproc=0.15.0.1-1build4 \ parallel=20161222-1.1 \ - perl=5.30.0-9ubuntu0.3 \ + perl=5.30.0-9ubuntu0.4 \ pkg-config=0.29.1-0ubuntu4 \ tar=1.30+dfsg-7ubuntu0.20.04.2 \ tofrodos=1.7.13+ds-4 \ unzip=6.0-25ubuntu1.1 \ - vim=2:8.1.2269-1ubuntu5.9 \ + vim=2:8.1.2269-1ubuntu5.17 \ wget=1.20.3-1ubuntu2 \ zlib1g-dev=1:1.2.11.dfsg-2ubuntu1.5 \ && \ diff --git a/docker/scripts/convert_docker_image_to_singularity.sh b/docker/scripts/convert_docker_image_to_singularity.sh index f4e19402..e36d20fa 100755 --- a/docker/scripts/convert_docker_image_to_singularity.sh +++ b/docker/scripts/convert_docker_image_to_singularity.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash +set -euo pipefail -set -e # This script documents how to build the singularity container # from the Dockerfile diff --git a/docker/scripts/install_bcftools.sh b/docker/scripts/install_bcftools.sh index 8487ef7c..c056e6a3 100644 --- a/docker/scripts/install_bcftools.sh +++ b/docker/scripts/install_bcftools.sh @@ -1,11 +1,10 @@ #!/bin/sh - -set -e +set -euo pipefail # deps apt-get update && apt-get install -y --no-install-recommends \ - libcurl4-gnutls-dev=7.68.0-1ubuntu2.14 \ - libperl-dev=5.30.0-9ubuntu0.3 \ + libcurl4-gnutls-dev=7.68.0-1ubuntu2.19 \ + libperl-dev=5.30.0-9ubuntu0.4 \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/scripts/install_beagle.sh b/docker/scripts/install_beagle.sh new file mode 100644 index 00000000..73f55a4a --- /dev/null +++ b/docker/scripts/install_beagle.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -euo pipefail + +# additional deps +apt-get update && apt-get install --no-install-recommends \ + default-jre=2:1.11-72 \ + -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +wget http://faculty.washington.edu/browning/beagle/beagle.22Jul22.46e.jar && \ + mv beagle.22Jul22.46e.jar beagle.jar + +cat stub.sh beagle.jar > /usr/bin/beagle && chmod +x /usr/bin/beagle diff --git a/docker/scripts/install_bgenix.sh b/docker/scripts/install_bgenix.sh index 890b4033..bb68dfc5 100644 --- a/docker/scripts/install_bgenix.sh +++ b/docker/scripts/install_bgenix.sh @@ -1,3 +1,6 @@ +#!/bin/sh +set -euo pipefail + # python appears to be a build time dependency apt-get update && apt-get install -y --no-install-recommends python3=3.8.2-0ubuntu2 && \ apt-get clean && \ diff --git a/docker/scripts/install_bolt.sh b/docker/scripts/install_bolt.sh index 64a65fdc..b9ff87c5 100644 --- a/docker/scripts/install_bolt.sh +++ b/docker/scripts/install_bolt.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -euo pipefail export VERSION="v2.3.5" diff --git a/docker/scripts/install_duohmm.sh b/docker/scripts/install_duohmm.sh new file mode 100644 index 00000000..f924af37 --- /dev/null +++ b/docker/scripts/install_duohmm.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -euo pipefail + +git clone https://github.com/jaredo/duohmm.git && \ + cd duohmm && \ + git checkout 95bd3958792aeaa43e9f301ead139e5691d7c165 && \ + make -j 4 && \ + cp bin/duohmm /usr/bin/ + + diff --git a/docker/scripts/install_eagle.sh b/docker/scripts/install_eagle.sh new file mode 100644 index 00000000..0fb438b7 --- /dev/null +++ b/docker/scripts/install_eagle.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -euo pipefail + +apt-get update && apt-get install --no-install-recommends \ + libopenblas-dev=0.3.8+ds-1ubuntu0.20.04.1 \ + -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +git clone --depth 1 -b v2.4.1 https://github.com/poruloh/Eagle.git && \ + cd Eagle && \ + git apply ../eagle.diff && \ + cd src && \ + make -j 4 && \ + cp eagle /usr/bin/ + + diff --git a/docker/scripts/install_flashpca.sh b/docker/scripts/install_flashpca.sh index 2c588d85..28fd6b09 100644 --- a/docker/scripts/install_flashpca.sh +++ b/docker/scripts/install_flashpca.sh @@ -1,3 +1,6 @@ +#!/bin/sh +set -euo pipefail + wget --no-check-certificate https://github.com/gabraham/flashpca/releases/download/v2.0/flashpca_x86-64.gz && \ gunzip flashpca_x86-64.gz && \ chmod +x flashpca_x86-64 && \ diff --git a/docker/scripts/install_gcta.sh b/docker/scripts/install_gcta.sh index 8589c54b..a519b171 100644 --- a/docker/scripts/install_gcta.sh +++ b/docker/scripts/install_gcta.sh @@ -1,8 +1,9 @@ #!/bin/sh +set -euo pipefail # gcta -curl -O -J -L https://yanglab.westlake.edu.cn/software/gcta/bin/gcta_1.93.2beta.zip && \ - unzip -j gcta_1.93.2beta.zip && \ - rm -rf gcta_1.93.2beta.zip +curl -O -J -L https://github.com/jianyangqt/gcta/releases/download/v1.93.3beta2/gcta_1.93.3beta2.zip && \ + unzip -j gcta_1.93.3beta2.zip && \ + rm -rf gcta_1.93.3beta2.zip cp gcta64 /bin diff --git a/docker/scripts/install_gctb.sh b/docker/scripts/install_gctb.sh index a6fbea41..aacef104 100644 --- a/docker/scripts/install_gctb.sh +++ b/docker/scripts/install_gctb.sh @@ -1,13 +1,11 @@ #!/bin/sh - +set -euo pipefail # gctb - wget --no-check-certificate https://cnsgenomics.com/software/gctb/download/gctb_2.02_Linux.zip && \ unzip gctb_2.02_Linux.zip && \ rm -rf gctb_2.02_Linux.zip - mv gctb_2.02_Linux/* . cp gctb /bin diff --git a/docker/scripts/install_gwama.sh b/docker/scripts/install_gwama.sh index 5fcaca37..90453027 100644 --- a/docker/scripts/install_gwama.sh +++ b/docker/scripts/install_gwama.sh @@ -1,3 +1,6 @@ +#!/bin/sh +set -euo pipefail + wget --no-check-certificate http://www.geenivaramu.ee/tools/GWAMA_v2.2.2.zip && \ unzip GWAMA_v2.2.2.zip && \ make && \ diff --git a/docker/scripts/install_htslib.sh b/docker/scripts/install_htslib.sh index 975ea5bc..50a1da53 100644 --- a/docker/scripts/install_htslib.sh +++ b/docker/scripts/install_htslib.sh @@ -1,12 +1,11 @@ #!/bin/sh - -set -e +set -euo pipefail # additional deps apt-get update && apt-get install --no-install-recommends \ libbz2-dev=1.0.8-2 \ liblzma-dev=5.2.4-1ubuntu1.1 \ - libssl-dev=1.1.1f-1ubuntu2.16 \ + libssl-dev=1.1.1f-1ubuntu2.19 \ -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/scripts/install_king.sh b/docker/scripts/install_king.sh index 7a60f66e..d829838f 100644 --- a/docker/scripts/install_king.sh +++ b/docker/scripts/install_king.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -euo pipefail # deps apt-get update && apt-get install --no-install-recommends \ diff --git a/docker/scripts/install_mambaforge.sh b/docker/scripts/install_mambaforge.sh index 79dc37a5..87ef37c8 100644 --- a/docker/scripts/install_mambaforge.sh +++ b/docker/scripts/install_mambaforge.sh @@ -1,3 +1,6 @@ +#!/bin/sh +set -euo pipefail + curl -sSL https://github.com/conda-forge/miniforge/releases/download/4.14.0-0/Mambaforge-4.14.0-0-$(uname)-$(uname -m).sh -o /tmp/mambaforge.sh \ && mkdir /root/.conda \ && bash /tmp/mambaforge.sh -bfp /usr/local \ diff --git a/docker/scripts/install_metal.sh b/docker/scripts/install_metal.sh index 3a17c782..78340444 100644 --- a/docker/scripts/install_metal.sh +++ b/docker/scripts/install_metal.sh @@ -1,5 +1,5 @@ #!/bin/sh - +set -euo pipefail # metal wget --no-check-certificate http://csg.sph.umich.edu/abecasis/metal/download/Linux-metal.tar.gz && \ diff --git a/docker/scripts/install_miniconda.sh b/docker/scripts/install_miniconda.sh deleted file mode 100644 index 859fcf0f..00000000 --- a/docker/scripts/install_miniconda.sh +++ /dev/null @@ -1,6 +0,0 @@ -curl -sSL https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -o /tmp/miniconda2.sh \ - && mkdir /root/.conda \ - && bash /tmp/miniconda2.sh -bfp /usr/local \ - && rm -rf /tmp/miniconda2.sh - -export PATH=$PATH:/opt/conda/bin diff --git a/docker/scripts/install_minimac4.sh b/docker/scripts/install_minimac4.sh index c42eadc1..f68f9aef 100644 --- a/docker/scripts/install_minimac4.sh +++ b/docker/scripts/install_minimac4.sh @@ -1,9 +1,10 @@ #!/bin/sh +set -euo pipefail # install some deps for installing cget apt-get update && \ apt-get install --no-install-recommends \ - python3-pip=20.0.2-5ubuntu1.6 \ + python3-pip=20.0.2-5ubuntu1.9 \ python3-click=7.0-3 \ python3-six=1.14.0-2 \ -y && \ diff --git a/docker/scripts/install_plink.sh b/docker/scripts/install_plink.sh index 964bb265..db8b219d 100644 --- a/docker/scripts/install_plink.sh +++ b/docker/scripts/install_plink.sh @@ -1,4 +1,6 @@ #!/bin/sh +set -euo pipefail + # plink wget --no-check-certificate https://s3.amazonaws.com/plink1-assets/plink_linux_x86_64_20200616.zip && \ unzip -j plink_linux_x86_64_20200616.zip && \ diff --git a/docker/scripts/install_plink2.sh b/docker/scripts/install_plink2.sh index 9ad58822..0f34cbea 100644 --- a/docker/scripts/install_plink2.sh +++ b/docker/scripts/install_plink2.sh @@ -1,4 +1,6 @@ #!/bin/sh +set -euo pipefail + wget --no-check-certificate https://s3.amazonaws.com/plink2-assets/alpha3/plink2_linux_x86_64_20220814.zip && \ unzip -j plink2_linux_x86_64_20220814.zip && \ rm -rf plink2_linux_x86_64_20220814.zip diff --git a/docker/scripts/install_plink2_avx2.sh b/docker/scripts/install_plink2_avx2.sh index c1805ed2..2d199da2 100644 --- a/docker/scripts/install_plink2_avx2.sh +++ b/docker/scripts/install_plink2_avx2.sh @@ -1,4 +1,6 @@ #!/bin/sh +set -euo pipefail + wget --no-check-certificate https://s3.amazonaws.com/plink2-assets/alpha3/plink2_linux_avx2_20220814.zip && \ unzip -j plink2_linux_avx2_20220814.zip && \ rm -rf plink2_linux_avx2_20220814.zip diff --git a/docker/scripts/install_prsice.sh b/docker/scripts/install_prsice.sh index 4e20b96e..42c52aa7 100644 --- a/docker/scripts/install_prsice.sh +++ b/docker/scripts/install_prsice.sh @@ -1,8 +1,7 @@ #!/bin/sh - +set -euo pipefail # Prsice - wget --no-check-certificate https://github.com/choishingwan/PRSice/releases/download/2.3.3/PRSice_linux.zip && \ unzip PRSice_linux.zip && \ rm -rf PRSice_linux.zip diff --git a/docker/scripts/install_qctools.sh b/docker/scripts/install_qctools.sh index ded8f0e7..e68931cf 100644 --- a/docker/scripts/install_qctools.sh +++ b/docker/scripts/install_qctools.sh @@ -1,8 +1,7 @@ #!/bin/sh - +set -euo pipefail # qctools - wget --no-check-certificate https://www.well.ox.ac.uk/~gav/resources/qctool_v2.0.6-Ubuntu16.04-x86_64.tgz && \ tar -xvzf qctool_v2.0.6-Ubuntu16.04-x86_64.tgz && \ rm -rf qctool_v2.0.6-Ubuntu16.04-x86_64.tgz diff --git a/docker/scripts/install_regenie.sh b/docker/scripts/install_regenie.sh index 80fb8b6b..7de593fb 100644 --- a/docker/scripts/install_regenie.sh +++ b/docker/scripts/install_regenie.sh @@ -1,4 +1,6 @@ #!/bin/sh +set -euo pipefail + wget --no-check-certificate https://github.com/rgcgithub/regenie/releases/download/v3.2.8/regenie_v3.2.8.gz_x86_64_Linux_mkl.zip && \ unzip -j regenie_v3.2.8.gz_x86_64_Linux_mkl.zip && \ rm -rf regenie_v3.2.8.gz_x86_64_Linux_mkl.zip && \ diff --git a/docker/scripts/install_shapeit4.sh b/docker/scripts/install_shapeit4.sh index 56f003bf..cc1cf69e 100644 --- a/docker/scripts/install_shapeit4.sh +++ b/docker/scripts/install_shapeit4.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -euo pipefail # additional deps apt-get update && apt-get install --no-install-recommends \ diff --git a/docker/scripts/install_shapeit5.sh b/docker/scripts/install_shapeit5.sh new file mode 100644 index 00000000..952d0acf --- /dev/null +++ b/docker/scripts/install_shapeit5.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -euo pipefail + +# build shapeit5 +git clone --depth 1 -b v5.1.1 https://github.com/odelaneau/shapeit5.git --recurse-submodules && \ + cd shapeit5 && \ + cd xcftools && \ + git apply ../../xcftools.diff && \ + cd .. && \ + git apply ../shapeit5.diff && \ + make -j 4 && \ + cp phase_common/bin/phase_common /usr/bin/ && \ + cp phase_rare/bin/phase_rare /usr/bin/ && \ + cp ligate/bin/ligate /usr/bin/ && \ + cp switch/bin/switch /usr/bin/ && \ + cp xcftools/bin/xcftools /usr/bin/ + diff --git a/docker/scripts/install_simu.sh b/docker/scripts/install_simu.sh index a5e31faa..298bc673 100644 --- a/docker/scripts/install_simu.sh +++ b/docker/scripts/install_simu.sh @@ -1,4 +1,6 @@ #!/bin/sh +set -euo pipefail + wget --no-check-certificate https://github.com/precimed/simu/releases/download/v0.9.4/simu_linux chmod +x simu_linux cp simu_linux /bin/ diff --git a/docker/scripts/install_switchError.sh b/docker/scripts/install_switchError.sh new file mode 100644 index 00000000..fb970c12 --- /dev/null +++ b/docker/scripts/install_switchError.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -euo pipefail + +apt-get update && apt-get install --no-install-recommends \ + r-mathlib=3.6.3-2 \ + libboost-iostreams1.71-dev=1.71.0-6ubuntu6 \ + -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +git clone https://github.com/ofrei/switcherror.git && \ + cd switcherror && \ + git checkout 6e688b1f03972b6bfb965ed8eb341b9309c75207 + git apply ../switchError.diff && \ + make -j 4 && \ + cp bin/switchError /usr/bin/. diff --git a/docker/scripts/install_vcftools.sh b/docker/scripts/install_vcftools.sh index 274a4c53..0fd28b21 100644 --- a/docker/scripts/install_vcftools.sh +++ b/docker/scripts/install_vcftools.sh @@ -1,8 +1,8 @@ #!/bin/sh - +set -euo pipefail # vcfools -git clone --depth 1 https://github.com/vcftools/vcftools.git && \ +git clone https://github.com/vcftools/vcftools.git && \ cd vcftools && \ git reset --hard 581c231991cb4db017b92eabc573e17128541ab5 && \ ./autogen.sh && \ diff --git a/docker/scripts/move_singularity_file.sh b/docker/scripts/move_singularity_file.sh index da0f59c5..4ed2c7e0 100755 --- a/docker/scripts/move_singularity_file.sh +++ b/docker/scripts/move_singularity_file.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash +set -euo pipefail + # helper script to change ownership to $USER of built container, # and moving file to the appropriate directory in repository -set -e - if [[ $(/usr/bin/id -u) -ne 0 ]]; then echo "Must run script with sudo or as root" exit diff --git a/docker/scripts/patches/eagle.diff b/docker/scripts/patches/eagle.diff new file mode 100644 index 00000000..6811ef46 --- /dev/null +++ b/docker/scripts/patches/eagle.diff @@ -0,0 +1,29 @@ +diff --git a/src/Makefile b/src/Makefile +index 08dfcb2..f623507 100644 +--- a/src/Makefile ++++ b/src/Makefile +@@ -9,12 +9,12 @@ + + ### modify these paths to local BLAS, Boost and htslib install directories + LBLAS = -lopenblas # alternatively -llapack (just need sgemm_) +-BLAS_DIR = /n/app/openblas/0.2.19/lib # or /n/groups/price/poru/external_software/openblas/0.2.14/lib (copied from Orchestra /opt/openblas/0.2.14/lib) ++BLAS_DIR = usr/lib # or /n/groups/price/poru/external_software/openblas/0.2.14/lib (copied from Orchestra /opt/openblas/0.2.14/lib) + +-BOOST_INSTALL_DIR = /home/pl88/boost_1_58_0/install ++BOOST_INSTALL_DIR = /usr/include/boost + # boost_1_58_0 was installed using GCC 4.8.5? + +-HTSLIB_DIR = /n/groups/price/poru/external_software/htslib/htslib-1.9 ++HTSLIB_DIR = /usr + ### NOTE: to reduce dependencies from htslib, configure htslib with: ./configure --disable-bz2 --disable-lzma --disable-plugins --disable-libcurl --disable-gcs --disable-s3 + + ### only for static linking (unnecessary in a local install; probably unnecessary on most systems) +@@ -96,7 +96,7 @@ endif + + + T = eagle +-O = DipTreePBWT.o Eagle.o EagleImpMiss.o EagleParams.o EaglePBWT.o FileUtils.o GenoData.o HapHedge.o MapInterpolater.o MemoryUtils.o NumericUtils.o StaticMultimap.o StringUtils.o SyncedVcfData.o Timer.o memcpy.o ++O = DipTreePBWT.o Eagle.o EagleImpMiss.o EagleParams.o EaglePBWT.o FileUtils.o GenoData.o HapHedge.o MapInterpolater.o MemoryUtils.o NumericUtils.o StaticMultimap.o StringUtils.o SyncedVcfData.o Timer.o + OMAIN = EagleMain.o $O + + .PHONY: clean diff --git a/docker/scripts/patches/shapeit5.diff b/docker/scripts/patches/shapeit5.diff new file mode 100644 index 00000000..af8de1f7 --- /dev/null +++ b/docker/scripts/patches/shapeit5.diff @@ -0,0 +1,75 @@ +diff --git a/ligate/makefile b/ligate/makefile +index 2d8c428..e063cc1 100644 +--- a/ligate/makefile ++++ b/ligate/makefile +@@ -29,8 +29,8 @@ BFILE=bin/$(NAME) + EXEFILE=bin/$(NAME)_static + + #CONDITIONAL PATH DEFINITON +-system: DYN_LIBS=-lz -lpthread -lbz2 -llzma +-system: HTSSRC=/usr/local ++system: DYN_LIBS=-lz -lpthread -lbz2 -llzma -lcurl -lcrypto ++system: HTSSRC=/usr + system: HTSLIB_INC=$(HTSSRC)/include/htslib + system: HTSLIB_LIB=$(HTSSRC)/lib/libhts.a + system: BOOST_INC=/usr/include +diff --git a/phase_common/makefile b/phase_common/makefile +index e2a051c..9b47951 100644 +--- a/phase_common/makefile ++++ b/phase_common/makefile +@@ -29,8 +29,8 @@ BFILE=bin/$(NAME) + EXEFILE=bin/$(NAME)_static + + #CONDITIONAL PATH DEFINITON +-system: DYN_LIBS=-lz -lpthread -lbz2 -llzma +-system: HTSSRC=/usr/local ++system: DYN_LIBS=-lz -lpthread -lbz2 -llzma -lcurl -lcrypto ++system: HTSSRC=/usr + system: HTSLIB_INC=$(HTSSRC)/include/htslib + system: HTSLIB_LIB=$(HTSSRC)/lib/libhts.a + system: BOOST_INC=/usr/include +diff --git a/phase_rare/makefile b/phase_rare/makefile +index e2a051c..9b47951 100644 +--- a/phase_rare/makefile ++++ b/phase_rare/makefile +@@ -29,8 +29,8 @@ BFILE=bin/$(NAME) + EXEFILE=bin/$(NAME)_static + + #CONDITIONAL PATH DEFINITON +-system: DYN_LIBS=-lz -lpthread -lbz2 -llzma +-system: HTSSRC=/usr/local ++system: DYN_LIBS=-lz -lpthread -lbz2 -llzma -lcurl -lcrypto ++system: HTSSRC=/usr + system: HTSLIB_INC=$(HTSSRC)/include/htslib + system: HTSLIB_LIB=$(HTSSRC)/lib/libhts.a + system: BOOST_INC=/usr/include +diff --git a/simulate/makefile b/simulate/makefile +index e2a051c..9b47951 100644 +--- a/simulate/makefile ++++ b/simulate/makefile +@@ -29,8 +29,8 @@ BFILE=bin/$(NAME) + EXEFILE=bin/$(NAME)_static + + #CONDITIONAL PATH DEFINITON +-system: DYN_LIBS=-lz -lpthread -lbz2 -llzma +-system: HTSSRC=/usr/local ++system: DYN_LIBS=-lz -lpthread -lbz2 -llzma -lcurl -lcrypto ++system: HTSSRC=/usr + system: HTSLIB_INC=$(HTSSRC)/include/htslib + system: HTSLIB_LIB=$(HTSSRC)/lib/libhts.a + system: BOOST_INC=/usr/include +diff --git a/switch/makefile b/switch/makefile +index 2d8c428..e063cc1 100644 +--- a/switch/makefile ++++ b/switch/makefile +@@ -29,8 +29,8 @@ BFILE=bin/$(NAME) + EXEFILE=bin/$(NAME)_static + + #CONDITIONAL PATH DEFINITON +-system: DYN_LIBS=-lz -lpthread -lbz2 -llzma +-system: HTSSRC=/usr/local ++system: DYN_LIBS=-lz -lpthread -lbz2 -llzma -lcurl -lcrypto ++system: HTSSRC=/usr + system: HTSLIB_INC=$(HTSSRC)/include/htslib + system: HTSLIB_LIB=$(HTSSRC)/lib/libhts.a + system: BOOST_INC=/usr/include diff --git a/docker/scripts/patches/switchError.diff b/docker/scripts/patches/switchError.diff new file mode 100644 index 00000000..3f8ef4cc --- /dev/null +++ b/docker/scripts/patches/switchError.diff @@ -0,0 +1,51 @@ +diff --git a/makefile b/makefile +index 29aa8b7..424b5f0 100644 +--- a/makefile ++++ b/makefile +@@ -1,12 +1,12 @@ + ########################################## + # SET CORRECTLY THESE 6 PATHS TO COMPILE # + ########################################## +-BOOST_INC= +-BOOST_LIB= +-RMATH_INC= +-RMATH_LIB= +-HTSLD_INC= +-HTSLD_LIB= ++BOOST_INC=/usr/include ++BOOST_LIB=/usr/lib/x86_64-linux-gnu ++RMATH_INC=/usr/include ++RMATH_LIB=/usr/lib ++HTSLD_INC=/usr/include ++HTSLD_LIB=/usr/lib + + #COMPILER MODE C++11 + CXX=g++ -std=c++0x +@@ -17,10 +17,10 @@ CXXFLAG_DBG=-g + CXXFLAG_WRN=-Wall -Wextra -Wno-sign-compare -Wno-unused-local-typedefs -Wno-deprecated -Wno-unused-parameter + + #BASE LIBRARIES +-LIB_FLAGS=-lz -lbz2 -lm -lpthread -llzma ++LIB_FLAGS=-lz -lbz2 -lm -lpthread -llzma -lcurl -lcrypto + + #FILE LISTS +-BFILE=bin/swithError ++BFILE=bin/switchError + HFILE=$(shell find src -name *.h) + TFILE=$(shell find lib -name *.h) + CFILE=$(shell find src -name *.cpp) +@@ -31,10 +31,10 @@ VPATH=$(shell for file in `find src -name *.cpp`; do echo $$(dirname $$file); do + all: desktop + + #UNIGE DESKTOP RELEASE VERSION +-desktop: RMATH_INC=$(HOME)/Tools/R-3.5.1/src/include +-desktop: RMATH_LIB=$(HOME)/Tools/R-3.5.1/src/nmath/standalone +-desktop: HTSLD_INC=$(HOME)/Tools/htslib-1.9 +-desktop: HTSLD_LIB=$(HOME)/Tools/htslib-1.9 ++desktop: RMATH_INC=/usr/include ++desktop: RMATH_LIB=/usr/lib ++desktop: HTSLD_INC=/usr/include ++desktop: HTSLD_LIB=/usr/lib + desktop: BOOST_INC=/usr/include + desktop: BOOST_LIB=/usr/lib/x86_64-linux-gnu + desktop: CXXFLAG=$(CXXFLAG_REL) $(CXXFLAG_WRN) diff --git a/docker/scripts/patches/xcftools.diff b/docker/scripts/patches/xcftools.diff new file mode 100644 index 00000000..2f86e8ce --- /dev/null +++ b/docker/scripts/patches/xcftools.diff @@ -0,0 +1,15 @@ +diff --git a/makefile b/makefile +index 97e0f6b..3a9631d 100644 +--- a/makefile ++++ b/makefile +@@ -29,8 +29,8 @@ BFILE=bin/$(NAME) + EXEFILE=bin/$(NAME)_static + + #CONDITIONAL PATH DEFINITON +-system: DYN_LIBS=-lz -lpthread -lbz2 -llzma +-system: HTSSRC=/usr/local ++system: DYN_LIBS=-lz -lpthread -lbz2 -llzma -lcurl -lcrypto ++system: HTSSRC=/usr + system: HTSLIB_INC=$(HTSSRC)/include/htslib + system: HTSLIB_LIB=$(HTSSRC)/lib/libhts.a + system: BOOST_INC=/usr/include diff --git a/docker/scripts/python_convert.sh b/docker/scripts/python_convert.sh index 03d682d2..14009d01 100644 --- a/docker/scripts/python_convert.sh +++ b/docker/scripts/python_convert.sh @@ -1,5 +1,4 @@ #!/bin/sh - - +set -euo pipefail git clone https://github.com/precimed/python_convert.git diff --git a/docker/scripts/stub.sh b/docker/scripts/stub.sh new file mode 100644 index 00000000..8b5f1c4a --- /dev/null +++ b/docker/scripts/stub.sh @@ -0,0 +1,10 @@ +#!/bin/sh +MYSELF=`which "$0" 2>/dev/null` +[ $? -gt 0 -a -f "$0" ] && MYSELF="./$0" +java=java +if test -n "$JAVA_HOME"; then + java="$JAVA_HOME/bin/java" +fi +java_args=-Xmx1g +exec "$java" $java_args -jar $MYSELF "$@" +exit 1 \ No newline at end of file diff --git a/singularity/gwas.sif b/singularity/gwas.sif index b32b068c..35cb9c66 100644 --- a/singularity/gwas.sif +++ b/singularity/gwas.sif @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:db7a9e7dc6413a1b2cc5a58ee0abe8188675e63a4f2774d4b4580741edeed452 -size 405778432 +oid sha256:6499b892f4bc586167cdcef28a7c7ba253a472080eed19c64b13d664e7ae511b +size 573915136 diff --git a/tests/test_gwas.py b/tests/test_gwas.py index dfa67d34..8763c4d7 100644 --- a/tests/test_gwas.py +++ b/tests/test_gwas.py @@ -7,6 +7,7 @@ import os import subprocess import tempfile +import pytest pth = os.path.join('singularity', 'gwas.sif') @@ -18,6 +19,12 @@ def test_gwas_bcftools(): assert out.returncode == 0 +def test_gwas_beagle(): + call = f'singularity run {pth} beagle' + out = subprocess.run(call.split(' ')) + assert out.returncode == 0 + + def test_gwas_bgenix(): for bin in ['bgenix', 'cat-bgen', 'edit-bgen']: call = f'singularity run {pth} {bin} -help' @@ -31,6 +38,12 @@ def test_gwas_bolt(): assert out.returncode == 0 +def test_gwas_eagle(): + call = f'singularity run {pth} eagle -h' + out = subprocess.run(call.split(' ')) + assert out.returncode == 0 + + def test_gwas_flashpca(): call = f'singularity run {pth} flashpca_x86-64 --version' out = subprocess.run(call.split(' ')) @@ -144,6 +157,21 @@ def test_gwas_shapeit4(): assert out.returncode == 0 +def test_gwas_shapeit5(): + for binary in ['phase_common', 'phase_rare', + 'ligate', 'switch', 'xcftools']: + call = f'singularity run {pth} {binary} --help' + out = subprocess.run(call.split(' ')) + assert out.returncode == 0 + + +@pytest.mark.xfail(reason="no help function for switchError") +def test_gwas_switchError(): + call = f'singularity run {pth} switchError --reg foo --gen foo --hap foo --fam foo --ps foo --out foo --maf 0.0' + out = subprocess.run(call.split(' ')) + assert out.returncode == 0 + + def test_gwas_simu(): cwd = os.getcwd() with tempfile.TemporaryDirectory() as d: @@ -162,3 +190,9 @@ def test_gwas_vcftools(): call = f'singularity run {pth} vcftools --version' out = subprocess.run(call.split(' ')) assert out.returncode == 0 + + +def test_gwas_duohmm(): + call = f'singularity run {pth} duohmm' + out = subprocess.run(call.split(' ')) + assert out.returncode == 0 diff --git a/version/version.py b/version/version.py index aeca514a..7a518021 100644 --- a/version/version.py +++ b/version/version.py @@ -1,8 +1,8 @@ _MAJOR = "1" -_MINOR = "3" +_MINOR = "4" # On main and in a nightly release the patch should be one ahead of the last # released build. -_PATCH = "9" +_PATCH = "0" # This is mainly for nightly builds which have the suffix ".dev$DATE". See # https://semver.org/#is-v123-a-semantic-version for the semantics. _SUFFIX = ""