gwas.sif: Include samtools, bedtools, liftOver; upgrade others

Fixes #219

CHANGELOG.md

@@ -43,7 +43,28 @@ If MD5 sum is not listed for a certain release then it means that the container
 
 ### Added
 
-* Added `samtools` to `gwas.sif` container
+* Added `samtools 1.19.2`, `bedtools 2.31.1`, `liftOver (latest)` to `gwas.sif` container
+
+### Updated
+
+* Updated the following binaries (not listing apt package updates)
+  * bcftools to 1.19
+  * bolt to 2.4.1
+  * gcta to 1.94.1
+  * gctb to 2.04.3
+  * htslib to 1.19.1
+  * king to 2.3.2
+  * minimac4 to 4.1.6
+  * plink to v1.90b7.2 64-bit (11 Dec 2023)
+  * plink2 to v2.00a5.10LM 64-bit Intel (5 Jan 2024)
+  * plink2_avx2 to v2.00a5.10LM AVX2 Intel (5 Jan 2024)
+  * PRSice_linux to 2.3.5
+  * regenie to 3.4
+  * vcftools to git SHA: d511f469e87c2ac9779bcdc3670b2b51667935fe
+
+### Fixed
+
+* Broken `docker/scripts/build_docker.sh` script
 
 ## [1.5.1] - 2023-10.20
 

docker/Makefile

@@ -8,13 +8,13 @@ all: hello.sif gwas.sif python3.sif r.sif saige.sif
 # Example commands executed by this make file
 # 
 # Run "make hello.sif" will trigger this:
-# docker build -t hello -f dockerfiles/hello/Dockerfile . && \  # build docker container
+# docker scripts/build_docker.sh hello && \  # build docker image
 #     scripts/convert_docker_image_to_singularity.sh hello && \         # produces hello.sif
 #     scripts/move_singularity_file.sh hello        # moves hello.sif to ../singularity/.
 #  
 # Run "make gwas.sif" will trigger this:
-# docker build -t gwas -f dockerfiles/gwas/Dockerfile . && \
-#     scripts/convert_docker_image_to_singularity.sh gwas && \       # produces gwas.sif
-#     scripts/move_singularity_file.sh gwas         # moves gwas.sif to ../singularity/.
+# docker scripts/build_docker.sh gwas && \  # build docker image
+#     scripts/convert_docker_image_to_singularity.sh gwas && \  # produces gwas.sif
+#     scripts/move_singularity_file.sh gwas   # moves gwas.sif to ../singularity/.
 #
 # Build logs are stored in the ./logs/ directory

docker/README.md

@@ -37,29 +37,31 @@ The detailed description of the available container [files](https://github.com/c
   | hello.sif         | ubuntu              | 20.04                                     | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy)
   | hello.sif         | plink               | v1.90b6.18 64-bit (16 Jun 2020)           | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
   | gwas.sif          | ubuntu              | 20.04                                     | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy)
-  | gwas.sif          | bcftools            | 1.12 (using htslib 1.12)                  | [MIT/Expat/GPLv3](https://github.com/samtools/bcftools/blob/develop/LICENSE)
+  | gwas.sif          | bcftools            | 1.19                                      | [MIT/Expat/GPLv3](https://github.com/samtools/bcftools/blob/develop/LICENSE)
+  | gwas.sif          | bedtools            | 2.31.1                                    | [MIT](https://opensource.org/licenses/MIT)
   | gwas.sif          | beagle              | 22Jul22.46e                               | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
   | gwas.sif          | bgenix              | 1.1.7                                     | [Boost](https://www.boost.org/LICENSE_1_0.txt)
-  | gwas.sif          | bolt                | v2.4 July 22, 2022                        | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
+  | gwas.sif          | bolt                | v2.4.1                                    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
   | gwas.sif          | cat-bgen            | same version as bgenix                    | [Boost](https://www.boost.org/LICENSE_1_0.txt)
   | gwas.sif          | duohmm              | 95bd395                                   | [MIT](https://opensource.org/licenses/MIT)
   | gwas.sif          | eagle               | v2.4.1                                    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
   | gwas.sif          | edit-bgen           | same version as bgenix                    | [Boost](https://www.boost.org/LICENSE_1_0.txt)
   | gwas.sif          | flashpca_x86-64     | 2.0                                       | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
-  | gwas.sif          | gcta64              | version 1.93.3 beta 2 Linux               | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
-  | gwas.sif          | gctb                | 2.02                                      | [MIT](https://opensource.org/licenses/MIT)
+  | gwas.sif          | gcta64              | 1.94.1                                    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
+  | gwas.sif          | gctb                | 2.04.3                                    | [MIT](https://opensource.org/licenses/MIT)
   | gwas.sif          | qctool              | 2.0.6, revision 18b8f17                   | [Boost](https://www.boost.org/LICENSE_1_0.txt)
   | gwas.sif          | GWAMA               | 2.2.2                                     | [BSD-3-Clause](https://opensource.org/licenses/BSD-3-Clause)
-  | gwas.sif          | HTSlib              | 1.12                                      | [MIT/Expat/Modified-BSD](https://github.com/samtools/htslib/blob/develop/LICENSE)
-  | gwas.sif          | king                | 2.2.9 - (c)                               | [permissive](https://www.kingrelatedness.com/Download.shtml)
+  | gwas.sif          | HTSlib              | 1.19.1                                    | [MIT/Expat/Modified-BSD](https://github.com/samtools/htslib/blob/develop/LICENSE)
+  | gwas.sif          | king                | 2.3.2                                     | [permissive](https://www.kingrelatedness.com/Download.shtml)
+  | gwas.sif          | liftOver            | latest                                    | [permissive](https://genome-store.ucsc.edu)
   | gwas.sif          | metal               | version released on 2011-03-25            | -
-  | gwas.sif          | minimac4            | v4.1.0                                    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
-  | gwas.sif          | plink               | v1.90b6.18 64-bit (16 Jun 2020)           | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
-  | gwas.sif          | plink2              | v2.00a3.6LM 64-bit Intel (14 Aug 2022)    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
-  | gwas.sif          | plink2_avx2         | v2.00a3.6LM AVX2 Intel (24 Jan 2020)      | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
-  | gwas.sif          | PRSice_linux        | 2.3.3 (2020-08-05)                        | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
-  | gwas.sif          | regenie             | v3.2.8                                    | [MIT/Boost](https://github.com/rgcgithub/regenie/blob/master/LICENSE)
-  | gwas.sif          | samtools            | v1.12                                     | [MIT/ExpatD](https://github.com/samtools/samtools/blob/develop/LICENSE)
+  | gwas.sif          | minimac4            | v4.1.6                                    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
+  | gwas.sif          | plink               | v1.90b7.2 64-bit (11 Dec 2023)            | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
+  | gwas.sif          | plink2              | v2.00a5.10LM 64-bit Intel (5 Jan 2024)    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
+  | gwas.sif          | plink2_avx2         | v2.00a5.10LM AVX2 Intel (5 Jan 2024)      | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
+  | gwas.sif          | PRSice_linux        | 2.3.5                                     | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
+  | gwas.sif          | regenie             | v3.4                                      | [MIT/Boost](https://github.com/rgcgithub/regenie/blob/master/LICENSE)
+  | gwas.sif          | samtools            | v1.19.2                                   | [MIT/ExpatD](https://github.com/samtools/samtools/blob/develop/LICENSE)
   | gwas.sif          | shapeit4.2          | v4.2.2                                    | [MIT](https://opensource.org/licenses/MIT)
   | gwas.sif          | shapeit5 phase_rare | v5.1.1                                    | [MIT](https://opensource.org/licenses/MIT)
   | gwas.sif          | shapeit5 phase_common | v5.1.1                                  | [MIT](https://opensource.org/licenses/MIT)
@@ -68,7 +70,7 @@ The detailed description of the available container [files](https://github.com/c
   | gwas.sif          | shapeit5 xcftools   | v5.1.1                                    | [MIT](https://opensource.org/licenses/MIT)
   | gwas.sif          | simu_linux          | v0.9.4                                    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
   | gwas.sif          | switchError         | 6e688b1                                   | [MIT](https://opensource.org/licenses/MIT)
-  | gwas.sif          | vcftools            | 0.1.17                                    | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
+  | gwas.sif          | vcftools            | 0.1.17 (git SHA: d511f469e)               | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)
   | python3.sif       | ubuntu              | 20.04 (LTS)                               | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy)
   | python3.sif       | LDpred              | 1.0.11                                    | [MIT](https://opensource.org/licenses/MIT)
   | python3.sif       | plink               | v1.90b6.18 64-bit (16 Jun 2020)           | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html)

docker/dockerfiles/gwas/Dockerfile

@@ -101,14 +101,6 @@ COPY /scripts/install_flashpca.sh /tmp/flashpca/
 RUN bash /tmp/flashpca/install_flashpca.sh && \
     rm -rf /tmp/flashpca
 
-# regenie
-WORKDIR /tmp/regenie
-COPY /scripts/install_regenie.sh /tmp/regenie/
-RUN bash /tmp/regenie/install_regenie.sh && \
-    rm -rf /tmp/regenie
-# https://github.com/potree/PotreeConverter/issues/281
-ENV LC_ALL="C"
-
 # gwama
 WORKDIR /tmp/gwama
 COPY /scripts/install_gwama.sh /tmp/gwama/
@@ -127,6 +119,14 @@ COPY /scripts/install_bgenix.sh /tmp/bgenix/
 RUN bash /tmp/bgenix/install_bgenix.sh && \
     rm -rf /tmp/bgenix
 
+# regenie (install after bgenix)
+WORKDIR /tmp/regenie
+COPY /scripts/install_regenie.sh /tmp/regenie/
+RUN bash /tmp/regenie/install_regenie.sh && \
+    rm -rf /tmp/regenie
+# https://github.com/potree/PotreeConverter/issues/281
+ENV LC_ALL="C"
+
 # shapeit4
 WORKDIR /tmp/shapeit4
 COPY /scripts/install_shapeit4.sh /tmp/shapeit4/
@@ -163,18 +163,28 @@ COPY /scripts/stub.sh /tmp/beagle/
 RUN bash /tmp/beagle/install_beagle.sh && \
     rm -rf /tmp/beagle
 
-
 # duohmm
 WORKDIR /tmp/duohmm
 COPY /scripts/install_duohmm.sh /tmp/duohmm/
 RUN bash /tmp/duohmm/install_duohmm.sh && \
     rm -rf /tmp/duohmm
 
-
 # samtools
 WORKDIR /tmp/samtools
 COPY /scripts/install_samtools.sh /tmp/samtools/
 RUN bash /tmp/samtools/install_samtools.sh && \
     rm -rf /tmp/samtools
 
+# bedtools
+WORKDIR /tmp/bedtools
+COPY /scripts/install_bedtools.sh /tmp/bedtools/
+RUN bash /tmp/bedtools/install_bedtools.sh && \
+    rm -rf /tmp/bedtools
+
+# liftOver
+WORKDIR /tmp/liftover
+COPY /scripts/install_liftover.sh /tmp/liftover/
+RUN bash /tmp/liftover/install_liftover.sh && \
+    rm -rf /tmp/liftover
+
 WORKDIR /tools

docker/scripts/apt_get_essential.sh

@@ -29,6 +29,7 @@ apt-get update && apt-get install -y --no-install-recommends \
    parallel=20161222-1.1 \
    perl=5.30.0-9ubuntu0.5 \
    pkg-config=0.29.1-0ubuntu4 \
+   python3=3.8.2-0ubuntu2 \
    tar=1.30+dfsg-7ubuntu0.20.04.4 \
    tofrodos=1.7.13+ds-4 \
    unzip=6.0-25ubuntu1.1 \
@@ -38,4 +39,6 @@ apt-get update && apt-get install -y --no-install-recommends \
    && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*
-
+
+# /usr/bin/python must exist for bgenix, qctool
+update-alternatives --install /usr/bin/python python /usr/bin/python3 10

docker/scripts/build_docker.sh

@@ -17,11 +17,14 @@ trap 'exit' ERR
 # this is a personal access token from github with read:packages scope and will have
 # to be updated every 30 days.
 if [ $1 == "r" ]; then
-    source ~/.bash_profile
+    source ~/.bashrc
     if [ -z "$GITHUB_PAT" ]; then
         echo "GITHUB_PAT not set"
         exit 1
     fi
+    # build docker image
+    docker build --build-arg GITHUB_PAT="${GITHUB_PAT}" -t $1 -f dockerfiles/$1/Dockerfile .
+else 
+    # build docker image
+    docker build -t $1 -f dockerfiles/$1/Dockerfile .
 fi
-# build docker image
-docker build --build-arg GITHUB_PAT=$GITHUB_PAT -t $1 -f dockerfiles/$1/Dockerfile .

docker/scripts/install_bcftools.sh

@@ -10,7 +10,7 @@ apt-get update && apt-get install -y --no-install-recommends \
     rm -rf /var/lib/apt/lists/*
 
 # bcftools
-git clone --recurse-submodules --depth 1 -b 1.12 https://github.com/samtools/bcftools.git && \
+git clone --recurse-submodules --depth 1 -b 1.19 https://github.com/samtools/bcftools.git && \
 cd bcftools && \
 autoheader && autoconf && ./configure --enable-libgsl --enable-perl-filters --with-htslib=/usr/ && \
 make -j12 && \

docker/scripts/install_bedtools.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -euo pipefail
+
+VERSION="2.31.1"
+wget https://github.com/arq5x/bedtools2/releases/download/v$VERSION/bedtools-$VERSION.tar.gz
+tar -zxvf bedtools-$VERSION.tar.gz
+cd bedtools2
+make -j4
+cp bin/* /bin
+

docker/scripts/install_bgenix.sh

@@ -1,25 +1,9 @@
 #!/bin/sh
 set -euo pipefail
 
-# python appears to be a build time dependency
-apt-get update && apt-get install -y  --no-install-recommends python3=3.8.2-0ubuntu2 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# /usr/bin/python must exist
-update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-
 # build bgen
-wget http://code.enkre.net/bgen/tarball/release/bgen.tgz && \
-    tar -xvzf bgen.tgz && mv bgen.tgz/* . && \
-    ./waf configure && \
-    ./waf -v
-# copy binaries
-cp build/apps/bgenix /bin && \
-    cp build/apps/cat-bgen /bin && \
-    cp build/apps/edit-bgen /bin
-
-# remove python
-apt-get purge \
-    python3 -y && \
-    apt-get autoremove --purge -y
+VERSION="1.1.7"
+wget http://code.enkre.net/bgen/tarball/release/v$VERSION.tgz && \
+    tar -xvzf v$VERSION.tgz && mv v$VERSION/* . && \
+    ./waf configure --prefix=/usr && \
+    ./waf install

docker/scripts/install_bolt.sh

@@ -1,9 +1,9 @@
 #!/bin/sh
 set -euo pipefail
 
-export VERSION="v2.3.5"
+export VERSION="v2.4.1"
 
-wget --no-check-certificate https://storage.googleapis.com/broad-alkesgroup-public/BOLT-LMM/downloads/old/BOLT-LMM_$VERSION.tar.gz
+wget --no-check-certificate https://storage.googleapis.com/broad-alkesgroup-public/BOLT-LMM/downloads/BOLT-LMM_$VERSION.tar.gz
 tar -xvzf BOLT-LMM_$VERSION.tar.gz
 rm -rf BOLT-LMM_$VERSION.tar.gz
 mv BOLT-LMM_$VERSION/* .

docker/scripts/install_gcta.sh

@@ -2,8 +2,8 @@
 set -euo pipefail
 
 # gcta
-curl -O -J -L https://github.com/jianyangqt/gcta/releases/download/v1.93.3beta2/gcta_1.93.3beta2.zip && \
-    unzip -j  gcta_1.93.3beta2.zip && \
-    rm -rf gcta_1.93.3beta2.zip
+VERSION="1.94.1"
+curl -O -J -L https://github.com/jianyangqt/gcta/releases/download/v$VERSION/gcta-$VERSION-linux-x86_64-static && \
+    mv gcta-$VERSION-linux-x86_64-static /bin/gcta64 && \
+    chmod +x /bin/gcta64
 
-cp gcta64  /bin

docker/scripts/install_gctb.sh

@@ -2,11 +2,12 @@
 set -euo pipefail
 
 # gctb
-wget --no-check-certificate https://cnsgenomics.com/software/gctb/download/gctb_2.02_Linux.zip  && \
-   unzip   gctb_2.02_Linux.zip && \
-   rm -rf gctb_2.02_Linux.zip
+VERSION="2.04.3"
+wget --no-check-certificate https://cnsgenomics.com/software/gctb/download/gctb_${VERSION}_Linux.zip  && \
+   unzip   gctb_${VERSION}_Linux.zip && \
+   rm -rf gctb_${VERSION}_Linux.zip
 
-mv gctb_2.02_Linux/* .
+mv gctb_${VERSION}_Linux/* .
 cp gctb /bin 
 
 chmod 755 /bin/gctb

docker/scripts/install_htslib.sh

@@ -11,7 +11,7 @@ apt-get update && apt-get install --no-install-recommends \
     rm -rf /var/lib/apt/lists/*
 
 # build and install HTSlib
-git clone --depth 1 -b 1.12 https://github.com/samtools/htslib.git --recurse-submodules && \
+git clone --depth 1 -b 1.19.1 https://github.com/samtools/htslib.git --recurse-submodules && \
     cd htslib && \
     autoreconf -i && \
     ./configure --prefix=/usr/ && \

docker/scripts/install_king.sh

@@ -9,9 +9,10 @@ apt-get update && apt-get install --no-install-recommends \
     rm -rf /var/lib/apt/lists/*
 
 # king
-wget --no-check-certificate https://www.kingrelatedness.com/executables/Linux-king229.tar.gz && \
-    tar -xvzf Linux-king229.tar.gz && \
-    rm -rf Linux-king229.tar.gz
+VERSION="232"
+wget --no-check-certificate https://www.kingrelatedness.com/executables/Linux-king$VERSION.tar.gz && \
+    tar -xvzf Linux-king$VERSION.tar.gz && \
+    rm -rf Linux-king$VERSION.tar.gz
 
 cp king  /bin
 

docker/scripts/install_liftover.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -euo pipefail
+
+wget https://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/liftOver
+chmod +x liftOver
+mv liftOver /bin

docker/scripts/install_minimac4.sh

@@ -15,7 +15,8 @@ apt-get update && \
 pip3 install --no-cache-dir cget==0.2.0
 
 # install Minimac4
-git clone --depth 1 -b v4.1.0 https://github.com/statgen/Minimac4.git && \
+VERSION="v4.1.6"
+git clone --depth 1 -b $VERSION https://github.com/statgen/Minimac4.git && \
 cd Minimac4 && \
 cget install -f ./requirements.txt && \
 mkdir build && cd build && \

docker/scripts/install_plink.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 # plink
-wget --no-check-certificate https://s3.amazonaws.com/plink1-assets/plink_linux_x86_64_20200616.zip && \
-    unzip -j plink_linux_x86_64_20200616.zip && \
-    rm -rf plink_linux_x86_64_20200616.zip
+wget --no-check-certificate https://s3.amazonaws.com/plink1-assets/plink_linux_x86_64_20231211.zip && \
+    unzip -j plink_linux_x86_64_20231211.zip && \
+    rm -rf plink_linux_x86_64_20231211.zip
 cp plink /bin

docker/scripts/install_plink2.sh

@@ -1,8 +1,8 @@
 #!/bin/sh
 set -euo pipefail
 
-wget --no-check-certificate https://s3.amazonaws.com/plink2-assets/alpha3/plink2_linux_x86_64_20220814.zip && \
-    unzip -j plink2_linux_x86_64_20220814.zip && \
-    rm -rf plink2_linux_x86_64_20220814.zip
+wget --no-check-certificate https://s3.amazonaws.com/plink2-assets/alpha5/plink2_linux_x86_64_20240105.zip && \
+    unzip -j plink2_linux_x86_64_20240105.zip && \
+    rm -rf plink2_linux_x86_64_20240105.zip
 
 cp plink2 /bin

docker/scripts/install_plink2_avx2.sh

@@ -1,9 +1,9 @@
 #!/bin/sh
 set -euo pipefail
 
-wget --no-check-certificate https://s3.amazonaws.com/plink2-assets/alpha3/plink2_linux_avx2_20220814.zip && \
-    unzip -j plink2_linux_avx2_20220814.zip && \
-    rm -rf plink2_linux_avx2_20220814.zip
+wget --no-check-certificate https://s3.amazonaws.com/plink2-assets/alpha5/plink2_linux_avx2_20240105.zip && \
+    unzip -j plink2_linux_avx2_20240105.zip && \
+    rm -rf plink2_linux_avx2_20240105.zip
 # we install non-avx plink2 so that default plink runs on any  CPU architecture
 
 

docker/scripts/install_prsice.sh

@@ -2,9 +2,9 @@
 set -euo pipefail
 
 # Prsice
- wget --no-check-certificate https://github.com/choishingwan/PRSice/releases/download/2.3.3/PRSice_linux.zip  && \
+ wget --no-check-certificate https://github.com/choishingwan/PRSice/releases/download/2.3.5/PRSice_linux.zip  && \
     unzip PRSice_linux.zip  && \
     rm -rf PRSice_linux.zip
 
 cp PRSice_linux  /bin
-#cp PRSice.R  /
+chmod +x /bin/PRSice_linux