From 7b3c4043d5b6788b95498a1527d9590e74ec1681 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 2 Oct 2017 18:57:02 -0400 Subject: [PATCH 01/15] enh: generate dockerfiles with neurodocker + rename base dockerfile Rename base.Dockerfile to Dockerfile.base to conform to moby style. For now, the master Neurodocker Docker image is used, but a versioned image will be used once a newer Neurodocker version is released. --- Dockerfile | 297 ++++++++++++++++++++++----------- docker/Dockerfile.base | 234 ++++++++++++++++++++++++++ docker/base.Dockerfile | 151 ----------------- docker/generate_dockerfiles.sh | 75 +++++++++ 4 files changed, 508 insertions(+), 249 deletions(-) create mode 100644 docker/Dockerfile.base delete mode 100644 docker/base.Dockerfile create mode 100755 docker/generate_dockerfiles.sh diff --git a/Dockerfile b/Dockerfile index 14a6dec135..073185f1cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,111 +1,212 @@ -# Copyright (c) 2016, The developers of the Stanford CRN -# All rights reserved. +# Generated by Neurodocker v0.3.1-2-g4dfcf56. # -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: +# Thank you for using Neurodocker. If you discover any issues +# or ways to improve this software, please submit an issue or +# pull request on our GitHub repository: +# https://github.com/kaczmarj/neurodocker # -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of crn_base nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Timestamp: 2017-10-02 22:55:57 +FROM kaczmarj/nipype:base -# -# Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile -# -FROM nipype/base:latest -MAINTAINER The nipype developers https://github.com/nipy/nipype - -ARG PYTHON_VERSION_MAJOR=3 - -# Installing and setting up miniconda -RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh && \ - bash Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \ - rm Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh - -ENV PATH=/usr/local/miniconda/bin:$PATH \ - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - ACCEPT_INTEL_PYTHON_EULA=yes \ - MKL_NUM_THREADS=1 \ - OMP_NUM_THREADS=1 -# MKL/OMP_NUM_THREADS: unless otherwise specified, each process should -# only use one thread - nipype will handle parallelization - -# Installing precomputed python packages -ARG PYTHON_VERSION_MINOR=6 -RUN conda config --add channels conda-forge; sync && \ - conda config --set always_yes yes --set changeps1 no; sync && \ - conda install -y python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} \ - mkl \ - numpy \ - scipy \ - scikit-learn \ - matplotlib \ - pandas \ - libxml2 \ - libxslt \ - traits=4.6.0 \ - psutil \ - icu=58.1 && \ - sync; - -# matplotlib cleanups: set default backend, precaching fonts -RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/matplotlib/mpl-data/matplotlibrc && \ - python -c "from matplotlib import font_manager" - -# Install CI scripts -COPY docker/files/run_* /usr/bin/ -RUN chmod +x /usr/bin/run_* - -# Replace imglob with a Python3 compatible version -COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py -RUN rm -rf ${FSLDIR}/bin/imglob && \ - chmod +x /usr/bin/fsl_imglob.py && \ - ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob - -# Installing dev requirements (packages that are not in pypi) -WORKDIR /src/ -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt && \ - rm -rf ~/.cache/pip - -RUN git clone https://github.com/INCF/pybids.git && \ - cd pybids && python setup.py develop - -# Installing nipype -COPY . /src/nipype -RUN cd /src/nipype && \ - pip install -e .[all] && \ - rm -rf ~/.cache/pip - -WORKDIR /work/ +ARG DEBIAN_FRONTEND=noninteractive + +#---------------------------------------------------------- +# Install common dependencies and create default entrypoint +#---------------------------------------------------------- +ENV LANG="en_US.UTF-8" \ + LC_ALL="C.UTF-8" \ + ND_ENTRYPOINT="/neurodocker/startup.sh" +RUN apt-get update -qq && apt-get install -yq --no-install-recommends \ + apt-utils bzip2 ca-certificates curl locales unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && localedef --force --inputfile=en_US --charmap=UTF-8 C.UTF-8 \ + && chmod 777 /opt && chmod a+s /opt \ + && mkdir -p /neurodocker \ + && if [ ! -f "$ND_ENTRYPOINT" ]; then \ + echo '#!/usr/bin/env bash' >> $ND_ENTRYPOINT \ + && echo 'set +x' >> $ND_ENTRYPOINT \ + && echo 'if [ -z "$*" ]; then /usr/bin/env bash; else $*; fi' >> $ND_ENTRYPOINT; \ + fi \ + && chmod -R 777 /neurodocker && chmod a+s /neurodocker +ENTRYPOINT ["/neurodocker/startup.sh"] + +LABEL maintainer="The nipype developers https://github.com/nipy/nipype" + +ENV MKL_NUM_THREADS="1" \ + OMP_NUM_THREADS="1" + +# Create new user: neuro +RUN useradd --no-user-group --create-home --shell /bin/bash neuro +USER neuro + +#------------------ +# Install Miniconda +#------------------ +ENV CONDA_DIR=/opt/conda \ + PATH=/opt/conda/bin:$PATH +RUN echo "Downloading Miniconda installer ..." \ + && miniconda_installer=/tmp/miniconda.sh \ + && curl -sSL -o $miniconda_installer https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && /bin/bash $miniconda_installer -b -p $CONDA_DIR \ + && rm -f $miniconda_installer \ + && conda config --system --prepend channels conda-forge \ + && conda config --system --set auto_update_conda false \ + && conda config --system --set show_channel_urls true \ + && conda update -y -q --all && sync \ + && conda clean -tipsy && sync +#------------------------- +# Create conda environment +#------------------------- +RUN conda create -y -q --name neuro \ + && sync && conda clean -tipsy && sync +ENV PATH=/opt/conda/envs/neuro/bin:$PATH + +COPY ["docker/files/run_builddocs.sh", "docker/files/run_examples.sh", "docker/files/run_pytests.sh", "nipype/external/fsl_imglob.py", "/usr/bin/"] + +COPY [".", "/src/nipype"] + +USER root + +# User-defined instruction +RUN chmod 777 -R /src/nipype + +USER neuro + +ARG PYTHON_VERSION_MAJOR="3" +ARG PYTHON_VERSION_MINOR="6" ARG BUILD_DATE ARG VCS_REF ARG VERSION -LABEL org.label-schema.build-date=$BUILD_DATE \ + +#------------------------- +# Update conda environment +#------------------------- +RUN conda install -y -q --name neuro python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} \ + icu=58.1 \ + libxml2 \ + libxslt \ + matplotlib \ + mkl \ + numpy \ + pandas \ + psutil \ + scikit-learn \ + scipy \ + traits=4.6.0 \ + && sync && conda clean -tipsy && sync \ + && /bin/bash -c "source activate neuro \ + && pip install -q --no-cache-dir -e /src/nipype[all]" \ + && sync + +LABEL org.label-schema.build-date="$BUILD_DATE" \ org.label-schema.name="NIPYPE" \ org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \ org.label-schema.url="http://nipype.readthedocs.io" \ - org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-ref="$VCS_REF" \ org.label-schema.vcs-url="https://github.com/nipy/nipype" \ - org.label-schema.version=$VERSION \ + org.label-schema.version="$VERSION" \ org.label-schema.schema-version="1.0" + +#-------------------------------------- +# Save container specifications to JSON +#-------------------------------------- +RUN echo '{ \ + \n "pkg_manager": "apt", \ + \n "check_urls": false, \ + \n "instructions": [ \ + \n [ \ + \n "base", \ + \n "kaczmarj/nipype:base" \ + \n ], \ + \n [ \ + \n "label", \ + \n { \ + \n "maintainer": "The nipype developers https://github.com/nipy/nipype" \ + \n } \ + \n ], \ + \n [ \ + \n "env", \ + \n { \ + \n "MKL_NUM_THREADS": "1", \ + \n "OMP_NUM_THREADS": "1" \ + \n } \ + \n ], \ + \n [ \ + \n "user", \ + \n "neuro" \ + \n ], \ + \n [ \ + \n "miniconda", \ + \n { \ + \n "env_name": "neuro", \ + \n "add_to_path": true \ + \n } \ + \n ], \ + \n [ \ + \n "copy", \ + \n [ \ + \n "docker/files/run_builddocs.sh", \ + \n "docker/files/run_examples.sh", \ + \n "docker/files/run_pytests.sh", \ + \n "nipype/external/fsl_imglob.py", \ + \n "/usr/bin/" \ + \n ] \ + \n ], \ + \n [ \ + \n "copy", \ + \n [ \ + \n ".", \ + \n "/src/nipype" \ + \n ] \ + \n ], \ + \n [ \ + \n "user", \ + \n "root" \ + \n ], \ + \n [ \ + \n "run", \ + \n "chmod 777 -R /src/nipype" \ + \n ], \ + \n [ \ + \n "user", \ + \n "neuro" \ + \n ], \ + \n [ \ + \n "arg", \ + \n { \ + \n "PYTHON_VERSION_MAJOR": "3", \ + \n "PYTHON_VERSION_MINOR": "6", \ + \n "BUILD_DATE": "", \ + \n "VCS_REF": "", \ + \n "VERSION": "" \ + \n } \ + \n ], \ + \n [ \ + \n "miniconda", \ + \n { \ + \n "env_name": "neuro", \ + \n "conda_install": "python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} icu=58.1 libxml2 libxslt matplotlib mkl numpy pandas psutil scikit-learn scipy traits=4.6.0", \ + \n "pip_opts": "-e", \ + \n "pip_install": "/src/nipype[all]" \ + \n } \ + \n ], \ + \n [ \ + \n "label", \ + \n { \ + \n "org.label-schema.build-date": "$BUILD_DATE", \ + \n "org.label-schema.name": "NIPYPE", \ + \n "org.label-schema.description": "NIPYPE - Neuroimaging in Python: Pipelines and Interfaces", \ + \n "org.label-schema.url": "http://nipype.readthedocs.io", \ + \n "org.label-schema.vcs-ref": "$VCS_REF", \ + \n "org.label-schema.vcs-url": "https://github.com/nipy/nipype", \ + \n "org.label-schema.version": "$VERSION", \ + \n "org.label-schema.schema-version": "1.0" \ + \n } \ + \n ] \ + \n ], \ + \n "generation_timestamp": "2017-10-02 22:55:57", \ + \n "neurodocker_version": "0.3.1-2-g4dfcf56" \ + \n}' > /neurodocker/neurodocker_specs.json diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 0000000000..5735c04b93 --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,234 @@ +# Generated by Neurodocker v0.3.1-2-g4dfcf56. +# +# Thank you for using Neurodocker. If you discover any issues +# or ways to improve this software, please submit an issue or +# pull request on our GitHub repository: +# https://github.com/kaczmarj/neurodocker +# +# Timestamp: 2017-10-02 22:55:55 + +FROM neurodebian@sha256:b09c09faa34bca0ea096b9360ee5121e048594cb8e2d7744d7d546ade88a2996 + +ARG DEBIAN_FRONTEND=noninteractive + +#---------------------------------------------------------- +# Install common dependencies and create default entrypoint +#---------------------------------------------------------- +ENV LANG="en_US.UTF-8" \ + LC_ALL="C.UTF-8" \ + ND_ENTRYPOINT="/neurodocker/startup.sh" +RUN apt-get update -qq && apt-get install -yq --no-install-recommends \ + apt-utils bzip2 ca-certificates curl locales unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && localedef --force --inputfile=en_US --charmap=UTF-8 C.UTF-8 \ + && chmod 777 /opt && chmod a+s /opt \ + && mkdir -p /neurodocker \ + && if [ ! -f "$ND_ENTRYPOINT" ]; then \ + echo '#!/usr/bin/env bash' >> $ND_ENTRYPOINT \ + && echo 'set +x' >> $ND_ENTRYPOINT \ + && echo 'if [ -z "$*" ]; then /usr/bin/env bash; else $*; fi' >> $ND_ENTRYPOINT; \ + fi \ + && chmod -R 777 /neurodocker && chmod a+s /neurodocker +ENTRYPOINT ["/neurodocker/startup.sh"] + +LABEL maintainer="The nipype developers https://github.com/nipy/nipype" + +#---------------------- +# Install MCR and SPM12 +#---------------------- +# Install MATLAB Compiler Runtime +RUN apt-get update -qq && apt-get install -yq --no-install-recommends libxext6 libxt6 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && echo "Downloading MATLAB Compiler Runtime ..." \ + && curl -sSL -o /tmp/mcr.zip https://www.mathworks.com/supportfiles/downloads/R2017a/deployment_files/R2017a/installers/glnxa64/MCR_R2017a_glnxa64_installer.zip \ + && unzip -q /tmp/mcr.zip -d /tmp/mcrtmp \ + && /tmp/mcrtmp/install -destinationFolder /opt/mcr -mode silent -agreeToLicense yes \ + && rm -rf /tmp/* + +# Install standalone SPM +RUN echo "Downloading standalone SPM ..." \ + && curl -sSL -o spm.zip http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_latest_Linux_R2017a.zip \ + && unzip -q spm.zip -d /opt \ + && chmod -R 777 /opt/spm* \ + && rm -rf spm.zip \ + && /opt/spm12/run_spm12.sh /opt/mcr/v92/ quit \ + && sed -i '$iexport SPMMCRCMD=\"/opt/spm12/run_spm12.sh /opt/mcr/v92/ script\"' $ND_ENTRYPOINT +ENV MATLABCMD=/opt/mcr/v92/toolbox/matlab \ + FORCE_SPMMCR=1 \ + LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/opt/mcr/v92/runtime/glnxa64:/opt/mcr/v92/bin/glnxa64:/opt/mcr/v92/sys/os/glnxa64:$LD_LIBRARY_PATH + +#-------------------- +# Install AFNI latest +#-------------------- +ENV PATH=/opt/afni:$PATH +RUN apt-get update -qq && apt-get install -yq --no-install-recommends ed gsl-bin libglu1-mesa-dev libglib2.0-0 libglw1-mesa \ + libgomp1 libjpeg62 libxm4 netpbm tcsh xfonts-base xvfb \ + && libs_path=/usr/lib/x86_64-linux-gnu \ + && if [ -f $libs_path/libgsl.so.19 ]; then \ + ln $libs_path/libgsl.so.19 $libs_path/libgsl.so.0; \ + fi \ + && echo "Install libxp (not in all ubuntu/debian repositories)" \ + && apt-get install -yq --no-install-recommends libxp6 \ + || /bin/bash -c " \ + curl --retry 5 -o /tmp/libxp6.deb -sSL http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb \ + && dpkg -i /tmp/libxp6.deb && rm -f /tmp/libxp6.deb" \ + && echo "Install libpng12 (not in all ubuntu/debian repositories" \ + && apt-get install -yq --no-install-recommends libpng12-0 \ + || /bin/bash -c " \ + curl -o /tmp/libpng12.deb -sSL http://mirrors.kernel.org/debian/pool/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb \ + && dpkg -i /tmp/libpng12.deb && rm -f /tmp/libpng12.deb" \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && echo "Downloading AFNI ..." \ + && mkdir -p /opt/afni \ + && curl -sSL --retry 5 https://afni.nimh.nih.gov/pub/dist/tgz/linux_openmp_64.tgz \ + | tar zx -C /opt/afni --strip-components=1 + +#-------------------------- +# Install FreeSurfer v6.0.0 +#-------------------------- +# Install version minimized for recon-all +# See https://github.com/freesurfer/freesurfer/issues/70 +RUN apt-get update -qq && apt-get install -yq --no-install-recommends bc libgomp1 libxmu6 libxt6 tcsh perl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && echo "Downloading minimized FreeSurfer ..." \ + && curl -sSL https://dl.dropbox.com/s/nnzcfttc41qvt31/recon-all-freesurfer6-3.min.tgz | tar xz -C /opt \ + && sed -i '$isource $FREESURFER_HOME/SetUpFreeSurfer.sh' $ND_ENTRYPOINT +ENV FREESURFER_HOME=/opt/freesurfer + +# User-defined instruction +RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh + +RUN apt-get update -qq \ + && apt-get install -y -q --no-install-recommends ants \ + apt-utils \ + bzip2 \ + file \ + fsl-core \ + fsl-mni152-templates \ + fusefat \ + g++ \ + git \ + graphviz \ + make \ + ruby \ + unzip \ + xvfb \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Add command(s) to entrypoint +RUN sed -i '$isource /etc/fsl/fsl.sh' $ND_ENTRYPOINT + +ENV ANTSPATH="/usr/lib/ants" \ + PATH="/usr/lib/ants:$PATH" + +#------------------------ +# Install Convert3D 1.0.0 +#------------------------ +RUN echo "Downloading C3D ..." \ + && mkdir /opt/c3d \ + && curl -sSL --retry 5 https://sourceforge.net/projects/c3d/files/c3d/1.0.0/c3d-1.0.0-Linux-x86_64.tar.gz/download \ + | tar -xzC /opt/c3d --strip-components=1 +ENV C3DPATH=/opt/c3d \ + PATH=/opt/c3d/bin:$PATH + +# User-defined instruction +RUN gem install fakes3 + +WORKDIR /work + +#-------------------------------------- +# Save container specifications to JSON +#-------------------------------------- +RUN echo '{ \ + \n "pkg_manager": "apt", \ + \n "check_urls": false, \ + \n "instructions": [ \ + \n [ \ + \n "base", \ + \n "neurodebian@sha256:b09c09faa34bca0ea096b9360ee5121e048594cb8e2d7744d7d546ade88a2996" \ + \n ], \ + \n [ \ + \n "label", \ + \n { \ + \n "maintainer": "The nipype developers https://github.com/nipy/nipype" \ + \n } \ + \n ], \ + \n [ \ + \n "spm", \ + \n { \ + \n "version": "12", \ + \n "matlab_version": "R2017a" \ + \n } \ + \n ], \ + \n [ \ + \n "afni", \ + \n { \ + \n "version": "latest" \ + \n } \ + \n ], \ + \n [ \ + \n "freesurfer", \ + \n { \ + \n "version": "6.0.0", \ + \n "min": true \ + \n } \ + \n ], \ + \n [ \ + \n "run", \ + \n "echo \"cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=\" | base64 -d | sh" \ + \n ], \ + \n [ \ + \n "install", \ + \n [ \ + \n "ants", \ + \n "apt-utils", \ + \n "bzip2", \ + \n "file", \ + \n "fsl-core", \ + \n "fsl-mni152-templates", \ + \n "fusefat", \ + \n "g++", \ + \n "git", \ + \n "graphviz", \ + \n "make", \ + \n "ruby", \ + \n "unzip", \ + \n "xvfb" \ + \n ] \ + \n ], \ + \n [ \ + \n "add_to_entrypoint", \ + \n [ \ + \n "source /etc/fsl/fsl.sh" \ + \n ] \ + \n ], \ + \n [ \ + \n "env", \ + \n { \ + \n "ANTSPATH": "/usr/lib/ants", \ + \n "PATH": "/usr/lib/ants:$PATH" \ + \n } \ + \n ], \ + \n [ \ + \n "c3d", \ + \n { \ + \n "version": "1.0.0" \ + \n } \ + \n ], \ + \n [ \ + \n "instruction", \ + \n "RUN gem install fakes3" \ + \n ], \ + \n [ \ + \n "workdir", \ + \n "/work" \ + \n ] \ + \n ], \ + \n "generation_timestamp": "2017-10-02 22:55:55", \ + \n "neurodocker_version": "0.3.1-2-g4dfcf56" \ + \n}' > /neurodocker/neurodocker_specs.json diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile deleted file mode 100644 index 25fbb36401..0000000000 --- a/docker/base.Dockerfile +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright (c) 2016, The developers of the Stanford CRN -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of crn_base nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -# -# Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile -# -FROM ubuntu:xenial-20161213 -MAINTAINER The nipype developers https://github.com/nipy/nipype - -# Set noninteractive -ENV DEBIAN_FRONTEND=noninteractive - -# Installing requirements for freesurfer installation -RUN apt-get update && \ - apt-get install -y --no-install-recommends curl ca-certificates && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -WORKDIR /opt -# Installing freesurfer -- do it first so that it is cached early -#----------------------------------------------------------------------------- -# 3. Install FreeSurfer v6.0 (minimized with reprozip): -# https://github.com/freesurfer/freesurfer/issues/70 -#----------------------------------------------------------------------------- -RUN curl -sSL https://dl.dropbox.com/s/pbaisn6m5qpi9uu/recon-all-freesurfer6-2.min.tgz?dl=0 | tar zx -C /opt -ENV FS_OVERRIDE=0 \ - OS=Linux \ - FSF_OUTPUT_FORMAT=nii.gz \ - FIX_VERTEX_AREA=\ - FREESURFER_HOME=/opt/freesurfer -ENV MNI_DIR=$FREESURFER_HOME/mni \ - SUBJECTS_DIR=$FREESURFER_HOME/subjects -ENV PERL5LIB=$MNI_DIR/share/perl5 \ - MNI_PERL5LIB=$MNI_DIR/share/perl5 \ - MINC_BIN_DIR=$MNI_DIR/bin \ - MINC_LIB_DIR=$MNI_DIR/lib \ - MNI_DATAPATH=$MNI_DIR/data -ENV PATH=$FREESURFER_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH -ENV FSL_DIR=/usr/share/fsl/5.0 -RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh - -# Enable neurodebian -COPY docker/files/neurodebian.gpg /etc/apt/neurodebian.gpg -RUN curl -sSL http://neuro.debian.net/lists/xenial.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ - apt-key add /etc/apt/neurodebian.gpg && \ - apt-key adv --refresh-keys --keyserver hkp://ha.pool.sks-keyservers.net 0xA5D32F012649A5A9 || true - -# Installing general Debian utilities and Neurodebian packages (FSL, AFNI, git) -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - fsl-core \ - fsl-mni152-templates \ - afni \ - ants \ - bzip2 \ - xvfb \ - git \ - graphviz \ - unzip \ - apt-utils \ - fusefat \ - make \ - file \ - # Added g++ to compile dipy in py3.6 - g++ \ - ruby && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -ENV FSLDIR=/usr/share/fsl/5.0 \ - FSLOUTPUTTYPE=NIFTI_GZ \ - FSLMULTIFILEQUIT=TRUE \ - POSSUMDIR=/usr/share/fsl/5.0 \ - LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH \ - FSLTCLSH=/usr/bin/tclsh \ - FSLWISH=/usr/bin/wish \ - AFNI_MODELPATH=/usr/lib/afni/models \ - AFNI_IMSAVE_WARNINGS=NO \ - AFNI_TTATLAS_DATASET=/usr/share/afni/atlases \ - AFNI_PLUGINPATH=/usr/lib/afni/plugins \ - ANTSPATH=/usr/lib/ants -ENV PATH=/usr/lib/fsl/5.0:/usr/lib/afni/bin:$ANTSPATH:$PATH - -# Installing and setting up c3d -RUN mkdir -p /opt/c3d && \ - curl -sSL "https://files.osf.io/v1/resources/nefdp/providers/osfstorage/59ca96a9b83f69025d6b8985?action=download&version=1&direct" \ - | tar -xzC /opt/c3d --strip-components 1 - -ENV C3DPATH=/opt/c3d/ -ENV PATH=$C3DPATH/bin:$PATH - -# Install fake-S3 -ENV GEM_HOME /usr/lib/ruby/gems/2.3 -ENV BUNDLE_PATH="$GEM_HOME" \ - BUNDLE_BIN="$GEM_HOME/bin" \ - BUNDLE_SILENCE_ROOT_WARNING=1 \ - BUNDLE_APP_CONFIG="$GEM_HOME" -ENV PATH $BUNDLE_BIN:$PATH -RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" && \ - chmod 777 "$GEM_HOME" "$BUNDLE_BIN" - -RUN gem install fakes3 - -# Install Matlab MCR: from the good old install_spm_mcr.sh of @chrisfilo -RUN echo "destinationFolder=/opt/mcr" > mcr_options.txt && \ - echo "agreeToLicense=yes" >> mcr_options.txt && \ - echo "outputFile=/tmp/matlabinstall_log" >> mcr_options.txt && \ - echo "mode=silent" >> mcr_options.txt && \ - mkdir -p matlab_installer && \ - curl -sSL http://www.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip \ - -o matlab_installer/installer.zip && \ - unzip matlab_installer/installer.zip -d matlab_installer/ && \ - matlab_installer/install -inputFile mcr_options.txt && \ - rm -rf matlab_installer mcr_options.txt - -# Install SPM -RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_r6472_Linux_R2015a.zip -o spm12.zip && \ - unzip spm12.zip && \ - rm -rf spm12.zip - -ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \ - SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \ - FORCE_SPMMCR=1 - -WORKDIR /work diff --git a/docker/generate_dockerfiles.sh b/docker/generate_dockerfiles.sh new file mode 100755 index 0000000000..07e1aa3774 --- /dev/null +++ b/docker/generate_dockerfiles.sh @@ -0,0 +1,75 @@ + #!/usr/bin/env bash + +# kaczmarj/neurodocker:master pulled on September 13, 2017. +NEURODOCKER_IMAGE="kaczmarj/neurodocker:master" +# neurodebian/stretch-non-free:latest pulled on September 13, 2017. +BASE_IMAGE="neurodebian@sha256:b09c09faa34bca0ea096b9360ee5121e048594cb8e2d7744d7d546ade88a2996" +NIPYPE_BASE_IMAGE="kaczmarj/nipype:base" +PKG_MANAGER="apt" + +# Save Dockerfiles relative to this path so that this script can be run from +# any directory. https://stackoverflow.com/a/246128/5666087 +DIR="$(dirname "$0")" + + +function generate_base_dockerfile() { + docker run --rm "$NEURODOCKER_IMAGE" generate \ + --base "$BASE_IMAGE" --pkg-manager "$PKG_MANAGER" \ + --label maintainer="The nipype developers https://github.com/nipy/nipype" \ + --spm version=12 matlab_version=R2017a \ + --afni version=latest \ + --freesurfer version=6.0.0 min=true \ + --run 'echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh' \ + --install ants apt-utils bzip2 file fsl-core fsl-mni152-templates \ + fusefat g++ git graphviz make ruby unzip xvfb \ + --add-to-entrypoint "source /etc/fsl/fsl.sh" \ + --env ANTSPATH='/usr/lib/ants' PATH='/usr/lib/ants:$PATH' \ + --c3d version=1.0.0 \ + --instruction "RUN gem install fakes3" \ + --workdir /work \ + --no-check-urls > "$DIR/Dockerfile.base" +} + + +# The Dockerfile ADD/COPY instructions do not honor the current user, so the +# owner of the directories has to be manually changed to user neuro. +# See https://github.com/moby/moby/issues/6119 for more information on this +# behavior. +# Docker plans on changing this behavior by added a `--chown` flag to the +# ADD/COPY commands. See https://github.com/moby/moby/pull/34263. + +function generate_main_dockerfile() { + docker run --rm "$NEURODOCKER_IMAGE" generate \ + --base "$NIPYPE_BASE_IMAGE" --pkg-manager "$PKG_MANAGER" \ + --label maintainer="The nipype developers https://github.com/nipy/nipype" \ + --env MKL_NUM_THREADS=1 OMP_NUM_THREADS=1 \ + --user neuro \ + --miniconda env_name=neuro \ + add_to_path=true \ + --copy docker/files/run_builddocs.sh docker/files/run_examples.sh \ + docker/files/run_pytests.sh nipype/external/fsl_imglob.py /usr/bin/ \ + --copy . /src/nipype \ + --user root \ + --run "chmod 777 -R /src/nipype" \ + --user neuro \ + --arg PYTHON_VERSION_MAJOR=3 PYTHON_VERSION_MINOR=6 BUILD_DATE VCS_REF VERSION \ + --miniconda env_name=neuro \ + conda_install='python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} + icu=58.1 libxml2 libxslt matplotlib mkl numpy + pandas psutil scikit-learn scipy traits=4.6.0' \ + pip_opts="-e" \ + pip_install="/src/nipype[all]" \ + --label org.label-schema.build-date='$BUILD_DATE' \ + org.label-schema.name="NIPYPE" \ + org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \ + org.label-schema.url="http://nipype.readthedocs.io" \ + org.label-schema.vcs-ref='$VCS_REF' \ + org.label-schema.vcs-url="https://github.com/nipy/nipype" \ + org.label-schema.version='$VERSION' \ + org.label-schema.schema-version="1.0" \ + --no-check-urls > "$DIR/../Dockerfile" +} + + +generate_base_dockerfile +generate_main_dockerfile From 7bc9a2fd39d6f2c8e6fcbba4c94723e4f1316203 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 2 Oct 2017 19:06:31 -0400 Subject: [PATCH 02/15] enh: migrate to circleci 2.0 + check base dockerfile against cache The specification for CircleCI 2.0 is stored in `.circleci/config.yml` instead of in `circle.yml`. Todo: - Run tests. For now, images are built and pushed, but no tests are run. - Minimize containers with neurodocker reprozip. This functionality will be updated soon. See discussion in ViDA-NYU/reprozip#274. --- .circleci/config.yml | 149 ++++++++++++++++++++++++++++++++ {.circle => .circleci}/tests.sh | 0 circle.yml | 86 ------------------ 3 files changed, 149 insertions(+), 86 deletions(-) create mode 100644 .circleci/config.yml rename {.circle => .circleci}/tests.sh (100%) delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..d0c9099617 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,149 @@ +# Examples: +# https://github.com/circleci/frontend/blob/master/.circleci/config.yml +# +# Questions +# --------- +# 1. Regarding the cache: what if the base Dockerfile is reverted to a previous +# version? The cache for that Dockerfile will exist, so it will pull the +# image, which is incorrect. Include a note in generate_dockerfiles.sh to +# increase the version of the cache. + +version: 2 +jobs: + + compare_base_dockerfiles: + docker: + - image: docker:17.06.2-ce-git # shell is /bin/ash (bash not available) + steps: + - checkout: + path: /home/circleci/nipype + - run: + name: Prune base Dockerfile in preparation for cache check + command: | + mkdir -p /tmp/docker + + # Remove empty lines, comments, and the timestamp from the base + # Dockerfile. Use the sha256 sum of this pruned Dockerfile as the + # cache key. + sed -e '/\s*#.*$/d' \ + -e '/^\s*$/d' \ + -e '/generation_timestamp/d' \ + /home/circleci/nipype/docker/Dockerfile.base \ + > /tmp/docker/Dockerfile.base-pruned + - restore_cache: + key: dftest-v4-master-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }} + - run: + name: Determine how to get base image + command: | + GET_BASE="/tmp/docker/get_base_image.sh" + + # This directory comes from the cache. + if [ -d /cache/base-dockerfile ]; then + echo 'echo Pulling base image ...' > "$GET_BASE" + echo 'docker pull kaczmarj/nipype:base' >> "$GET_BASE" + else + echo 'echo Building base image ...' > "$GET_BASE" + echo 'docker build -t kaczmarj/nipype:base - < /home/circleci/nipype/docker/Dockerfile.base' >> "$GET_BASE" + fi + - persist_to_workspace: + root: /tmp + paths: + - docker/* + + + build_and_test: + parallelism: 1 + # Ideally, we could test inside the main docker image. + machine: + # Ubuntu 14.04 with Docker 17.03.0-ce + image: circleci/classic:201703-01 + steps: + - checkout: + path: /home/circleci/nipype + - attach_workspace: + at: /tmp + - run: + name: Get base image (pull or build) + no_output_timeout: 60m + command: | + bash /tmp/docker/get_base_image.sh + - run: + name: Build main image (latest & py36) + no_output_timeout: 60m + command: | + cd /home/circleci/nipype + + docker build --rm=false \ + --tag kaczmarj/nipype:latest \ + --tag kaczmarj/nipype:py36 \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg VCS_REF=`git rev-parse --short HEAD` \ + --build-arg VERSION=$CIRCLE_TAG . + - run: + name: Build main image (py27) + no_output_timeout: 60m + command: | + cd /home/circleci/nipype + + docker build --rm=false \ + --tag kaczmarj/nipype:py27 \ + --build-arg PYTHON_VERSION_MAJOR=2 \ + --build-arg PYTHON_VERSION_MINOR=7 \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg VCS_REF=`git rev-parse --short HEAD` \ + --build-arg VERSION=$CIRCLE_TAG-py27 /home/circleci/nipype + - run: + name: Run tests + command: | + echo "This is node $CIRCLE_NODE_INDEX" + echo "No tests to run yet." + - run: + name: Save Docker images to workspace + no_output_timeout: 60m + command: | + if [ "$CIRCLE_NODE_INDEX" -eq "0" ]; then + echo "Saving Docker images to tar.gz files ..." + docker save kaczmarj/nipype:latest kaczmarj/nipype:py36 | gzip > /tmp/docker/nipype-latest-py36.tar.gz + fi + - persist_to_workspace: + root: /tmp + paths: + - docker/* + + + deploy: + docker: + - image: docker:17.06.2-ce-git + steps: + - checkout + - setup_remote_docker + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-latest-py36.tar.gz + - run: + name: Push to DockerHub + no_output_timeout: 60m + command: | + if [ "${CIRCLE_BRANCH}" == "enh/circleci-neurodocker" ]; then + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker push kaczmarj/nipype:latest + docker push kaczmarj/nipype:py36 + fi +# TODO: write pruned Dockerfile to cache here. Make a shell script that will +# prune Dockerfiles + +workflows: + version: 2 + build_test_deply: + jobs: + - compare_base_dockerfiles + - build_and_test: + requires: + - compare_base_dockerfiles + - deploy: + requires: + - build_and_test diff --git a/.circle/tests.sh b/.circleci/tests.sh similarity index 100% rename from .circle/tests.sh rename to .circleci/tests.sh diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 5624dbb7f8..0000000000 --- a/circle.yml +++ /dev/null @@ -1,86 +0,0 @@ -machine: - pre: - - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - DATA_NIPYPE_TUTORIAL_URL: "${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - DATA_NIPYPE_FSL_COURSE: "${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - DATA_NIPYPE_FSL_FEEDS: "${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - WORKDIR: "$HOME/work" - CODECOV_TOKEN: "ac172a50-8e66-42e5-8822-5373fcf54686" - services: - - docker - -dependencies: - cache_directories: - - "~/docker" - - "~/examples" - - "~/.apt-cache" - - pre: - # Let CircleCI cache the apt archive - - mkdir -p ~/.apt-cache/partial && sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives - - sudo apt-get -y update && sudo apt-get install -y wget bzip2 - # Create work folder and force group permissions - - mkdir -p $WORKDIR && sudo setfacl -d -m group:ubuntu:rwx $WORKDIR && sudo setfacl -m group:ubuntu:rwx $WORKDIR - - mkdir -p $HOME/docker $HOME/examples $WORKDIR/tests $WORKDIR/logs $WORKDIR/crashfiles ${CIRCLE_TEST_REPORTS}/tests/ - - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi - - (cd $HOME/docker && gzip -d cache.tar.gz && docker load --input $HOME/docker/cache.tar) || true : - timeout: 6000 - override: - # Get data - - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi - - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi - - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi - - if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py; fi - # Docker - - docker images - - ? | - e=1 && for i in {1..5}; do - docker build --rm=false -f docker/base.Dockerfile -t nipype/base:latest . && e=0 && break || sleep 15; - done && [ "$e" -eq "0" ] - : - timeout: 21600 - - ? | - e=1 && for i in {1..5}; do - docker build --rm=false -t nipype/nipype:latest -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; - done && [ "$e" -eq "0" ] - : - timeout: 6000 - - ? | - e=1 && for i in {1..5}; do - docker build --rm=false -t nipype/nipype:py27 --build-arg PYTHON_VERSION_MAJOR=2 --build-arg PYTHON_VERSION_MINOR=7 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG-py27 . && e=0 && break || sleep 15; - done && [ "$e" -eq "0" ] - : - timeout: 6000 - - docker save -o $HOME/docker/cache.tar ubuntu:xenial-20161213 nipype/base:latest nipype/nipype:py36 && (cd $HOME/docker && gzip cache.tar) : - timeout: 6000 - -test: - override: - - bash .circle/tests.sh : - timeout: 7200 - parallel: true - -general: - artifacts: - - "~/work/docs" - - "~/work/logs" - - "~/work/tests" - - "~/work/crashfiles" - -deployment: - production: - tag: /.*/ - commands: - # Deploy to docker hub - - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/base:latest; fi : - timeout: 21600 - - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/nipype:latest; fi : - timeout: 21600 - - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag nipype/nipype nipype/nipype:$CIRCLE_TAG && docker push nipype/nipype:$CIRCLE_TAG; fi : - timeout: 21600 - -# Automatic deployment to Pypi: -# - printf "[distutils]\nindex-servers =\n pypi\n\n[pypi]\nusername:$PYPI_USER\npassword:$PYPI_PASS\n" > ~/.pypirc -# - python setup.py sdist upload -r pypi From 222b15ff29bfdafc352e555f47600a43e829e197 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 6 Nov 2017 15:48:49 -0500 Subject: [PATCH 03/15] add: cli to generate dockerfiles --- Makefile | 14 ++++- docker/generate_dockerfiles.sh | 102 +++++++++++++++++++++++++++------ 2 files changed, 95 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 31f67bf500..0e1e927232 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ PYTHON ?= python NOSETESTS=`which nosetests` -.PHONY: zipdoc sdist egg upload_to_pypi trailing-spaces clean-pyc clean-so clean-build clean-ctags clean in inplace test-code test-coverage test html specs check-before-commit check +.PHONY: zipdoc sdist egg upload_to_pypi trailing-spaces clean-pyc clean-so clean-build clean-ctags clean in inplace test-code test-coverage test html specs check-before-commit check gen-base-dockerfile gen-main-dockerfile gen-dockerfiles zipdoc: html zip documentation.zip doc/_build/html @@ -61,7 +61,7 @@ test-code: in test-coverage: clean-tests in py.test --doctest-modules --cov-config .coveragerc --cov=nipype nipype - + test: tests # just another name tests: clean test-code @@ -79,3 +79,13 @@ check-before-commit: specs trailing-spaces html test @echo "built docs" @echo "ran test" @echo "generated spec tests" + +gen-base-dockerfile: + @echo "Generating base Dockerfile" + bash docker/generate_dockerfiles.sh -b + +gen-main-dockerfile: + @echo "Generating main Dockerfile" + bash docker/generate_dockerfiles.sh -m + +gen-dockerfiles: gen-base-dockerfile gen-main-dockerfile diff --git a/docker/generate_dockerfiles.sh b/docker/generate_dockerfiles.sh index 07e1aa3774..cec0f19f80 100755 --- a/docker/generate_dockerfiles.sh +++ b/docker/generate_dockerfiles.sh @@ -1,32 +1,81 @@ - #!/usr/bin/env bash +#!/usr/bin/env bash +# +# Generate base and main Dockerfiles for Nipype. + +set -e + +USAGE="usage: $(basename $0) [-h] [-b] [-m]" + +function Help { + cat <&2 + exit 1 + ;; + esac +done + + +# neurodocker version 0.3.1-19-g8d02eb4 +NEURODOCKER_IMAGE="kaczmarj/neurodocker@sha256:6b5f92f413b9710b7581e62293a8f74438b14ce7e4ab1ce68db2a09f7c64375a" + +# neurodebian:stretch-non-free pulled on November 3, 2017 +BASE_IMAGE="neurodebian@sha256:7590552afd0e7a481a33314724ae27f76ccedd05ffd7ac06ec38638872427b9b" -# kaczmarj/neurodocker:master pulled on September 13, 2017. -NEURODOCKER_IMAGE="kaczmarj/neurodocker:master" -# neurodebian/stretch-non-free:latest pulled on September 13, 2017. -BASE_IMAGE="neurodebian@sha256:b09c09faa34bca0ea096b9360ee5121e048594cb8e2d7744d7d546ade88a2996" NIPYPE_BASE_IMAGE="kaczmarj/nipype:base" PKG_MANAGER="apt" - -# Save Dockerfiles relative to this path so that this script can be run from -# any directory. https://stackoverflow.com/a/246128/5666087 DIR="$(dirname "$0")" - function generate_base_dockerfile() { docker run --rm "$NEURODOCKER_IMAGE" generate \ --base "$BASE_IMAGE" --pkg-manager "$PKG_MANAGER" \ --label maintainer="The nipype developers https://github.com/nipy/nipype" \ --spm version=12 matlab_version=R2017a \ - --afni version=latest \ + --afni version=latest install_python2=true \ --freesurfer version=6.0.0 min=true \ --run 'echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh' \ - --install ants apt-utils bzip2 file fsl-core fsl-mni152-templates \ + --install ants apt-utils bzip2 convert3d file fsl-core fsl-mni152-templates \ fusefat g++ git graphviz make ruby unzip xvfb \ --add-to-entrypoint "source /etc/fsl/fsl.sh" \ --env ANTSPATH='/usr/lib/ants' PATH='/usr/lib/ants:$PATH' \ - --c3d version=1.0.0 \ - --instruction "RUN gem install fakes3" \ - --workdir /work \ + --run "gem install fakes3" \ --no-check-urls > "$DIR/Dockerfile.base" } @@ -45,12 +94,17 @@ function generate_main_dockerfile() { --env MKL_NUM_THREADS=1 OMP_NUM_THREADS=1 \ --user neuro \ --miniconda env_name=neuro \ - add_to_path=true \ + activate=true \ --copy docker/files/run_builddocs.sh docker/files/run_examples.sh \ docker/files/run_pytests.sh nipype/external/fsl_imglob.py /usr/bin/ \ --copy . /src/nipype \ --user root \ - --run "chmod 777 -R /src/nipype" \ + --run 'chown -R neuro /src +&& chmod +x /usr/bin/fsl_imglob.py /usr/bin/run_*.sh +&& . /etc/fsl/fsl.sh +&& ln -sf /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob +&& mkdir /work +&& chown neuro /work' \ --user neuro \ --arg PYTHON_VERSION_MAJOR=3 PYTHON_VERSION_MINOR=6 BUILD_DATE VCS_REF VERSION \ --miniconda env_name=neuro \ @@ -59,6 +113,12 @@ function generate_main_dockerfile() { pandas psutil scikit-learn scipy traits=4.6.0' \ pip_opts="-e" \ pip_install="/src/nipype[all]" \ + --run-bash "mkdir -p /src/pybids + && curl -sSL --retry 5 https://github.com/INCF/pybids/tarball/master + | tar -xz -C /src/pybids --strip-components 1 + && source activate neuro + && pip install --no-cache-dir -e /src/pybids" \ + --workdir /work \ --label org.label-schema.build-date='$BUILD_DATE' \ org.label-schema.name="NIPYPE" \ org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \ @@ -67,9 +127,13 @@ function generate_main_dockerfile() { org.label-schema.vcs-url="https://github.com/nipy/nipype" \ org.label-schema.version='$VERSION' \ org.label-schema.schema-version="1.0" \ - --no-check-urls > "$DIR/../Dockerfile" + --no-check-urls } -generate_base_dockerfile -generate_main_dockerfile +if [ "$GENERATE_BASE" == 1 ]; then + generate_base_dockerfile > "$DIR/Dockerfile.base" +fi +if [ "$GENERATE_MAIN" == 1 ]; then + generate_main_dockerfile > "$DIR/../Dockerfile" +fi From 3b81bcd838672c4a3a06bcb8f91cee62f0aa8d6f Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 6 Nov 2017 15:49:23 -0500 Subject: [PATCH 04/15] Change default TESTPATH to /src/nipype/nipype from /src/nipype --- docker/files/run_pytests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/files/run_pytests.sh b/docker/files/run_pytests.sh index 19b6fcab87..76935b42f8 100644 --- a/docker/files/run_pytests.sh +++ b/docker/files/run_pytests.sh @@ -4,7 +4,7 @@ set -x set -u -TESTPATH=${1:-/src/nipype/} +TESTPATH=${1:-/src/nipype/nipype} WORKDIR=${WORK:-/work} PYTHON_VERSION=$( python -c "import sys; print('{}{}'.format(sys.version_info[0], sys.version_info[1]))" ) @@ -34,4 +34,3 @@ find ${WORKDIR} -maxdepth 1 -name "crash-*" -exec mv {} ${WORKDIR}/crashfiles/ \ echo "Unit tests finished with exit code ${exit_code}" exit ${exit_code} - From 86d70f50feaf2c7809905209f47d22bfee2f57f5 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 6 Nov 2017 16:06:16 -0500 Subject: [PATCH 05/15] enh: use svg travis shield + rm whitespace --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 85d34a704d..8831d11b2e 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ NIPYPE: Neuroimaging in Python: Pipelines and Interfaces ======================================================== -.. image:: https://travis-ci.org/nipy/nipype.png?branch=master +.. image:: https://travis-ci.org/nipy/nipype.svg?branch=master :target: https://travis-ci.org/nipy/nipype .. image:: https://circleci.com/gh/nipy/nipype/tree/master.svg?style=svg @@ -94,4 +94,3 @@ Contributing to the project --------------------------- If you'd like to contribute to the project please read our `guidelines `_. Please also read through our `code of conduct `_. - From 2d15a1ea0d4fd954218d4dca9b5e98ad981ba4b7 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 6 Nov 2017 16:06:41 -0500 Subject: [PATCH 06/15] update circleci config.yml - install dependency codecov - update nipype/info.py if CIRCLE_TAG is set - retry docker image builds - download test data - fix workdir permission denied issue - save all docker images to single tarball - pipe dockerhub password to `docker login` - push all docker images to docker hub on success --- .circleci/config.yml | 157 +++++++++++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 58 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0c9099617..5734e2274b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,37 +1,22 @@ -# Examples: -# https://github.com/circleci/frontend/blob/master/.circleci/config.yml -# -# Questions -# --------- -# 1. Regarding the cache: what if the base Dockerfile is reverted to a previous -# version? The cache for that Dockerfile will exist, so it will pull the -# image, which is incorrect. Include a note in generate_dockerfiles.sh to -# increase the version of the cache. - version: 2 jobs: compare_base_dockerfiles: docker: - - image: docker:17.06.2-ce-git # shell is /bin/ash (bash not available) + - image: docker:17.09.0-ce-git steps: - checkout: path: /home/circleci/nipype - run: name: Prune base Dockerfile in preparation for cache check + working_directory: /home/circleci/nipype/docker command: | mkdir -p /tmp/docker - - # Remove empty lines, comments, and the timestamp from the base - # Dockerfile. Use the sha256 sum of this pruned Dockerfile as the - # cache key. - sed -e '/\s*#.*$/d' \ - -e '/^\s*$/d' \ - -e '/generation_timestamp/d' \ - /home/circleci/nipype/docker/Dockerfile.base \ - > /tmp/docker/Dockerfile.base-pruned + # Use the sha256 sum of the pruned Dockerfile as the cache key. + ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned - restore_cache: - key: dftest-v4-master-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }} + # TODO: change this to 'master' after we are sure this works. + key: dftest-v5-enh/circleci-neurodocker-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }} - run: name: Determine how to get base image command: | @@ -39,11 +24,11 @@ jobs: # This directory comes from the cache. if [ -d /cache/base-dockerfile ]; then - echo 'echo Pulling base image ...' > "$GET_BASE" - echo 'docker pull kaczmarj/nipype:base' >> "$GET_BASE" + echo "echo Pulling base image ..." > "$GET_BASE" + echo "docker pull kaczmarj/nipype:base" >> "$GET_BASE" else - echo 'echo Building base image ...' > "$GET_BASE" - echo 'docker build -t kaczmarj/nipype:base - < /home/circleci/nipype/docker/Dockerfile.base' >> "$GET_BASE" + echo "echo Building base image ..." > "$GET_BASE" + echo "docker build -t kaczmarj/nipype:base - < /home/circleci/nipype/docker/Dockerfile.base" >> "$GET_BASE" fi - persist_to_workspace: root: /tmp @@ -52,8 +37,7 @@ jobs: build_and_test: - parallelism: 1 - # Ideally, we could test inside the main docker image. + parallelism: 4 machine: # Ubuntu 14.04 with Docker 17.03.0-ce image: circleci/classic:201703-01 @@ -62,48 +46,92 @@ jobs: path: /home/circleci/nipype - attach_workspace: at: /tmp + - run: + name: Get test dependencies + command: | + pip install --no-cache-dir codecov + - run: + name: Modify Nipype version if necessary + working_directory: /home/circleci/nipype + command: | + if [ "$CIRCLE_TAG" != "" ]; then + sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py + fi - run: name: Get base image (pull or build) no_output_timeout: 60m + # TODO: remove `docker pull` once once caching works. command: | - bash /tmp/docker/get_base_image.sh + # bash /tmp/docker/get_base_image.sh + docker pull kaczmarj/nipype:base - run: - name: Build main image (latest & py36) + name: Build main image (py36) no_output_timeout: 60m + working_directory: /home/circleci/nipype command: | - cd /home/circleci/nipype - - docker build --rm=false \ - --tag kaczmarj/nipype:latest \ - --tag kaczmarj/nipype:py36 \ - --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ - --build-arg VCS_REF=`git rev-parse --short HEAD` \ - --build-arg VERSION=$CIRCLE_TAG . + e=1 && for i in {1..5}; do + docker build \ + --rm=false \ + --tag kaczmarj/nipype:latest \ + --tag kaczmarj/nipype:py36 \ + --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ + --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] - run: name: Build main image (py27) no_output_timeout: 60m + working_directory: /home/circleci/nipype command: | - cd /home/circleci/nipype + e=1 && for i in {1..5}; do + docker build \ + --rm=false \ + --tag kaczmarj/nipype:py27 \ + --build-arg PYTHON_VERSION_MAJOR=2 \ + --build-arg PYTHON_VERSION_MINOR=7 \ + --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ + --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - docker build --rm=false \ - --tag kaczmarj/nipype:py27 \ - --build-arg PYTHON_VERSION_MAJOR=2 \ - --build-arg PYTHON_VERSION_MINOR=7 \ - --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ - --build-arg VCS_REF=`git rev-parse --short HEAD` \ - --build-arg VERSION=$CIRCLE_TAG-py27 /home/circleci/nipype + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - run: name: Run tests + no_output_timeout: 4h + environment: + WORKDIR: /home/circleci/work command: | - echo "This is node $CIRCLE_NODE_INDEX" - echo "No tests to run yet." + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/tests.sh + - store_artifacts: + path: /home/circleci/work/tests - run: name: Save Docker images to workspace no_output_timeout: 60m command: | if [ "$CIRCLE_NODE_INDEX" -eq "0" ]; then echo "Saving Docker images to tar.gz files ..." - docker save kaczmarj/nipype:latest kaczmarj/nipype:py36 | gzip > /tmp/docker/nipype-latest-py36.tar.gz + docker save kaczmarj/nipype:base \ + kaczmarj/nipype:latest \ + kaczmarj/nipype:py36 \ + kaczmarj/nipype:py27 > /tmp/docker/nipype-base-latest-py36-py27.tar + echo "$(du -h /tmp/docker/nipype-base-latest-py36-py27.tar)" fi - persist_to_workspace: root: /tmp @@ -113,7 +141,7 @@ jobs: deploy: docker: - - image: docker:17.06.2-ce-git + - image: docker:17.09.0-ce-git steps: - checkout - setup_remote_docker @@ -123,18 +151,27 @@ jobs: name: Load saved Docker images. no_output_timeout: 60m command: | - docker load < /tmp/docker/nipype-latest-py36.tar.gz + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar - run: name: Push to DockerHub - no_output_timeout: 60m + no_output_timeout: 120m command: | - if [ "${CIRCLE_BRANCH}" == "enh/circleci-neurodocker" ]; then - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker push kaczmarj/nipype:latest - docker push kaczmarj/nipype:py36 - fi -# TODO: write pruned Dockerfile to cache here. Make a shell script that will -# prune Dockerfiles + echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin + docker push kaczmarj/nipype:base + docker push kaczmarj/nipype:latest + docker push kaczmarj/nipype:py36 + docker push kaczmarj/nipype:py27 + - run: + name: Prune base Dockerfile to update cache + command: | + cd /home/circleci/nipype/docker + # Use the sha256 sum of the pruned Dockerfile as the cache key. + ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned + - save_cache: + paths: + - /tmp/docker/Dockerfile.base-pruned + key: dftest-v5-{{ .Branch }}-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }} + workflows: version: 2 @@ -145,5 +182,9 @@ workflows: requires: - compare_base_dockerfiles - deploy: + filters: + branches: + # TODO: change this to master after we are sure this works. + only: enh/circleci-neurodocker requires: - build_and_test From 70cf23074af279a07192dde6e453386f2bd33e8f Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 6 Nov 2017 16:13:16 -0500 Subject: [PATCH 07/15] add: file to prune dockerfiles Use this file to prune Dockerfiles before getting their hash. - remove empty lines, comments, and timestamp --- docker/prune_dockerfile.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docker/prune_dockerfile.sh diff --git a/docker/prune_dockerfile.sh b/docker/prune_dockerfile.sh new file mode 100644 index 0000000000..e6b05ebbcf --- /dev/null +++ b/docker/prune_dockerfile.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [ -z "$1" ]; then + echo "Usage: $(basename $0) " + exit 1 +fi + +# Remove empty lines, comments, and timestamp. +sed -e '/\s*#.*$/d' -e '/^\s*$/d' -e '/generation_timestamp/d' "$1" From 1a2a3c446ac1ecf231b87a9bc6f83fb00a0f6e20 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 6 Nov 2017 16:15:37 -0500 Subject: [PATCH 08/15] regenerate dockerfiles --- Dockerfile | 41 +++++++++++++++++++++++-------- docker/Dockerfile.base | 45 ++++++++++------------------------ docker/generate_dockerfiles.sh | 7 ------ 3 files changed, 44 insertions(+), 49 deletions(-) diff --git a/Dockerfile b/Dockerfile index 073185f1cc..9ea5e11017 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -# Generated by Neurodocker v0.3.1-2-g4dfcf56. +# Generated by Neurodocker v0.3.1-19-g8d02eb4. # # Thank you for using Neurodocker. If you discover any issues # or ways to improve this software, please submit an issue or # pull request on our GitHub repository: # https://github.com/kaczmarj/neurodocker # -# Timestamp: 2017-10-02 22:55:57 +# Timestamp: 2017-11-06 21:15:09 FROM kaczmarj/nipype:base @@ -54,15 +54,14 @@ RUN echo "Downloading Miniconda installer ..." \ && conda config --system --prepend channels conda-forge \ && conda config --system --set auto_update_conda false \ && conda config --system --set show_channel_urls true \ - && conda update -y -q --all && sync \ && conda clean -tipsy && sync #------------------------- # Create conda environment #------------------------- RUN conda create -y -q --name neuro \ - && sync && conda clean -tipsy && sync -ENV PATH=/opt/conda/envs/neuro/bin:$PATH + && sync && conda clean -tipsy && sync \ + && sed -i '$isource activate neuro' $ND_ENTRYPOINT COPY ["docker/files/run_builddocs.sh", "docker/files/run_examples.sh", "docker/files/run_pytests.sh", "nipype/external/fsl_imglob.py", "/usr/bin/"] @@ -71,7 +70,12 @@ COPY [".", "/src/nipype"] USER root # User-defined instruction -RUN chmod 777 -R /src/nipype +RUN chown -R neuro /src \ + && chmod +x /usr/bin/fsl_imglob.py /usr/bin/run_*.sh \ + && . /etc/fsl/fsl.sh \ + && ln -sf /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob \ + && mkdir /work \ + && chown neuro /work USER neuro @@ -101,6 +105,15 @@ RUN conda install -y -q --name neuro python=${PYTHON_VERSION_MAJOR}.${PYTHON_VER && pip install -q --no-cache-dir -e /src/nipype[all]" \ && sync +# User-defined BASH instruction +RUN bash -c "mkdir -p /src/pybids \ + && curl -sSL --retry 5 https://github.com/INCF/pybids/tarball/master \ + | tar -xz -C /src/pybids --strip-components 1 \ + && source activate neuro \ + && pip install --no-cache-dir -e /src/pybids" + +WORKDIR /work + LABEL org.label-schema.build-date="$BUILD_DATE" \ org.label-schema.name="NIPYPE" \ org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \ @@ -142,7 +155,7 @@ RUN echo '{ \ \n "miniconda", \ \n { \ \n "env_name": "neuro", \ - \n "add_to_path": true \ + \n "activate": "true" \ \n } \ \n ], \ \n [ \ @@ -168,7 +181,7 @@ RUN echo '{ \ \n ], \ \n [ \ \n "run", \ - \n "chmod 777 -R /src/nipype" \ + \n "chown -R neuro /src\\n&& chmod +x /usr/bin/fsl_imglob.py /usr/bin/run_*.sh\\n&& . /etc/fsl/fsl.sh\\n&& ln -sf /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob\\n&& mkdir /work\\n&& chown neuro /work" \ \n ], \ \n [ \ \n "user", \ @@ -194,6 +207,14 @@ RUN echo '{ \ \n } \ \n ], \ \n [ \ + \n "run_bash", \ + \n "mkdir -p /src/pybids\\n && curl -sSL --retry 5 https://github.com/INCF/pybids/tarball/master\\n | tar -xz -C /src/pybids --strip-components 1\\n && source activate neuro\\n && pip install --no-cache-dir -e /src/pybids" \ + \n ], \ + \n [ \ + \n "workdir", \ + \n "/work" \ + \n ], \ + \n [ \ \n "label", \ \n { \ \n "org.label-schema.build-date": "$BUILD_DATE", \ @@ -207,6 +228,6 @@ RUN echo '{ \ \n } \ \n ] \ \n ], \ - \n "generation_timestamp": "2017-10-02 22:55:57", \ - \n "neurodocker_version": "0.3.1-2-g4dfcf56" \ + \n "generation_timestamp": "2017-11-06 21:15:09", \ + \n "neurodocker_version": "0.3.1-19-g8d02eb4" \ \n}' > /neurodocker/neurodocker_specs.json diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 5735c04b93..429930ca66 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -1,13 +1,13 @@ -# Generated by Neurodocker v0.3.1-2-g4dfcf56. +# Generated by Neurodocker v0.3.1-19-g8d02eb4. # # Thank you for using Neurodocker. If you discover any issues # or ways to improve this software, please submit an issue or # pull request on our GitHub repository: # https://github.com/kaczmarj/neurodocker # -# Timestamp: 2017-10-02 22:55:55 +# Timestamp: 2017-11-06 21:15:07 -FROM neurodebian@sha256:b09c09faa34bca0ea096b9360ee5121e048594cb8e2d7744d7d546ade88a2996 +FROM neurodebian@sha256:7590552afd0e7a481a33314724ae27f76ccedd05ffd7ac06ec38638872427b9b ARG DEBIAN_FRONTEND=noninteractive @@ -64,7 +64,7 @@ ENV MATLABCMD=/opt/mcr/v92/toolbox/matlab \ #-------------------- ENV PATH=/opt/afni:$PATH RUN apt-get update -qq && apt-get install -yq --no-install-recommends ed gsl-bin libglu1-mesa-dev libglib2.0-0 libglw1-mesa \ - libgomp1 libjpeg62 libxm4 netpbm tcsh xfonts-base xvfb \ + libgomp1 libjpeg62 libxm4 netpbm tcsh xfonts-base xvfb python \ && libs_path=/usr/lib/x86_64-linux-gnu \ && if [ -f $libs_path/libgsl.so.19 ]; then \ ln $libs_path/libgsl.so.19 $libs_path/libgsl.so.0; \ @@ -106,6 +106,7 @@ RUN apt-get update -qq \ && apt-get install -y -q --no-install-recommends ants \ apt-utils \ bzip2 \ + convert3d \ file \ fsl-core \ fsl-mni152-templates \ @@ -126,21 +127,9 @@ RUN sed -i '$isource /etc/fsl/fsl.sh' $ND_ENTRYPOINT ENV ANTSPATH="/usr/lib/ants" \ PATH="/usr/lib/ants:$PATH" -#------------------------ -# Install Convert3D 1.0.0 -#------------------------ -RUN echo "Downloading C3D ..." \ - && mkdir /opt/c3d \ - && curl -sSL --retry 5 https://sourceforge.net/projects/c3d/files/c3d/1.0.0/c3d-1.0.0-Linux-x86_64.tar.gz/download \ - | tar -xzC /opt/c3d --strip-components=1 -ENV C3DPATH=/opt/c3d \ - PATH=/opt/c3d/bin:$PATH - # User-defined instruction RUN gem install fakes3 -WORKDIR /work - #-------------------------------------- # Save container specifications to JSON #-------------------------------------- @@ -150,7 +139,7 @@ RUN echo '{ \ \n "instructions": [ \ \n [ \ \n "base", \ - \n "neurodebian@sha256:b09c09faa34bca0ea096b9360ee5121e048594cb8e2d7744d7d546ade88a2996" \ + \n "neurodebian@sha256:7590552afd0e7a481a33314724ae27f76ccedd05ffd7ac06ec38638872427b9b" \ \n ], \ \n [ \ \n "label", \ @@ -168,7 +157,8 @@ RUN echo '{ \ \n [ \ \n "afni", \ \n { \ - \n "version": "latest" \ + \n "version": "latest", \ + \n "install_python2": "true" \ \n } \ \n ], \ \n [ \ @@ -188,6 +178,7 @@ RUN echo '{ \ \n "ants", \ \n "apt-utils", \ \n "bzip2", \ + \n "convert3d", \ \n "file", \ \n "fsl-core", \ \n "fsl-mni152-templates", \ @@ -215,20 +206,10 @@ RUN echo '{ \ \n } \ \n ], \ \n [ \ - \n "c3d", \ - \n { \ - \n "version": "1.0.0" \ - \n } \ - \n ], \ - \n [ \ - \n "instruction", \ - \n "RUN gem install fakes3" \ - \n ], \ - \n [ \ - \n "workdir", \ - \n "/work" \ + \n "run", \ + \n "gem install fakes3" \ \n ] \ \n ], \ - \n "generation_timestamp": "2017-10-02 22:55:55", \ - \n "neurodocker_version": "0.3.1-2-g4dfcf56" \ + \n "generation_timestamp": "2017-11-06 21:15:07", \ + \n "neurodocker_version": "0.3.1-19-g8d02eb4" \ \n}' > /neurodocker/neurodocker_specs.json diff --git a/docker/generate_dockerfiles.sh b/docker/generate_dockerfiles.sh index cec0f19f80..4478bca4a1 100755 --- a/docker/generate_dockerfiles.sh +++ b/docker/generate_dockerfiles.sh @@ -80,13 +80,6 @@ function generate_base_dockerfile() { } -# The Dockerfile ADD/COPY instructions do not honor the current user, so the -# owner of the directories has to be manually changed to user neuro. -# See https://github.com/moby/moby/issues/6119 for more information on this -# behavior. -# Docker plans on changing this behavior by added a `--chown` flag to the -# ADD/COPY commands. See https://github.com/moby/moby/pull/34263. - function generate_main_dockerfile() { docker run --rm "$NEURODOCKER_IMAGE" generate \ --base "$NIPYPE_BASE_IMAGE" --pkg-manager "$PKG_MANAGER" \ From 9e0571c815b0be5e61f0ba270d4be23ce5695363 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 6 Nov 2017 16:32:36 -0500 Subject: [PATCH 09/15] enh: update codecov call + reformat --- .circleci/tests.sh | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/.circleci/tests.sh b/.circleci/tests.sh index 0178ab91dd..0eaffcce93 100644 --- a/.circleci/tests.sh +++ b/.circleci/tests.sh @@ -13,39 +13,43 @@ if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then exit 1 fi -# These tests are manually balanced based on previous build timings. +# TODO: change this image name +DOCKER_IMAGE="kaczmarj/nipype" + +# These tests are manually balanced based on previous build timings. # They may need to be rebalanced in the future. case ${CIRCLE_NODE_INDEX} in 0) - docker run --rm=false -it -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_pytests.sh && \ - docker run --rm=false -it -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_pytests.sh && \ - docker run --rm=false -it -v $WORKDIR:/work -w /src/nipype/doc --entrypoint=/usr/bin/run_builddocs.sh nipype/nipype:py36 /usr/bin/run_builddocs.sh && \ - docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d && \ - docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d + docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh \ + && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh \ + && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh \ + && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d \ + && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d exitcode=$? ;; 1) - docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 && \ - docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline + docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 \ + && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline exitcode=$? ;; 2) - docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \ - docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline + docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \ + && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline exitcode=$? ;; 3) - docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \ - docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline && \ - docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow + docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \ + && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline \ + && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow exitcode=$? ;; esac -cp ${WORKDIR}/tests/*.xml ${CIRCLE_TEST_REPORTS}/tests/ - # Exit with error if any of the tests failed if [ "$exitcode" != "0" ]; then exit 1; fi -codecov -f "coverage*.xml" -s "${WORKDIR}/tests/" -R "${HOME}/nipype/" -F unittests -e CIRCLE_NODE_INDEX -codecov -f "smoketest*.xml" -s "${WORKDIR}/tests/" -R "${HOME}/nipype/" -F smoketests -e CIRCLE_NODE_INDEX +codecov --file "${WORKDIR}/tests/coverage*.xml" \ + --root "${HOME}/nipype/" --flags unittests -e CIRCLE_NODE_INDEX + +codecov --file "${WORKDIR}/tests/smoketest*.xml" \ + --root "${HOME}/nipype/" --flags smoketests -e CIRCLE_NODE_INDEX From 97b4048135068b90260caf13f8bea628f97b4f46 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 6 Nov 2017 18:43:34 -0500 Subject: [PATCH 10/15] fix path when updating dockerfile cache --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5734e2274b..22ba0bfbd4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,12 +126,10 @@ jobs: no_output_timeout: 60m command: | if [ "$CIRCLE_NODE_INDEX" -eq "0" ]; then - echo "Saving Docker images to tar.gz files ..." docker save kaczmarj/nipype:base \ kaczmarj/nipype:latest \ kaczmarj/nipype:py36 \ kaczmarj/nipype:py27 > /tmp/docker/nipype-base-latest-py36-py27.tar - echo "$(du -h /tmp/docker/nipype-base-latest-py36-py27.tar)" fi - persist_to_workspace: root: /tmp @@ -143,7 +141,8 @@ jobs: docker: - image: docker:17.09.0-ce-git steps: - - checkout + - checkout: + path: /home/circleci/nipype - setup_remote_docker - attach_workspace: at: /tmp @@ -163,8 +162,9 @@ jobs: docker push kaczmarj/nipype:py27 - run: name: Prune base Dockerfile to update cache + working_directory: /home/circleci/nipype/docker command: | - cd /home/circleci/nipype/docker + mkdir -p /tmp/docker # Use the sha256 sum of the pruned Dockerfile as the cache key. ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned - save_cache: From 637f0335e0e02d96f53fbad7f13f409892196ea9 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 13 Nov 2017 11:27:08 -0500 Subject: [PATCH 11/15] regenerate dockerfiles with base nipype/nipype:base --- Dockerfile | 8 ++++---- docker/Dockerfile.base | 4 ++-- docker/generate_dockerfiles.sh | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9ea5e11017..65ebfac9ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,9 @@ # pull request on our GitHub repository: # https://github.com/kaczmarj/neurodocker # -# Timestamp: 2017-11-06 21:15:09 +# Timestamp: 2017-11-13 16:22:04 -FROM kaczmarj/nipype:base +FROM nipype/nipype:base ARG DEBIAN_FRONTEND=noninteractive @@ -132,7 +132,7 @@ RUN echo '{ \ \n "instructions": [ \ \n [ \ \n "base", \ - \n "kaczmarj/nipype:base" \ + \n "nipype/nipype:base" \ \n ], \ \n [ \ \n "label", \ @@ -228,6 +228,6 @@ RUN echo '{ \ \n } \ \n ] \ \n ], \ - \n "generation_timestamp": "2017-11-06 21:15:09", \ + \n "generation_timestamp": "2017-11-13 16:22:04", \ \n "neurodocker_version": "0.3.1-19-g8d02eb4" \ \n}' > /neurodocker/neurodocker_specs.json diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 429930ca66..de82e111e8 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -5,7 +5,7 @@ # pull request on our GitHub repository: # https://github.com/kaczmarj/neurodocker # -# Timestamp: 2017-11-06 21:15:07 +# Timestamp: 2017-11-13 16:22:02 FROM neurodebian@sha256:7590552afd0e7a481a33314724ae27f76ccedd05ffd7ac06ec38638872427b9b @@ -210,6 +210,6 @@ RUN echo '{ \ \n "gem install fakes3" \ \n ] \ \n ], \ - \n "generation_timestamp": "2017-11-06 21:15:07", \ + \n "generation_timestamp": "2017-11-13 16:22:02", \ \n "neurodocker_version": "0.3.1-19-g8d02eb4" \ \n}' > /neurodocker/neurodocker_specs.json diff --git a/docker/generate_dockerfiles.sh b/docker/generate_dockerfiles.sh index 4478bca4a1..52eee8a1e6 100755 --- a/docker/generate_dockerfiles.sh +++ b/docker/generate_dockerfiles.sh @@ -59,7 +59,7 @@ NEURODOCKER_IMAGE="kaczmarj/neurodocker@sha256:6b5f92f413b9710b7581e62293a8f7443 # neurodebian:stretch-non-free pulled on November 3, 2017 BASE_IMAGE="neurodebian@sha256:7590552afd0e7a481a33314724ae27f76ccedd05ffd7ac06ec38638872427b9b" -NIPYPE_BASE_IMAGE="kaczmarj/nipype:base" +NIPYPE_BASE_IMAGE="nipype/nipype:base" PKG_MANAGER="apt" DIR="$(dirname "$0")" From a29400b66137d6842d4376ea874afcdf8a7b5fb6 Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 13 Nov 2017 11:27:32 -0500 Subject: [PATCH 12/15] enh: optimize + use nipype/nipype dockerhub repo - use master branch for caching - replace kaczmarj/nipype with nipype/nipype - reset cache prefixes - use latest docker (v17.10.0-ce) containers and machine - use environment variable to determine whether to pull or build base image - save docker images to tar.gz if on master branch (deploying) - use fastest gzip compression (gives good ratio of speed/compression) --- .circleci/config.yml | 85 ++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 22ba0bfbd4..2a7f11698e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ jobs: compare_base_dockerfiles: docker: - - image: docker:17.09.0-ce-git + - image: docker:17.10.0-ce-git steps: - checkout: path: /home/circleci/nipype @@ -15,32 +15,28 @@ jobs: # Use the sha256 sum of the pruned Dockerfile as the cache key. ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned - restore_cache: - # TODO: change this to 'master' after we are sure this works. - key: dftest-v5-enh/circleci-neurodocker-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }} + key: dockerfile-cache-v1-master-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }} - run: name: Determine how to get base image command: | - GET_BASE="/tmp/docker/get_base_image.sh" - - # This directory comes from the cache. - if [ -d /cache/base-dockerfile ]; then - echo "echo Pulling base image ..." > "$GET_BASE" - echo "docker pull kaczmarj/nipype:base" >> "$GET_BASE" + if [ -f /tmp/docker/cache/Dockerfile.base-pruned ]; then + echo "Cache found. Will pull base image." + echo 'export GET_BASE=PULL' > /tmp/docker/get_base_image.sh else - echo "echo Building base image ..." > "$GET_BASE" - echo "docker build -t kaczmarj/nipype:base - < /home/circleci/nipype/docker/Dockerfile.base" >> "$GET_BASE" + echo "Cache not found. Will build base image." + echo 'export GET_BASE=BUILD' > /tmp/docker/get_base_image.sh fi - persist_to_workspace: root: /tmp paths: - - docker/* + - docker/get_base_image.sh build_and_test: parallelism: 4 machine: - # Ubuntu 14.04 with Docker 17.03.0-ce - image: circleci/classic:201703-01 + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 steps: - checkout: path: /home/circleci/nipype @@ -60,10 +56,19 @@ jobs: - run: name: Get base image (pull or build) no_output_timeout: 60m - # TODO: remove `docker pull` once once caching works. + working_directory: /home/circleci/nipype command: | - # bash /tmp/docker/get_base_image.sh - docker pull kaczmarj/nipype:base + source /tmp/docker/get_base_image.sh + if [ "$GET_BASE" == "PULL" ]; then + echo "Pulling base image ..." + docker pull nipype/nipype:base + elif [ "$GET_BASE" == "BUILD" ]; then + echo "Building base image ..." + docker build -t nipype/nipype:base - < docker/Dockerfile.base + else + echo "Error: method to get base image not understood" + exit 1 + fi - run: name: Build main image (py36) no_output_timeout: 60m @@ -72,8 +77,8 @@ jobs: e=1 && for i in {1..5}; do docker build \ --rm=false \ - --tag kaczmarj/nipype:latest \ - --tag kaczmarj/nipype:py36 \ + --tag nipype/nipype:latest \ + --tag nipype/nipype:py36 \ --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ @@ -87,7 +92,7 @@ jobs: e=1 && for i in {1..5}; do docker build \ --rm=false \ - --tag kaczmarj/nipype:py27 \ + --tag nipype/nipype:py27 \ --build-arg PYTHON_VERSION_MAJOR=2 \ --build-arg PYTHON_VERSION_MINOR=7 \ --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ @@ -125,21 +130,26 @@ jobs: name: Save Docker images to workspace no_output_timeout: 60m command: | - if [ "$CIRCLE_NODE_INDEX" -eq "0" ]; then - docker save kaczmarj/nipype:base \ - kaczmarj/nipype:latest \ - kaczmarj/nipype:py36 \ - kaczmarj/nipype:py27 > /tmp/docker/nipype-base-latest-py36-py27.tar + if [ "$CIRCLE_NODE_INDEX" -eq "0" ] && [ "$CIRCLE_BRANCH" == "master" ]; then + docker save nipype/nipype:base \ + nipype/nipype:latest \ + nipype/nipype:py36 \ + nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz + du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz + else + # Workaround for `persist_to_workspace` to succeed when we are + # not deploying Docker images. + touch /tmp/docker/nipype-base-latest-py36-py27.tar.gz fi - persist_to_workspace: root: /tmp paths: - - docker/* + - docker/nipype-base-latest-py36-py27.tar.gz deploy: docker: - - image: docker:17.09.0-ce-git + - image: docker:17.10.0-ce-git steps: - checkout: path: /home/circleci/nipype @@ -150,27 +160,27 @@ jobs: name: Load saved Docker images. no_output_timeout: 60m command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - run: name: Push to DockerHub no_output_timeout: 120m command: | echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin - docker push kaczmarj/nipype:base - docker push kaczmarj/nipype:latest - docker push kaczmarj/nipype:py36 - docker push kaczmarj/nipype:py27 + docker push nipype/nipype:base + docker push nipype/nipype:latest + docker push nipype/nipype:py36 + docker push nipype/nipype:py27 - run: name: Prune base Dockerfile to update cache working_directory: /home/circleci/nipype/docker command: | - mkdir -p /tmp/docker + mkdir -p /tmp/docker/cache # Use the sha256 sum of the pruned Dockerfile as the cache key. - ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned + ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/cache/Dockerfile.base-pruned - save_cache: paths: - - /tmp/docker/Dockerfile.base-pruned - key: dftest-v5-{{ .Branch }}-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }} + - /tmp/docker/cache/Dockerfile.base-pruned + key: dockerfile-cache-v1-{{ .Branch }}-{{ checksum "/tmp/docker/cache/Dockerfile.base-pruned" }} workflows: @@ -184,7 +194,6 @@ workflows: - deploy: filters: branches: - # TODO: change this to master after we are sure this works. - only: enh/circleci-neurodocker + only: master requires: - build_and_test From 399fe166a9565cd642a765bcc855e79e4122ca1b Mon Sep 17 00:00:00 2001 From: jakubk Date: Mon, 13 Nov 2017 11:33:05 -0500 Subject: [PATCH 13/15] use nipype/nipype repo instead of kaczmarj/nipype --- .circleci/tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/tests.sh b/.circleci/tests.sh index 0eaffcce93..f55a3249d7 100644 --- a/.circleci/tests.sh +++ b/.circleci/tests.sh @@ -13,8 +13,7 @@ if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then exit 1 fi -# TODO: change this image name -DOCKER_IMAGE="kaczmarj/nipype" +DOCKER_IMAGE="nipype/nipype" # These tests are manually balanced based on previous build timings. # They may need to be rebalanced in the future. From 28547ccf8d3d9ddd068ba3d57ee8bc217f61de54 Mon Sep 17 00:00:00 2001 From: jakubk Date: Tue, 14 Nov 2017 21:23:47 -0500 Subject: [PATCH 14/15] generate dockerfiles in ci + rm dockerfiles - do not store dockerfiles in repo - cache base dockerfile in deploy step that was generated in `compare_base_dockerfiles` step - use global `working_directory` --- .circleci/config.yml | 24 ++--- Dockerfile | 233 ----------------------------------------- docker/Dockerfile.base | 215 ------------------------------------- 3 files changed, 11 insertions(+), 461 deletions(-) delete mode 100644 Dockerfile delete mode 100644 docker/Dockerfile.base diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a7f11698e..bd09e99e84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,11 +7,14 @@ jobs: steps: - checkout: path: /home/circleci/nipype + - setup_remote_docker - run: - name: Prune base Dockerfile in preparation for cache check + name: Generate and prune base Dockerfile in preparation for cache check working_directory: /home/circleci/nipype/docker command: | mkdir -p /tmp/docker + ash ./generate_dockerfiles.sh -b + # Use the sha256 sum of the pruned Dockerfile as the cache key. ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned - restore_cache: @@ -29,6 +32,7 @@ jobs: - persist_to_workspace: root: /tmp paths: + - docker/Dockerfile.base-pruned - docker/get_base_image.sh @@ -37,18 +41,19 @@ jobs: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype steps: - checkout: path: /home/circleci/nipype - attach_workspace: at: /tmp - run: - name: Get test dependencies + name: Get test dependencies and generate Dockerfiles command: | pip install --no-cache-dir codecov + make gen-dockerfiles - run: name: Modify Nipype version if necessary - working_directory: /home/circleci/nipype command: | if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py @@ -56,7 +61,6 @@ jobs: - run: name: Get base image (pull or build) no_output_timeout: 60m - working_directory: /home/circleci/nipype command: | source /tmp/docker/get_base_image.sh if [ "$GET_BASE" == "PULL" ]; then @@ -72,7 +76,6 @@ jobs: - run: name: Build main image (py36) no_output_timeout: 60m - working_directory: /home/circleci/nipype command: | e=1 && for i in {1..5}; do docker build \ @@ -87,7 +90,6 @@ jobs: - run: name: Build main image (py27) no_output_timeout: 60m - working_directory: /home/circleci/nipype command: | e=1 && for i in {1..5}; do docker build \ @@ -151,8 +153,6 @@ jobs: docker: - image: docker:17.10.0-ce-git steps: - - checkout: - path: /home/circleci/nipype - setup_remote_docker - attach_workspace: at: /tmp @@ -171,12 +171,10 @@ jobs: docker push nipype/nipype:py36 docker push nipype/nipype:py27 - run: - name: Prune base Dockerfile to update cache - working_directory: /home/circleci/nipype/docker + name: Move pruned Dockerfile to /tmp/docker/cache directory command: | - mkdir -p /tmp/docker/cache - # Use the sha256 sum of the pruned Dockerfile as the cache key. - ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/cache/Dockerfile.base-pruned + mkdir -p /tmp/docker/cache/ + mv /tmp/docker/Dockerfile.base-pruned /tmp/docker/cache/Dockerfile.base-pruned - save_cache: paths: - /tmp/docker/cache/Dockerfile.base-pruned diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 65ebfac9ca..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,233 +0,0 @@ -# Generated by Neurodocker v0.3.1-19-g8d02eb4. -# -# Thank you for using Neurodocker. If you discover any issues -# or ways to improve this software, please submit an issue or -# pull request on our GitHub repository: -# https://github.com/kaczmarj/neurodocker -# -# Timestamp: 2017-11-13 16:22:04 - -FROM nipype/nipype:base - -ARG DEBIAN_FRONTEND=noninteractive - -#---------------------------------------------------------- -# Install common dependencies and create default entrypoint -#---------------------------------------------------------- -ENV LANG="en_US.UTF-8" \ - LC_ALL="C.UTF-8" \ - ND_ENTRYPOINT="/neurodocker/startup.sh" -RUN apt-get update -qq && apt-get install -yq --no-install-recommends \ - apt-utils bzip2 ca-certificates curl locales unzip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && localedef --force --inputfile=en_US --charmap=UTF-8 C.UTF-8 \ - && chmod 777 /opt && chmod a+s /opt \ - && mkdir -p /neurodocker \ - && if [ ! -f "$ND_ENTRYPOINT" ]; then \ - echo '#!/usr/bin/env bash' >> $ND_ENTRYPOINT \ - && echo 'set +x' >> $ND_ENTRYPOINT \ - && echo 'if [ -z "$*" ]; then /usr/bin/env bash; else $*; fi' >> $ND_ENTRYPOINT; \ - fi \ - && chmod -R 777 /neurodocker && chmod a+s /neurodocker -ENTRYPOINT ["/neurodocker/startup.sh"] - -LABEL maintainer="The nipype developers https://github.com/nipy/nipype" - -ENV MKL_NUM_THREADS="1" \ - OMP_NUM_THREADS="1" - -# Create new user: neuro -RUN useradd --no-user-group --create-home --shell /bin/bash neuro -USER neuro - -#------------------ -# Install Miniconda -#------------------ -ENV CONDA_DIR=/opt/conda \ - PATH=/opt/conda/bin:$PATH -RUN echo "Downloading Miniconda installer ..." \ - && miniconda_installer=/tmp/miniconda.sh \ - && curl -sSL -o $miniconda_installer https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && /bin/bash $miniconda_installer -b -p $CONDA_DIR \ - && rm -f $miniconda_installer \ - && conda config --system --prepend channels conda-forge \ - && conda config --system --set auto_update_conda false \ - && conda config --system --set show_channel_urls true \ - && conda clean -tipsy && sync - -#------------------------- -# Create conda environment -#------------------------- -RUN conda create -y -q --name neuro \ - && sync && conda clean -tipsy && sync \ - && sed -i '$isource activate neuro' $ND_ENTRYPOINT - -COPY ["docker/files/run_builddocs.sh", "docker/files/run_examples.sh", "docker/files/run_pytests.sh", "nipype/external/fsl_imglob.py", "/usr/bin/"] - -COPY [".", "/src/nipype"] - -USER root - -# User-defined instruction -RUN chown -R neuro /src \ - && chmod +x /usr/bin/fsl_imglob.py /usr/bin/run_*.sh \ - && . /etc/fsl/fsl.sh \ - && ln -sf /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob \ - && mkdir /work \ - && chown neuro /work - -USER neuro - -ARG PYTHON_VERSION_MAJOR="3" -ARG PYTHON_VERSION_MINOR="6" -ARG BUILD_DATE -ARG VCS_REF -ARG VERSION - -#------------------------- -# Update conda environment -#------------------------- -RUN conda install -y -q --name neuro python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} \ - icu=58.1 \ - libxml2 \ - libxslt \ - matplotlib \ - mkl \ - numpy \ - pandas \ - psutil \ - scikit-learn \ - scipy \ - traits=4.6.0 \ - && sync && conda clean -tipsy && sync \ - && /bin/bash -c "source activate neuro \ - && pip install -q --no-cache-dir -e /src/nipype[all]" \ - && sync - -# User-defined BASH instruction -RUN bash -c "mkdir -p /src/pybids \ - && curl -sSL --retry 5 https://github.com/INCF/pybids/tarball/master \ - | tar -xz -C /src/pybids --strip-components 1 \ - && source activate neuro \ - && pip install --no-cache-dir -e /src/pybids" - -WORKDIR /work - -LABEL org.label-schema.build-date="$BUILD_DATE" \ - org.label-schema.name="NIPYPE" \ - org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \ - org.label-schema.url="http://nipype.readthedocs.io" \ - org.label-schema.vcs-ref="$VCS_REF" \ - org.label-schema.vcs-url="https://github.com/nipy/nipype" \ - org.label-schema.version="$VERSION" \ - org.label-schema.schema-version="1.0" - -#-------------------------------------- -# Save container specifications to JSON -#-------------------------------------- -RUN echo '{ \ - \n "pkg_manager": "apt", \ - \n "check_urls": false, \ - \n "instructions": [ \ - \n [ \ - \n "base", \ - \n "nipype/nipype:base" \ - \n ], \ - \n [ \ - \n "label", \ - \n { \ - \n "maintainer": "The nipype developers https://github.com/nipy/nipype" \ - \n } \ - \n ], \ - \n [ \ - \n "env", \ - \n { \ - \n "MKL_NUM_THREADS": "1", \ - \n "OMP_NUM_THREADS": "1" \ - \n } \ - \n ], \ - \n [ \ - \n "user", \ - \n "neuro" \ - \n ], \ - \n [ \ - \n "miniconda", \ - \n { \ - \n "env_name": "neuro", \ - \n "activate": "true" \ - \n } \ - \n ], \ - \n [ \ - \n "copy", \ - \n [ \ - \n "docker/files/run_builddocs.sh", \ - \n "docker/files/run_examples.sh", \ - \n "docker/files/run_pytests.sh", \ - \n "nipype/external/fsl_imglob.py", \ - \n "/usr/bin/" \ - \n ] \ - \n ], \ - \n [ \ - \n "copy", \ - \n [ \ - \n ".", \ - \n "/src/nipype" \ - \n ] \ - \n ], \ - \n [ \ - \n "user", \ - \n "root" \ - \n ], \ - \n [ \ - \n "run", \ - \n "chown -R neuro /src\\n&& chmod +x /usr/bin/fsl_imglob.py /usr/bin/run_*.sh\\n&& . /etc/fsl/fsl.sh\\n&& ln -sf /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob\\n&& mkdir /work\\n&& chown neuro /work" \ - \n ], \ - \n [ \ - \n "user", \ - \n "neuro" \ - \n ], \ - \n [ \ - \n "arg", \ - \n { \ - \n "PYTHON_VERSION_MAJOR": "3", \ - \n "PYTHON_VERSION_MINOR": "6", \ - \n "BUILD_DATE": "", \ - \n "VCS_REF": "", \ - \n "VERSION": "" \ - \n } \ - \n ], \ - \n [ \ - \n "miniconda", \ - \n { \ - \n "env_name": "neuro", \ - \n "conda_install": "python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} icu=58.1 libxml2 libxslt matplotlib mkl numpy pandas psutil scikit-learn scipy traits=4.6.0", \ - \n "pip_opts": "-e", \ - \n "pip_install": "/src/nipype[all]" \ - \n } \ - \n ], \ - \n [ \ - \n "run_bash", \ - \n "mkdir -p /src/pybids\\n && curl -sSL --retry 5 https://github.com/INCF/pybids/tarball/master\\n | tar -xz -C /src/pybids --strip-components 1\\n && source activate neuro\\n && pip install --no-cache-dir -e /src/pybids" \ - \n ], \ - \n [ \ - \n "workdir", \ - \n "/work" \ - \n ], \ - \n [ \ - \n "label", \ - \n { \ - \n "org.label-schema.build-date": "$BUILD_DATE", \ - \n "org.label-schema.name": "NIPYPE", \ - \n "org.label-schema.description": "NIPYPE - Neuroimaging in Python: Pipelines and Interfaces", \ - \n "org.label-schema.url": "http://nipype.readthedocs.io", \ - \n "org.label-schema.vcs-ref": "$VCS_REF", \ - \n "org.label-schema.vcs-url": "https://github.com/nipy/nipype", \ - \n "org.label-schema.version": "$VERSION", \ - \n "org.label-schema.schema-version": "1.0" \ - \n } \ - \n ] \ - \n ], \ - \n "generation_timestamp": "2017-11-13 16:22:04", \ - \n "neurodocker_version": "0.3.1-19-g8d02eb4" \ - \n}' > /neurodocker/neurodocker_specs.json diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base deleted file mode 100644 index de82e111e8..0000000000 --- a/docker/Dockerfile.base +++ /dev/null @@ -1,215 +0,0 @@ -# Generated by Neurodocker v0.3.1-19-g8d02eb4. -# -# Thank you for using Neurodocker. If you discover any issues -# or ways to improve this software, please submit an issue or -# pull request on our GitHub repository: -# https://github.com/kaczmarj/neurodocker -# -# Timestamp: 2017-11-13 16:22:02 - -FROM neurodebian@sha256:7590552afd0e7a481a33314724ae27f76ccedd05ffd7ac06ec38638872427b9b - -ARG DEBIAN_FRONTEND=noninteractive - -#---------------------------------------------------------- -# Install common dependencies and create default entrypoint -#---------------------------------------------------------- -ENV LANG="en_US.UTF-8" \ - LC_ALL="C.UTF-8" \ - ND_ENTRYPOINT="/neurodocker/startup.sh" -RUN apt-get update -qq && apt-get install -yq --no-install-recommends \ - apt-utils bzip2 ca-certificates curl locales unzip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && localedef --force --inputfile=en_US --charmap=UTF-8 C.UTF-8 \ - && chmod 777 /opt && chmod a+s /opt \ - && mkdir -p /neurodocker \ - && if [ ! -f "$ND_ENTRYPOINT" ]; then \ - echo '#!/usr/bin/env bash' >> $ND_ENTRYPOINT \ - && echo 'set +x' >> $ND_ENTRYPOINT \ - && echo 'if [ -z "$*" ]; then /usr/bin/env bash; else $*; fi' >> $ND_ENTRYPOINT; \ - fi \ - && chmod -R 777 /neurodocker && chmod a+s /neurodocker -ENTRYPOINT ["/neurodocker/startup.sh"] - -LABEL maintainer="The nipype developers https://github.com/nipy/nipype" - -#---------------------- -# Install MCR and SPM12 -#---------------------- -# Install MATLAB Compiler Runtime -RUN apt-get update -qq && apt-get install -yq --no-install-recommends libxext6 libxt6 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && echo "Downloading MATLAB Compiler Runtime ..." \ - && curl -sSL -o /tmp/mcr.zip https://www.mathworks.com/supportfiles/downloads/R2017a/deployment_files/R2017a/installers/glnxa64/MCR_R2017a_glnxa64_installer.zip \ - && unzip -q /tmp/mcr.zip -d /tmp/mcrtmp \ - && /tmp/mcrtmp/install -destinationFolder /opt/mcr -mode silent -agreeToLicense yes \ - && rm -rf /tmp/* - -# Install standalone SPM -RUN echo "Downloading standalone SPM ..." \ - && curl -sSL -o spm.zip http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_latest_Linux_R2017a.zip \ - && unzip -q spm.zip -d /opt \ - && chmod -R 777 /opt/spm* \ - && rm -rf spm.zip \ - && /opt/spm12/run_spm12.sh /opt/mcr/v92/ quit \ - && sed -i '$iexport SPMMCRCMD=\"/opt/spm12/run_spm12.sh /opt/mcr/v92/ script\"' $ND_ENTRYPOINT -ENV MATLABCMD=/opt/mcr/v92/toolbox/matlab \ - FORCE_SPMMCR=1 \ - LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/opt/mcr/v92/runtime/glnxa64:/opt/mcr/v92/bin/glnxa64:/opt/mcr/v92/sys/os/glnxa64:$LD_LIBRARY_PATH - -#-------------------- -# Install AFNI latest -#-------------------- -ENV PATH=/opt/afni:$PATH -RUN apt-get update -qq && apt-get install -yq --no-install-recommends ed gsl-bin libglu1-mesa-dev libglib2.0-0 libglw1-mesa \ - libgomp1 libjpeg62 libxm4 netpbm tcsh xfonts-base xvfb python \ - && libs_path=/usr/lib/x86_64-linux-gnu \ - && if [ -f $libs_path/libgsl.so.19 ]; then \ - ln $libs_path/libgsl.so.19 $libs_path/libgsl.so.0; \ - fi \ - && echo "Install libxp (not in all ubuntu/debian repositories)" \ - && apt-get install -yq --no-install-recommends libxp6 \ - || /bin/bash -c " \ - curl --retry 5 -o /tmp/libxp6.deb -sSL http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb \ - && dpkg -i /tmp/libxp6.deb && rm -f /tmp/libxp6.deb" \ - && echo "Install libpng12 (not in all ubuntu/debian repositories" \ - && apt-get install -yq --no-install-recommends libpng12-0 \ - || /bin/bash -c " \ - curl -o /tmp/libpng12.deb -sSL http://mirrors.kernel.org/debian/pool/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb \ - && dpkg -i /tmp/libpng12.deb && rm -f /tmp/libpng12.deb" \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && echo "Downloading AFNI ..." \ - && mkdir -p /opt/afni \ - && curl -sSL --retry 5 https://afni.nimh.nih.gov/pub/dist/tgz/linux_openmp_64.tgz \ - | tar zx -C /opt/afni --strip-components=1 - -#-------------------------- -# Install FreeSurfer v6.0.0 -#-------------------------- -# Install version minimized for recon-all -# See https://github.com/freesurfer/freesurfer/issues/70 -RUN apt-get update -qq && apt-get install -yq --no-install-recommends bc libgomp1 libxmu6 libxt6 tcsh perl \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && echo "Downloading minimized FreeSurfer ..." \ - && curl -sSL https://dl.dropbox.com/s/nnzcfttc41qvt31/recon-all-freesurfer6-3.min.tgz | tar xz -C /opt \ - && sed -i '$isource $FREESURFER_HOME/SetUpFreeSurfer.sh' $ND_ENTRYPOINT -ENV FREESURFER_HOME=/opt/freesurfer - -# User-defined instruction -RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh - -RUN apt-get update -qq \ - && apt-get install -y -q --no-install-recommends ants \ - apt-utils \ - bzip2 \ - convert3d \ - file \ - fsl-core \ - fsl-mni152-templates \ - fusefat \ - g++ \ - git \ - graphviz \ - make \ - ruby \ - unzip \ - xvfb \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Add command(s) to entrypoint -RUN sed -i '$isource /etc/fsl/fsl.sh' $ND_ENTRYPOINT - -ENV ANTSPATH="/usr/lib/ants" \ - PATH="/usr/lib/ants:$PATH" - -# User-defined instruction -RUN gem install fakes3 - -#-------------------------------------- -# Save container specifications to JSON -#-------------------------------------- -RUN echo '{ \ - \n "pkg_manager": "apt", \ - \n "check_urls": false, \ - \n "instructions": [ \ - \n [ \ - \n "base", \ - \n "neurodebian@sha256:7590552afd0e7a481a33314724ae27f76ccedd05ffd7ac06ec38638872427b9b" \ - \n ], \ - \n [ \ - \n "label", \ - \n { \ - \n "maintainer": "The nipype developers https://github.com/nipy/nipype" \ - \n } \ - \n ], \ - \n [ \ - \n "spm", \ - \n { \ - \n "version": "12", \ - \n "matlab_version": "R2017a" \ - \n } \ - \n ], \ - \n [ \ - \n "afni", \ - \n { \ - \n "version": "latest", \ - \n "install_python2": "true" \ - \n } \ - \n ], \ - \n [ \ - \n "freesurfer", \ - \n { \ - \n "version": "6.0.0", \ - \n "min": true \ - \n } \ - \n ], \ - \n [ \ - \n "run", \ - \n "echo \"cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=\" | base64 -d | sh" \ - \n ], \ - \n [ \ - \n "install", \ - \n [ \ - \n "ants", \ - \n "apt-utils", \ - \n "bzip2", \ - \n "convert3d", \ - \n "file", \ - \n "fsl-core", \ - \n "fsl-mni152-templates", \ - \n "fusefat", \ - \n "g++", \ - \n "git", \ - \n "graphviz", \ - \n "make", \ - \n "ruby", \ - \n "unzip", \ - \n "xvfb" \ - \n ] \ - \n ], \ - \n [ \ - \n "add_to_entrypoint", \ - \n [ \ - \n "source /etc/fsl/fsl.sh" \ - \n ] \ - \n ], \ - \n [ \ - \n "env", \ - \n { \ - \n "ANTSPATH": "/usr/lib/ants", \ - \n "PATH": "/usr/lib/ants:$PATH" \ - \n } \ - \n ], \ - \n [ \ - \n "run", \ - \n "gem install fakes3" \ - \n ] \ - \n ], \ - \n "generation_timestamp": "2017-11-13 16:22:02", \ - \n "neurodocker_version": "0.3.1-19-g8d02eb4" \ - \n}' > /neurodocker/neurodocker_specs.json From 56838b98e9470f7d38284c3f4481d8d28593ba19 Mon Sep 17 00:00:00 2001 From: jakubk Date: Wed, 15 Nov 2017 10:37:24 -0500 Subject: [PATCH 15/15] add jakub kaczmarzyk (mit) --- .zenodo.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index 2fb6b63d61..41497da6d8 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -533,6 +533,11 @@ "affiliation": "University of Texas at Austin", "name": "De La Vega, Alejandro", "orcid": "0000-0001-9062-3778" + }, + { + "affiliation": "MIT", + "name": "Kaczmarzyk, Jakub", + "orcid": "0000-0002-5544-7577" } ], "keywords": [