diff --git a/.autorc b/.autorc new file mode 100644 index 00000000..eaf43f22 --- /dev/null +++ b/.autorc @@ -0,0 +1,7 @@ +{ + "onlyPublishWithReleaseLabel": true, + "baseBranch": "master", + "author": "Repronim Bot ", + "noVersionPrefix": true, + "plugins": ["git-tag"] +} diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000..aa9f1312 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,4 @@ +[codespell] +skip = .git,*.pdf,*.svg,versioneer.py,_version.py +# didi -- some name Dear to someone +ignore-words-list = didi diff --git a/.dockerignore b/.dockerignore index 9a36a2ab..c85911a8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,3 @@ -# Sphinx documentation. -# The Docker image is not intended to be used to build docs. -/docs - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/.et b/.et new file mode 100644 index 00000000..77a1472c --- /dev/null +++ b/.et @@ -0,0 +1,2 @@ +{ "bad_versions" : ["0.9.2", "0.9.3"] +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..17e26374 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +--- +# Documentation +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +version: 2 +updates: +- package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml new file mode 100644 index 00000000..fa4f34d5 --- /dev/null +++ b/.github/workflows/bootstrap.yml @@ -0,0 +1,85 @@ +# this workflow bootstraps the testing of the build the docker images +# +# - this will run the python script used to generate the workflows +# based on a the jinja template +# - commit and push the generated workflows to the branch test_docker_build +# where they will be executed + +name: bootstrap + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [ "master" ] + paths: + - .github/workflows/bootstrap.yml + - .github/workflows/create_workflows.py + - neurodocker/** + +# Uses the cron schedule for github actions +# +# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#scheduled-events +# +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) +# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) +# │ │ │ │ │ +# │ │ │ │ │ +# │ │ │ │ │ +# * * * * * + + schedule: + - cron: 0 0 1,15 * * + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + inputs: + software_name: + description: 'software to test' + required: true + default: 'all' + +permissions: + contents: write + actions: write +jobs: + bootstrap: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CI_FLOW }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: python -m pip install jinja2 pyyaml + + - name: Create workflows + run: | + software_name=${{ inputs.software_name }} + if [ -z "$software_name" ]; then + software_name="all" + fi + if [ "$software_name" = "all" ]; then + echo "testing all software" + else + echo "testing ${software_name}" + fi + git checkout -b test_docker_build + python .github/workflows/create_workflows.py --software_name ${software_name} + ls -l .github/workflows + git add . + git config --global user.email "no-reply@repronim.org" + git config --global user.name "Repronim neurodocker bot" + git commit -am "added new workflows" + git push origin --force test_docker_build diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 00000000..48ad5f28 --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,26 @@ +--- +name: Codespell + +on: + push: + branches: [master] + pull_request: + branches: [master] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Codespell + uses: codespell-project/actions-codespell@v2 diff --git a/.github/workflows/create_workflows.py b/.github/workflows/create_workflows.py new file mode 100644 index 00000000..e7d562b5 --- /dev/null +++ b/.github/workflows/create_workflows.py @@ -0,0 +1,218 @@ +""" +This scripts uses a jinja template to create CI workflows to test. + + - different linux distributions (split by the package manager they use) + - different software that neurodocker supports + - different install method for a given software + +All of those are defined in a python dictionary. + +Versions to install are read from the neurodocker template for a given software. +It is possible to skip a version by adding a "skip_versions" key to the software. + +Each workflow: + + - installs the latest version of neurodocker + - builds a dockerfile for a combination of OS / software / version / install method + - cat the dockerfile + - attempts to build the corresponding image + +This script will also create a "dashboard" saved in docs/README.md +to be picked up to be rendered by the github pages. +This requires for you to build the pages from the docs folder +and on the branch where the workflows are pushed to (currently "test_docker_build"). + +""" +import argparse +from pathlib import Path + +import yaml # type: ignore +from jinja2 import Environment, FileSystemLoader, select_autoescape + +apt_based = [ + "ubuntu:22.04", + "ubuntu:18.04", + "debian:bullseye-slim", + "debian:buster-slim", +] +yum_based = ["fedora:36", "centos:7"] + +""" +Add a "skip_versions" key to the software dictionary if you want to skip +testing a specific version. For example, if you want to skip testing +version 1.0.0 of afni, add the following to the software dictionary: + + "afni": { + "skip_versions": ["1.0.0"], + "methods": ["binaries", "source"], + "afni_python": ["true", "false"], + }, + +""" +output_dir = Path(__file__).parent + +template_folder = Path(__file__).parents[2].joinpath("neurodocker", "templates") + +build_dashboard_file = Path(__file__).parents[2].joinpath("docs", "README.md") + +# this has to match the name of the branch where the workflows are pushed to +# see .github/workflows/bootstrap.yml +branch = "test_docker_build" + +# Update to match your username and repo name if you are testing things on your fork +# "ReproNim/neurodocker" +repo = "ReproNim/neurodocker" + + +def software() -> dict[str, dict[str, list[str]]]: + return { + "afni": { + "methods": ["binaries", "source"], + "afni_python": ["true", "false"], + }, + "ants": { + "methods": ["binaries", "source"], + }, + "cat12": {"methods": ["binaries"]}, + "convert3d": {"methods": ["binaries"]}, + "dcm2niix": { + "methods": ["binaries", "source"], + }, + "freesurfer": {"methods": []}, + "fsl": { + "methods": ["binaries"], + }, + "matlabmcr": { + "methods": ["binaries"], + }, + "mricron": {"methods": ["binaries"]}, + "mrtrix3": { + "methods": ["binaries", "source"], + }, + "spm12": {"methods": ["binaries"]}, + "miniconda": {}, + } + + +def create_dashboard_file(): + """Create a build dashboard file.""" + + print("creating build dashboard file...") + print(build_dashboard_file) + + gh_actions_url = "http://github-actions.40ants.com/" + + with open(build_dashboard_file, "w") as f: + image_base_url = f"{gh_actions_url}{repo}/matrix.svg?branch={branch}" + print( + """ +# Build dashboard +""", + file=f, + ) + + # table of content + for software_, _ in software().items(): + print(f"""- [{software_}](#{software_})""", file=f) + + print("", file=f) + + # link to the github actions workflow and image of the build status + for software_, _ in software().items(): + image_url = f"{image_base_url}&only={software_}" + print( + f"""## {software_} + +[{software_} workflow](https://github.com/{repo}/actions/workflows/{software_}.yml) + +![{software_} build status]({image_url}) +""", + file=f, + ) + + +def get_versions_from_neurodocker_template(software: str) -> list[str]: + """Load the list of versions to test from the software template.""" + template = template_folder.joinpath(software).with_suffix(".yaml") + with open(template, "r") as f: + data = yaml.load(f, Loader=yaml.FullLoader) + return list(data["binaries"]["urls"].keys()) + + +def stringify(some_list: list[str]) -> str: + if len(some_list) == 1: + return f"'{some_list[0]}'" + return "'" + "', '".join(some_list) + "'" + + +def main(software_name="all"): + env = Environment( + loader=FileSystemLoader(Path(__file__).parent), + autoescape=select_autoescape(), + lstrip_blocks=True, + trim_blocks=True, + ) + + template = env.get_template("docker_build.jinja") + + os = { + "apt_based": stringify(apt_based), + "yum_based": stringify(yum_based), + "all": stringify(apt_based + yum_based), + } + + # only keep relevant software + software_to_test = software() + if software_name in software_to_test: + software_to_test = {software_name: software_to_test[software_name]} + + for software_, spec in software_to_test.items(): + wf = { + "header": "# This is file is automatically generated. Do not edit.", + "os": os, + "software": software_, + } + + versions = get_versions_from_neurodocker_template(software_) + for i in spec.get("skip_versions", []): + versions.remove(i) + if software_ == "miniconda": + versions = ["latest"] + + if versions is not None and len(versions) > 0: + wf["add_version"] = True + wf["versions"] = stringify(versions) + + if spec.get("methods") is not None and len(spec["methods"]) > 0: + wf["add_method"] = True + wf["methods"] = stringify(spec["methods"]) + + if spec.get("afni_python") is not None and len(spec["afni_python"]) > 0: + wf["add_afni_python"] = True + wf["afni_python"] = stringify(spec["afni_python"]) + + output_file = output_dir.joinpath(software_).with_suffix(".yml") + print("creating workflow") + print(f"{output_file}") + with open(output_file, "w") as f: + print(template.render(wf=wf), file=f) + + create_dashboard_file() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + choices = list(software().keys()) + choices.append("all") + + parser.add_argument( + "--software_name", + required=False, + default="all", + choices=choices, + nargs=1, + ) + args = parser.parse_args() + + main(software_name=args.software_name[0]) diff --git a/.github/workflows/docker_build.jinja b/.github/workflows/docker_build.jinja new file mode 100644 index 00000000..940d5946 --- /dev/null +++ b/.github/workflows/docker_build.jinja @@ -0,0 +1,78 @@ +{{ wf.header }} +name: '{{ wf.software }}' + +concurrency: + group: ${{ '{{' }} github.workflow {{ '}}' }}-${{ '{{' }} github.ref {{ '}}' }} + cancel-in-progress: true + +on: + push: + branches: ["test_docker_build"] + +jobs: + + {{ wf.software }}: + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + base_image: [{{ wf.os.all }}] + {% if wf.add_version %} + version: [{{ wf.versions }}] + {% endif %} + {% if wf.add_method %} + method: [{{ wf.methods }}] + {% endif %} + {% if wf.add_afni_python %} + afni_python: [{{ wf.afni_python }}] + {% endif %} + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install neurodocker + run: python -m pip install --editable .[dev] + + - name: Generate Dockerfile + run: | + + apt_based=({{ wf.os.apt_based }}) + if [[ " ${apt_based[*]} " =~ ${{ '{{' }} matrix.base_image {{ '}}' }} ]]; then + pkg_manager="apt" + fi + + yum_based=({{ wf.os.yum_based }}) + if [[ " ${yum_based[*]} " =~ ${{ '{{' }} matrix.base_image {{ '}}' }} ]]; then + pkg_manager="yum" + fi + + neurodocker \ + generate docker \ + --base-image=${{ '{{' }} matrix.base_image {{ '}}' }} \ + --pkg-manager=${pkg_manager} \ + --yes \ +{% if wf.software == 'cat12' %} + --matlabmcr method='binaries' version='2017b' \ +{% endif %} + --{{ wf.software }} \ +{% if wf.add_version %} + version=${{ '{{' }} matrix.version {{ '}}' }} \ +{% endif %} +{% if wf.add_method %} + method=${{ '{{' }} matrix.method {{ '}}' }} \ +{% endif %} +{% if wf.add_afni_python %} + install_python3=${{ '{{' }} matrix.afni_python {{ '}}' }} \ +{% endif %} > Dockerfile_tmp + + cat Dockerfile_tmp + + - name: Build the Docker image + run: docker build -f Dockerfile_tmp . diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..0481cdc1 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,25 @@ +on: + push: + branches: + - master +permissions: + contents: write +jobs: + build_docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install neurodocker + run: python -m pip install --editable .[all] + - name: build docs + run: | + make -C docs cli + sphinx-build docs docs-build + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: docs-build # The folder the action should deploy. diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..bb1b8952 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,27 @@ +name: Publish to pypi on Github release + +on: + release: + types: [published] + +jobs: + pypi-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install build & twine + run: python -m pip install build twine + + - name: Publish to pypi + run: | + python -m build + twine upload dist/* + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 68d50e89..fb0880a9 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -1,49 +1,42 @@ name: CI +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: - push: - branches: [ master ] pull_request: branches: [ master ] jobs: run-tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.10', '3.9', '3.8', '3.7'] + python-version: ['3.11', '3.10', '3.9', '3.8'] + fail-fast: false steps: - - name: Install Singularity + - name: Install Apptainer + env: + VERSION: 1.1.5 run: | sudo apt-get update - sudo apt-get install --yes \ - libssl-dev \ - uuid-dev \ - libgpgme11-dev \ - squashfs-tools - curl -fsSL https://github.com/hpcng/singularity/releases/download/v3.6.4/singularity-3.6.4.tar.gz | tar xz - cd singularity - ./mconfig -p $HOME/opt/singularity - cd builddir - make - sudo make install - - name: Set Singularity environment variables + sudo apt-get install -y wget + wget https://github.com/apptainer/apptainer/releases/download/v${VERSION}/apptainer_${VERSION}_amd64.deb + wget https://github.com/apptainer/apptainer/releases/download/v${VERSION}/apptainer-suid_${VERSION}_amd64.deb + sudo apt-get install --yes ./apptainer* + - name: Set Apptainer/Singularity environment variables run: | - echo $HOME/opt/singularity/bin >> $GITHUB_PATH - # Give reproenv the full path to singularity, so it still works with `sudo`. - echo REPROENV_SINGULARITY_PROGRAM=$HOME/opt/singularity/bin/singularity >> $GITHUB_ENV - echo SINGULARITY_CACHEDIR=/dev/shm/$(whoami)/singularity >> $GITHUB_ENV - - uses: actions/checkout@v2 + # Give reproenv the full path to apptainer, so it still works with `sudo`. + echo REPROENV_APPTAINER_PROGRAM=apptainer >> $GITHUB_ENV + echo APPTAINER_CACHEDIR=/dev/shm/$(whoami)/singularity >> $GITHUB_ENV + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install neurodocker run: python -m pip install --editable .[dev] - - name: Check types - run: mypy neurodocker - - name: Check style - run: flake8 neurodocker - name: Run python tests run: pytest --numprocesses auto - name: Get code coverage diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..fff25414 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Auto-release on PR merge + +on: + # ATM, this is the closest trigger to a PR merging + push: + branches: + - master + +env: + AUTO_VERSION: v10.37.6 + +jobs: + auto-release: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')" + steps: + - uses: actions/checkout@v4 + + - name: Prepare repository + # Fetch full git history and tags + run: git fetch --unshallow --tags + + - name: Unset header + # checkout@v2 adds a header that makes branch protection report errors + # because the Github action bot is not a collaborator on the repo + run: git config --local --unset http.https://github.com/.extraheader + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Download auto + run: | + auto_download_url="$(curl -fsSL https://api.github.com/repos/intuit/auto/releases/tags/$AUTO_VERSION | jq -r '.assets[] | select(.name == "auto-linux.gz") | .browser_download_url')" + wget -O- "$auto_download_url" | gunzip > ~/auto + chmod a+x ~/auto + + - name: Create release + run: | + ~/auto shipit -vv + env: + GH_TOKEN: ${{ secrets.AUTO_USER_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..3c5e18b5 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,25 @@ +# CRON job to close inactive issues. +# https://docs.github.com/en/actions/managing-issues-and-pull-requests/closing-inactive-issues + +name: Close inactive issues +on: + schedule: + - cron: "30 1 * * *" + +jobs: + close-issues: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v8 + with: + days-before-issue-stale: 60 + days-before-issue-close: 60 + stale-issue-label: "stale" + stale-issue-message: "This issue is stale because it has been open for 30 days with no activity." + close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." + days-before-pr-stale: -1 + days-before-pr-close: -1 + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test_doc.yml b/.github/workflows/test_doc.yml new file mode 100644 index 00000000..3986d80c --- /dev/null +++ b/.github/workflows/test_doc.yml @@ -0,0 +1,40 @@ +name: test examples + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + + schedule: + - cron: 0 0 1 * * + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + test_examples: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install neurodocker + run: python -m pip install --editable . + + - name: test common uses + run: bash < docs/common_uses/conda_multiple_env.txt + + - name: "test nipype_tuto: failure expected" + run: bash < docs/examples/nipype_tuto.txt + # do not fail the workflow if the example fails + continue-on-error: true diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..afe4801d --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,56 @@ +name: linters, formatters and type checking + +on: + pull_request: + branches: [ master ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + format: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install neurodocker + run: python -m pip install --editable .[dev] + + - name: Check style + run: flake8 neurodocker + + - name: Check black formatting + run: black --check neurodocker + + - name: Run isort + run: isort --diff --check --settings-path pyproject.toml . + + typecheck: + + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ['3.11', '3.10', '3.9', '3.8'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install neurodocker + run: python -m pip install --editable .[dev] + + - name: Check types + run: mypy --install-types --non-interactive neurodocker diff --git a/.gitignore b/.gitignore index c85911a8..18ff2a7d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# MACOS related +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -137,3 +140,6 @@ dmypy.json # Cython debug symbols cython_debug/ + +# Pycharm +.idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d5c5a4a5..e9bab3b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,32 @@ repos: - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files + - repo: https://github.com/psf/black - rev: 20.8b1 + rev: "23.1.0" hooks: - id: black + +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + args: [--profile, black] + +- repo: https://github.com/pycqa/flake8 + rev: "6.0.0" + hooks: + - id: flake8 + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: "v0.991" + hooks: + - id: mypy + additional_dependencies: [types-all] + files: neurodocker + args: ["--config-file", "setup.cfg"] + +- repo: https://github.com/codespell-project/codespell + rev: v2.2.5 + hooks: + - id: codespell diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..31d3beb3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,104 @@ +# 0.9.5 (Fri May 12 2023) + +#### 🐛 Bug Fix + +- Afni missing dependencies for suma [#512](https://github.com/ReproNim/neurodocker/pull/512) ([@stebo85](https://github.com/stebo85)) +- modifying value for entry point to allow -arg or --arg [#515](https://github.com/ReproNim/neurodocker/pull/515) ([@djarecka](https://github.com/djarecka) [@kaczmarj](https://github.com/kaczmarj)) +- Mcr missing deb libxp6 [#526](https://github.com/ReproNim/neurodocker/pull/526) ([@stebo85](https://github.com/stebo85)) +- NEW: Add support for FSL version 6.0.6 onwards [#527](https://github.com/ReproNim/neurodocker/pull/527) ([@ghisvail](https://github.com/ghisvail)) +- Enable build of docs with Sphinx 6 [#525](https://github.com/ReproNim/neurodocker/pull/525) ([@ghisvail](https://github.com/ghisvail)) +- Add FSL version 6.0.5.2 [#523](https://github.com/ReproNim/neurodocker/pull/523) ([@ghisvail](https://github.com/ghisvail)) +- Update ants.yaml [#521](https://github.com/ReproNim/neurodocker/pull/521) ([@araikes](https://github.com/araikes) [@kaczmarj](https://github.com/kaczmarj)) +- [FIX] fix link to build dashboard [#517](https://github.com/ReproNim/neurodocker/pull/517) ([@Remi-Gau](https://github.com/Remi-Gau)) +- Update cli.rst [#514](https://github.com/ReproNim/neurodocker/pull/514) ([@sooyounga](https://github.com/sooyounga) [@djarecka](https://github.com/djarecka)) +- updated version tags and added latest tag clarification to docs [#516](https://github.com/ReproNim/neurodocker/pull/516) ([@sooyounga](https://github.com/sooyounga)) +- Minc install from deb and rpm [#509](https://github.com/ReproNim/neurodocker/pull/509) ([@stebo85](https://github.com/stebo85)) +- fix: repo info ([@satra](https://github.com/satra)) +- [INFRA] test docker builds in CI [#487](https://github.com/ReproNim/neurodocker/pull/487) ([@Remi-Gau](https://github.com/Remi-Gau) [@pre-commit-ci[bot]](https://github.com/pre-commit-ci[bot]) [@satra](https://github.com/satra)) +- do not install sphinx 6.x [#505](https://github.com/ReproNim/neurodocker/pull/505) ([@kaczmarj](https://github.com/kaczmarj)) +- add bad versions to et file [#502](https://github.com/ReproNim/neurodocker/pull/502) ([@satra](https://github.com/satra)) +- [TESTS] check black style in github actions [#501](https://github.com/ReproNim/neurodocker/pull/501) ([@kaczmarj](https://github.com/kaczmarj)) + +#### ⚠️ Pushed to `master` + +- Update README.md ([@djarecka](https://github.com/djarecka)) +- add workflow token ([@satra](https://github.com/satra)) +- add commit agent ([@satra](https://github.com/satra)) +- add all changed files ([@satra](https://github.com/satra)) +- Update bootstrap.yml ([@satra](https://github.com/satra)) +- simplify git commit ([@satra](https://github.com/satra)) +- allow writing actions ([@satra](https://github.com/satra)) +- remove on demand ([@satra](https://github.com/satra)) +- fix docs build ([@satra](https://github.com/satra)) +- fix: syntax ([@satra](https://github.com/satra)) +- testing sphinx build ([@satra](https://github.com/satra)) + +#### 📝 Documentation + +- changed base image for AFNI to fedora:35 [#520](https://github.com/ReproNim/neurodocker/pull/520) ([@stebo85](https://github.com/stebo85) [@kaczmarj](https://github.com/kaczmarj)) + +#### Authors: 9 + +- [@araikes](https://github.com/araikes) +- [@pre-commit-ci[bot]](https://github.com/pre-commit-ci[bot]) +- Dorota Jarecka ([@djarecka](https://github.com/djarecka)) +- Ghislain Vaillant ([@ghisvail](https://github.com/ghisvail)) +- Jakub Kaczmarzyk ([@kaczmarj](https://github.com/kaczmarj)) +- Remi Gau ([@Remi-Gau](https://github.com/Remi-Gau)) +- Satrajit Ghosh ([@satra](https://github.com/satra)) +- Sooyoung Ahn ([@sooyounga](https://github.com/sooyounga)) +- Steffen Bollmann ([@stebo85](https://github.com/stebo85)) + +--- + +# 0.9.4 (Wed Jan 18 2023) + +#### 🐛 Bug Fix + +- fix the types in --copy and --entrypoint [#500](https://github.com/ReproNim/neurodocker/pull/500) ([@kaczmarj](https://github.com/kaczmarj)) +- add long_description to setup.cfg [#465](https://github.com/ReproNim/neurodocker/pull/465) ([@kaczmarj](https://github.com/kaczmarj)) +- [FIX] add regression test for 498 [#499](https://github.com/ReproNim/neurodocker/pull/499) ([@Remi-Gau](https://github.com/Remi-Gau)) + +#### Authors: 2 + +- Jakub Kaczmarzyk ([@kaczmarj](https://github.com/kaczmarj)) +- Remi Gau ([@Remi-Gau](https://github.com/Remi-Gau)) + +--- + +# 0.9.3 (Mon Jan 16 2023) + +#### 🐛 Bug Fix + +- FIX: skip minification tests on M1/M2 macs and run them otherwise [#497](https://github.com/ReproNim/neurodocker/pull/497) ([@kaczmarj](https://github.com/kaczmarj)) +- Add/ants24x [#473](https://github.com/ReproNim/neurodocker/pull/473) ([@araikes](https://github.com/araikes) [@kaczmarj](https://github.com/kaczmarj)) + +#### Authors: 2 + +- [@araikes](https://github.com/araikes) +- Jakub Kaczmarzyk ([@kaczmarj](https://github.com/kaczmarj)) + +--- + +# 0.9.2 (Sat Jan 14 2023) + +#### 🐛 Bug Fix + +- fix: auto setup [#496](https://github.com/ReproNim/neurodocker/pull/496) ([@satra](https://github.com/satra)) +- enh: add release workflow [#495](https://github.com/ReproNim/neurodocker/pull/495) ([@satra](https://github.com/satra)) +- remove empty lines [#488](https://github.com/ReproNim/neurodocker/pull/488) ([@satra](https://github.com/satra)) +- FIX: --version output in containers [#493](https://github.com/ReproNim/neurodocker/pull/493) ([@kaczmarj](https://github.com/kaczmarj)) +- fix: remove py 3.7 and add apptainer 1.1.5 [#490](https://github.com/ReproNim/neurodocker/pull/490) ([@satra](https://github.com/satra)) +- fix: adjust optionEatAll for click >= 8 [#492](https://github.com/ReproNim/neurodocker/pull/492) ([@satra](https://github.com/satra)) +- update pre-commit [#482](https://github.com/ReproNim/neurodocker/pull/482) ([@Remi-Gau](https://github.com/Remi-Gau)) + +#### ⚠️ Pushed to `master` + +- fix: install mypy stubs ([@satra](https://github.com/satra)) +- fix: mypy configuration ([@satra](https://github.com/satra)) + +#### Authors: 3 + +- Jakub Kaczmarzyk ([@kaczmarj](https://github.com/kaczmarj)) +- Remi Gau ([@Remi-Gau](https://github.com/Remi-Gau)) +- Satrajit Ghosh ([@satra](https://github.com/satra)) diff --git a/README.md b/README.md index e2bb63ce..6b5ff122 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![build status](https://github.com/ReproNim/neurodocker/actions/workflows/pull-request.yml/badge.svg)](https://github.com/ReproNim/neurodocker/actions/workflows/pull-request.yml) [![docker pulls](https://img.shields.io/docker/pulls/repronim/neurodocker.svg)](https://hub.docker.com/r/repronim/neurodocker/) -[![docker pulls](https://img.shields.io/docker/pulls/kaczmarj/neurodocker.svg)](https://hub.docker.com/r/kaczmarj/neurodocker/) [![python versions](https://img.shields.io/pypi/pyversions/neurodocker.svg)](https://pypi.org/project/neurodocker/) [![DOI](https://zenodo.org/badge/88654995.svg)](https://zenodo.org/badge/latestdoi/88654995) @@ -15,11 +14,12 @@ Please see our website https://www.repronim.org/neurodocker for more information Use the _Neurodocker_ Docker image (recommended): ```shell -docker run --rm repronim/neurodocker:0.7.0 --help +docker run --rm repronim/neurodocker:latest --help ``` The Docker images were moved to [repronim/neurodocker](https://hub.docker.com/r/repronim/neurodocker) from [kaczmarj/neurodocker](https://hub.docker.com/r/kaczmarj/neurodocker). + This project can also be installed with `pip`: ```shell @@ -47,3 +47,10 @@ python -m pip install --no-cache-dir --editable .[all] ``` Before committing changes, initialize `pre-commit` with `pre-commit install`. This will format code with each commit to keep the style consistent. _Neurodocker_ uses `black` for formatting. + + +## Build status + +You can check the status of the build of the Docker images +for several of the neuroimaging software packages that are supported by _Neurodocker_ +on [this page](https://github.com/ReproNim/neurodocker/blob/test_docker_build/docs/README.md). diff --git a/docs/.nojekyll b/docs/.nojekyll deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/Makefile b/docs/Makefile index d4bb2cbb..5942c131 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -14,7 +14,10 @@ help: .PHONY: help Makefile +cli: + bash generate_cli_help.sh + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile +%: Makefile cli @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/common_uses/conda_multiple_env.txt b/docs/common_uses/conda_multiple_env.txt new file mode 100644 index 00000000..53ad8102 --- /dev/null +++ b/docs/common_uses/conda_multiple_env.txt @@ -0,0 +1,17 @@ +neurodocker generate docker \ + --pkg-manager apt \ + --base-image debian:bullseye-slim \ + --miniconda \ + version=latest \ + env_name=envA \ + env_exists=false \ + conda_install=pandas \ + --miniconda \ + version=latest \ + installed=true \ + env_name=envB \ + env_exists=false \ + conda_install=scipy \ +> multi-conda-env.Dockerfile + +docker build --tag multi-conda-env --file multi-conda-env.Dockerfile . diff --git a/docs/conf.py b/docs/conf.py index ebe09bec..311edc11 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,12 +6,13 @@ # -- Path setup -------------------------------------------------------------- +import sys + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # from pathlib import Path -import sys _here = Path(__file__).parent.absolute() sys.path.insert(0, str(_here / "..")) @@ -23,7 +24,7 @@ # -- Project information ----------------------------------------------------- project = "Neurodocker" -copyright = "2021, Neurodocker Developers" +copyright = "2017-2023, Neurodocker Developers" author = "Neurodocker Developers" @@ -38,12 +39,9 @@ "sphinx.ext.githubpages", "sphinx.ext.intersphinx", "sphinx.ext.coverage", - # "sphinx.ext.mathjax", - # "sphinx.ext.ifconfig", "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.viewcode", - # "sphinx.ext.linkcode", "sphinxcontrib.apidoc", ] @@ -62,10 +60,34 @@ # html_theme = "pydata_sphinx_theme" +html_theme_options = { + "use_edit_page_button": True, + "icon_links": [ + { + "name": "GitHub", + "url": "https://github.com/ReproNim/neurodocker", + "icon": "fa-brands fa-github", + }, + { + "name": "Docker Hub", + "url": "https://hub.docker.com/r/repronim/neurodocker", + "icon": "fa-brands fa-docker", + }, + ], +} + + +html_context = { + "github_user": "ReproNim", + "github_repo": "neurodocker", + "github_version": "master", + "doc_path": "docs", +} + # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +# html_static_path = ["_static"] # -- Options for extensions --------------------------------------------------- diff --git a/docs/examples/nipype_tuto.txt b/docs/examples/nipype_tuto.txt new file mode 100644 index 00000000..c25e6db0 --- /dev/null +++ b/docs/examples/nipype_tuto.txt @@ -0,0 +1,38 @@ +neurodocker generate docker \ + --pkg-manager apt \ + --base-image debian:bullseye-slim \ + --yes \ + --ants version=2.4.3 \ + --fsl version=6.0.7.1 \ + --convert3d version=1.0.0 \ + --install gcc g++ graphviz tree git-annex vim emacs-nox nano less ncdu tig octave netbase \ + --miniconda \ + version=latest \ + mamba=true \ + conda_install="python=3.11 nipype pybids=0.16.3 pytest jupyterlab jupyter_contrib_nbextensions traits scikit-image seaborn nbformat nb_conda" \ + pip_install="nilearn=0.10.1 datalad[full] nipy duecredit nbval" \ + --run 'jupyter nbextension enable exercise2/main && jupyter nbextension enable spellchecker/main' \ + --run 'mkdir /data && chmod 777 /data && chmod a+s /data' \ + --run 'mkdir /output && chmod 777 /output && chmod a+s /output' \ + --spm12 version=r7771 \ + --user neuro \ + --run-bash 'cd /data + && datalad install -r ///workshops/nih-2017/ds000114 + && cd ds000114 + && datalad update -r + && datalad get -r sub-01/ses-test/anat sub-01/ses-test/func/*fingerfootlips*' \ + --run 'curl -fL https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/580705089ad5a101f17944a9 -o /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz + && tar xf /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz -C /data/ds000114/derivatives/fmriprep/. + && rm /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz + && find /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c -type f -not -name ?mm_T1.nii.gz -not -name ?mm_brainmask.nii.gz -not -name ?mm_tpm*.nii.gz -delete' \ + --copy . "/home/neuro/nipype_tutorial" \ + --user root \ + --run 'chown -R neuro /home/neuro/nipype_tutorial' \ + --run 'rm -rf /opt/conda/pkgs/*' \ + --user neuro \ + --run 'mkdir -p ~/.jupyter && echo c.NotebookApp.ip = \"0.0.0.0\" > ~/.jupyter/jupyter_notebook_config.py' \ + --workdir /home/neuro/nipype_tutorial \ + --entrypoint jupyter-notebook \ +> nipype-tutorial.Dockerfile + +docker build --tag nipype-tutorial --file nipype-tutorial.Dockerfile . diff --git a/docs/generate_cli_help.sh b/docs/generate_cli_help.sh new file mode 100644 index 00000000..9abc6843 --- /dev/null +++ b/docs/generate_cli_help.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +# quick and dirty way to make sure the CLI help is up to date + +echo "Generating CLI help for Neurodocker and its subcommands..." + +neurodocker --help > user_guide/cli_help.txt +neurodocker generate --help > user_guide/generate_cli_help.txt +neurodocker generate docker --help > user_guide/generate_docker_cli_help.txt +neurodocker generate singularity --help > user_guide/generate_singularity_cli_help.txt +neurodocker minify --help > user_guide/minify_cli_help.txt diff --git a/docs/index.rst b/docs/index.rst index 939a497d..16e11e18 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,6 @@ This website is in progress. Is there is something you would like to see here, p :caption: Contents: user_guide/index - build_results api diff --git a/docs/sphinxext/github_link.py b/docs/sphinxext/github_link.py index fd4b7049..c215e5bb 100644 --- a/docs/sphinxext/github_link.py +++ b/docs/sphinxext/github_link.py @@ -2,12 +2,12 @@ https://github.com/scikit-learn/scikit-learn/blob/master/doc/sphinxext/github_link.py """ -from operator import attrgetter import inspect -import subprocess import os +import subprocess import sys from functools import partial +from operator import attrgetter REVISION_CMD = "git rev-parse --short HEAD" diff --git a/docs/user_guide/.gitignore b/docs/user_guide/.gitignore new file mode 100644 index 00000000..2211df63 --- /dev/null +++ b/docs/user_guide/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/docs/user_guide/cli.rst b/docs/user_guide/cli.rst index 3dea2ff3..9b4c3667 100644 --- a/docs/user_guide/cli.rst +++ b/docs/user_guide/cli.rst @@ -7,430 +7,25 @@ This program has two subcommands: :code:`generate` and :code:`minify`. neurodocker ----------- - .. code-block:: - - Usage: neurodocker [OPTIONS] COMMAND [ARGS]... - - Generate custom containers, and minify existing containers. - - Options: - --version Show the version and exit. - --help Show this message and exit. - - Commands: - generate Generate a container. - minify Minify a container. +.. literalinclude:: cli_help.txt neurodocker generate ~~~~~~~~~~~~~~~~~~~~ -.. code-block:: - - Usage: neurodocker generate [OPTIONS] COMMAND [ARGS]... - - Generate a container. - - Options: - --template-path DIRECTORY Path to directories with templates to register - [env var: REPROENV_TEMPLATE_PATH] - - --help Show this message and exit. +.. literalinclude:: generate_cli_help.txt - Commands: - docker Generate a Dockerfile. - singularity Generate a Singularity recipe. - -The :code: `neurodocker generate` command has two subcommands: `docker` and `singularity`. Most of the arguments for these subcommands are identical, but please check the details below. +The ``neurodocker generate`` command has two subcommands: `docker` and `singularity`. +Most of the arguments for these subcommands are identical, but please check the details below. neurodocker generate docker ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: - - Usage: neurodocker generate docker [OPTIONS] - - Generate a Dockerfile. - - Options: - -p, --pkg-manager [apt|yum] System package manager [required] - -b, --base-image TEXT Base image [required] - --arg KEY=VALUE Build-time variables (do not persist after - container is built) - - --copy TEXT Copy files into the container. Provide at least - two paths. The last path is always the - destination path in the container. - - --env KEY=VALUE Set persistent environment variables - --install TEXT Install packages with system package manager - --label KEY=VALUE Set labels on the container - --run TEXT Run commands in /bin/sh - --run-bash TEXT Run commands in a bash shell - --user TEXT Switch to a different user (create user if it - does not exist) - - --workdir TEXT Set the working directory - --_header KEY=VALUE Add _header - method=[source] - options for method=source - - --afni KEY=VALUE Add afni - method=[binaries|source] - options for method=binaries - - install_path [default: /opt/afni-{{ self.version }}] - - version [default: latest] - - install_r_pkgs [default: false] - - install_python3 [default: false] - options for method=source - - version [required] - - install_path [default: /opt/afni-{{ self.version }}] - - install_r_pkgs [default: false] - - install_python3 [default: false] - - --ants KEY=VALUE Add ants - method=[binaries|source] - options for method=binaries - - version [required] - version=[2.3.4|2.3.2|2.3.1|2.3.0|2.2.0|2.1.0|2.0.3|2.0.0] - - install_path [default: /opt/ants-{{ self.version }}] - options for method=source - - version [required] - - install_path [default: /opt/ants-{{ self.version }}] - - cmake_opts [default: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF] - - make_opts [default: -j1] - - --cat12 KEY=VALUE Add cat12 - method=[binaries] - options for method=binaries - - version [required] - version=[r1933_R2017b] - - install_path [default: /opt/CAT12-{{ self.version }}] - - --convert3d KEY=VALUE Add convert3d - method=[binaries] - options for method=binaries - - version [required] - version=[nightly|1.0.0] - - install_path [default: /opt/convert3d-{{ self.version }}] - - --dcm2niix KEY=VALUE Add dcm2niix - method=[binaries|source] - options for method=binaries - - version [required] - version=[v1.0.20201102|v1.0.20200331|v1.0.20190902|latest] - - install_path [default: /opt/dcm2niix-{{ self.version }}] - options for method=source - - version [required] - - install_path [default: /opt/dcm2niix-{{ self.version }}] - - cmake_opts [default: ] - - make_opts [default: -j1] - - --freesurfer KEY=VALUE Add freesurfer - method=[binaries] - options for method=binaries - - version [required] - version=[7.1.1-min|7.1.1|7.1.0|6.0.1|6.0.0-min|6.0.0] - - install_path [default: /opt/freesurfer-{{ self.version }}] - - exclude_paths [default: average/mult-comp-cor - lib/cuda - lib/qt - subjects/V1_average - subjects/bert - subjects/cvs_avg35 - subjects/cvs_avg35_inMNI152 - subjects/fsaverage3 - subjects/fsaverage4 - subjects/fsaverage5 - subjects/fsaverage6 - subjects/fsaverage_sym - trctrain - ] - - --fsl KEY=VALUE Add fsl - method=[binaries] - options for method=binaries - - version [required] - version=[6.0.4|6.0.3|6.0.2|6.0.1|6.0.0|5.0.9|5.0.8|5.0.11|5.0.10] - - install_path [default: /opt/fsl-{{ self.version }}] - - exclude_paths [default: ] - **Note**: FSL is non-free. If you are considering commercial use of FSL, please consult the relevant license(s). - - --jq KEY=VALUE Add jq - method=[binaries|source] - options for method=binaries - - version [required] - version=[1.6|1.5] - options for method=source - - version [required] - - --minc KEY=VALUE Add minc - method=[binaries] - options for method=binaries - - version [required] - version=[1.9.15] - - install_path [default: /opt/minc-{{ self.version }}] - - --miniconda KEY=VALUE Add miniconda - method=[binaries] - options for method=binaries - - version [required] - version=[latest|*] - - install_path [default: /opt/miniconda-{{ self.version }}] - - installed [default: false] - - env_name [default: base] - - env_exists [default: true] - - conda_install [default: ] - - pip_install [default: ] - - conda_opts [default: ] - - pip_opts [default: ] - - yaml_file [default: ] - - --mricron KEY=VALUE Add mricron - method=[binaries] - options for method=binaries - - version [required] - version=[1.0.20190902|1.0.20190410|1.0.20181114|1.0.20180614|1.0.20180404|1.0.20171220] - - install_path [default: /opt/mricron-{{ self.version }}] - - --mrtrix3 KEY=VALUE Add mrtrix3 - method=[binaries|source] - options for method=binaries - - version [required] - version=[3.0.2|3.0.1|3.0.0] - - install_path [default: /opt/mrtrix3-{{ self.version }}] - - build_processes [default: 1] - options for method=source - - version [required] - - install_path [default: /opt/mrtrix3-{{ self.version }}] - - build_processes [default: ] - - --ndfreeze KEY=VALUE Add ndfreeze - method=[source] - options for method=source - - date [required] - - opts [default: ] - - --neurodebian KEY=VALUE Add neurodebian - method=[binaries] - options for method=binaries - - version [required] - version=[usa-tn|usa-nh|usa-ca|japan|greece|germany-munich|germany-magdeburg|china-zhejiang|china-tsinghua|china-scitech|australia] - - os_codename [required] - - full_or_libre [default: full] - - --petpvc KEY=VALUE Add petpvc - method=[binaries] - options for method=binaries - - version [required] - version=[1.2.4|1.2.2|1.2.1|1.2.0-b|1.2.0-a|1.1.0|1.0.0] - - install_path [default: /opt/petpvc-{{ self.version }}] - - --spm12 KEY=VALUE Add spm12 - method=[binaries] - options for method=binaries - - version [required] - version=[r7771|r7487|r7219|r6914|r6685|r6472|r6225|dev] - - install_path [default: /opt/spm12-{{ self.version }}] - - matlab_install_path [default: /opt/matlab-compiler-runtime-2010a] - - --vnc KEY=VALUE Add vnc - method=[source] - options for method=source - - passwd [required] - - --help Show this message and exit. +.. literalinclude:: generate_docker_cli_help.txt neurodocker generate singularity ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: - - Usage: neurodocker generate singularity [OPTIONS] - - Generate a Singularity recipe. - - Options: - -p, --pkg-manager [apt|yum] System package manager [required] - -b, --base-image TEXT Base image [required] - --arg KEY=VALUE Build-time variables (do not persist after - container is built) - - --copy TEXT Copy files into the container. Provide at least - two paths. The last path is always the - destination path in the container. - - --env KEY=VALUE Set persistent environment variables - --install TEXT Install packages with system package manager - --label KEY=VALUE Set labels on the container - --run TEXT Run commands in /bin/sh - --run-bash TEXT Run commands in a bash shell - --user TEXT Switch to a different user (create user if it - does not exist) - - --workdir TEXT Set the working directory - --_header KEY=VALUE Add _header - method=[source] - options for method=source - - --afni KEY=VALUE Add afni - method=[binaries|source] - options for method=binaries - - install_path [default: /opt/afni-{{ self.version }}] - - version [default: latest] - - install_r_pkgs [default: false] - - install_python3 [default: false] - options for method=source - - version [required] - - install_path [default: /opt/afni-{{ self.version }}] - - install_r_pkgs [default: false] - - install_python3 [default: false] - - --ants KEY=VALUE Add ants - method=[binaries|source] - options for method=binaries - - version [required] - version=[2.3.4|2.3.2|2.3.1|2.3.0|2.2.0|2.1.0|2.0.3|2.0.0] - - install_path [default: /opt/ants-{{ self.version }}] - options for method=source - - version [required] - - install_path [default: /opt/ants-{{ self.version }}] - - cmake_opts [default: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF] - - make_opts [default: -j1] - - --convert3d KEY=VALUE Add convert3d - method=[binaries] - options for method=binaries - - version [required] - version=[nightly|1.0.0] - - install_path [default: /opt/convert3d-{{ self.version }}] - - --dcm2niix KEY=VALUE Add dcm2niix - method=[binaries|source] - options for method=binaries - - version [required] - version=[v1.0.20201102|v1.0.20200331|v1.0.20190902|latest] - - install_path [default: /opt/dcm2niix-{{ self.version }}] - options for method=source - - version [required] - - install_path [default: /opt/dcm2niix-{{ self.version }}] - - cmake_opts [default: ] - - make_opts [default: -j1] - - --freesurfer KEY=VALUE Add freesurfer - method=[binaries] - options for method=binaries - - version [required] - version=[7.1.1-min|7.1.1|7.1.0|6.0.1|6.0.0-min|6.0.0] - - install_path [default: /opt/freesurfer-{{ self.version }}] - - exclude_paths [default: average/mult-comp-cor - lib/cuda - lib/qt - subjects/V1_average - subjects/bert - subjects/cvs_avg35 - subjects/cvs_avg35_inMNI152 - subjects/fsaverage3 - subjects/fsaverage4 - subjects/fsaverage5 - subjects/fsaverage6 - subjects/fsaverage_sym - trctrain - ] - - --fsl KEY=VALUE Add fsl - method=[binaries] - options for method=binaries - - version [required] - version=[6.0.4|6.0.3|6.0.2|6.0.1|6.0.0|5.0.9|5.0.8|5.0.11|5.0.10] - - install_path [default: /opt/fsl-{{ self.version }}] - - exclude_paths [default: ] - **Note**: FSL is non-free. If you are considering commercial use of FSL, please consult the relevant license(s). - - --jq KEY=VALUE Add jq - method=[binaries|source] - options for method=binaries - - version [required] - version=[1.6|1.5] - options for method=source - - version [required] - - --minc KEY=VALUE Add minc - method=[binaries] - options for method=binaries - - version [required] - version=[1.9.15] - - install_path [default: /opt/minc-{{ self.version }}] - - --miniconda KEY=VALUE Add miniconda - method=[binaries] - options for method=binaries - - version [required] - version=[latest|*] - - install_path [default: /opt/miniconda-{{ self.version }}] - - installed [default: false] - - env_name [default: base] - - env_exists [default: true] - - conda_install [default: ] - - pip_install [default: ] - - conda_opts [default: ] - - pip_opts [default: ] - - yaml_file [default: ] - - --mricron KEY=VALUE Add mricron - method=[binaries] - options for method=binaries - - version [required] - version=[1.0.20190902|1.0.20190410|1.0.20181114|1.0.20180614|1.0.20180404|1.0.20171220] - - install_path [default: /opt/mricron-{{ self.version }}] - - --mrtrix3 KEY=VALUE Add mrtrix3 - method=[binaries|source] - options for method=binaries - - version [required] - version=[3.0.2|3.0.1|3.0.0] - - install_path [default: /opt/mrtrix3-{{ self.version }}] - - build_processes [default: 1] - options for method=source - - version [required] - - install_path [default: /opt/mrtrix3-{{ self.version }}] - - build_processes [default: ] - - --ndfreeze KEY=VALUE Add ndfreeze - method=[source] - options for method=source - - date [required] - - opts [default: ] - - --neurodebian KEY=VALUE Add neurodebian - method=[binaries] - options for method=binaries - - version [required] - version=[usa-tn|usa-nh|usa-ca|japan|greece|germany-munich|germany-magdeburg|china-zhejiang|china-tsinghua|china-scitech|australia] - - os_codename [required] - - full_or_libre [default: full] - - --petpvc KEY=VALUE Add petpvc - method=[binaries] - options for method=binaries - - version [required] - version=[1.2.4|1.2.2|1.2.1|1.2.0-b|1.2.0-a|1.1.0|1.0.0] - - install_path [default: /opt/petpvc-{{ self.version }}] - - --spm12 KEY=VALUE Add spm12 - method=[binaries] - options for method=binaries - - version [required] - version=[r7771|r7487|r7219|r6914|r6685|r6472|r6225|dev] - - install_path [default: /opt/spm12-{{ self.version }}] - - matlab_install_path [default: /opt/matlab-compiler-runtime-2010a] - - --vnc KEY=VALUE Add vnc - method=[source] - options for method=source - - passwd [required] - - --help Show this message and exit. - +.. literalinclude:: generate_singularity_cli_help.txt neurodocker minify ~~~~~~~~~~~~~~~~~~ @@ -443,27 +38,4 @@ neurodocker minify pip install neurodocker[minify] - -.. code-block:: - - Usage: neurodocker minify [OPTIONS] COMMAND... - - Minify a container. - - Trace COMMAND... in the container, and remove all files in `--dirs-to- - prune` that were not used by the commands. - - Examples - -------- - docker run --rm -itd --name to-minify python:3.9-slim bash - neurodocker minify \ - --container to-minify \ - --dir /usr/local \ - "python -c 'a = 1 + 1; print(a)'" - - Options: - -c, --container TEXT ID or name of running Docker container [required] - -d, --dir TEXT Directories in container to prune. Data will be lost - in these directories [required] - - --help Show this message and exit. +.. literalinclude:: minify_cli_help.txt diff --git a/docs/user_guide/common_uses.rst b/docs/user_guide/common_uses.rst index 7c78550b..c1ddac31 100644 --- a/docs/user_guide/common_uses.rst +++ b/docs/user_guide/common_uses.rst @@ -31,7 +31,7 @@ This example demonstrates how to build and run an image with Jupyter Notebook. neurodocker generate docker \ --pkg-manager apt \ - --base-image debian:buster-slim \ + --base-image debian:bullseye-slim \ --miniconda \ version=latest \ conda_install="matplotlib notebook numpy pandas seaborn" \ @@ -44,7 +44,9 @@ This example demonstrates how to build and run an image with Jupyter Notebook. # Run the image. The current directory is mounted to the working directory of the # Docker image, so our notebooks are saved to the current directory. - docker run --rm -it --publish 8888:8888 --volume $(pwd):/work notebook \ + docker run --rm -it \ + --publish 8888:8888 \ + --volume $(pwd):/work notebook \ jupyter-notebook --no-browser --ip 0.0.0.0 @@ -53,25 +55,7 @@ Multiple Conda Environments This example demonstrates how to create a Docker image with multiple conda environments. -.. code-block:: bash - - neurodocker generate docker \ - --pkg-manager apt \ - --base-image debian:buster-slim \ - --miniconda \ - version=latest \ - env_name=envA \ - env_exists=false \ - conda_install=pandas \ - --miniconda \ - version=latest \ - installed=true \ - env_name=envB \ - env_exists=false \ - conda_install=scipy \ - > multi-conda-env.Dockerfile - - docker build --tag multi-conda-env --file multi-conda-env.Dockerfile . +.. literalinclude:: common_uses/conda_multiple_env.txt One can use the image in the following way: diff --git a/docs/user_guide/examples.rst b/docs/user_guide/examples.rst index 2a0aaa50..638f3fff 100644 --- a/docs/user_guide/examples.rst +++ b/docs/user_guide/examples.rst @@ -44,11 +44,30 @@ FSL Docker ~~~~~~ +.. code-block:: bash + + neurodocker generate docker \ + --pkg-manager apt \ + --base-image debian:bullseye-slim \ + --fsl version=6.0.7.1 \ + > fsl6071.Dockerfile + + docker build --tag fsl:6.0.7.1 --file fsl6071.Dockerfile . + +This will ask the following question interactively: + +.. code-block:: bash + + FSL is non-free. If you are considering commercial use of FSL, please consult the relevant license(s). Proceed? [y/N] + +If you are using neurodocker non-interactively, this problem can be avoided using: + .. code-block:: bash neurodocker generate docker \ --pkg-manager apt \ --base-image debian:buster-slim \ + --yes \ --fsl version=6.0.4 \ > fsl604.Dockerfile @@ -68,8 +87,8 @@ Docker .. code-block:: bash neurodocker generate docker \ - --pkg-manager apt \ - --base-image debian:buster-slim \ + --pkg-manager yum \ + --base-image fedora:36 \ --afni method=binaries version=latest \ > afni-binaries.Dockerfile @@ -81,26 +100,31 @@ This does not install AFNI's R packages. To install relevant R things, use the f .. code-block:: bash neurodocker generate docker \ - --pkg-manager apt \ - --base-image debian:buster-slim \ + --pkg-manager yum \ + --base-image fedora:36 \ --afni method=binaries version=latest install_r_pkgs=true \ > afni-binaries-r.Dockerfile docker build --tag afni:latest-with-r --file afni-binaries-r.Dockerfile . +.. todo:: -One can also build AFNI from source. The code below builds the current master branch. -Beware that this is AFNI's bleeding edge! + Building AFNI from source is currently failing on most tested distributions. -.. code-block:: bash +.. https://github.com/ReproNim/neurodocker/blob/test_docker_build/docs/README.md#afni - neurodocker generate docker \ - --pkg-manager apt \ - --base-image debian:buster-slim \ - --afni method=source version=master \ - > afni-source.Dockerfile +.. One can also build AFNI from source. The code below builds the current master branch. +.. Beware that this is AFNI's bleeding edge! + +.. .. code-block:: bash - docker build --tag afni:master --file afni-source.Dockerfile . +.. neurodocker generate docker \ +.. --pkg-manager yum \ +.. --base-image fedora:36 \ +.. --afni method=source version=master \ +.. > afni-source.Dockerfile + +.. docker build --tag afni:master --file afni-source.Dockerfile . FreeSurfer ---------- @@ -111,19 +135,25 @@ FreeSurfer Docker ~~~~~~ -The FreeSurfer installation is several gigabytes in size, but sometimes, users just -the pieces for :code:`recon-all`. For this reason, Neurodocker provides a FreeSurfer -minified for :code:`recon-all`. - .. code-block:: bash neurodocker generate docker \ --pkg-manager apt \ - --base-image debian:buster-slim \ - --freesurfer version=7.1.1-min \ - > freesurfer7-min.Dockerfile + --base-image debian:bullseye-slim \ + --freesurfer version=7.4.1 \ + > freesurfer741.Dockerfile + + docker build --tag freesurfer:7.4.1 --file freesurfer741.Dockerfile . + +.. todo:: - docker build --tag freesurfer:7.1.1-min --file freesurfer7-min.Dockerfile . + The minified version on Freesurfer currently fails to build on all tested distributions. + +.. https://github.com/ReproNim/neurodocker/blob/test_docker_build/docs/README.md#freesurfer + +.. The FreeSurfer installation is several gigabytes in size, but sometimes, users just +.. the pieces for :code:`recon-all`. For this reason, Neurodocker provides a FreeSurfer +.. minified for :code:`recon-all`. ANTS ---- @@ -132,51 +162,59 @@ ANTS neurodocker generate docker \ --pkg-manager apt \ - --base-image debian:buster-slim \ - --ants version=2.3.4 \ + --base-image debian:bullseye-slim \ + --ants version=2.4.3 \ > ants-234.Dockerfile - docker build --tag ants:2.3.4 --file ants-234.Dockerfile . + docker build --tag ants:2.4.3 --file ants-243.Dockerfile . + +.. note:: + Building docker images of ANTS from source fails on most tested distributions. +.. https://github.com/ReproNim/neurodocker/blob/test_docker_build/docs/README.md#ants CAT12 ---- +----- -CAT12 requires the MCR in the correction version. Miniconda and nipype is optional but recommended to use CAT12 from NiPype. +CAT12 requires the MCR in the correction version. +Miniconda and nipype is optional but recommended to use CAT12 from NiPype. .. code-block:: bash neurodocker generate docker \ - --base-image ubuntu:16.04 \ + --base-image ubuntu:22.04 \ --pkg-manager apt \ --mcr 2017b \ - --cat12 version=r1933_R2017b \ + --cat12 version=r2166_R2017b \ --miniconda \ version=latest \ - conda_install='python=3.8 traits nipype numpy scipy h5py scikit-image' \ - > cat12-r1933_R2017b.Dockerfile + conda_install='python=3.11 traits nipype numpy scipy h5py scikit-image' \ + > cat12-r2166_R2017b.Dockerfile - docker build --tag cat12:r1933_R2017b --file cat12-r1933_R2017b.Dockerfile . + docker build --tag cat12:r2166_R2017b --file cat12-r2166_R2017b.Dockerfile . SPM --- -.. note:: - - Due to the version of the Matlab Compiler Runtime used, SPM12 should be used with - a Debian Stretch base image. +.. Due to the version of the Matlab Compiler Runtime used, +.. SPM12 should be used with a Debian Stretch base image. .. code-block:: bash neurodocker generate docker \ --pkg-manager apt \ - --base-image debian:stretch-slim \ + --base-image centos:7 \ --spm12 version=r7771 \ > spm12-r7771.Dockerfile docker build --tag spm12:r7771 --file spm12-r7771.Dockerfile . +.. note:: + + Building docker images of SPM12 from source fails on most tested distributions. + +.. https://github.com/ReproNim/neurodocker/blob/test_docker_build/docs/README.md#spm12 Miniconda --------- @@ -187,7 +225,7 @@ Docker with new :code:`conda` environment, python packages installed with :code: neurodocker generate docker \ --pkg-manager apt \ - --base-image debian:buster-slim \ + --base-image debian:bullseye-slim \ --miniconda \ version=latest \ env_name=env_scipy \ @@ -199,53 +237,12 @@ Docker with new :code:`conda` environment, python packages installed with :code: docker build --tag conda-env --file conda-env.Dockerfile . -Nipype tutorial ---------------- +.. Nipype tutorial +.. --------------- -.. _nipype_tutorial_docker: +.. .. _nipype_tutorial_docker: Docker ~~~~~~ -.. code-block:: bash - - neurodocker generate docker \ - --pkg-manager apt \ - --base-image neurodebian:stretch-non-free \ - --arg DEBIAN_FRONTEND=noninteractive \ - --install convert3d ants fsl gcc g++ graphviz tree \ - git-annex-standalone vim emacs-nox nano less ncdu \ - tig git-annex-remote-rclone octave netbase \ - --spm12 version=r7771 \ - --miniconda \ - version=latest \ - conda_install="python=3.8 pytest jupyter jupyterlab jupyter_contrib_nbextensions - traits pandas matplotlib scikit-learn scikit-image seaborn nbformat - nb_conda" \ - pip_install="https://github.com/nipy/nipype/tarball/master - https://github.com/INCF/pybids/tarball/master - nilearn datalad[full] nipy duecredit nbval" \ - --run 'jupyter nbextension enable exercise2/main && jupyter nbextension enable spellchecker/main' \ - --run 'mkdir /data && chmod 777 /data && chmod a+s /data' \ - --run 'mkdir /output && chmod 777 /output && chmod a+s /output' \ - --user neuro \ - --run-bash 'cd /data - && datalad install -r ///workshops/nih-2017/ds000114 - && cd ds000114 - && datalad update -r - && datalad get -r sub-01/ses-test/anat sub-01/ses-test/func/*fingerfootlips*' \ - --run 'curl -fL https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/580705089ad5a101f17944a9 -o /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz - && tar xf /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz -C /data/ds000114/derivatives/fmriprep/. - && rm /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c.tar.gz - && find /data/ds000114/derivatives/fmriprep/mni_icbm152_nlin_asym_09c -type f -not -name ?mm_T1.nii.gz -not -name ?mm_brainmask.nii.gz -not -name ?mm_tpm*.nii.gz -delete' \ - --copy . "/home/neuro/nipype_tutorial" \ - --user root \ - --run 'chown -R neuro /home/neuro/nipype_tutorial' \ - --run 'rm -rf /opt/conda/pkgs/*' \ - --user neuro \ - --run 'mkdir -p ~/.jupyter && echo c.NotebookApp.ip = \"0.0.0.0\" > ~/.jupyter/jupyter_notebook_config.py' \ - --workdir /home/neuro/nipype_tutorial \ - --entrypoint jupyter-notebook \ - > nipype-tutorial.Dockerfile - - docker build --tag nipype-tutorial . +.. literalinclude:: examples/nipype_tuto.txt diff --git a/docs/user_guide/installation.rst b/docs/user_guide/installation.rst index bf161777..4f435786 100644 --- a/docs/user_guide/installation.rst +++ b/docs/user_guide/installation.rst @@ -9,11 +9,33 @@ Docker or Singularity. .. code-block:: bash - docker run --rm repronim/neurodocker:0.7.0 --help + docker run --rm repronim/neurodocker:latest --help + +Note: Some tools require an interactive input during installation (e.g. FSL). This can either be handled using the Neurodocker `--yes` option (see examples -> FSL) or running the container interactively will also allow to answer this question: + +.. code-block:: bash + + docker run -i --rm + +Alternatively, a singularity container: + +.. code-block:: bash + + singularity run docker://repronim/neurodocker:latest --help + +Note: The version tag `latest` is a moving target and points to the latest stable release. .. code-block:: bash - singularity run docker://repronim/neurodocker:0.7.0 --help + repronim/neurodocker:latest -> latest release (0.9.4 now) + repronim/neurodocker:master -> master branch + repronim/neurodocker:0.9.4 + repronim/neurodocker:0.9.2 + repronim/neurodocker:0.9.1 + repronim/neurodocker:0.9.0 + repronim/neurodocker:0.8.0 + repronim/neurodocker:0.7.0 + ... pip --- @@ -29,7 +51,7 @@ the Neurodocker Python API. Python 3.7 or newer is required. conda ----- -We recommend using a virtual environment or a :code:`conda` environment. +We recommend using a virtual environment or a :code:`conda` environment. In order to create a new :code:`conda` environment and install Neurodocker: .. code-block:: bash diff --git a/docs/user_guide/quickstart.rst b/docs/user_guide/quickstart.rst index a98cec1e..07a1682e 100644 --- a/docs/user_guide/quickstart.rst +++ b/docs/user_guide/quickstart.rst @@ -20,9 +20,10 @@ This is a file that defines how to build a Docker image. .. code-block:: bash - neurodocker generate docker --pkg-manager apt \ - --base-image neurodebian:buster \ - --ants version=2.3.4 \ + neurodocker generate docker \ + --pkg-manager apt \ + --base-image neurodebian:bullseye \ + --ants version=2.4.3 \ --miniconda version=latest conda_install="nipype notebook" \ --user nonroot @@ -37,9 +38,10 @@ file in an empty directory, and build with :code:`docker build`: mkdir docker-example cd docker-example # saving the output of neurodocker command in a file: Dockerfile - neurodocker generate docker --pkg-manager apt \ - --base-image neurodebian:buster \ - --ants version=2.3.4 \ + neurodocker generate docker \ + --pkg-manager apt \ + --base-image neurodebian:bullseye \ + --ants version=2.4.3 \ --miniconda version=latest conda_install="nipype notebook" \ --user nonroot > Dockerfile # building a new image using the Dockerfile (use --file option if other name is used) @@ -54,7 +56,10 @@ created in :code:`/work` would be gone after the container was stopped. .. code-block:: bash - docker run --rm -it --workdir /work --volume $PWD:/work --publish 8888:8888 \ + docker run --rm -it \ + --workdir /work \ + --volume $PWD:/work \ + --publish 8888:8888 \ nipype-ants jupyter-notebook --ip 0.0.0.0 --port 8888 Feel free to create a new notebook and :code:`import nipype`. @@ -62,17 +67,18 @@ Feel free to create a new notebook and :code:`import nipype`. Singularity ~~~~~~~~~~~ -In most cases the only difference between generating Dockerfile and -`Singularity definition file `_ (the file that is used to create a Singularity container) is in +In most cases the only difference between generating Dockerfile and +`Singularity definition file `_ (the file that is used to create a Singularity container) is in a form of :code:`neurodocker generate` command, `neurodocker generate singularity` has to be used instead of :code:`neurodocker generate docker`. **This requires having `Singularity `_ installed first.** .. code-block:: bash - neurodocker generate singularity --pkg-manager apt \ - --base-image neurodebian:buster \ - --ants version=2.3.4 \ + neurodocker generate singularity \ + --pkg-manager apt \ + --base-image neurodebian:bullseye\ + --ants version=2.4.3 \ --miniconda version=latest conda_install="nipype notebook" \ --user nonroot @@ -87,9 +93,10 @@ will not be able to run this on a shared computing environment, like a high perf mkdir singularity-example cd singularity-example # saving the output of the Neurodocker command in the Singularity file - neurodocker generate singularity --pkg-manager apt \ - --base-image neurodebian:buster \ - --ants version=2.3.4 \ + neurodocker generate singularity \ + --pkg-manager apt \ + --base-image neurodebian:bullseye\ + --ants version=2.4.3 \ --miniconda version=latest conda_install="nipype notebook" \ --user nonroot > Singularity # building a new image using the Singularity file diff --git a/neurodocker/__init__.py b/neurodocker/__init__.py index bedceb43..4c0cb0b2 100644 --- a/neurodocker/__init__.py +++ b/neurodocker/__init__.py @@ -5,8 +5,8 @@ from pathlib import Path -from neurodocker._version import get_versions from neurodocker import reproenv # noqa: F401 +from neurodocker._version import get_versions __version__ = get_versions()["version"] del get_versions diff --git a/neurodocker/cli/cli.py b/neurodocker/cli/cli.py index 995cc3d6..9316547e 100644 --- a/neurodocker/cli/cli.py +++ b/neurodocker/cli/cli.py @@ -1,29 +1,41 @@ import click from neurodocker import __version__ -from neurodocker.cli.generate import generate -from neurodocker.cli.generate import genfromjson +from neurodocker.cli.generate import generate, genfromjson @click.group() @click.version_option(__version__, message="%(prog)s version %(version)s") def cli(): - """Generate custom containers, and minify existing containers. - - The minify command is available only if Docker is installed and running. - """ + """Generate custom containers, and minify existing containers.""" cli.add_command(generate) cli.add_command(genfromjson) + +def _arm_on_mac() -> bool: + """Return True if on an ARM processor (M1/M2) in macos operating system.""" + import platform + + is_mac = platform.system().lower() == "darwin" + is_arm = platform.processor().lower() == "arm" + return is_mac and is_arm + + +# If dockerpy is installed, the Docker client can be instantiated, and if we are not +# running on an ARM-based mac computer, then we add the minification command. +# See https://github.com/docker/for-mac/issues/5191 for more information about why +# we skip ARM-based macs. +# # `docker-py` is required for minification but is not installed by default. # We also pass if there is an error retrieving the Docker client. # Say, for example, the Docker engine is not running (or it is not installed). try: from neurodocker.cli.minify.trace import minify - cli.add_command(minify) + if not _arm_on_mac(): + cli.add_command(minify) except (ImportError, RuntimeError): # TODO: should we log a debug message? We don't even have a logger at the # time of writing, so probably not. diff --git a/neurodocker/cli/generate.py b/neurodocker/cli/generate.py index 08141f73..3e1cd6a9 100644 --- a/neurodocker/cli/generate.py +++ b/neurodocker/cli/generate.py @@ -5,20 +5,26 @@ # TODO: add a dedicated class for key=value in the eat-all class. +from __future__ import annotations + import json as json_lib -from pathlib import Path import sys -import typing as ty +from pathlib import Path +from typing import IO, Any, Optional, Type, cast import click -from neurodocker.reproenv.renderers import _Renderer -from neurodocker.reproenv.renderers import DockerRenderer -from neurodocker.reproenv.renderers import SingularityRenderer -from neurodocker.reproenv.state import get_template -from neurodocker.reproenv.state import register_template -from neurodocker.reproenv.state import registered_templates -from neurodocker.reproenv.state import registered_templates_items +from neurodocker.reproenv.renderers import ( + DockerRenderer, + SingularityRenderer, + _Renderer, +) +from neurodocker.reproenv.state import ( + get_template, + register_template, + registered_templates, + registered_templates_items, +) from neurodocker.reproenv.template import Template from neurodocker.reproenv.types import allowed_pkg_managers @@ -44,15 +50,15 @@ def __init__(self, *args, **kwds): ) ] - def get_command(self, ctx: click.Context, name: str) -> ty.Optional[click.Command]: + def get_command(self, ctx: click.Context, name: str) -> Optional[click.Command]: command = self.commands.get(name) if command is None: return command # return immediately to error can be logged # This is only set if a subcommand is called. Calling --help on the group # does not set --template-path. - template_path: ty.Tuple[str] = ctx.params.get("template_path", tuple()) - yamls: ty.List[Path] = [] + template_path: tuple[str] = ctx.params.get("template_path", tuple()) + yamls: list[Path] = [] for p in template_path: path = Path(p) for pattern in ("*.yaml", "*.yml"): @@ -61,7 +67,7 @@ def get_command(self, ctx: click.Context, name: str) -> ty.Optional[click.Comman for path in yamls: _ = register_template(path) - params: ty.List[click.Parameter] = [ + params: list[click.Parameter] = [ click.Option( ["-p", "--pkg-manager"], type=click.Choice(list(allowed_pkg_managers), case_sensitive=False), @@ -82,11 +88,11 @@ class OrderedParamsCommand(click.Command): parameters. """ - def parse_args(self, ctx: click.Context, args: ty.List[str]): - self._options: ty.List[ty.Tuple[click.Parameter, ty.Any]] = [] + def parse_args(self, ctx: click.Context, args: list[str]): + self._options: list[tuple[click.Parameter, Any]] = [] # run the parser for ourselves to preserve the passed order parser = self.make_parser(ctx) - param_order: ty.List[click.Parameter] + param_order: list[click.Parameter] opts, _, param_order = parser.parse_args(args=list(args)) for param in param_order: # We need the parameter name... so if it's None, let's panic. @@ -122,11 +128,16 @@ def __init__(self, *args, **kwargs): nargs = kwargs.pop("nargs", -1) assert nargs == -1, "nargs, if set, must be -1 not {}".format(nargs) super(OptionEatAll, self).__init__(*args, **kwargs) + if self.type is click.STRING: + raise ValueError( + "in the current implementation of OptionEatAll, `type` cannot be a" + " string." + ) self._previous_parser_process = None self._eat_all_parser = None def add_to_parser(self, parser, ctx): - def parser_process(value, state): + def parser_process(value, state: click.parser.ParsingState): # method to hook to the parser.process done = False value = [value] @@ -138,7 +149,6 @@ def parser_process(value, state): if not done: value.append(state.rargs.pop(0)) value = tuple(value) - # call the actual process self._previous_parser_process(value, state) @@ -178,8 +188,8 @@ def fn(v: str): return fn(value) -def _get_common_renderer_params() -> ty.List[click.Parameter]: - params: ty.List[click.Parameter] = [ +def _get_common_renderer_params() -> list[click.Parameter]: + params: list[click.Parameter] = [ click.Option( ["-p", "--pkg-manager"], type=click.Choice(list(allowed_pkg_managers), case_sensitive=False), @@ -202,6 +212,7 @@ def _get_common_renderer_params() -> ty.List[click.Parameter]: OptionEatAll( ["--copy"], multiple=True, + type=tuple, help=( "Copy files into the container. Provide at least two paths." " The last path is always the destination path in the container." @@ -216,11 +227,13 @@ def _get_common_renderer_params() -> ty.List[click.Parameter]: OptionEatAll( ["--entrypoint"], multiple=True, + type=tuple, help="Set entrypoint of the container", ), OptionEatAll( ["--install"], multiple=True, + type=tuple, help="Install packages with system package manager", ), OptionEatAll( @@ -279,9 +292,9 @@ def _create_help_for_template(template: Template) -> str: return h -def _get_params_for_registered_templates() -> ty.List[click.Parameter]: +def _get_params_for_registered_templates() -> list[click.Parameter]: """Return list of click parameters for registered templates.""" - params: ty.List[click.Parameter] = [] + params: list[click.Parameter] = [] names_tmpls = list(registered_templates_items()) names_tmpls.sort(key=lambda r: r[0]) # sort by name for name, tmpl in names_tmpls: @@ -297,7 +310,7 @@ def _params_to_renderer_dict(ctx: click.Context, pkg_manager) -> dict: """Return dictionary compatible with compatible with `_Renderer.from_dict()`.""" renderer_dict = {"pkg_manager": pkg_manager, "instructions": []} cmd = ctx.command - cmd = ty.cast(OrderedParamsCommand, cmd) + cmd = cast(OrderedParamsCommand, cmd) for param, value in cmd._options: d = _get_instruction_for_param(ctx=ctx, param=param, value=value) # TODO: what happens if `d is None`? @@ -308,21 +321,23 @@ def _params_to_renderer_dict(ctx: click.Context, pkg_manager) -> dict: return renderer_dict -def _get_instruction_for_param( - ctx: click.Context, param: click.Parameter, value: ty.Any -): +def _get_instruction_for_param(ctx: click.Context, param: click.Parameter, value: Any): # TODO: clean this up. d = None if param.name == "from_": d = {"name": param.name, "kwds": {"base_image": value}} # arg elif param.name == "arg": - assert len(value) == 2, "expected key=value pair for --arg" + if len(value) != 2: + raise click.ClickException("expected key=value pair for --arg") k, v = value d = {"name": param.name, "kwds": {"key": k, "value": v}} # copy elif param.name == "copy": - assert len(value) > 1, "expected at least two values for --copy" + if not isinstance(value, tuple): + raise ValueError("expected this value to be a tuple (contact developers)") + if len(value) < 2: + raise click.ClickException("expected at least two values for --copy") source, destination = list(value[:-1]), value[-1] d = {"name": param.name, "kwds": {"source": source, "destination": destination}} # env @@ -331,11 +346,12 @@ def _get_instruction_for_param( d = {"name": param.name, "kwds": {**value}} # entrypoint elif param.name == "entrypoint": - if isinstance(value, str): - value = [value] - else: - value = list(value) - d = {"name": param.name, "kwds": {"args": value}} + if not isinstance(value, tuple): + raise ValueError("expected this value to be a tuple (contact developers)") + value_spl = [] + for el in value: + value_spl += el.split() + d = {"name": param.name, "kwds": {"args": value_spl}} # install elif param.name == "install": opts = None @@ -399,7 +415,7 @@ def generate(*, template_path): def _base_generate( - ctx: click.Context, renderer: ty.Type[_Renderer], pkg_manager: str, **kwds + ctx: click.Context, renderer: Type[_Renderer], pkg_manager: str, **kwds ): """Function that does all of the work of `generate docker` and `generate singularity`. The difference between those two is the renderer used. @@ -459,14 +475,14 @@ def singularity(ctx: click.Context, pkg_manager: str, **kwds): type=click.File("r"), default=sys.stdin, ) -def genfromjson(*, container_type: str, input: ty.IO): +def genfromjson(*, container_type: str, input: IO): """Generate a container from a ReproEnv JSON file. INPUT is standard input by default or a path to a JSON file. """ d = json_lib.load(input) - renderer: ty.Type[_Renderer] + renderer: Type[_Renderer] if container_type.lower() == "docker": renderer = DockerRenderer elif container_type.lower() == "singularity": diff --git a/neurodocker/cli/minify/_prune.py b/neurodocker/cli/minify/_prune.py index 027fd04b..533eebc7 100644 --- a/neurodocker/cli/minify/_prune.py +++ b/neurodocker/cli/minify/_prune.py @@ -1,7 +1,7 @@ """Remove all files under a directory but not caught by `reprozip trace`.""" +from __future__ import annotations from pathlib import Path -import typing as ty import yaml @@ -18,10 +18,9 @@ def _in_docker() -> bool: def main( *, - yaml_file: ty.Union[str, Path], - directories_to_prune: ty.Union[ty.List[str], ty.List[Path]], + yaml_file: str | Path, + directories_to_prune: list[str] | list[Path], ): - if not _in_docker(): raise RuntimeError( "Not running in a Docker container. This script should only be used within" @@ -51,7 +50,7 @@ def main( if not d.is_dir(): raise ValueError(f"Directory does not exist: {d}") - all_files: ty.Set[Path] = set() + all_files: set[Path] = set() for d in directories_to_prune: all_files.update(Path(d).rglob("*")) diff --git a/neurodocker/cli/minify/_trace.sh b/neurodocker/cli/minify/_trace.sh index 256deb0d..f9f04861 100644 --- a/neurodocker/cli/minify/_trace.sh +++ b/neurodocker/cli/minify/_trace.sh @@ -40,14 +40,12 @@ function install_missing_dependencies() { function install_conda_reprozip() { TMP_CONDA_INSTALLER=/tmp/miniconda.sh ls /tmp - curl -sSL -o "$TMP_CONDA_INSTALLER" "$CONDA_URL" - ls /tmp + curl -sSL -o "$TMP_CONDA_INSTALLER" "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" bash $TMP_CONDA_INSTALLER -b -f -p $REPROZIP_CONDA rm -f $TMP_CONDA_INSTALLER - ${REPROZIP_CONDA}/bin/python -m pip install --no-cache-dir reprozip + ${REPROZIP_CONDA}/bin/mamba install -c conda-forge -y reprozip } - function run_reprozip_trace() { # https://askubuntu.com/a/674347 cmds=("$@") diff --git a/neurodocker/cli/minify/tests/test_minify.py b/neurodocker/cli/minify/tests/test_minify.py index 70969cec..ee8859c8 100644 --- a/neurodocker/cli/minify/tests/test_minify.py +++ b/neurodocker/cli/minify/tests/test_minify.py @@ -1,13 +1,19 @@ from pathlib import Path -from click.testing import CliRunner import pytest +from click.testing import CliRunner +from neurodocker.cli.cli import _arm_on_mac from neurodocker.cli.minify.trace import minify docker = pytest.importorskip("docker", reason="docker-py not found") +skip_arm_on_mac = pytest.mark.skipif( + _arm_on_mac(), reason="minification does not work on M1/M2 macs" +) + +@skip_arm_on_mac def test_minify(): client = docker.from_env() container = client.containers.run("python:3.9-slim", detach=True, tty=True) @@ -37,6 +43,7 @@ def test_minify(): container.remove() +@skip_arm_on_mac def test_minify_abort(): client = docker.from_env() container = client.containers.run("python:3.9-slim", detach=True, tty=True) @@ -65,6 +72,7 @@ def test_minify_abort(): container.remove() +@skip_arm_on_mac def test_minify_with_mounted_volume(tmp_path: Path): client = docker.from_env() diff --git a/neurodocker/cli/minify/trace.py b/neurodocker/cli/minify/trace.py index a900626b..33234f2b 100644 --- a/neurodocker/cli/minify/trace.py +++ b/neurodocker/cli/minify/trace.py @@ -7,11 +7,13 @@ # TODO: consider implementing custom types for Docker container and paths within a # Docker container. +from __future__ import annotations + import io import logging -from pathlib import Path import tarfile -import typing as ty +from pathlib import Path +from typing import Generator, cast import click @@ -36,9 +38,9 @@ def copy_file_to_container( - container: ty.Union[str, docker.models.containers.Container], - src: ty.Union[str, Path], - dest: ty.Union[str, Path], + container: str | docker.models.containers.Container, + src: str | Path, + dest: str | Path, ) -> bool: """Copy `local_filepath` into `container`:`container_path`. @@ -98,10 +100,10 @@ def _get_mounts(container: docker.models.containers.Container) -> dict: @click.option("--yes", is_flag=True, help="Reply yes to all prompts.") @click.argument("command", nargs=-1, required=True) def minify( - container: ty.Union[str, docker.models.containers.Container], - directories_to_prune: ty.Tuple[str], + container: str | docker.models.containers.Container, + directories_to_prune: tuple[str], yes: bool, - command: ty.Tuple[str], + command: tuple[str], ) -> None: """Minify a container. @@ -118,7 +120,7 @@ def minify( "python -c 'a = 1 + 1; print(a)'" """ container = client.containers.get(container) - container = ty.cast(docker.models.containers.Container, container) + container = cast(docker.models.containers.Container, container) cmds = " ".join(f'"{c}"' for c in command) @@ -132,7 +134,7 @@ def minify( # iteration. exec_dict: dict = container.client.api.exec_create(container.id, cmd=trace_cmd) exec_id: str = exec_dict["Id"] - log_gen: ty.Generator[bytes, None, None] = container.client.api.exec_start( + log_gen: Generator[bytes, None, None] = container.client.api.exec_start( exec_id, stream=True ) for log in log_gen: diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index b6066713..f5444535 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -1,14 +1,15 @@ from pathlib import Path -from click.testing import CliRunner import pytest +from click.testing import CliRunner -from neurodocker.cli.cli import generate -from neurodocker.cli.cli import genfromjson +from neurodocker.cli.cli import generate, genfromjson from neurodocker.reproenv.state import _TemplateRegistry -from neurodocker.reproenv.tests.utils import get_build_and_run_fns -from neurodocker.reproenv.tests.utils import skip_if_no_docker -from neurodocker.reproenv.tests.utils import skip_if_no_singularity +from neurodocker.reproenv.tests.utils import ( + get_build_and_run_fns, + skip_if_no_docker, + skip_if_no_singularity, +) # Test that a template can be rendered # We need to use `reproenv generate` as the entrypoint here because the generate command diff --git a/neurodocker/cli/tests/test_cli.py b/neurodocker/cli/tests/test_cli.py index 1b108779..ba54339e 100644 --- a/neurodocker/cli/tests/test_cli.py +++ b/neurodocker/cli/tests/test_cli.py @@ -2,10 +2,11 @@ from pathlib import Path -from click.testing import CliRunner import pytest +from click.testing import CliRunner from neurodocker.cli.cli import generate +from neurodocker.cli.generate import OptionEatAll _cmds = ["docker", "singularity"] @@ -42,6 +43,69 @@ def test_minimal_args(cmd: str, pkg_manager: str): assert result.exit_code == 0, result.output +def test_copy_issue_498(): + runner = CliRunner() + result = runner.invoke( + generate, + [ + "docker", + "--pkg-manager", + "apt", + "--base-image", + "debian", + # copy + "--copy", + "file1", + "file2", + ], + ) + assert "file1" in result.output + assert '\nCOPY ["file1", \\\n "file2"]\n' in result.output + + # Fail if given fewer than two inputs to --copy. + result = runner.invoke( + generate, + [ + "docker", + "--pkg-manager", + "apt", + "--base-image", + "debian", + # copy + "--copy", + "file1", + ], + ) + assert "Error: expected at least two values for --copy" in result.output + assert result.exit_code != 0 + + +# Issue #498 references --copy but the same broken behavior is seen in --entrypoint. +def test_entrypoint_issue_498(): + runner = CliRunner() + result = runner.invoke( + generate, + [ + "docker", + "--pkg-manager", + "apt", + "--base-image", + "debian", + "--entrypoint", + "printf", + "this", + "that", + ], + ) + assert '\nENTRYPOINT ["printf", "this", "that"]\n' in result.output + + +def test_optioneatall_type_issue_498(): + with pytest.raises(ValueError): + OptionEatAll(["--foo"], type=str) + OptionEatAll(["--foo"], type=tuple) + + @pytest.mark.parametrize("cmd", _cmds) @pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) def test_all_args(cmd: str, pkg_manager: str): diff --git a/neurodocker/reproenv/__init__.py b/neurodocker/reproenv/__init__.py index e13a5700..93f37002 100644 --- a/neurodocker/reproenv/__init__.py +++ b/neurodocker/reproenv/__init__.py @@ -4,7 +4,7 @@ from neurodocker.reproenv.renderers import DockerRenderer # noqa: F401 from neurodocker.reproenv.renderers import SingularityRenderer # noqa: F401 +from neurodocker.reproenv.state import get_template # noqa: F401 from neurodocker.reproenv.state import register_template # noqa: F401 from neurodocker.reproenv.state import registered_templates # noqa: F401 -from neurodocker.reproenv.state import get_template # noqa: F401 from neurodocker.reproenv.template import Template # noqa: F401 diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index 68d2a35c..4217d75a 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -8,22 +8,21 @@ import os import pathlib import types -import typing as ty +from typing import Callable, Mapping, NoReturn, Optional, Union import jinja2 -from neurodocker.reproenv.exceptions import RendererError -from neurodocker.reproenv.exceptions import TemplateError -from neurodocker.reproenv.state import _TemplateRegistry -from neurodocker.reproenv.state import _validate_renderer -from neurodocker.reproenv.template import _BaseInstallationTemplate -from neurodocker.reproenv.template import Template -from neurodocker.reproenv.types import _SingularityHeaderType -from neurodocker.reproenv.types import REPROENV_SPEC_FILE_IN_CONTAINER -from neurodocker.reproenv.types import allowed_pkg_managers -from neurodocker.reproenv.types import allowed_installation_methods -from neurodocker.reproenv.types import installation_methods_type -from neurodocker.reproenv.types import pkg_managers_type +from neurodocker.reproenv.exceptions import RendererError, TemplateError +from neurodocker.reproenv.state import _TemplateRegistry, _validate_renderer +from neurodocker.reproenv.template import Template, _BaseInstallationTemplate +from neurodocker.reproenv.types import ( + REPROENV_SPEC_FILE_IN_CONTAINER, + _SingularityHeaderType, + allowed_installation_methods, + allowed_pkg_managers, + installation_methods_type, + pkg_managers_type, +) # All jinja2 templates are instantiated from this environment object. It is # configured to dislike undefined attributes. For example, if a template is @@ -35,7 +34,7 @@ # template. -def _raise_helper(msg: str) -> ty.NoReturn: +def _raise_helper(msg: str) -> NoReturn: raise RendererError(msg) @@ -44,13 +43,13 @@ def _raise_helper(msg: str) -> ty.NoReturn: # TODO: add a flag that avoids buggy behavior when basing a new container on # one created with ReproEnv. -PathType = ty.Union[str, pathlib.Path, os.PathLike] +PathType = Union[str, pathlib.Path, os.PathLike] def _render_string_from_template( source: str, template: _BaseInstallationTemplate ) -> str: - """Take a string from a template and render """ + """Take a string from a template and render""" # TODO: we could use a while loop or recursive function to render the template until # there are no jinja-specific things. At this point, we support one level of # nesting. @@ -87,7 +86,7 @@ def _render_string_from_template( return source -def _log_instruction(func: ty.Callable): +def _log_instruction(func: Callable): """Decorator that logs instructions passed to a Renderer. This adds the logs to the `_instructions` attribute of the Renderer instance. @@ -132,7 +131,7 @@ def with_logging(self, *args, **kwds): class _Renderer: def __init__( - self, pkg_manager: pkg_managers_type, users: ty.Optional[ty.Set[str]] = None + self, pkg_manager: pkg_managers_type, users: Optional[set[str]] = None ) -> None: if pkg_manager not in allowed_pkg_managers: raise RendererError( @@ -146,7 +145,7 @@ def __init__( # specification to JSON, because if we are not root, we can change to root, # write the file, and return to whichever user we were. self._current_user = "root" - self._instructions: ty.Mapping = { + self._instructions: Mapping = { "pkg_manager": self.pkg_manager, "existing_users": list(self._users), "instructions": [], @@ -179,11 +178,11 @@ def __str__(self) -> str: return f"{masthead}\n\n{image_spec}" @property - def users(self) -> ty.Set[str]: + def users(self) -> set[str]: return self._users @classmethod - def from_dict(cls, d: ty.Mapping) -> _Renderer: + def from_dict(cls, d: Mapping) -> _Renderer: """Instantiate a new renderer from a dictionary of instructions.""" # raise error if invalid _validate_renderer(d) @@ -275,7 +274,7 @@ def add_template( # Add environment (render any jinja templates). if template_method.env: - d: ty.Mapping[str, str] = { + d: Mapping[str, str] = { _render_string_from_template( k, template_method ): _render_string_from_template(v, template_method) @@ -286,7 +285,7 @@ def add_template( # Patch the `template_method.install_dependencies` instance method so it can be # used (ie rendered) in a template and have access to the pkg_manager requested. def install_patch( - inner_self: _BaseInstallationTemplate, pkgs: ty.List[str], opts: str = None + inner_self: _BaseInstallationTemplate, pkgs: list[str], opts: str = None ) -> str: return _install(pkgs=pkgs, pkg_manager=self.pkg_manager) @@ -339,7 +338,6 @@ def install_dependencies_patch( def add_registered_template( self, name: str, method: installation_methods_type = None, **kwds ) -> _Renderer: - # Template was validated at registration time. template_dict = _TemplateRegistry.get(name) @@ -371,26 +369,32 @@ def arg(self, key: str, value: str = None): def copy( self, - source: ty.Union[PathType, ty.List[PathType]], - destination: ty.Union[PathType, ty.List[PathType]], + source: PathType | list[PathType], + destination: PathType | list[PathType], ) -> _Renderer: raise NotImplementedError() def env(self, **kwds: str) -> _Renderer: raise NotImplementedError() - def entrypoint(self, args: ty.List[str]) -> _Renderer: + def entrypoint(self, args: list[str]) -> _Renderer: raise NotImplementedError() def from_(self, base_image: str) -> _Renderer: raise NotImplementedError() - def install(self, pkgs: ty.List[str], opts: str = None) -> _Renderer: + def install(self, pkgs: list[str], opts: str = None) -> _Renderer: raise NotImplementedError() def label(self, **kwds: str) -> _Renderer: raise NotImplementedError() + def labels(self, labels_dict: dict) -> _Renderer: + """Adds a set of labels to the dockerfile from a dict. This permits + labels that can include special chars (e.g. '.').""" + self.label(**labels_dict) + return self + def run(self, command: str) -> _Renderer: raise NotImplementedError() @@ -425,16 +429,16 @@ def _get_instructions(self) -> str: j = " \\\n".join(j.splitlines()) # Escape the % characters so printf does not interpret them as delimiters. j = j.replace("%", "%%") + # Escape single quotes with '"'"' + j = j.replace("'", "'\"'\"'") cmd = f"printf '{j}' > {REPROENV_SPEC_FILE_IN_CONTAINER}" return cmd class DockerRenderer(_Renderer): - def __init__( - self, pkg_manager: pkg_managers_type, users: ty.Set[str] = None - ) -> None: + def __init__(self, pkg_manager: pkg_managers_type, users: set[str] = None) -> None: super().__init__(pkg_manager=pkg_manager, users=users) - self._parts: ty.List[str] = [] + self._parts: list[str] = [] def render(self) -> str: """Return the rendered Dockerfile.""" @@ -461,7 +465,7 @@ def arg(self, key: str, value: str = None) -> DockerRenderer: @_log_instruction def copy( self, - source: ty.Union[PathType, ty.List[PathType]], + source: PathType | list[PathType], destination: PathType, from_: str = None, chown: str = None, @@ -488,7 +492,7 @@ def env(self, **kwds: str) -> DockerRenderer: return self @_log_instruction - def entrypoint(self, args: ty.List[str]) -> DockerRenderer: + def entrypoint(self, args: list[str]) -> DockerRenderer: s = 'ENTRYPOINT ["{}"]'.format('", "'.join(args)) self._parts.append(s) return self @@ -504,7 +508,7 @@ def from_(self, base_image: str, as_: str = None) -> DockerRenderer: return self @_log_instruction - def install(self, pkgs: ty.List[str], opts=None) -> DockerRenderer: + def install(self, pkgs: list[str], opts=None) -> DockerRenderer: """Install system packages.""" command = _install(pkgs, pkg_manager=self.pkg_manager, opts=opts) command = _indent_run_instruction(command) @@ -557,18 +561,18 @@ def workdir(self, path: PathType) -> DockerRenderer: class SingularityRenderer(_Renderer): def __init__( - self, pkg_manager: pkg_managers_type, users: ty.Optional[ty.Set[str]] = None + self, pkg_manager: pkg_managers_type, users: Optional[set[str]] = None ) -> None: super().__init__(pkg_manager=pkg_manager, users=users) self._header: _SingularityHeaderType = {} - # The '%setup' section is intentionally ommitted. - self._files: ty.List[str] = [] - self._environment: ty.List[ty.Tuple[str, str]] = [] - self._post: ty.List[str] = [] + # The '%setup' section is intentionally omitted. + self._files: list[str] = [] + self._environment: list[tuple[str, str]] = [] + self._post: list[str] = [] self._runscript = "" # TODO: is it OK to use a dict here? Labels could be overwritten. - self._labels: ty.Dict[str, str] = {} + self._labels: dict[str, str] = {} def render(self) -> str: s = "" @@ -610,9 +614,9 @@ def render(self) -> str: # Add labels. if self._labels: - s += "\n\n%labels\n" + s += "\n\n%labels" for kv in self._labels.items(): - s += " ".join(kv) + s += "\n" + " ".join(kv) return s @@ -627,7 +631,7 @@ def arg(self, key: str, value: str = None) -> SingularityRenderer: @_log_instruction def copy( self, - source: ty.Union[PathType, ty.List[PathType]], + source: PathType | list[PathType], destination: PathType, ) -> SingularityRenderer: if not isinstance(source, (list, tuple)): @@ -643,7 +647,7 @@ def env(self, **kwds: str) -> SingularityRenderer: return self @_log_instruction - def entrypoint(self, args: ty.List[str]) -> SingularityRenderer: + def entrypoint(self, args: list[str]) -> SingularityRenderer: self._runscript = " ".join(args) return self @@ -665,7 +669,7 @@ def from_(self, base_image: str) -> SingularityRenderer: return self @_log_instruction - def install(self, pkgs: ty.List[str], opts=None) -> SingularityRenderer: + def install(self, pkgs: list[str], opts=None) -> SingularityRenderer: """Install system packages.""" command = _install(pkgs, pkg_manager=self.pkg_manager, opts=opts) self.run(command) @@ -709,6 +713,8 @@ def _indent_run_instruction(string: str, indent=4) -> str: lines = string.splitlines() for ii, line in enumerate(lines): line = line.rstrip() + if not line: + continue is_last_line = ii == len(lines) - 1 already_cont = line.startswith(("&&", "&", "||", "|", "fi")) is_comment = line.startswith("#") @@ -726,7 +732,7 @@ def _indent_run_instruction(string: str, indent=4) -> str: return "\n".join(out) -def _install(pkgs: ty.List[str], pkg_manager: str, opts: str = None) -> str: +def _install(pkgs: list[str], pkg_manager: str, opts: str = None) -> str: if pkg_manager == "apt": return _apt_install(pkgs, opts) elif pkg_manager == "yum": @@ -736,7 +742,7 @@ def _install(pkgs: ty.List[str], pkg_manager: str, opts: str = None) -> str: raise RendererError(f"Unknown package manager '{pkg_manager}'.") -def _apt_install(pkgs: ty.List[str], opts: str = None, sort=True) -> str: +def _apt_install(pkgs: list[str], opts: str = None, sort=True) -> str: """Return command to install deb packages with `apt-get` (Debian-based distros). `opts` are options passed to `yum install`. Default is "-q --no-install-recommends". @@ -754,7 +760,7 @@ def _apt_install(pkgs: ty.List[str], opts: str = None, sort=True) -> str: return s.strip() -def _apt_install_debs(urls: ty.List[str], opts: str = None, sort=True) -> str: +def _apt_install_debs(urls: list[str], opts: str = None, sort=True) -> str: """Return command to install deb packages with `apt-get` (Debian-based distros). `opts` are options passed to `yum install`. Default is "-q". @@ -778,7 +784,7 @@ def install_one(url: str): return s -def _yum_install(pkgs: ty.List[str], opts: str = None, sort=True) -> str: +def _yum_install(pkgs: list[str], opts: str = None, sort=True) -> str: """Return command to install packages with `yum` (CentOS, Fedora). `opts` are options passed to `yum install`. Default is "-q". diff --git a/neurodocker/reproenv/schemas/renderer.json b/neurodocker/reproenv/schemas/renderer.json index 7e5073e6..d01880de 100644 --- a/neurodocker/reproenv/schemas/renderer.json +++ b/neurodocker/reproenv/schemas/renderer.json @@ -394,4 +394,4 @@ "additionalProperties": false } } -} \ No newline at end of file +} diff --git a/neurodocker/reproenv/state.py b/neurodocker/reproenv/state.py index 090424a9..6601c068 100644 --- a/neurodocker/reproenv/state.py +++ b/neurodocker/reproenv/state.py @@ -1,10 +1,11 @@ """Stateful objects in reproenv runtime.""" +from __future__ import annotations import copy import json import os from pathlib import Path -import typing as ty +from typing import ItemsView, KeysView import jsonschema import yaml @@ -16,18 +17,20 @@ except ImportError: # pragma: no cover from yaml import SafeLoader # type: ignore # pragma: no cover -from neurodocker.reproenv.exceptions import RendererError -from neurodocker.reproenv.exceptions import TemplateError -from neurodocker.reproenv.exceptions import TemplateNotFound +from neurodocker.reproenv.exceptions import ( + RendererError, + TemplateError, + TemplateNotFound, +) from neurodocker.reproenv.types import TemplateType _schemas_path = Path(__file__).parent / "schemas" with (_schemas_path / "template.json").open("r") as f: - _TEMPLATE_SCHEMA: ty.Dict = json.load(f) + _TEMPLATE_SCHEMA: dict = json.load(f) with (_schemas_path / "renderer.json").open("r") as f: - _RENDERER_SCHEMA: ty.Dict = json.load(f) + _RENDERER_SCHEMA: dict = json.load(f) def _validate_template(template: TemplateType): @@ -57,7 +60,7 @@ def _validate_renderer(d): class _TemplateRegistry: """Object to hold templates in memory.""" - _templates: ty.Dict[str, TemplateType] = {} + _templates: dict[str, TemplateType] = {} @classmethod def _reset(cls): @@ -67,7 +70,7 @@ def _reset(cls): @classmethod def register( cls, - path_or_template: ty.Union[str, os.PathLike, TemplateType], + path_or_template: str | os.PathLike | TemplateType, name: str = None, ) -> TemplateType: """Register a template. This will overwrite an existing template with the @@ -148,12 +151,12 @@ def get(cls, name: str) -> TemplateType: ) @classmethod - def keys(cls) -> ty.KeysView[str]: + def keys(cls) -> KeysView[str]: """Return names of registered templates.""" return cls._templates.keys() @classmethod - def items(cls) -> ty.ItemsView[str, TemplateType]: + def items(cls) -> ItemsView[str, TemplateType]: return cls._templates.items() diff --git a/neurodocker/reproenv/template.py b/neurodocker/reproenv/template.py index 62fe94fb..1d365ef6 100644 --- a/neurodocker/reproenv/template.py +++ b/neurodocker/reproenv/template.py @@ -3,13 +3,15 @@ from __future__ import annotations import copy -import typing as ty +from typing import Mapping, Optional, cast from neurodocker.reproenv.exceptions import TemplateKeywordArgumentError from neurodocker.reproenv.state import _validate_template -from neurodocker.reproenv.types import _BinariesTemplateType -from neurodocker.reproenv.types import _SourceTemplateType -from neurodocker.reproenv.types import TemplateType +from neurodocker.reproenv.types import ( + TemplateType, + _BinariesTemplateType, + _SourceTemplateType, +) class Template: @@ -39,8 +41,8 @@ class Template: def __init__( self, template: TemplateType, - binaries_kwds: ty.Mapping[str, str] = None, - source_kwds: ty.Mapping[str, str] = None, + binaries_kwds: Mapping[str, str] = None, + source_kwds: Mapping[str, str] = None, ): # Validate against JSON schema. Registered templates were already validated at # registration time, but if we do not validate here, then in-memory templates @@ -48,9 +50,9 @@ def __init__( _validate_template(template) self._template = copy.deepcopy(template) - self._binaries: ty.Optional[_BinariesTemplate] = None + self._binaries: Optional[_BinariesTemplate] = None self._binaries_kwds = {} if binaries_kwds is None else binaries_kwds - self._source: ty.Optional[_SourceTemplate] = None + self._source: Optional[_SourceTemplate] = None self._source_kwds = {} if source_kwds is None else source_kwds if "binaries" in self._template: @@ -67,11 +69,11 @@ def name(self) -> str: return self._template["name"] @property - def binaries(self) -> ty.Union[None, _BinariesTemplate]: + def binaries(self) -> None | _BinariesTemplate: return self._binaries @property - def source(self) -> ty.Union[None, _SourceTemplate]: + def source(self) -> None | _SourceTemplate: return self._source @property @@ -102,7 +104,7 @@ class _BaseInstallationTemplate: def __init__( self, - template: ty.Union[_BinariesTemplateType, _SourceTemplateType], + template: _BinariesTemplateType | _SourceTemplateType, **kwds: str, ) -> None: self._template = copy.deepcopy(template) @@ -190,7 +192,7 @@ def template(self): return self._template @property - def env(self) -> ty.Mapping[str, str]: + def env(self) -> Mapping[str, str]: return self._template.get("env", {}) @property @@ -198,29 +200,29 @@ def instructions(self) -> str: return self._template.get("instructions", "") @property - def arguments(self) -> ty.Mapping: + def arguments(self) -> Mapping: return self._template.get("arguments", {}) @property - def required_arguments(self) -> ty.Set[str]: + def required_arguments(self) -> set[str]: args = self.arguments.get("required", None) return set(args) if args is not None else set() @property - def optional_arguments(self) -> ty.Dict[str, str]: + def optional_arguments(self) -> dict[str, str]: args = self.arguments.get("optional", None) return args if args is not None else {} @property - def versions(self) -> ty.Set[str]: + def versions(self) -> set[str]: raise NotImplementedError() - def dependencies(self, pkg_manager: str) -> ty.List[str]: + def dependencies(self, pkg_manager: str) -> list[str]: deps_dict = self._template.get("dependencies", {}) # TODO: not sure why the following line raises a type error in mypy. return deps_dict.get(pkg_manager, []) # type: ignore - def install(self, pkgs: ty.List[str], opts: str = None) -> str: + def install(self, pkgs: list[str], opts: str = None) -> str: raise NotImplementedError( "This method is meant to be patched by renderer objects, so it can be used" " in templates and have access to the pkg_manager being used." @@ -238,15 +240,15 @@ def __init__(self, template: _BinariesTemplateType, **kwds: str): super().__init__(template=template, **kwds) @property - def urls(self) -> ty.Mapping[str, str]: + def urls(self) -> Mapping[str, str]: # TODO: how can the code be changed so this cast is not necessary? - self._template = ty.cast(_BinariesTemplateType, self._template) + self._template = cast(_BinariesTemplateType, self._template) return self._template.get("urls", {}) @property - def versions(self) -> ty.Set[str]: + def versions(self) -> set[str]: # TODO: how can the code be changed so this cast is not necessary? - self._template = ty.cast(_BinariesTemplateType, self._template) + self._template = cast(_BinariesTemplateType, self._template) return set(self.urls.keys()) @@ -255,5 +257,5 @@ def __init__(self, template: _SourceTemplateType, **kwds: str): super().__init__(template=template, **kwds) @property - def versions(self) -> ty.Set[str]: + def versions(self) -> set[str]: return {"ANY"} diff --git a/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py b/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py index 79fad9ef..b616af46 100644 --- a/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py +++ b/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py @@ -1,18 +1,18 @@ # TODO: add more tests for `from_dict` method. from pathlib import Path -import typing as ty +from typing import cast import pytest -from neurodocker.reproenv.renderers import DockerRenderer -from neurodocker.reproenv.renderers import SingularityRenderer +from neurodocker.reproenv.renderers import DockerRenderer, SingularityRenderer from neurodocker.reproenv.state import _TemplateRegistry -from neurodocker.reproenv.tests.utils import get_build_and_run_fns -from neurodocker.reproenv.tests.utils import skip_if_no_docker -from neurodocker.reproenv.tests.utils import skip_if_no_singularity -from neurodocker.reproenv.types import installation_methods_type -from neurodocker.reproenv.types import pkg_managers_type +from neurodocker.reproenv.tests.utils import ( + get_build_and_run_fns, + skip_if_no_docker, + skip_if_no_singularity, +) +from neurodocker.reproenv.types import installation_methods_type, pkg_managers_type _template_filepath = Path(__file__).parent / "sample-template-jq.yaml" @@ -41,7 +41,6 @@ def test_build_using_renderer_from_dict( fd_version_startswith: str, tmp_path: Path, ): - _TemplateRegistry._reset() _TemplateRegistry.register(_template_filepath) @@ -102,8 +101,8 @@ def test_build_using_renderer_instance_methods( _TemplateRegistry._reset() _TemplateRegistry.register(_template_filepath) - pkg_manager = ty.cast(pkg_managers_type, pkg_manager) - method = ty.cast(installation_methods_type, method) + pkg_manager = cast(pkg_managers_type, pkg_manager) + method = cast(installation_methods_type, method) fd_exe = "fdfind" if pkg_manager == "apt" else "fd" diff --git a/neurodocker/reproenv/tests/test_build_images_simple.py b/neurodocker/reproenv/tests/test_build_images_simple.py index 8870d304..535e8a82 100644 --- a/neurodocker/reproenv/tests/test_build_images_simple.py +++ b/neurodocker/reproenv/tests/test_build_images_simple.py @@ -1,10 +1,11 @@ import pytest -from neurodocker.reproenv.renderers import DockerRenderer -from neurodocker.reproenv.renderers import SingularityRenderer -from neurodocker.reproenv.tests.utils import get_build_and_run_fns -from neurodocker.reproenv.tests.utils import skip_if_no_docker -from neurodocker.reproenv.tests.utils import skip_if_no_singularity +from neurodocker.reproenv.renderers import DockerRenderer, SingularityRenderer +from neurodocker.reproenv.tests.utils import ( + get_build_and_run_fns, + skip_if_no_docker, + skip_if_no_singularity, +) @pytest.mark.parametrize( @@ -15,7 +16,6 @@ ], ) def test_build_simple(cmd: str, tmp_path): - rcls = DockerRenderer if cmd == "docker" else SingularityRenderer # Create a Dockerfile. diff --git a/neurodocker/reproenv/tests/test_renderers.py b/neurodocker/reproenv/tests/test_renderers.py index 31f93cb8..c0a7cfd4 100644 --- a/neurodocker/reproenv/tests/test_renderers.py +++ b/neurodocker/reproenv/tests/test_renderers.py @@ -1,9 +1,11 @@ import pytest from neurodocker.reproenv.exceptions import RendererError -from neurodocker.reproenv.renderers import _Renderer -from neurodocker.reproenv.renderers import DockerRenderer -from neurodocker.reproenv.renderers import SingularityRenderer +from neurodocker.reproenv.renderers import ( + DockerRenderer, + SingularityRenderer, + _Renderer, +) def test_renderer(): diff --git a/neurodocker/reproenv/tests/test_renderers_docker.py b/neurodocker/reproenv/tests/test_renderers_docker.py index 5f3a3c13..e033214f 100644 --- a/neurodocker/reproenv/tests/test_renderers_docker.py +++ b/neurodocker/reproenv/tests/test_renderers_docker.py @@ -177,6 +177,7 @@ def test_docker_render_from_instance_methods(): ) d.env(PATH="$PATH:/opt/foo/bin") d.label(ORG="myorg") + d.labels({"org.test.label": "another label"}) rendered = str(d) rendered = prune_rendered(rendered).strip() assert ( @@ -188,7 +189,8 @@ def test_docker_render_from_instance_methods(): "foo/baz/cat.txt", \\ "/opt/"] ENV PATH="$PATH:/opt/foo/bin" -LABEL ORG="myorg\"""" +LABEL ORG="myorg" +LABEL org.test.label="another label\"""" ) d = DockerRenderer("apt") diff --git a/neurodocker/reproenv/tests/test_renderers_singularity.py b/neurodocker/reproenv/tests/test_renderers_singularity.py index e122373b..ceb560e4 100644 --- a/neurodocker/reproenv/tests/test_renderers_singularity.py +++ b/neurodocker/reproenv/tests/test_renderers_singularity.py @@ -164,6 +164,7 @@ def test_singularity_render_from_instance_methods(): s.copy(["foo/bar/baz.txt", "foo/baz/cat.txt"], "/opt/") s.env(FOO="BAR") s.label(ORG="BAZ") + s.labels({"org.test.label": "BAX"}) s.run("echo foobar") rendered = str(s) rendered = prune_rendered(rendered).strip() @@ -184,7 +185,8 @@ def test_singularity_render_from_instance_methods(): echo foobar %labels -ORG BAZ""" +ORG BAZ +org.test.label BAX""" ) # User diff --git a/neurodocker/reproenv/tests/test_state.py b/neurodocker/reproenv/tests/test_state.py index 1bb97385..a201a92e 100644 --- a/neurodocker/reproenv/tests/test_state.py +++ b/neurodocker/reproenv/tests/test_state.py @@ -3,18 +3,15 @@ import pytest import yaml -from neurodocker.reproenv import exceptions +from neurodocker.reproenv import exceptions, types from neurodocker.reproenv.state import _TemplateRegistry, _validate_template -from neurodocker.reproenv import types def test_validate_template_invalid_templates(): with pytest.raises(exceptions.TemplateError, match="'name' is a required property"): _validate_template({}) - with pytest.raises( - exceptions.TemplateError, match="'binaries' is a required property" - ): + with pytest.raises(exceptions.TemplateError, match="{'name': 'bar'} is not valid"): _validate_template({"name": "bar"}) # missing 'name' top-level key diff --git a/neurodocker/reproenv/tests/test_template.py b/neurodocker/reproenv/tests/test_template.py index 20a130dd..30f0b186 100644 --- a/neurodocker/reproenv/tests/test_template.py +++ b/neurodocker/reproenv/tests/test_template.py @@ -1,8 +1,6 @@ import pytest -from neurodocker.reproenv import exceptions -from neurodocker.reproenv import template -from neurodocker.reproenv import types +from neurodocker.reproenv import exceptions, template, types def test_template(): diff --git a/neurodocker/reproenv/tests/utils.py b/neurodocker/reproenv/tests/utils.py index c3c67e14..d1111150 100644 --- a/neurodocker/reproenv/tests/utils.py +++ b/neurodocker/reproenv/tests/utils.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import contextlib import getpass import os -from pathlib import Path import subprocess -import typing as ty import uuid +from pathlib import Path +from typing import Generator import pytest @@ -49,7 +51,7 @@ def _singularity_available(): @contextlib.contextmanager -def build_docker_image(context: Path, remove=False) -> ty.Generator[str, None, None]: +def build_docker_image(context: Path, remove=False) -> Generator[str, None, None]: """Context manager that builds a Docker image and removes it on exit. The argument `remove` is `False` by default because we clean up all images at the @@ -64,7 +66,7 @@ def build_docker_image(context: Path, remove=False) -> ty.Generator[str, None, N if not df.exists(): raise FileNotFoundError(f"Dockerfile not found: {df}") tag = "reproenv-pytest-" + uuid.uuid4().hex - cmd: ty.List[str] = ["docker", "build", "--tag", tag, str(context)] + cmd: list[str] = ["docker", "build", "--tag", tag, str(context)] try: _ = subprocess.check_output(cmd, cwd=context) yield tag @@ -80,13 +82,11 @@ def build_docker_image(context: Path, remove=False) -> ty.Generator[str, None, N @contextlib.contextmanager -def build_singularity_image( - context: Path, remove=True -) -> ty.Generator[str, None, None]: - """Context manager that builds a Singularity image and removes it on exit. +def build_singularity_image(context: Path, remove=True) -> Generator[str, None, None]: + """Context manager that builds a Apptainer image and removes it on exit. - If `sudo singularity` is not available, the full path to `singularity` can be set - with the environment variable `REPROENV_SINGULARITY_PROGRAM`. + If `sudo singularity` is not available, the full path to `apptainer` can be set + with the environment variable `REPROENV_APPTAINER_PROGRAM`. Yields ------ @@ -95,15 +95,15 @@ def build_singularity_image( """ recipe = context / "Singularity" if not recipe.exists(): - raise FileNotFoundError(f"Singularity recipe not found: {recipe}") + raise FileNotFoundError(f"Apptainer recipe not found: {recipe}") sif = context / f"reproenv-pytest-{uuid.uuid4().hex}.sif" - # Set singularity cache to /dev/shm + # Set apptainer cache to /dev/shm user = getpass.getuser() - cachedir = Path("/") / "dev" / "shm" / user / "singularity" - singularity = os.environ.get("REPROENV_SINGULARITY_PROGRAM", "singularity") - cmd: ty.List[str] = [ + cachedir = Path("/") / "dev" / "shm" / user / "apptainer" + singularity = os.environ.get("REPROENV_APPTAINER_PROGRAM", "apptainer") + cmd: list[str] = [ "sudo", - f"SINGULARITY_CACHEDIR={cachedir}", + f"APPTAINER_CACHEDIR={cachedir}", singularity, "build", str(sif), @@ -120,9 +120,7 @@ def build_singularity_image( pass -def run_docker_image( - img: str, args: ty.List[str] = None, entrypoint: ty.List[str] = None -): +def run_docker_image(img: str, args: list[str] = None, entrypoint: list[str] = None): """Wrapper for `docker run`. Returns @@ -144,7 +142,7 @@ def run_docker_image( def run_singularity_image( - img: str, args: ty.List[str] = None, entrypoint: ty.List[str] = None + img: str, args: list[str] = None, entrypoint: list[str] = None ): """Wrapper for `singularity run` or `singularity exec`. @@ -155,7 +153,7 @@ def run_singularity_image( """ scmd = "run" if entrypoint is None else "exec" # sudo not required - cmd: ty.List[str] = ["singularity", scmd, "--cleanenv", img] + cmd: list[str] = ["singularity", scmd, "--cleanenv", img] if entrypoint is not None: cmd.extend(entrypoint) if args is not None: diff --git a/neurodocker/reproenv/types.py b/neurodocker/reproenv/types.py index def4c76c..d74e5954 100644 --- a/neurodocker/reproenv/types.py +++ b/neurodocker/reproenv/types.py @@ -1,6 +1,7 @@ """Define types used in ReproEnv.""" +from __future__ import annotations -import typing as ty +from typing import Mapping from mypy_extensions import TypedDict from typing_extensions import Literal @@ -28,23 +29,23 @@ class _InstallationDependenciesType(TypedDict, total=False): `yum`, and Debian and Ubuntu use `apt` and `dpkg`. """ - apt: ty.List[str] - debs: ty.List[str] - yum: ty.List[str] + apt: list[str] + debs: list[str] + yum: list[str] class _TemplateArgumentsType(TypedDict): """Arguments (i.e., variables) that are used in the template.""" - required: ty.List[str] - optional: ty.Mapping[str, str] + required: list[str] + optional: Mapping[str, str] class _BaseTemplateType(TypedDict, total=False): """Keys common to both types of templates: binaries and source.""" arguments: _TemplateArgumentsType - env: ty.Mapping[str, str] + env: Mapping[str, str] dependencies: _InstallationDependenciesType instructions: str @@ -58,7 +59,7 @@ class _SourceTemplateType(_BaseTemplateType): class _BinariesTemplateType(_BaseTemplateType): """Template that defines how to install software from pre-compiled binaries.""" - urls: ty.Mapping[str, str] + urls: Mapping[str, str] class TemplateType(TypedDict, total=False): diff --git a/neurodocker/templates/afni.yaml b/neurodocker/templates/afni.yaml index c40acdf4..717986f8 100644 --- a/neurodocker/templates/afni.yaml +++ b/neurodocker/templates/afni.yaml @@ -18,46 +18,67 @@ binaries: dependencies: apt: - ca-certificates + - cmake - curl - ed - gsl-bin + - libcurl4-openssl-dev - libgl1-mesa-dri + - libjpeg-turbo8-dev - libglu1-mesa-dev - libglib2.0-0 - libglw1-mesa - libgomp1 - libjpeg62 + - libssl-dev + - libudunits2-dev - libxm4 - multiarch-support - netpbm + - python-is-python3 + - python3-pip - tcsh - xfonts-base - xvfb yum: + - cmake - curl - ed - gsl + - libcurl-devel - libGLU - libgomp + - libjpeg-turbo-devel - libpng12 - libXp - libXpm - mesa-dri-drivers - netpbm-progs - openmotif + - openssl-devel + - python-is-python3 + - python3-pip - R - tcsh + - udunits2-devel - which - xorg-x11-fonts-misc - xorg-x11-server-Xvfb + - wget + - mesa-dri-drivers + - mesa-libGLw + - which + - unzip + - ncurses-compat-libs debs: - http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb - http://snapshot.debian.org/archive/debian-security/20160113T213056Z/pool/updates/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb instructions: | {{ self.install_dependencies() }} - {%- if self.install_python3.lower() in ["true", "1", "y"] %} + {% if self.install_python3.lower() in ["true", "1", "y"] -%} {{ self.install(["python3"]) }} - {% endif %} + pip3 install matplotlib + {%- endif %} gsl_path="$(find / -name 'libgsl.so.??' || printf '')" if [ -n "$gsl_path" ]; then \ ln -sfv "$gsl_path" "$(dirname $gsl_path)/libgsl.so.0"; \ diff --git a/neurodocker/templates/ants.yaml b/neurodocker/templates/ants.yaml index e6e3d2b9..ad50a89a 100644 --- a/neurodocker/templates/ants.yaml +++ b/neurodocker/templates/ants.yaml @@ -8,7 +8,11 @@ binaries: optional: install_path: /opt/ants-{{ self.version }} urls: - # Binaries were compiled by Jakub Kaczmarzyk (https://github.com/kaczmarj) + # Official binaries are provided as of 2.4.1 (https://github.com/ANTsX/ANTs/releases) + "2.4.3": https://github.com/ANTsX/ANTs/releases/download/v2.4.3/ants-2.4.3-centos7-X64-gcc.zip + "2.4.2": https://github.com/ANTsX/ANTs/releases/download/v2.4.2/ants-2.4.2-centos7-X64-gcc.zip + "2.4.1": https://github.com/ANTsX/ANTs/releases/download/v2.4.1/ants-2.4.1-centos7-X64-gcc.zip + # Binaries prior to 2.4.x were compiled by Jakub Kaczmarzyk (https://github.com/kaczmarj) "2.3.4": https://dl.dropbox.com/s/gwf51ykkk5bifyj/ants-Linux-centos6_x86_64-v2.3.4.tar.gz "2.3.2": https://dl.dropbox.com/s/hrm530kcqe3zo68/ants-Linux-centos6_x86_64-v2.3.2.tar.gz "2.3.1": https://dl.dropbox.com/s/1xfhydsf4t4qoxg/ants-Linux-centos6_x86_64-v2.3.1.tar.gz @@ -21,17 +25,26 @@ binaries: apt: - ca-certificates - curl + - unzip yum: - curl + - unzip env: ANTSPATH: "{{ self.install_path }}/" PATH: "{{ self.install_path }}:$PATH" instructions: | {{ self.install_dependencies() }} echo "Downloading ANTs ..." + {% if (self.version == "2.4.1" or self.version == "2.4.2" or self.version == "2.4.3") -%} + curl -fsSL -o ants.zip {{ self.urls[self.version] }} + unzip ants.zip -d /opt + mv {{ self.install_path }}/bin/* {{ self.install_path }} + rm ants.zip + {% elif self.version != "2.4.1" -%} mkdir -p {{ self.install_path }} curl -fsSL {{ self.urls[self.version] }} \ | tar -xz -C {{ self.install_path }} --strip-components 1 + {% endif -%} source: arguments: @@ -85,5 +98,3 @@ source: mv ../Scripts/* {{ self.install_path }} ; \ fi rm -rf /tmp/ants - - \ No newline at end of file diff --git a/neurodocker/templates/cat12.yaml b/neurodocker/templates/cat12.yaml index 09991037..2eb85ef9 100644 --- a/neurodocker/templates/cat12.yaml +++ b/neurodocker/templates/cat12.yaml @@ -13,6 +13,7 @@ binaries: install_path: /opt/CAT12-{{ self.version }} urls: r1933_R2017b: http://www.neuro.uni-jena.de/cat12/CAT12.8_r1933_R2017b_MCR_Linux.zip + r2166_R2017b: http://www.neuro.uni-jena.de/cat12/CAT12.8.2_r2166_R2017b_MCR_Linux.zip dependencies: apt: - ca-certificates @@ -25,9 +26,7 @@ binaries: FORCE_SPMMCR: "1" SPM_HTML_BROWSER: "0" MCR_INHIBIT_CTF_LOCK: "1" - SPM_HTML_BROWSER: "0" SPMROOT: "{{ self.install_path }}" - MCR_INHIBIT_CTF_LOCK: "1" PATH: "{{ self.install_path }}:$PATH" instructions: | @@ -42,3 +41,6 @@ binaries: chmod -R 777 {{ self.install_path }} # Test {{ self.install_path }}/spm12 function exit + # Fix m file + rm {{ self.install_path }}/spm12_mcr/home/gaser/gaser/spm/spm12/toolbox/cat12/cat_long_main.m + cp {{ self.install_path }}/spm12_mcr/home/gaser/gaser/spm/spm12/toolbox/cat12/cat_long_main.txt {{ self.install_path }}/spm12_mcr/home/gaser/gaser/spm/spm12/toolbox/cat12/cat_long_main.m diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 77dceb6a..21f6ddbe 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -25,6 +25,10 @@ binaries: subjects/fsaverage_sym trctrain urls: + "7.4.1": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.4.1/freesurfer-linux-centos7_x86_64-7.4.1.tar.gz + "7.3.2": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.3.2/freesurfer-linux-centos7_x86_64-7.3.2.tar.gz + "7.3.1": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.3.1/freesurfer-linux-centos7_x86_64-7.3.1.tar.gz + "7.3.0": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.3.0/freesurfer-linux-centos7_x86_64-7.3.0.tar.gz "7.2.0-centos6": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.2.0/freesurfer-CentOS6-7.2.0-1.x86_64.rpm "7.2.0-centos7": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.2.0/freesurfer-CentOS7-7.2.0-1.x86_64.rpm "7.2.0-centos8": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.2.0/freesurfer-CentOS8-7.2.0-1.x86_64.rpm diff --git a/neurodocker/templates/fsl.yaml b/neurodocker/templates/fsl.yaml index 68b7bfc5..8e284842 100644 --- a/neurodocker/templates/fsl.yaml +++ b/neurodocker/templates/fsl.yaml @@ -19,6 +19,14 @@ binaries: install_path: /opt/fsl-{{ self.version }} exclude_paths: "" urls: + "6.0.7.4": https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fslinstaller.py + "6.0.7.1": https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fslinstaller.py + "6.0.6.4": https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fslinstaller.py + "6.0.6.3": https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fslinstaller.py + "6.0.6.2": https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fslinstaller.py + "6.0.6.1": https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fslinstaller.py + "6.0.6": https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fslinstaller.py + "6.0.5.2": https://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-6.0.5.2-centos7_64.tar.gz "6.0.5.1": https://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-6.0.5.1-centos7_64.tar.gz "6.0.5": https://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-6.0.5-centos7_64.tar.gz "6.0.4": https://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-6.0.4-centos6_64.tar.gz @@ -52,6 +60,7 @@ binaries: - libxrender1 - libxt6 - nano + - python3 - sudo - wget yum: @@ -75,6 +84,7 @@ binaries: - libpng12 - nano - openblas-serial + - python3 - sudo - wget env: @@ -90,6 +100,10 @@ binaries: FSLGECUDAQ: "cuda.q" instructions: | {{ self.install_dependencies() }} + {% if self.version.split('.') | map('int') | list >= [6, 0, 6] %} + echo "Installing FSL ..." + curl -fsSL {{ self.urls[self.version] }} | python3 - -d {{ self.install_path }} -V {{ self.version }} + {% else %} echo "Downloading FSL ..." mkdir -p {{ self.install_path }} curl -fL {{ self.urls[self.version] }} \ @@ -112,3 +126,4 @@ binaries: echo "Removing bundled with FSLeyes libz likely incompatible with the one from OS" rm -f {{ self.install_path }}/bin/FSLeyes/libz.so.1 {% endif -%} + {% endif -%} diff --git a/neurodocker/templates/matlabmcr.yaml b/neurodocker/templates/matlabmcr.yaml index 7d21662a..ec81a3d3 100644 --- a/neurodocker/templates/matlabmcr.yaml +++ b/neurodocker/templates/matlabmcr.yaml @@ -12,6 +12,7 @@ binaries: curl_opts: "" install_path: /opt/MCR-{{ self.version }} urls: + "2023a": https://ssd.mathworks.com/supportfiles/downloads/R2023a/Release/4/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2023a_Update_4_glnxa64.zip "2021b": https://ssd.mathworks.com/supportfiles/downloads/R2021b/Release/2/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021b_Update_2_glnxa64.zip "2021a": https://ssd.mathworks.com/supportfiles/downloads/R2021a/Release/5/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021a_Update_5_glnxa64.zip "2020b": https://ssd.mathworks.com/supportfiles/downloads/R2020b/Release/6/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2020b_Update_6_glnxa64.zip @@ -43,7 +44,6 @@ binaries: - libxmu6 - libxpm-dev - libxt6 - - multiarch-support - unzip - openjdk-8-jre - dbus-x11 @@ -57,15 +57,13 @@ binaries: - unzip - java-1.8.0-openjdk - dbus-x11 - debs: - - http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb env: LD_LIBRARY_PATH: | - {% set versionTovXX = {"2021b": "v911", "2021a": "v910", "2020b": "v99", "2020a": "v98", "2019b": "v97", "2019a": "v96", "2018b": "v95", "2018a": "v94", "2017b": "v93", "2017a": "v92", "2016b": "v91", "2016a": "v901", "2015b": "v90", "2015aSP1": "v851", "2015a": "v85", "2014b": "v84", "2014a": "v83", "2013b": "v82", "2013a": "v81", "2012b": "v80", "2012a": "v717"} -%} + {% set versionTovXX = {"2023a": "v914", "2021b": "v911", "2021a": "v910", "2020b": "v99", "2020a": "v98", "2019b": "v97", "2019a": "v96", "2018b": "v95", "2018a": "v94", "2017b": "v93", "2017a": "v92", "2016b": "v91", "2016a": "v901", "2015b": "v90", "2015aSP1": "v851", "2015a": "v85", "2014b": "v84", "2014a": "v83", "2013b": "v82", "2013a": "v81", "2012b": "v80", "2012a": "v717"} -%} $LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:{{ self.install_path }}/{{ versionTovXX[self.version] }}/runtime/glnxa64:{{ self.install_path }}/{{ versionTovXX[self.version] }}/bin/glnxa64:{{ self.install_path }}/{{ versionTovXX[self.version] }}/sys/os/glnxa64:{{ self.install_path }}/{{ versionTovXX[self.version] }}/extern/bin/glnxa64 MATLABCMD: "{{ self.install_path }}/{{ self.version }}/toolbox/matlab" XAPPLRESDIR: | - {% set versionTovXX = {"2021b": "v911", "2021a": "v910", "2020b": "v99", "2020a": "v98", "2019b": "v97", "2019a": "v96", "2018b": "v95", "2018a": "v94", "2017b": "v93", "2017a": "v92", "2016b": "v91", "2016a": "v901", "2015b": "v90", "2015aSP1": "v851", "2015a": "v85", "2014b": "v84", "2014a": "v83", "2013b": "v82", "2013a": "v81", "2012b": "v80", "2012a": "v717"} -%} + {% set versionTovXX = {"2023a": "v914", "2021b": "v911", "2021a": "v910", "2020b": "v99", "2020a": "v98", "2019b": "v97", "2019a": "v96", "2018b": "v95", "2018a": "v94", "2017b": "v93", "2017a": "v92", "2016b": "v91", "2016a": "v901", "2015b": "v90", "2015aSP1": "v851", "2015a": "v85", "2014b": "v84", "2014a": "v83", "2013b": "v82", "2013a": "v81", "2012b": "v80", "2012a": "v717"} -%} /opt/{{ self.install_path }}/{{ versionTovXX[self.version] }}/x11/app-defaults MCRROOT: "{{ self.install_path }}/{{ self.version }}" @@ -74,7 +72,6 @@ binaries: {{ self.install_dependencies() }} echo "Downloading MATLAB Compiler Runtime ..." {% if self.version == "2010a" -%} - {{ self.install_debs() }} curl {{ self.curl_opts }} -o "$TMPDIR/MCRInstaller.bin" {{ self.urls[self.version] }} chmod +x "$TMPDIR/MCRInstaller.bin" "$TMPDIR/MCRInstaller.bin" -silent -P installLocation="{{ self.install_path }}" diff --git a/neurodocker/templates/minc.yaml b/neurodocker/templates/minc.yaml index 181d39cb..d57da09e 100644 --- a/neurodocker/templates/minc.yaml +++ b/neurodocker/templates/minc.yaml @@ -2,9 +2,8 @@ # # Repository: https://github.com/BIC-MNI/minc-toolkit-v2 # -# Binaries are compiled in a CentOS 6.9 Docker container, based on this -# Dockerfile: -# https://github.com/BIC-MNI/build_packages/blob/master/build_centos_6.9_x64/Dockerfile +# Instructions: http://bic-mni.github.io/#on-debianubuntu +# The Debian packages are unpacked for yum based distributions as well name: minc binaries: @@ -14,7 +13,10 @@ binaries: optional: install_path: /opt/minc-{{ self.version }} urls: - "1.9.15": https://dl.dropbox.com/s/40hjzizaqi91373/minc-toolkit-1.9.15-20170529-CentOS_6.9-x86_64.tar.gz + "1.9.15": https://packages.bic.mni.mcgill.ca/minc-toolkit/Debian/minc-toolkit-1.9.15-20170529-Ubuntu_16.04-x86_64.deb + "1.9.16": https://packages.bic.mni.mcgill.ca/minc-toolkit/Debian/minc-toolkit-1.9.16-20180117-Ubuntu_18.04-x86_64.deb + "1.9.17": https://packages.bic.mni.mcgill.ca/minc-toolkit/Debian/minc-toolkit-1.9.17-20190313-Ubuntu_18.04-x86_64.deb + "1.9.18": https://packages.bic.mni.mcgill.ca/minc-toolkit/Debian/minc-toolkit-1.9.18-20200813-Ubuntu_18.04-x86_64.deb dependencies: apt: - ca-certificates @@ -29,6 +31,18 @@ binaries: - libgomp1 - libjpeg62 - unzip + - octave + - libglu1-mesa + - libgl1-mesa-glx + - perl + - imagemagick + - bc + - ed + - libc6 + - libstdc++6 + - gdebi-core + - binutils + - git yum: - curl - libICE @@ -41,9 +55,19 @@ binaries: - libjpeg-turbo - mesa-libGL-devel - unzip + - octave + - mesa-dri-drivers + - epel-release + - glibc + - libstdc++ + - ImageMagick + - perl + - binutils + - git env: MINC_TOOLKIT: "{{ self.install_path }}" - PATH: "$PATH:{{ self.install_path }}/bin:{{ self.install_path }}/pipeline" + MINC_TOOLKIT_VERSION: "{{ self.install_path }}" + PATH: "{{ self.install_path }}/bin:{{ self.install_path }}/volgenmodel-nipype/extra-scripts:{{ self.install_path }}/pipeline:$PATH" PERL5LIB: "{{ self.install_path }}/perl:{{ self.install_path }}/pipeline:${PERL5LIB}" LD_LIBRARY_PATH: "{{ self.install_path }}/lib:{{ self.install_path }}/lib/InsightToolkit:${LD_LIBRARY_PATH}" MNI_DATAPATH: "{{ self.install_path }}/share" @@ -54,14 +78,15 @@ binaries: instructions: | {{ self.install_dependencies() }} echo "Downloading MINC, BEASTLIB, and MODELS..." - mkdir -p {{ self.install_path }} - curl -fL {{ self.urls[self.version] }} \ - | tar -xz -C {{ self.install_path }} --strip-components 1 + cd / + # ar allows to extract the debian package so we can also install this in centos based OSs. + curl {{ self.urls[self.version] }} -o minc.deb && ar p minc.deb data.tar.gz | tar zx && rm minc.deb + ln -s /opt/minc/{{ self.version }} {{ self.install_path }} + git clone https://github.com/CAIsr/volgenmodel-nipype.git {{ self.install_path }}/volgenmodel-nipype/ curl -fL http://packages.bic.mni.mcgill.ca/tgz/beast-library-1.1.tar.gz \ | tar -xz -C {{ self.install_path }}/share curl -fL -o /tmp/mni_90a.zip http://www.bic.mni.mcgill.ca/~vfonov/icbm/2009/mni_icbm152_nlin_sym_09a_minc2.zip unzip /tmp/mni_90a.zip -d {{ self.install_path }}/share/icbm152_model_09a curl -fL -o /tmp/mni_90c.zip http://www.bic.mni.mcgill.ca/~vfonov/icbm/2009/mni_icbm152_nlin_sym_09c_minc2.zip unzip /tmp/mni_90c.zip -d {{ self.install_path }}/share/icbm152_model_09c - sed -i 's+MINC_TOOLKIT=/opt/minc+MINC_TOOLKIT={{ self.install_path }}+g' {{ self.install_path }}/minc-toolkit-config.sh rm -rf /tmp/mni* diff --git a/neurodocker/templates/miniconda.yaml b/neurodocker/templates/miniconda.yaml index a18fb693..770a33c5 100644 --- a/neurodocker/templates/miniconda.yaml +++ b/neurodocker/templates/miniconda.yaml @@ -31,6 +31,7 @@ binaries: conda_opts: "" pip_opts: "" yaml_file: "" + mamba: "false" instructions: | {% if not self.installed.lower() in ["true", "y", "1"] -%} {{ self.install_dependencies() }} @@ -44,6 +45,10 @@ binaries: {% if self.version == "latest" -%} conda update -yq -nbase conda {% endif -%} + {% if self.mamba == "true" -%} + conda install -yq -nbase conda-libmamba-solver + conda config --set solver libmamba + {% endif -%} # Prefer packages in conda-forge conda config --system --prepend channels conda-forge # Packages in lower-priority channels not considered if a package with the same diff --git a/neurodocker/templates/mrtrix3.yaml b/neurodocker/templates/mrtrix3.yaml index 81b02a4a..4d8345d2 100644 --- a/neurodocker/templates/mrtrix3.yaml +++ b/neurodocker/templates/mrtrix3.yaml @@ -22,6 +22,8 @@ binaries: - libpng - libtiff urls: + "3.0.4": https://github.com/MRtrix3/mrtrix3/releases/download/3.0.4/conda-linux-mrtrix3-3.0.4-h2bc3f7f_0.tar.bz2 + "3.0.3": https://github.com/MRtrix3/mrtrix3/releases/download/3.0.3/conda-linux-mrtrix3-3.0.3-h2bc3f7f_0.tar.bz2 "3.0.2": https://github.com/MRtrix3/mrtrix3/releases/download/3.0.2/conda-linux-mrtrix3-3.0.2-h6bb024c_0.tar.bz2 "3.0.1": https://github.com/MRtrix3/mrtrix3/releases/download/3.0.1/conda-linux-mrtrix3-3.0.1-h6bb024c_0.tar.bz2 "3.0.0": https://github.com/MRtrix3/mrtrix3/releases/download/3.0.0/conda-linux-mrtrix3-3.0.0-h6bb024c_0.tar.bz2 @@ -42,7 +44,7 @@ source: optional: repo: https://github.com/MRtrix3/mrtrix3.git install_path: /opt/mrtrix3-{{ self.version }} - build_processes: "" + build_processes: "1" dependencies: apt: - ca-certificates diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..6d3d6e6c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[tool.isort] +combine_as_imports = true +line_length = 88 +profile = "black" +skip_gitignore = true +src_paths = [ + 'neurodocker', + 'docs' +] diff --git a/setup.cfg b/setup.cfg index 8806d74e..96e4bfcc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,6 +4,8 @@ url = https://github.com/repronim/neurodocker author = Neurodocker Developers author_email = jakub.kaczmarzyk@gmail.com description = A generic generator of Dockerfiles and Singularity recipes +long_description = file: README.md +long_description_content_type = text/markdown license = Apache License, 2.0 classifiers = Development Status :: 4 - Beta @@ -14,17 +16,18 @@ classifiers = Operating System :: OS Independent Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Software Development Topic :: Software Development :: Libraries :: Python Modules [options] packages = find: -python_requires = >= 3.7 +python_requires = >= 3.8 install_requires = - click >= 7.0, <8.0 + click etelemetry >= 0.2.0 jinja2 >= 2.0 jsonschema >= 3.0 @@ -37,18 +40,19 @@ minify = docker >= 4.4.1 dev = %(minify)s - black <= 21.12b0 + black >= 23.1.0 + isort codecov flake8 - mypy == 0.790 + mypy pre-commit pytest >= 6.0 pytest-cov >= 2.0.0 pytest-reportlog >= 0.1.2 pytest-xdist >= 2.2.0 docs = - sphinx >= 3.4 - pydata-sphinx-theme + sphinx <7 + pydata-sphinx-theme >= 0.13 sphinxcontrib.apidoc >= 0.3 all = %(minify)s @@ -63,7 +67,7 @@ console_scripts = neurodocker = templates/*.yaml reproenv/schemas/*.json - reprozip/_trace.sh + cli/minify/_trace.sh [versioneer] VCS = git @@ -78,9 +82,19 @@ max-line-length = 88 extend-ignore = E203 exclude = neurodocker/_version.py +[mypy] +exclude = reproenv/tests +no_implicit_optional=False + [mypy-neurodocker._version] ignore_errors = True +[mypy-neurodocker.reproenv.tests.*] +ignore_errors = True + +[mypy-pytest] +ignore_missing_imports = True + [mypy-docker] ignore_missing_imports = True diff --git a/setup.py b/setup.py index 2bda439a..1a9bb8e1 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,7 @@ """Setup script for neurodocker.""" -from setuptools import setup - import versioneer +from setuptools import setup version = versioneer.get_version() cmdclass = versioneer.get_cmdclass()