diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b72ae5b15..2ac96d2fcb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,26 +2,34 @@ version: 2 jobs: build_docs: docker: - - image: circleci/python:3.8 + - image: cimg/python:3.8 steps: # checkout code to default ~/project - checkout - run: name: install dependencies command: | + python -m venv env + source env/bin/activate python -m pip install --upgrade pip pip install -r requirements.txt + pip install -e ~/project/tools/schemacode/ - run: name: generate docs - command: mkdocs build --clean --strict --verbose + command: | + source env/bin/activate + mkdocs build --clean --strict --verbose - persist_to_workspace: # the mkdocs build outputs are in ~/project/site root: ~/project paths: site + - store_artifacts: + path: ~/project/site/ + destination: dev_docs linkchecker: docker: - - image: circleci/python:3.8 + - image: cimg/python:3.8 steps: # checkout code to default ~/project - checkout @@ -31,11 +39,14 @@ jobs: - run: name: install linkchecker command: | - python -m ensurepip + python -m venv env + source env/bin/activate + python -m pip install --upgrade pip python -m pip install linkchecker - run: name: check links command: | + source env/bin/activate git status if (! git log -1 --pretty=oneline | grep REL:) ; then chmod a+rX -R ~ @@ -44,9 +55,11 @@ jobs: # failures for local file:/// -- yoh found no better way, linkchecker -t 1 --check-extern \ --ignore-url 'file:///.*' \ - --ignore-url https://fonts.gstatic.com \ - --ignore-url "https://github.com/bids-standard/bids-specification/(pull|tree)/.*" \ - --ignore-url "https://github.com/[^/]*" \ + --ignore-url 'https://fonts.gstatic.com' \ + --ignore-url 'https://github.com/bids-standard/bids-specification/(pull|tree)/.*' \ + --ignore-url 'https://github.com/[^/]*' \ + --ignore-url 'https://doi.org/.*' \ + --ignore-url 'https://bids-specification.readthedocs.io/en/stable/.*' \ ~/project/site/*html ~/project/site/*/*.html else echo "Release PR - do nothing" @@ -117,7 +130,7 @@ jobs: # Run remark on the auto generated changes.md file remark: docker: - - image: node:latest + - image: cimg/node:lts steps: # checkout code to default ~/project - checkout @@ -150,7 +163,7 @@ jobs: # Push built changelog to repo Changelog-bot: docker: - - image: circleci/openjdk:8-jdk + - image: cimg/base:stable steps: - setup_remote_docker: version: 17.11.0-ce diff --git a/.gitattributes b/.gitattributes index 2bd791b40d..d2563f43d1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,4 @@ *.png -text *.jpg -text *.webm -text +tools/schemacode/schemacode/_version.py export-subst diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 3609c21cfd..db5f92e7d6 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: codespell-project/actions-codespell@master with: skip: js,*.svg diff --git a/.github/workflows/markdown_style.yml b/.github/workflows/markdown_style.yml index 578c6e8cf5..ecc5d6403f 100644 --- a/.github/workflows/markdown_style.yml +++ b/.github/workflows/markdown_style.yml @@ -6,7 +6,7 @@ jobs: markdown-style: runs-on : ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup NodeJS uses: actions/setup-node@v2 with: diff --git a/.github/workflows/no-bad-latin.yml b/.github/workflows/no-bad-latin.yml index 8a67e45ae3..da5bf72ac7 100644 --- a/.github/workflows/no-bad-latin.yml +++ b/.github/workflows/no-bad-latin.yml @@ -31,10 +31,10 @@ jobs: # This section collects together the steps involved in running the test steps: # Checkout the repository. Relies on another GH-Action. - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # Set up the Python version. Relies on another GH-Action. - name: Setup Python 3.7 - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.7 # Install Python dependencies diff --git a/.github/workflows/schemacode_ci.yml b/.github/workflows/schemacode_ci.yml new file mode 100644 index 0000000000..2bcb2c96d3 --- /dev/null +++ b/.github/workflows/schemacode_ci.yml @@ -0,0 +1,94 @@ +name: "schemacode_ci" + +on: + push: + branches: + - "master" + paths: + - "tools/schemacode/**" + - "src/schema/**" + pull_request: + branches: + - "*" + paths: + - "tools/schemacode/**" + - "src/schema/**" + +jobs: + check_skip: + runs-on: ubuntu-latest + outputs: + skip: ${{ steps.result_step.outputs.ci-skip }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - id: result_step + uses: mstachniuk/ci-skip@master + with: + commit-filter: "[skip ci];[ci skip];[skip github]" + commit-filter-separator: ";" + + run_tests: + needs: check_skip + if: ${{ needs.check_skip.outputs.skip == 'false' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + name: ${{ matrix.os }} with Python ${{ matrix.python-version }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v3 + + - name: "Set up Python" + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: "Display Python version" + shell: bash {0} + run: python -c "import sys; print(sys.version)" + + - name: "Install the schemacode package" + shell: bash {0} + run: | + python -m pip install --progress-bar off --upgrade pip setuptools wheel + python -m pip install -e ./tools/schemacode[tests] + + - name: "Run tests" + shell: bash {0} + run: | + python -m pytest --pyargs schemacode --cov=schemacode ./tools/schemacode/ + + - name: "Upload coverage to CodeCov" + uses: codecov/codecov-action@v1 + if: success() + + flake8-lint: + runs-on: ubuntu-latest + name: Lint schemacode + steps: + - name: Check out source repository + uses: actions/checkout@v3 + + - name: Set up Python environment + uses: actions/setup-python@v3 + with: + python-version: "3.7" + + - name: "Install the schemacode package" + shell: bash {0} + run: | + python -m pip install --progress-bar off --upgrade pip setuptools wheel + python -m pip install -e ./tools/schemacode[tests] + + - name: "Run flake8" + working-directory: ./tools/schemacode/ + shell: bash {0} + run: | + flake8 . diff --git a/.github/workflows/yml_lint.yml b/.github/workflows/yml_lint.yml index 9f5609afa6..99e534c78b 100644 --- a/.github/workflows/yml_lint.yml +++ b/.github/workflows/yml_lint.yml @@ -6,9 +6,9 @@ jobs: yml-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.9 - name: Install dependencies diff --git a/.gitignore b/.gitignore index 6e97a859ac..50f8037199 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ site/ .DS_Store .idea venvs +.vscode/ pdf_build_src/bids-spec.pdf pdf_build_src/bids-spec_pandoc_log.json diff --git a/BIDS_logo/README.md b/BIDS_logo/README.md index 4605ec1542..a6f19ed9cc 100644 --- a/BIDS_logo/README.md +++ b/BIDS_logo/README.md @@ -1,11 +1,21 @@ -The BIDS logo originates from a [design competition](https://en.99designs.de/logo-design/contests/design-brain-imaging-data-standard-logo-916110). +# BIDS logo + +The BIDS logo originates from a +[design competition](https://en.99designs.de/logo-design/contests/design-brain-imaging-data-standard-logo-916110). See the related discussion here: https://github.com/bids-standard/bids-specification/issues/216 -The sources of the animated version of the logo can be found at +All static versions of the logo are shared under a CC BY 4.0 license, see +[LICENSE](../LICENSE) +in the root of the repository. + +The animated version of the logo is shared under a CC BY-SA 4.0 license, +copyright Adina Wagner, +please see the source: [github.com/adswa/animated-bids-logo](https://github.com/adswa/animated-bids-logo). # Files + - Standard formats - `BIDS_logo.jpg` - Transparent background diff --git a/BIDS_logo/favicon_package_v0.16.zip b/BIDS_logo/favicon_package_v0.16.zip new file mode 100644 index 0000000000..cf09ddc88b Binary files /dev/null and b/BIDS_logo/favicon_package_v0.16.zip differ diff --git a/CODEOWNERS b/CODEOWNERS index 5dbad3bf60..6fd6504405 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,3 +18,7 @@ /src/05-derivatives/05-functional-derivatives.md @effigies /src/05-derivatives/06-diffusion-derivatives.md @francopestilli @oesteban @Lestropie /src/99-appendices/06-meg-file-formats.md @monkeyman192 + +# The schema +/src/schema/ @tsalo @erdalkaraca +/tools/schemacode/ @tsalo @erdalkaraca diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cef99f9fa8..a8f740e19b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -225,88 +225,44 @@ That would look like this: ## Using macros We use [mkdocs-macros](https://mkdocs-macros-plugin.readthedocs.io/en/latest/) -to render parts of the BIDS specification from the BIDS schema. -Macros make it easy to achieve a consistent style throughout the specification, -and changing a given macro will automatically change all appropriate paragraphs in the specification. - -For example, all tables on BIDS metadata are generated via macros that make use of data in the -[yaml files](src/schema/metadata) in the [schema](src/schema/README.md). - -These macros are written in Python -(see the folders [tools/schemacode](tools/schemacode) and [tools/mkdocs_macros_bids](tools/mkdocs_macros_bids)), -and are called directly in the Markdown document where you want the output of the macro to be inserted. - -For example: - -```Text -{{ MACROS___make_metadata_table( - { - "SamplingFrequency": "REQUIRED", - "StartTime": "RECOMMENDED, but REQUIRED for sparse sequences", - } -) }} -``` - -This macro will create a table for the "SamplingFrequency" and "StartTime" metadata, -filling the table with the content specified in their respective yaml files -(see [SamplingFrequency.yaml](src/schema/metadata/SamplingFrequency.yaml) and [StartTime.yaml](src/schema/metadata/StartTime.yaml)). +to standardize how some aspects of the BIDS specification are rendered in HTML. -Some of the content created by the macro can be specified in the macro call itself, as opposed to in the yaml files. -Here the `"REQUIRED"`, `"RECOMMENDED, but REQUIRED for sparse sequences"` -specify the content of the requirement level column for each piece of metadata. + +We have dedicated documentation for [this](./macro_doc.md). -This macro also allows you to append extra content to the description of that metadata -by specifying it in the macro call: - -```Text -{{ MACROS___make_metadata_table( - { - "SamplingFrequency": ("REQUIRED", "This extra content will be added to the description") - "StartTime": "RECOMMENDED, but REQUIRED for sparse sequences", - } -) }} -``` +## Building the specification using mkdocs -### Writing folder content examples +We are using mkdocs to render our specification. +Please follow these instructions if you would like to build the specification locally. -We also use macros to have a consistent style to render the examples of folder contents. +#### 1. Download the BIDS specification [repository](https://github.com/bids-standard/bids-specification/tree/master) onto your computer -These code for these macros are in the folder [tools/schemacode](tools/schemacode). +This can be done by clicking the green button on the right titled "Clone or +download" +or using [this link](https://github.com/bids-standard/bids-specification/archive/master.zip). -To insert examples in the code you have make calls to the macro like this: +Or you can use the following `git` command in a terminal: +```bash +git clone https://github.com/bids-standard/bids-specification.git ``` -{{ MACROS___make_filetree_example( - { - "sub-01": { - "func": { - "sub-control01_task-nback_bold.json": "", - }, - } - } +#### 2. In the terminal (command line) navigate to your local version of the specification -) }} -``` +This location will have the same files you see on our +[main specification page](https://github.com/bids-standard/bids-specification). +Note that a file browser window may not show the hidden files +(those that start with a period, like `.remarkrc`). -And this will be turned into this. +If you cloned the repository using the `git` command above, you can then just do: -```Text -└─ sub-01/ - └─ func/ - └─ sub-control01_task-nback_bold.json +```bash +cd bids-specification ``` -When you have complex files and folder structure, we suggest you use this -[Jupyter notebook](tools/filetree_example.ipynb) for sandboxing your example -before you insert the macro call into the markdown document. +Enter all commands below from the command line prompt located at the root of the local version of the specification. -## Building the specification using mkdocs - -We are using mkdocs to render our specification. -Please follow these instructions if you would like to build the specification locally. - -#### 1. Install mkdocs, the material theme and the required extensions +#### 3. Install mkdocs, the material theme and the required extensions In the following links, you can find more information about @@ -318,40 +274,41 @@ You will also need several other mkdocs plugins, like `branchcustomization` and To install all of this make sure you have a recent version of Python on your computer. The [DataLad Handbook](http://handbook.datalad.org/en/latest/intro/installation.html#python-3-all-operating-systems) provides helpful instructions for setting up Python. -An easy way to install the correct version of mkdocs and all the other required extensions -is to use the `requirements.txt` file contained in this repository, -by using the following command: +In general, we strongly recommend that you install all dependencies in an isolated Python environment. +For example using `conda`, as described in this [Geohackweek tutorial](https://geohackweek.github.io/Introductory/01-conda-tutorial/). ```bash -pip install -r requirements.txt +conda create --name bids-spec +conda activate bids-spec ``` -However this will also install some other packages you might not want to have (like `numpy`). -So if you only want to install what you need to build the specification, -use the following command: +Or alternatively using `venv`, as described in this [Real Python tutorial](https://realpython.com/python-virtual-environments-a-primer/). + +A short version of the commands needed to create and activate your `venv` virtual environment would look like: ```bash -pip install \ - mkdocs \ - mkdocs-material \ - pymdown-extensions \ - mkdocs-branchcustomization-plugin \ - mkdocs-macros-plugin \ - tabulate +python -m venv env +source env/bin/activate ``` -#### 2. Download the BIDS specification [repository](https://github.com/bids-standard/bids-specification/tree/master) onto your computer +Note that this will create a local directory called `env` within the bids-specification directory +but that its content will not be tracked by `git` because it is listed in the `.gitignore` file. -This can be done by clicking the green button on the right titled "Clone or -download" -or using [this link](https://github.com/bids-standard/bids-specification/archive/master.zip). +Once you have activated your isolated Python environment, +an easy way to install the correct version of mkdocs and all the other required extensions +is to use the `requirements.txt` file contained in this repository as follows: -#### 3. In the terminal (command line) navigate to your local version of the specification +```bash +pip install -U pip +pip install -r requirements.txt +pip install -e tools/schemacode/ +``` -This location will have the same files you see on our -[main specification page](https://github.com/bids-standard/bids-specification). -Note: A finder window may not show the hidden files (those that start with a -period, like `.remarkrc`) +The first command ensures you are using an up to date version of `pip`, +and the second command installs all dependencies. +The third command ensures to install the BIDS schema code as an "editable" install, +so that if you make changes to the schema files, +these are automatically reflected in the sourcecode. #### 4. Ready to build! @@ -458,7 +415,7 @@ specification, do not hesitate to make a suggestion by showing a draft in a GitH After discussion and approval by the community, you can then submit your image in a pull request. -Images should be added to an `images` folder that is at the same level as the Markdown file +Images should be added to an `images` directory that is at the same level as the Markdown file where your image will be added. For example if you want to add a figure `figure01.png` to `src/05-derivatives/01-introduction.md` then your image should go to `src/05-derivatives/images/figure01.png`. @@ -672,7 +629,7 @@ reviewer as a co-author. ## Making a change to the BIDS-schema Several aspects of the specification are defined in a set of YAML files in the -`src/schema` folder. The content of those files is described in a dedicated +`src/schema` directory. The content of those files is described in a dedicated [README file](./src/schema/README.md). ### 1. Ensure that changes to the specification are matched in the schema diff --git a/DECISION-MAKING.md b/DECISION-MAKING.md index a2bc9d8de6..f52f7e2763 100644 --- a/DECISION-MAKING.md +++ b/DECISION-MAKING.md @@ -20,6 +20,12 @@ BIDS governance. | Melanie Ganz ([@melanieganz](https://github.com/melanieganz)) | | Robert Oostenveld ([@robertoostenveld](https://github.com/robertoostenveld)) | | Russell Poldrack ([@poldrack](https://github.com/poldrack)) | +| Ariel Rokem ([@arokem](https://github.com/arokem)) | + +#### Past steering group members + +| Name | +|------------------------------------------------------------------------------| | Kirstie Whitaker ([@KirstieJane](https://github.com/KirstieJane)) | ### Maintainers Group @@ -32,6 +38,7 @@ BIDS governance. | Taylor Salo ([@tsalo](https://github.com/tsalo)) | 3h/week | MRI | | Remi Gau ([@Remi-Gau](https://github.com/Remi-Gau)) | 3h/week | Community development, MRI | | Anthony Galassi ([@bendhouseart](https://github.com/bendhouseart)) | 3h/week | PET, Community development | +| Eric Earl ([@ericearl](https://github.com/ericearl)) | 2h/week | | In addition to the [BIDS Governance](https://bids.neuroimaging.io/governance.html#bids-maintainers-group) classification of a maintainer, maintainers may declare a limited scope of responsibility. diff --git a/LICENSE b/LICENSE index d4466ac392..d591978380 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Attribution 4.0 International -Copyright (c) 2020, BIDS Contributors. +Copyright (c) 2018-2022, BIDS Contributors. ======================================================================= diff --git a/README.md b/README.md index 41136e0fd3..5bf0f2b55b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ [![@BIDSstandard](http://img.shields.io/twitter/follow/bidsstandard.svg?style=social)](https://twitter.com/BIDSstandard) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3686061.svg)](https://doi.org/10.5281/zenodo.3686061) -bids-logo +bids-logo +bids-logo The [Brain Imaging Data Structure (BIDS)](https://bids.neuroimaging.io) is an emerging standard for the organisation of neuroimaging data. @@ -25,6 +26,7 @@ BIDS currently supports the following data modalities with more to come in the f - behavioral - physiological - PET +- microscopy # Formatting your data with BIDS diff --git a/macros_doc.md b/macros_doc.md new file mode 100644 index 0000000000..c202891717 --- /dev/null +++ b/macros_doc.md @@ -0,0 +1,382 @@ +# Using MkDocs macros in the BIDS specification + +We use [mkdocs-macros](https://mkdocs-macros-plugin.readthedocs.io/en/latest/) +to standardize how some aspects of the BIDS specification are rendered in HTML. +Macros make it easy to achieve a consistent style throughout the specification, +and changing a given macro will automatically change all appropriate paragraphs +in the specification. + +Below you will find answers to frequently asked questions regarding using macros +in the BIDS specification. + +- [Using MkDocs macros in the BIDS specification](#using-mkdocs-macros-in-the-bids-specification) + - [What are macros and why use them?](#what-are-macros-and-why-use-them) + - [What kind of input information are required by macros?](#what-kind-of-input-information-are-required-by-macros) + - [What macros are available?](#what-macros-are-available) + - [When should I use a macro?](#when-should-i-use-a-macro) + - [Do I need learn how to program to use those macros?](#do-i-need-learn-how-to-program-to-use-those-macros) + - [Anything else I need to know if I need to insert a new macro call?](#anything-else-i-need-to-know-if-i-need-to-insert-a-new-macro-call) + - [How-To and Examples](#how-to-and-examples) + - [Writing directory content examples](#writing-directory-content-examples) + - [Generating tables](#generating-tables) + - [Modifying a term in the table](#modifying-a-term-in-the-table) + - [Why would you NOT want to modify the content of the yml file directly ?](#why-would-you-not-want-to-modify-the-content-of-the-yml-file-directly-) + - [Adding a new term to the table](#adding-a-new-term-to-the-table) + - [Should I create a macro if I need a new kind of table?](#should-i-create-a-macro-if-i-need-a-new-kind-of-table) + - [Why use macros at all?](#why-use-macros-at-all) + - [Links and references](#links-and-references) + +## What are macros and why use them? + +A macro is a rule or pattern that specifies how an input should be mapped to +output. Macros are very useful for standardizing the output format for items +such as tables. You might already be familiar with using macros from other tools +such as Excel. + +MkDocs (the tool we use to turn the markdown version of the BIDS specification +into HTML pages) supports macros. In the specification document, we use these +macros to standardize the format of items such as tables and examples. + +The following is an example of a macro used to create consistent "file tree" +layouts in the documentation. The macro takes a single parameter, the directory +tree to be displayed in JSON format. If you insert the following in the BIDS +markdown document: + +```python +{{ MACROS___make_filetree_example( + + { + "sub-01": { + "func": { + "sub-control01_task-nback_bold.json": "", + }, + } + } + +) }} +``` + +The result would be rendered in the specification document as: + +```bash +└─ sub-01/ + └─ func/ + └─ sub-control01_task-nback_bold.json +``` + +## What kind of input information are required by macros? + +Some macros only use the arguments you directly supply in the macro call. + +Other macros use information (such as metadata terms) from external sources, and +you will need to provide links to this information as part of the call. For +example, parts of the BIDS specification are formalized into a "schema" so that +requirements in the specification can be automatically checked by validators. +Several of the macros incorporate information from this schema to assure +consistency. + +## What macros are available? + +All the macros we use are in listed in this +[python file](https://github.com/bids-standard/bids-specification/blob/master/tools/mkdocs_macros_bids/macros.py). + +| Name | Purpose | Uses schema | Example | +| ----------------------- | -------------------------------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| make_columns_table | Generate a markdown table of TSV column information. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/03-modality-agnostic-files.md?plain=1#L202) | +| make_entity_table | Generate an entity table from the schema, based on specific filters. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/99-appendices/04-entity-table.md?plain=1#L23) | +| make_entity_definitions | Generate definitions and other relevant information for entities in the specification. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/99-appendices/09-entities.md?plain=1#L16) | +| make_filename_template | Generate a filename template from the schema, based on specific filters. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/04-modality-specific-files/10-microscopy.md?plain=1#L21) | +| make_filetree_example | Generate a filetree snippet from example content. | No | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/02-common-principles.md?plain=1#L268) | +| make_glossary | | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/99-appendices/14-glossary.md?plain=1#L9) | +| make_metadata_table | Generate a markdown table of metadata field information. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/02-common-principles.md?plain=1#L462) | +| make_suffix_table | Generate a markdown table of suffix information. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/04-modality-specific-files/01-magnetic-resonance-imaging-data.md?plain=1#L199) | + +## When should I use a macro? + +Most typo corrections and minor changes to the BIDS specification do not require +you to use macros. Even adding a table may not require you to use macros unless +the table falls into one of the categories listed in the macros table. + +If you want to add content with a macro and need help, do not hesitate to +contact a member of the bids-maintainers for help. To do this, you can either +mention an individual maintainer by their GitHub username or mention the whole +team (`@bids-standard/maintainers`). + +## Do I need learn how to program to use those macros? + +Macros don't require programming knowledge to use. You do need to know what +arguments the macro expects. The examples linked in the above table provide +useful guidance in this respect. + +Macros that extract information from the schema also require you to use the +correct terms in the schema. This process is illustrated in the next section. + +Note that under the hood the macros themselves call python code that can be +found in the +[`tools` directory](https://github.com/bids-standard/bids-specification/tree/master/tools). +If you are interested in creating a new macro for users, this would be useful. + +## Anything else I need to know if I need to insert a new macro call? + +One nice thing for the people who will come after you (or yourself in 6 months +when you get back to the document you just edited) is to leave a comment before +the macro to quickly explain what it does and where to find more information +about it. + + + +It could for example look like this: + +```markdown + + +{{ MACROS\_\_\_make_metadata_table( { "AcquisitionMode": "REQUIRED", +"MoonPhase": "OPTIONAL", "ImageDecayCorrected": "REQUIRED", +"ImageDecayCorrectionTime": "REQUIRED", + + ... + +} ) }} +``` + +## How-To and Examples + +### Writing directory content examples + +One of the simplest macro we use helps us create consistent "file tree" examples +that would look like this in the final document: + +```Text +└─ sub-01/ + └─ func/ + └─ sub-control01_task-nback_bold.json +``` + +To do this get this output, your macro call would look like this: + +``` +{{ MACROS___make_filetree_example( + + { + "sub-01": { + "func": { + "sub-control01_task-nback_bold.json": "", + }, + } + } + +) }} +``` + +When you have complex files and directory structure, we suggest you use this +[Jupyter notebook](tools/filetree_example.ipynb) for sandboxing your example +before you insert the macro call into the markdown document. + +### Generating tables + +Say you want to edit the content of table of the `Reconstruction` section for +the PET page. + +The HTML version of the this section is here: + +https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/09-positron-emission-tomography.html#reconstruction + +In the markdown document, it is here: + +https://github.com/bids-standard/bids-specification/blob/master/src/04-modality-specific-files/09-positron-emission-tomography.md#reconstruction + +GitHub's way of directly rendering markdown documents makes it a bit harder to +read, so if you opened the markdown document in your code editor it would look +like this. + +```markdown +#### Reconstruction + +{{ MACROS___make_metadata_table( + { + "AcquisitionMode": "REQUIRED", + "ImageDecayCorrected": "REQUIRED", + "ImageDecayCorrectionTime": "REQUIRED", + "ReconMethodName": "REQUIRED", + "ReconMethodParameterLabels": "REQUIRED", + "ReconMethodParameterUnits": "REQUIRED", + "ReconMethodParameterValues": "REQUIRED", + "ReconFilterType": "REQUIRED", + "ReconFilterSize": "REQUIRED", + "AttenuationCorrection": "REQUIRED", + "ReconMethodImplementationVersion": "RECOMMENDED", + "AttenuationCorrectionMethodReference": "RECOMMENDED", + "ScaleFactor": "RECOMMENDED", + "ScatterFraction": "RECOMMENDED", + "DecayCorrectionFactor": "RECOMMENDED", + "DoseCalibrationFactor": "RECOMMENDED", + "PromptRate": "RECOMMENDED", + "RandomRate": "RECOMMENDED", + "SinglesRate": "RECOMMENDED", + } +) }} +``` + +--- + +**HINT:** if you want to see the "raw" content of a file on Github you can +always press the `raw` button that is on the top right of the document you are +browsing on Github. + +--- + +This section calls the macro `make_metadata_table` to create the table when +building the HTML version of that page. + +The macro will create the different columns of the table: + +- Key name +- Requirement level +- Data type +- Description + +A general description of that macro call would look like this: + +```python +{{ MACROS___make_metadata_table( + { + "TermToRender": "REQUIREMENT_LEVEL plus anything else after", "Extra content you want to append after the description of that term." + } +) }} +``` + +To know what to put in the different columns, the macro will go and look into +the +[`metadata.yaml`](https://github.com/bids-standard/bids-specification/blob/master/src/schema/objects/metadata.yaml) +file in the BIDS schema and find the entry that correspond to the term you want +to add. + +And in the above example, all the information about `AcquisitionMode` would be +read from +[that section](https://github.com/bids-standard/bids-specification/blob/master/src/schema/objects/metadata.yaml#L20). + +If you had to write the markdown equivalent of the general example for the macro +call above it would give a table that would look like this: + +| Key name | Requirement level | Data type | Description | +| ------------ | ------------------------------------------ | --------- | -------------------------------------------- | +| TermToRender | REQUIREMENT_LEVEL plus anything else after | string | whatever description was in the metadata.yml | + +#### Modifying a term in the table + +So if you want to change the content of what will appear in the HTML table, you +need to edit this `metadata.yml` file. + +If you wanted to add some extra content to that table, but without modifying the +definition in the schema, then you could just add some extra content into the +macro call. + +```python +#### Reconstruction + +{{ MACROS___make_metadata_table( + { + "AcquisitionMode": "REQUIRED", "But only when the acquisition was done on a full moon." + + ... +``` + +#### Why would you NOT want to modify the content of the yml file directly ? + +Well the same term can be used in different parts of the BIDS specification and +some details that might apply to, say, PET might not apply to how the term is +used for MRI. So we can use the schema for the common part and add extra content +where necessary in the macro call. + +So always better to check if that term is not used somewhere else before making +a change in the yml file. When in doubt add the change directly in the macro +call and ask the BIDS maintainers for help. + +#### Adding a new term to the table + +Say you wanted to add a new term `MoonPhase` to the table, on the second row. +You would do it like this. But this would only work if the `metadata.yml` file +contains an entry for `MoonPhase`. If this is the case because the term already +exists and is used somewhere in the BIDS specification, you are in luck and you +can just stop there. + +```python +#### Reconstruction + +{{ MACROS___make_metadata_table( + { + "AcquisitionMode": "REQUIRED", + "MoonPhase": "OPTIONAL", + "ImageDecayCorrected": "REQUIRED", + "ImageDecayCorrectionTime": "REQUIRED", + + ... + + } +) }} +``` + +If the term does not exist, you need to add it. `YML` files have a fairly strict +syntax where spaces and indentation matter a lot. You can a mini intro to YML +files in the Turing way: +https://the-turing-way.netlify.app/reproducible-research/renv/renv-yaml.html + +In practice, you should try to use a code editor that tells you when your syntax +is wrong. + +#### Should I create a macro if I need a new kind of table? + +As a rule of thumb, no, unless it is clear that this kind of table will reappear +many times in the future in the specification. But this is usually hard to +predict so better start with a table in Markdown. + +If later we see that the same type of table keeps reoccuring the specification +we could create a macro to generate them. + +## Why use macros at all? + +> Seriously why did you have to make it so complicated just to have pretty +> tables? Are macros that necessary ? Couldn't we just have everything in +> Markdown? + +In principle we could, and before we started working on the schema that's +exactly what we did. But there are several good reasons to use macros. + +When a definition gets repeated in different places, we could just copy-paste +it. But when you start having several copies of that definition, if you have to +modify it, you then need to edit several files and never forget any of them. So +this becomes very error prone. + +So it becomes better to have one central place for that definition and grab that +definition every time we need to reuse it. + +In practice this applies the +[DRY principle ("Don't Repeat Yourself")](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself) +to the specification: + +> "Every piece of knowledge must have a single, unambiguous, authoritative +> representation within a system". + +Having one centralized place where we put all our definitions can be useful when +we want other tools (the BIDS validator, bids-matlab...) to use the content of +the specification. + +This is where the BIDS schema (those .yml files we talked about above) comes in +as it is meant to be a machine readable version of the specification. + +And so to avoid having to maintain the SAME definition in both the schema and +specification, we started using macros to generate the specification from the +schema. + +## Links and references + +- [documentation for mkdocs](https://www.mkdocs.org) and how to install it + locally, +- [documentation for the material theme](https://squidfunk.github.io/mkdocs-material/) + we use. +- [documentation for the `macros` plugin](https://mkdocs-macros-plugin.readthedocs.io/en/latest/) diff --git a/mkdocs.yml b/mkdocs.yml index 8cfc8853d9..01eb96dd5c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,12 +1,21 @@ -site_name: Brain Imaging Data Structure v1.6.1-dev +site_name: Brain Imaging Data Structure v1.7.1-dev site_url: https://bids-specification.readthedocs.io/en/stable/ theme: name: material - custom_dir: theme_customizations/ favicon: images/favicon.png logo: images/logo.png features: - navigation.sections +copyright: Copyright © 2018-2022, BIDS Contributors - CC BY 4.0 +extra: + generator: false + social: + - icon: fontawesome/brands/twitter + link: https://twitter.com/BIDSstandard/ + - icon: fontawesome/brands/github + link: https://github.com/bids-standard/bids-specification/ + - icon: fontawesome/brands/google + link: https://groups.google.com/g/bids-discussion extra_javascript: - js/jquery-3.6.0.min.js markdown_extensions: @@ -39,6 +48,7 @@ nav: - Behavioral experiments (with no neural recordings): 04-modality-specific-files/07-behavioral-experiments.md - Genetic Descriptor: 04-modality-specific-files/08-genetic-descriptor.md - Positron Emission Tomography: 04-modality-specific-files/09-positron-emission-tomography.md + - Microscopy: 04-modality-specific-files/10-microscopy.md - Derivatives: - BIDS Derivatives: 05-derivatives/01-introduction.md - Common data types and metadata: 05-derivatives/02-common-data-types.md @@ -59,8 +69,9 @@ nav: - Quantitative MRI: 99-appendices/11-qmri.md - Arterial Spin Labeling: 99-appendices/12-arterial-spin-labeling.md - Cross modality correspondence: 99-appendices/13-cross-modality-correspondence.md + - Glossary: 99-appendices/14-glossary.md - Changelog: CHANGES.md - The BIDS Starter Kit: + - Website: https://bids-standard.github.io/bids-starter-kit + - Tutorials: https://bids-standard.github.io/bids-starter-kit/tutorials/tutorials.html - GitHub repository: https://github.com/bids-standard/bids-starter-kit - - Tutorials: https://github.com/bids-standard/bids-starter-kit/wiki/Tutorials - - Wiki: https://github.com/bids-standard/bids-starter-kit/wiki diff --git a/requirements.txt b/requirements.txt index 806c28a3c1..edc1538e9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,4 @@ pymdown-extensions>=7.0.0 mkdocs-branchcustomization-plugin~=0.1.3 mkdocs-macros-plugin numpy -pandas -PYYaml -tabulate +tools/schemacode/ diff --git a/src/01-introduction.md b/src/01-introduction.md index 79ce0831d2..f3342ba00a 100644 --- a/src/01-introduction.md +++ b/src/01-introduction.md @@ -40,7 +40,7 @@ and the INCF Neuroimaging Data Sharing (NIDASH) Task Force. While working on BIDS we consulted many neuroscientists to make sure it covers most common experiments, but at the same time is intuitive and easy to adopt. The specification is intentionally -based on simple file formats and folder structures to reflect current lab +based on simple file formats and directory structures to reflect current lab practices and make it accessible to a wide range of scientists coming from different backgrounds. @@ -113,12 +113,22 @@ For example: #### PET +- Norgaard, M., Matheson, G.J., Hansen, H.D., Thomas, A., Searle, G., Rizzo, G., + Veronese, M., Giacomel, A., Yaqub, M., Tonietto, M., Funck, T., Gillman, A., Boniface, + H., Routier, A., Dalenberg, J.R.., Betthauser, T., Feingold, F., Markiewicz, C.J., + Gorgolewski, K.J., Blair, R.W., Appelhoff, S., Gau, R., Salo, T., Niso, G., Pernet, C., + Phillips, C., Oostenveld, R., Gallezot, J-D., Carson, R.E., Knudsen, G.M., + Innis R.B. & Ganz M. (2021). + **PET-BIDS, an extension to the brain imaging data structure for positron emission tomography**. + Scientific Data, 9 (65). + [doi:10.1038/s41597-022-01164-1](https://doi.org/10.1038/s41597-022-01164-1) + - Knudsen GM, Ganz M, Appelhoff S, Boellaard R, Bormans G, Carson RE, Catana C, Doudet D, Gee AD, Greve DN, Gunn RN, Halldin C, Herscovitch P, Huang H, Keller SH, Lammertsma AA, Lanzenberger R, Liow JS, Lohith TG, Lubberink M, Lyoo CH, Mann JJ, Matheson GJ, Nichols TE, Nørgaard M, Ogden T, Parsey R, Pike VW, Price J, Rizzo G, - Rosa-Neto P, Schain M, Scott PJH, Searle G, Slifstein M, Suhara T, Talbot PS, - Thomas A, Veronese M, Wong DF, Yaqub M, Zanderigo F, Zoghbi S, Innis RB. (2020). + Rosa-Neto P, Schain M, Scott PJH, Searle G, Slifstein M, Suhara T, Talbot PS, Thomas A, + Veronese M, Wong DF, Yaqub M, Zanderigo F, Zoghbi S, Innis RB. (2020). **Guidelines for Content and Format of PET Brain Data in Publications and in Archives: A Consensus Paper**. Journal of Cerebral Blood Flow and Metabolism, 2020 Aug; 40(8): 1576-1585. [doi:10.1177/0271678X20905433](https://doi.org/10.1177/0271678X20905433) diff --git a/src/02-common-principles.md b/src/02-common-principles.md index 62da6e2fb1..528e6c61ce 100644 --- a/src/02-common-principles.md +++ b/src/02-common-principles.md @@ -54,6 +54,7 @@ misunderstanding we clarify them here. 1. `ieeg` (intracranial electroencephalography) 1. `beh` (behavioral) 1. `pet` (positron emission tomography) + 1. `micr` (microscopy) Data files are contained in a directory named for the data type. In raw datasets, the data type directory is nested inside subject and @@ -103,11 +104,11 @@ misunderstanding we clarify them here. The modality may overlap with, but should not be confused with the **data type**. -1. **Suffix** - an alphanumeric string that forms part of a file name, located +1. **Suffix** - an alphanumeric string that forms part of a filename, located after all [entities](#entities) and following a final `_`, right before the **file extension**; for example, it is `eeg` in `sub-05_task-matchingpennies_eeg.vhdr`. -1. **File extension** - a portion of the file name after the left-most +1. **File extension** - a portion of the filename after the left-most period (`.`) preceded by any other alphanumeric. For example, `.gitignore` does not have a file extension, but the file extension of `test.nii.gz` is `.nii.gz`. Note that the left-most period is included in the file extension. @@ -121,7 +122,7 @@ misunderstanding we clarify them here. ## Entities An "entity" is an attribute that can be associated with a file, contributing -to the identification of that file as a component of its file name in the +to the identification of that file as a component of its filename in the form of a hyphen-separated key-value pair. Each entity has the following attributes: @@ -130,10 +131,10 @@ Each entity has the following attributes: to be provided via the entity. 1. *Key*: A short string, typically a compression of the entity name, - which uniquely identifies the entity when part of a file name. + which uniquely identifies the entity when part of a filename. 1. *Value type*: The requisite form of the value that gets specified - alongside the key whenever the entity appears in a file name. + alongside the key whenever the entity appears in a filename. For each entity, the value is of one of two possible types: 1. *Index*: A non-negative integer, potentially zero-padded for @@ -144,7 +145,7 @@ Each entity has the following attributes: (see [Case collision intolerance](#case-collision-intolerance)). The entity *format* is a string that prescribes how the entity appears within -any given file name. +any given filename. For a hypothetical entity with key "`key`", the format can be either "`key-`" or "`key-