diff --git a/.appveyor.yml b/.appveyor.yml index 36bdefd..53b2253 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -61,12 +61,13 @@ environment: # is a need for debugging # Ubuntu core tests - - ID: Ubu20 + - ID: Ubu22 DTS: datalad_neuroimaging - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004 + APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2204 + PY: 3.9 INSTALL_SYSPKGS: python3-virtualenv dcm2niix # system git-annex is way too old, use better one - INSTALL_GITANNEX: git-annex -m deb-url --url http://snapshot.debian.org/archive/debian/20210906T204127Z/pool/main/g/git-annex/git-annex_8.20210903-1_amd64.deb + INSTALL_GITANNEX: git-annex -m snapshot CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov # Windows core tests - ID: WinP39core @@ -76,13 +77,13 @@ environment: PY: 39-x64 INSTALL_GITANNEX: git-annex -m datalad/packages # MacOS core tests - - ID: MacP38core + - ID: MacP39core DTS: datalad_neuroimaging APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey - PY: 3.8 - INSTALL_GITANNEX: git-annex + PY: 3.9 + INSTALL_GITANNEX: git-annex -m datalad/packages DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets - CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov + CODECOV_BINARY: https://cli.codecov.io/v0.7.4/macos/codecov matrix: allow_failures: @@ -141,6 +142,7 @@ install: - sh: "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || virtualenv -p 3 ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\"" - cmd: "set PATH=C:\\Python%PY%;C:\\Python%PY%\\Scripts;%PATH%" # deploy the datalad installer, override version via DATALAD_INSTALLER_VERSION + - python -m pip install --upgrade pip - cmd: IF DEFINED DATALAD_INSTALLER_VERSION ( python -m pip install "datalad-installer%DATALAD_INSTALLER_VERSION%" @@ -149,11 +151,11 @@ install: ) - sh: python -m pip install datalad-installer${DATALAD_INSTALLER_VERSION:-} # Missing system software - - sh: "[ -n \"$INSTALL_SYSPKGS\" ] && ( [ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacOS\" ] && brew install -q ${INSTALL_SYSPKGS} || { sudo apt-get update -y && sudo apt-get install --no-install-recommends -y ${INSTALL_SYSPKGS}; } ) || true" + - sh: tools/appveyor/install-syspkgs $INSTALL_SYSPKGS # Install git-annex on windows, otherwise INSTALL_SYSPKGS can be used # deploy git-annex, if desired - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX% - - sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer --sudo ok ${INSTALL_GITANNEX}" + - sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer -E ${HOME}/dlinstaller_env.sh --sudo ok ${INSTALL_GITANNEX}" # in case of a snapshot installation, use the following approach to adjust # the PATH as necessary #- sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer -E ${HOME}/dlinstaller_env.sh --sudo ok ${INSTALL_GITANNEX}" @@ -175,6 +177,7 @@ build_script: before_test: + - sh: source ${HOME}/dlinstaller_env.sh # simple call to see if datalad and git-annex are installed properly - datalad wtf diff --git a/.github/workflows/docbuild.yml b/.github/workflows/docbuild.yml index 127c330..af8244e 100644 --- a/.github/workflows/docbuild.yml +++ b/.github/workflows/docbuild.yml @@ -13,10 +13,10 @@ jobs: git config --global user.email "test@github.land" git config --global user.name "GitHub Almighty" - uses: actions/checkout@v1 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test_crippledfs.yml b/.github/workflows/test_crippledfs.yml index 329529f..eb36b8d 100644 --- a/.github/workflows/test_crippledfs.yml +++ b/.github/workflows/test_crippledfs.yml @@ -28,10 +28,10 @@ jobs: git config --global user.email "test@github.land" git config --global user.name "GitHub Almighty" - uses: actions/checkout@v1 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | pip install -r requirements-devel.txt diff --git a/datalad_neuroimaging/extractors/dicom.py b/datalad_neuroimaging/extractors/dicom.py index 81e970f..59dfe19 100644 --- a/datalad_neuroimaging/extractors/dicom.py +++ b/datalad_neuroimaging/extractors/dicom.py @@ -14,30 +14,29 @@ import logging lgr = logging.getLogger('datalad.metadata.extractors.dicom') from datalad.log import log_progress +from datalad.support.exceptions import CapturedException +from datalad.support.external_versions import external_versions -try: - # renamed for 1.0 release - import pydicom as dcm - from pydicom.errors import InvalidDicomError +import pydicom as dcm +from pydicom.errors import InvalidDicomError + +NOT_IMPLEMENTED_TYPES = tuple() # (FileDataset,) +if external_versions["pydicom"] >= "3": + # everything is a FileDataset now, so we will decide based on have a UID + pass +else: from pydicom.dicomdir import DicomDir -except ImportError: # pragma: no cover - import dicom as dcm - from dicom.errors import InvalidDicomError - from dicom.dicomdir import DicomDir + NOT_IMPLEMENTED_TYPES = (DicomDir,) try: from collections.abc import MutableSequence except ImportError: from collections import MutableSequence -from distutils.version import LooseVersion from datalad_deprecated.metadata.definitions import vocabulary_id from datalad_deprecated.metadata.extractors.base import BaseMetadataExtractor - -# pydicom 2.0.0 renamed PersonName3 to PersonName: -PersonName = dcm.valuerep.PersonName3 \ - if LooseVersion(dcm.__version__) < "2.0.0" else dcm.valuerep.PersonName +PersonName = dcm.valuerep.PersonName # Data types we care to extract/handle _SCALAR_TYPES = ( int, float, string_types, dcm.valuerep.DSfloat, dcm.valuerep.IS, @@ -156,17 +155,21 @@ def get_metadata(self, dataset, content): continue try: - d = dcm.read_file(absfp, defer_size=1000, stop_before_pixels=True) - except InvalidDicomError: + d = dcm.dcmread(absfp, defer_size=1000, stop_before_pixels=True) + except InvalidDicomError as exc: # we can only ignore - lgr.debug('"%s" does not look like a DICOM file, skipped', f) + lgr.debug('"%s" does not look like a DICOM file, skipped: %s', + absfp, + CapturedException(exc)) continue - if isinstance(d, DicomDir): - lgr.debug("%s appears to be a DICOMDIR file. Extraction not yet" - " implemented, skipped", f) + if NOT_IMPLEMENTED_TYPES and isinstance(d, NOT_IMPLEMENTED_TYPES): + lgr.debug("%s appears to be a DICOMDIR or alike: got %s. Extraction not yet" + " implemented, skipped", f, d) + continue + elif not hasattr(d, 'SeriesInstanceUID'): + lgr.debug("%s does not have SeriesInstanceUID, skipped", f) continue - ddict = None if content: ddict = _struct2dict(d) diff --git a/datalad_neuroimaging/extractors/tests/test_nidm.py b/datalad_neuroimaging/extractors/tests/test_nidm.py index 30e34d4..574e3bb 100644 --- a/datalad_neuroimaging/extractors/tests/test_nidm.py +++ b/datalad_neuroimaging/extractors/tests/test_nidm.py @@ -46,7 +46,7 @@ def test_nidm(path=None): # aggregation done without whining assert_status('ok', res) res = ds.metadata(reporton='datasets') - # ATM we do not forsee file-based metadata to come back from NIDM + # ATM we do not foresee file-based metadata to come back from NIDM assert_result_count(res, 1) # kill version info core = res[0]['metadata']['datalad_core'] diff --git a/datalad_neuroimaging/tests/test_dicomconv.py b/datalad_neuroimaging/tests/test_dicomconv.py index 227162a..9d2ee98 100644 --- a/datalad_neuroimaging/tests/test_dicomconv.py +++ b/datalad_neuroimaging/tests/test_dicomconv.py @@ -56,4 +56,6 @@ def test_dicom_metadata_aggregation(path=None): def test_validate_bids_fixture(): bids_ds = get_bids_dataset() # dicom source dataset is absent - eq_(len(bids_ds.subdatasets(fulfilled=True, return_type='list')), 0) + # yoh: disabled since makes little sense (now?) since dataset is subdataset + # as of 978772e9468a5ae30de309cc6ac4370795de75cc etc in 2018 + # eq_(len(bids_ds.subdatasets(fulfilled=True, return_type='list')), 0) diff --git a/setup.cfg b/setup.cfg index 03820cd..3d26336 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,11 +14,11 @@ classifiers = [options] zip_safe = False -python_requires = >= 3.8 +python_requires = >= 3.9 install_requires = datalad >= 0.16.7 datalad-deprecated >= 0.2.7 - pydicom # DICOM metadata + pydicom >= 2.0.0 # DICOM metadata pybids >= 0.15.1 # BIDS metadata nibabel # NIfTI metadata pandas # bids2scidata export diff --git a/tools/appveyor/install-syspkgs b/tools/appveyor/install-syspkgs new file mode 100755 index 0000000..bfb84a9 --- /dev/null +++ b/tools/appveyor/install-syspkgs @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# no install requested -> exit +[ -z "$1" ] && exit 0 || true + +if (which apt-get > /dev/null ); then + sudo apt-get update -qq -y --allow-releaseinfo-change + sudo apt-get install -q --no-install-recommends -y eatmydata + sudo eatmydata apt-get install -q --no-install-recommends -y $* +else + brew install -q $* +fi