Merge pull request #133 from datalad/bf-testing

Test against 3.9 not EOLed 3.8, fix typo, compatibility with recent pydicom 3.x

.appveyor.yml

@@ -61,12 +61,13 @@ environment:
     # is a need for debugging
 
     # Ubuntu core tests
-    - ID: Ubu20
+    - ID: Ubu22
       DTS: datalad_neuroimaging
-      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
+      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2204
+      PY: 3.9
       INSTALL_SYSPKGS: python3-virtualenv dcm2niix
       # system git-annex is way too old, use better one
-      INSTALL_GITANNEX: git-annex -m deb-url --url http://snapshot.debian.org/archive/debian/20210906T204127Z/pool/main/g/git-annex/git-annex_8.20210903-1_amd64.deb
+      INSTALL_GITANNEX: git-annex -m snapshot
       CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
     # Windows core tests
     - ID: WinP39core
@@ -76,13 +77,13 @@ environment:
       PY: 39-x64
       INSTALL_GITANNEX: git-annex -m datalad/packages
     # MacOS core tests
-    - ID: MacP38core
+    - ID: MacP39core
       DTS: datalad_neuroimaging
       APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
-      PY: 3.8
-      INSTALL_GITANNEX: git-annex
+      PY: 3.9
+      INSTALL_GITANNEX: git-annex -m datalad/packages
       DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets
-      CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov
+      CODECOV_BINARY: https://cli.codecov.io/v0.7.4/macos/codecov
 
 matrix:
   allow_failures:
@@ -141,6 +142,7 @@ install:
   - sh: "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || virtualenv -p 3 ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\""
   - cmd: "set PATH=C:\\Python%PY%;C:\\Python%PY%\\Scripts;%PATH%"
   # deploy the datalad installer, override version via DATALAD_INSTALLER_VERSION
+  - python -m pip install --upgrade pip
   - cmd:
       IF DEFINED DATALAD_INSTALLER_VERSION (
       python -m pip install "datalad-installer%DATALAD_INSTALLER_VERSION%"
@@ -149,11 +151,11 @@ install:
       )
   - sh: python -m pip install datalad-installer${DATALAD_INSTALLER_VERSION:-}
   # Missing system software
-  - sh: "[ -n \"$INSTALL_SYSPKGS\" ] && ( [ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacOS\" ] && brew install -q ${INSTALL_SYSPKGS} || { sudo apt-get update -y && sudo apt-get install --no-install-recommends -y ${INSTALL_SYSPKGS}; } ) || true"
+  - sh: tools/appveyor/install-syspkgs $INSTALL_SYSPKGS
   # Install git-annex on windows, otherwise INSTALL_SYSPKGS can be used
   # deploy git-annex, if desired
   - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX%
-  - sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer --sudo ok ${INSTALL_GITANNEX}"
+  - sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer -E ${HOME}/dlinstaller_env.sh --sudo ok ${INSTALL_GITANNEX}"
   # in case of a snapshot installation, use the following approach to adjust
   # the PATH as necessary
   #- sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer -E ${HOME}/dlinstaller_env.sh --sudo ok ${INSTALL_GITANNEX}"
@@ -175,6 +177,7 @@ build_script:
 
 
 before_test:
+  - sh: source ${HOME}/dlinstaller_env.sh 
   # simple call to see if datalad and git-annex are installed properly
   - datalad wtf
 

.github/workflows/docbuild.yml

@@ -13,10 +13,10 @@ jobs:
         git config --global user.email "test@github.land"
         git config --global user.name "GitHub Almighty"
     - uses: actions/checkout@v1
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/setup-python@v1
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/test_crippledfs.yml

@@ -28,10 +28,10 @@ jobs:
         git config --global user.email "test@github.land"
         git config --global user.name "GitHub Almighty"
     - uses: actions/checkout@v1
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/setup-python@v1
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install dependencies
       run: |
         pip install -r requirements-devel.txt

datalad_neuroimaging/extractors/dicom.py

@@ -14,30 +14,29 @@
 import logging
 lgr = logging.getLogger('datalad.metadata.extractors.dicom')
 from datalad.log import log_progress
+from datalad.support.exceptions import CapturedException
+from datalad.support.external_versions import external_versions
 
-try:
-    # renamed for 1.0 release
-    import pydicom as dcm
-    from pydicom.errors import InvalidDicomError
+import pydicom as dcm
+from pydicom.errors import InvalidDicomError
+
+NOT_IMPLEMENTED_TYPES = tuple() # (FileDataset,)
+if external_versions["pydicom"] >= "3":
+    # everything is a FileDataset now, so we will decide based on have a UID
+    pass
+else:
     from pydicom.dicomdir import DicomDir
-except ImportError:  # pragma: no cover
-    import dicom as dcm
-    from dicom.errors import InvalidDicomError
-    from dicom.dicomdir import DicomDir
+    NOT_IMPLEMENTED_TYPES = (DicomDir,)
 
 try:
     from collections.abc import MutableSequence
 except ImportError:
     from collections import MutableSequence
 
-from distutils.version import LooseVersion
 from datalad_deprecated.metadata.definitions import vocabulary_id
 from datalad_deprecated.metadata.extractors.base import BaseMetadataExtractor
 
-
-# pydicom 2.0.0 renamed PersonName3 to PersonName:
-PersonName = dcm.valuerep.PersonName3 \
-    if LooseVersion(dcm.__version__) < "2.0.0" else dcm.valuerep.PersonName
+PersonName = dcm.valuerep.PersonName
 # Data types we care to extract/handle
 _SCALAR_TYPES = (
     int, float, string_types, dcm.valuerep.DSfloat, dcm.valuerep.IS,
@@ -156,17 +155,21 @@ def get_metadata(self, dataset, content):
                 continue
 
             try:
-                d = dcm.read_file(absfp, defer_size=1000, stop_before_pixels=True)
-            except InvalidDicomError:
+                d = dcm.dcmread(absfp, defer_size=1000, stop_before_pixels=True)
+            except InvalidDicomError as exc:
                 # we can only ignore
-                lgr.debug('"%s" does not look like a DICOM file, skipped', f)
+                lgr.debug('"%s" does not look like a DICOM file, skipped: %s',
+                          absfp,
+                          CapturedException(exc))
                 continue
 
-            if isinstance(d, DicomDir):
-                lgr.debug("%s appears to be a DICOMDIR file. Extraction not yet"
-                          " implemented, skipped", f)
+            if NOT_IMPLEMENTED_TYPES and isinstance(d, NOT_IMPLEMENTED_TYPES):
+                lgr.debug("%s appears to be a DICOMDIR or alike: got %s. Extraction not yet"
+                          " implemented, skipped", f, d)
+                continue
+            elif not hasattr(d, 'SeriesInstanceUID'):
+                lgr.debug("%s does not have SeriesInstanceUID, skipped", f)
                 continue
-
             ddict = None
             if content:
                 ddict = _struct2dict(d)

datalad_neuroimaging/extractors/tests/test_nidm.py

@@ -46,7 +46,7 @@ def test_nidm(path=None):
     # aggregation done without whining
     assert_status('ok', res)
     res = ds.metadata(reporton='datasets')
-    # ATM we do not forsee file-based metadata to come back from NIDM
+    # ATM we do not foresee file-based metadata to come back from NIDM
     assert_result_count(res, 1)
     # kill version info
     core = res[0]['metadata']['datalad_core']

datalad_neuroimaging/tests/test_dicomconv.py

@@ -56,4 +56,6 @@ def test_dicom_metadata_aggregation(path=None):
 def test_validate_bids_fixture():
     bids_ds = get_bids_dataset()
     # dicom source dataset is absent
-    eq_(len(bids_ds.subdatasets(fulfilled=True, return_type='list')), 0)
+    # yoh: disabled since makes little sense (now?) since dataset is subdataset
+    # as of 978772e9468a5ae30de309cc6ac4370795de75cc etc in 2018
+    # eq_(len(bids_ds.subdatasets(fulfilled=True, return_type='list')), 0)

setup.cfg

@@ -14,11 +14,11 @@ classifiers =
 
 [options]
 zip_safe = False
-python_requires = >= 3.8
+python_requires = >= 3.9
 install_requires =
     datalad >= 0.16.7
     datalad-deprecated >= 0.2.7
-    pydicom  # DICOM metadata
+    pydicom >= 2.0.0  # DICOM metadata
     pybids >= 0.15.1  # BIDS metadata
     nibabel  # NIfTI metadata
     pandas  # bids2scidata export

tools/appveyor/install-syspkgs

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# no install requested -> exit
+[ -z "$1" ] && exit 0 || true
+
+if (which apt-get > /dev/null ); then
+	sudo apt-get update -qq -y --allow-releaseinfo-change
+	sudo apt-get install -q --no-install-recommends -y eatmydata
+	sudo eatmydata apt-get install -q --no-install-recommends -y $*
+else
+	brew install -q $*
+fi