Add authentication to DICOMweb testing

This adds an additional docker-compose yaml, `auth-docker-compose.yml`, which
spins up a dcm4chee server that requires authentication via keycloak. This
authenticated server is now being used for the CircleCI testing.

This PR also adds a couple of scripts to work with this server, including an
`update_access_token_lifespan.py` script (with the default access token
lifespan of 10 minutes, the token would expire before it would be tested
in the CI), and a `create_keycloak_token.py` script, which is used to create
the access token that we provide to the assetstore.

Since we may want to perform local testing without authentication, the
non-authenticated dcm4chee docker-compose is still there. The testing code
checks for a `DICOMWEB_TEST_TOKEN` environment variable, where it expects
to find a token. If that environment variable is set, then it uses token
authentication with that token. If it is not set, then it assumes authentication
is not required and attempts to run the tests without authentication.

Signed-off-by: Patrick Avery <patrick.avery@kitware.com>

.circleci/config.yml

@@ -71,12 +71,29 @@ commands:
       - run:
           name: start dcm4chee and upload example data (for DICOMweb tests)
           command: |
-            docker-compose -f ./.circleci/dcm4chee/docker-compose.yml up -d
+            docker-compose -f ./.circleci/dcm4chee/auth-docker-compose.yml up -d
             export DICOMWEB_TEST_URL=http://localhost:8008/dcm4chee-arc/aets/DCM4CHEE/rs
             echo "export DICOMWEB_TEST_URL=$DICOMWEB_TEST_URL" >> $BASH_ENV
-            pip install dicomweb_client
+            pip install dicomweb_client python-keycloak
+
+            # Wait up to 60 seconds for keycloak to be ready
+            echo 'Waiting for keycloak to start...'
+            KEYCLOAK_URL=https://localhost:8843
+            curl -k --retry 60 -f --retry-all-errors --retry-delay 1 -s -o /dev/null $KEYCLOAK_URL
+            echo 'Updating keycloak token lifespan...'
+            python -W ignore ./.circleci/dcm4chee/update_access_token_lifespan.py
+
+            echo 'Creating keycloak access token...'
+            # Now create the token
+            export DICOMWEB_TEST_TOKEN=$(python -W ignore ./.circleci/dcm4chee/create_keycloak_token.py)
+            echo "export DICOMWEB_TEST_TOKEN=$DICOMWEB_TEST_TOKEN" >> $BASH_ENV
+
             # Wait up to 30 seconds for the server if it isn't ready
-            curl --retry 30 -f --retry-all-errors --retry-delay 1 -s -o /dev/null $DICOMWEB_TEST_URL/studies
+            echo 'Waiting for dcm4chee to start...'
+            curl --header "Authorization: Bearer $DICOMWEB_TEST_TOKEN" --retry 30 -f --retry-all-errors --retry-delay 1 -s -o /dev/null $DICOMWEB_TEST_URL/studies
+
+            # Upload the example data
+            echo 'Uploading example data...'
             python ./.circleci/dcm4chee/upload_example_data.py
       - run:
           name: start rabbitmq

.circleci/dcm4chee/auth-docker-compose.yml

@@ -0,0 +1,100 @@
+version: "3"
+
+volumes:
+  db_data: {}
+  arc_data: {}
+  ldap_data: {}
+  ldap_config: {}
+  mysql: {}
+  keycloak: {}
+
+services:
+  ldap:
+    image: dcm4che/slapd-dcm4chee:2.6.5-31.2
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+    expose:
+      - 389
+    environment:
+      STORAGE_DIR: /storage/fs1
+    volumes:
+      - ldap_data:/var/lib/openldap/openldap-data
+      - ldap_config:/etc/openldap/slapd.d
+  mariadb:
+    image: mariadb:10.11.4
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+    ports:
+      - "3306:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: secret
+      MYSQL_DATABASE: keycloak
+      MYSQL_USER: keycloak
+      MYSQL_PASSWORD: keycloak
+    volumes:
+      - mysql:/var/lib/mysql
+  keycloak:
+    image: dcm4che/keycloak:23.0.3
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+    ports:
+      - "8843:8843"
+    environment:
+      KC_HTTPS_PORT: 8843
+      KC_HOSTNAME: localhost
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: changeit
+      KC_DB: mariadb
+      KC_DB_URL_DATABASE: keycloak
+      KC_DB_URL_HOST: mariadb
+      KC_DB_USERNAME: keycloak
+      KC_DB_PASSWORD: keycloak
+      KC_LOG: file
+      ARCHIVE_HOST: localhost
+      KEYCLOAK_WAIT_FOR: ldap:389 mariadb:3306
+    depends_on:
+      - ldap
+      - mariadb
+    volumes:
+      - keycloak:/opt/keycloak/data
+  db:
+    image: dcm4che/postgres-dcm4chee:15.4-31
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+    expose:
+      - 5432
+    environment:
+      POSTGRES_DB: pacsdb
+      POSTGRES_USER: pacs
+      POSTGRES_PASSWORD: pacs
+    volumes:
+      - db_data:/var/lib/postgresql/data
+  arc:
+    image: dcm4che/dcm4chee-arc-psql:5.31.2-secure
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+    ports:
+      - "8008:8080"
+    environment:
+      POSTGRES_DB: pacsdb
+      POSTGRES_USER: pacs
+      POSTGRES_PASSWORD: pacs
+      AUTH_SERVER_URL: https://keycloak:8843
+      WILDFLY_CHOWN: /opt/wildfly/standalone /storage
+      WILDFLY_WAIT_FOR: ldap:389 db:5432 keycloak:8843
+    depends_on:
+      - ldap
+      - keycloak
+      - db
+    volumes:
+      - arc_data:/storage

.circleci/dcm4chee/create_keycloak_token.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+# This script can be used to create a keycloak token for the
+# dcm4chee server via the python-keycloak API. python-keycloak
+# must be installed.
+
+from keycloak import KeycloakOpenID
+
+keycloack_openid = KeycloakOpenID(
+    server_url='https://localhost:8843',
+    client_id='dcm4chee-arc-rs',
+    realm_name='dcm4che',
+    client_secret_key='changeit',
+    # Certificate is not working, just don't verify...
+    verify=False,
+)
+
+token_dict = keycloack_openid.token('user', 'changeit')
+print(token_dict['access_token'])

.circleci/dcm4chee/docker-compose.yml

@@ -8,40 +8,44 @@ volumes:
 
 services:
   ldap:
-    image: dcm4che/slapd-dcm4chee:2.6.0-26.0
+    image: dcm4che/slapd-dcm4chee:2.6.5-31.2
     logging:
       driver: json-file
       options:
         max-size: "10m"
     expose:
       - 389
-    env_file: docker-compose.env
+    environment:
+      STORAGE_DIR: /storage/fs1
     volumes:
       - ldap_data:/var/lib/openldap/openldap-data
       - ldap_config:/etc/openldap/slapd.d
-
   db:
-    image: dcm4che/postgres-dcm4chee:14.2-26
+    image: dcm4che/postgres-dcm4chee:15.4-31
     logging:
       driver: json-file
       options:
         max-size: "10m"
     expose:
       - 5432
-    env_file: docker-compose.env
+    environment:
+      POSTGRES_DB: pacsdb
+      POSTGRES_USER: pacs
+      POSTGRES_PASSWORD: pacs
     volumes:
       - db_data:/var/lib/postgresql/data
-
   arc:
-    image: dcm4che/dcm4chee-arc-psql:5.26.0
+    image: dcm4che/dcm4chee-arc-psql:5.31.2
     logging:
       driver: json-file
       options:
         max-size: "10m"
     ports:
       - "8008:8080"
-    env_file: docker-compose.env
     environment:
+      POSTGRES_DB: pacsdb
+      POSTGRES_USER: pacs
+      POSTGRES_PASSWORD: pacs
       WILDFLY_CHOWN: /opt/wildfly/standalone /storage
       WILDFLY_WAIT_FOR: ldap:389 db:5432
     depends_on:

.circleci/dcm4chee/update_access_token_lifespan.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+# Change the access token life span in the realm settings
+
+import requests
+from keycloak import KeycloakOpenIDConnection
+
+
+def create_openid_admin_token():
+    # Get an admin OpenID access token. This expires after 60 seconds.
+    return KeycloakOpenIDConnection(
+        server_url='https://localhost:8843',
+        username='admin',
+        password='changeit',
+        realm_name='master',
+        verify=False,
+    ).token['access_token']
+
+
+def set_access_token_life_span(token, lifespan):
+    # curl command looks like this:
+    # curl 'https://localhost:8843/admin/realms/dcm4che' \
+    #   -X 'PUT' \
+    #   -H 'Content-Type: application/json' \
+    #   -H 'authorization: Bearer $TOKEN' \
+    #   -d '{"accessTokenLifespan":6000}' \
+    #   --insecure
+    session = requests.Session()
+    session.headers.update({'Authorization': f'Bearer {token}'})
+
+    url = 'https://localhost:8843/admin/realms/dcm4che'
+    r = session.put(url, json={'accessTokenLifespan': lifespan}, verify=False)
+    r.raise_for_status()
+
+
+if __name__ == '__main__':
+    token = create_openid_admin_token()
+
+    # Set default timetout to be 1 hour
+    set_access_token_life_span(token, 3600)

.circleci/dcm4chee/upload_example_data.py

@@ -5,9 +5,10 @@
 
 from dicomweb_client import DICOMwebClient
 from pydicom import dcmread
+from requests import Session
 
 
-def upload_example_data(server_url):
+def upload_example_data(server_url, token=None):
 
     # This is TCGA-AA-3697
     sha512s = [
@@ -26,7 +27,13 @@ def upload_example_data(server_url):
         dataset = dcmread(BytesIO(data))
         datasets.append(dataset)
 
-    client = DICOMwebClient(server_url)
+    if token is not None:
+        session = Session()
+        session.headers.update({'Authorization': f'Bearer {token}'})
+    else:
+        session = None
+
+    client = DICOMwebClient(server_url, session=session)
     client.store_instances(datasets)
 
 
@@ -38,4 +45,5 @@ def upload_example_data(server_url):
         msg = 'DICOMWEB_TEST_URL must be set'
         raise Exception(msg)
 
-    upload_example_data(url)
+    token = os.getenv('DICOMWEB_TEST_TOKEN')
+    upload_example_data(url, token=token)

sources/dicom/test_dicom/test_web_client.py

@@ -28,6 +28,13 @@ def testDICOMWebClient(boundServer, fsAssetstore, db):
     dicomweb_test_url = os.environ['DICOMWEB_TEST_URL']
     data = data.replace('DICOMWEB_TEST_URL', f"'{dicomweb_test_url}'")
 
+    dicomweb_test_token = os.getenv('DICOMWEB_TEST_TOKEN')
+    if dicomweb_test_token:
+        dicomweb_test_token = f"'{dicomweb_test_token}'"
+    else:
+        dicomweb_test_token = 'null'
+    data = data.replace('DICOMWEB_TEST_TOKEN', dicomweb_test_token)
+
     # Need to avoid context manager for this to work on Windows
     tf = tempfile.NamedTemporaryFile(delete=False)
     try:

sources/dicom/test_dicom/web_client_specs/dicomWebSpec.js

@@ -1,5 +1,6 @@
 // These will be replaced by templating
 const url = DICOMWEB_TEST_URL;
+const token = DICOMWEB_TEST_TOKEN;
 
 girderTest.importPlugin('jobs', 'large_image', 'dicomweb');
 
@@ -56,8 +57,13 @@ describe('DICOMWeb assetstore', function () {
         }, 'No token provided check');
 
         runs(function () {
-            // Change the auth type back to None
-            $('#g-new-dwas-auth-type').val(null);
+            if (token == null) {
+                // Change the auth type back to None
+                $('#g-new-dwas-auth-type').val(null);
+            } else {
+                // Set the token
+                $('#g-new-dwas-auth-token').val(token);
+            }
 
             // This should work now
             $('#g-new-dwas-form input.btn-primary').click();

test/test_source_dicomweb.py

@@ -2,6 +2,9 @@
 import sys
 
 import pytest
+import requests
+
+from large_image.exceptions import TileSourceError
 
 from . import utilities
 
@@ -23,6 +26,19 @@ def testTilesFromDICOMweb():
         'series_uid': '1.3.6.1.4.1.5962.99.1.3205815762.381594633.1639588388306.2.0',
     }
 
+    # Use a token if we were provided with one.
+    token = os.getenv('DICOMWEB_TEST_TOKEN')
+    if token:
+        # First, verify that we receive an authorization error without the token
+        match_message = '401 Client Error: Unauthorized for url'
+        with pytest.raises(TileSourceError, match=match_message):
+            large_image_source_dicom.open(dicomweb_file)
+
+        # Create a session, add the token, and try again
+        session = requests.Session()
+        session.headers.update({'Authorization': f'Bearer {token}'})
+        dicomweb_file['session'] = session
+
     source = large_image_source_dicom.open(dicomweb_file)
 
     tileMetadata = source.getMetadata()

tox.ini

@@ -10,7 +10,7 @@ skip_missing_interpreters = true
 toxworkdir = {toxinidir}/build/tox
 
 [testenv]
-passenv = PYTEST_*,DICOMWEB_TEST_URL
+passenv = PYTEST_*,DICOMWEB_TEST_URL,DICOMWEB_TEST_TOKEN
 extras =
   memcached
   performance