Release/1.3.20240619

.github/workflows/dockerized-tests.yml

@@ -1,6 +1,6 @@
-name: Built docker image and run tests
+name: Docker - build, test, push
 
-# This workflow will built docker image and run tests inside the container.
+# This workflow will build docker image and run tests inside the container.
 # This workflow is only executed if there is pull request with change in pyproject.toml dependencies, 
 # or in Dockerfile, or in docker workflow. 
 
@@ -10,22 +10,118 @@ on:
       - '.github/workflows/docker-image.yml'
       - 'pyproject.toml'
       - 'Dockerfile'
+
+  push:
+    branches:
+      - 'develop'
+
+  release:
+    types: [published]
 
   # allows to manually start a workflow run from the GitHub UI or using the GitHub API.    
   workflow_dispatch:
+    inputs:
+      push-image:
+        description: "Push image to docker hub"
+        required: false
+        type: boolean
+        default: false
+      push-description:
+        description: "Update docker hub description"
+        required: false
+        type: boolean
+        default: false
+      tag:
+        description: "Tag for the docker image"
+        required: false
+        default: "workflow-dispatch"
+
 
 jobs:
-  built:
+  build:
     runs-on: ubuntu-latest
-    permissions:
-      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      # We do not bother with setup-qemu-action since we don't care about emulation right now
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          tags: aiod/metadata_catalogue:ci
+          outputs: type=docker,dest=/tmp/aiod_mc_image.tar
+          cache-from: type=gha
+          cache-to: type=gha,mode=min
+      # We store the image as an artifact, so it can be used by the `test` step
+      # and inspected manually if needed (download it through Github Actions UI)
+      - name: Store Image
+        uses: actions/upload-artifact@v4
+        with:
+          name: aiod_mc_image
+          path: /tmp/aiod_mc_image.tar
 
+  test:
+    runs-on: ubuntu-latest
+    needs: [build]
     steps:
-      - uses: actions/checkout@v2
-      - name: Build the docker image
-        run: docker build --tag aiod_metadata_catalogue:latest -f Dockerfile .
-
-      - name: Run docker container and pytest tests
+      # We need to check out the repository, so that we have the `scripts` directory to mount.
+      # This is required to run the backup script tests.
+      - uses: actions/checkout@v4
+      - name: Retrieve Image
+        uses: actions/download-artifact@v4
+        with:
+          name: aiod_mc_image
+          path: /tmp
+      - name: Load Image
+        run: |
+          docker load --input /tmp/aiod_mc_image.tar
+          docker image ls -a
+      - name: Run pytest from docker
         run: |
-          docker run -v ./scripts:/scripts -e KEYCLOAK_CLIENT_SECRET="mocked_secret" --entrypoint "" aiod_metadata_catalogue sh -c "pip install \".[dev]\" && pytest tests -s"
+          docker run -v ./scripts:/scripts -e KEYCLOAK_CLIENT_SECRET="mocked_secret" --entrypoint "" aiod/metadata_catalogue:ci sh -c "pip install \".[dev]\" && pytest tests -s"
 
+  publish:
+    needs: [test]
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    steps:
+      # The correct tag depends on how this workflow was invoked, see also docker-description.md
+      - name: Set Develop Tag
+        if: github.ref == 'refs/heads/develop'
+        run: echo "IMAGE_TAGS=aiod/metadata_catalogue:develop" >> "$GITHUB_ENV"
+      - name: Set Release Tag
+        if: github.event_name == 'release'
+        run: echo "IMAGE_TAGS=aiod/metadata_catalogue:latest,aiod/metadata_catalogue:${{ github.event.release.tag_name }}" >> "$GITHUB_ENV"
+      - name: Set Dispatch Tag
+        if: github.event_name == 'workflow_dispatch'
+        run: echo "IMAGE_TAGS=aiod/metadata_catalogue:${{ inputs.tag }}" >> "$GITHUB_ENV"
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.AIOD_DOCKER_PAT }}
+      - name: Echo tags
+        run: echo $IMAGE_TAGS
+      - name: Build
+        if: (github.event_name != 'workflow_dispatch') || inputs.push-image
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          file: ./Dockerfile
+          tags: ${{ env.IMAGE_TAGS }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=min
+      - name: Update repository description
+        if: (github.event_name == 'release') || inputs.push-description
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.AIOD_DOCKER_PAT }}
+          repository: aiod/metadata_catalogue
+          readme-filepath: ./docker-description.md
+          short-description: "Metadata catalogue REST API for AI on Demand."

.github/workflows/pytest-tests.yml

@@ -41,10 +41,7 @@ jobs:
         source venv/bin/activate
         pre-commit run --all
 
-    - name: Test with pytest
-      run: |
-        source venv/bin/activate
-        pytest ./src/tests/
+
 
 
 

README.md

@@ -77,7 +77,7 @@ Information on how to install Docker is found in [their documentation](https://d
 docker compose --profile examples up -d
 ```
 
-starts the MYSQL Server, the REST API, Keycloak for Identy and access management and Nginx for reverse proxing. \
+starts the MYSQL Server, the REST API, Keycloak for Identity and access management and Nginx for reverse proxying. \
 Once started, you should be able to visit the REST API server at: http://localhost and Keycloak at http://localhost/aiod-auth \
 To authenticate to the REST API swagger interface the predefined user is: user, and password: password \
 To authenticate as admin to Keycloak the predefined user is: admin and password: password \

authentication/Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/keycloak/keycloak:latest as builder
+FROM quay.io/keycloak/keycloak:24.0.4 as builder
 
 # Enable health and metrics support
 ENV KC_HEALTH_ENABLED=true
@@ -12,7 +12,7 @@ WORKDIR /opt/keycloak
 #RUN keytool -genkeypair -storepass password -storetype PKCS12 -keyalg RSA -keysize 2048 -dname "CN=server" -alias server -ext "SAN:c=DNS:localhost,IP:127.0.0.1" -keystore conf/server.keystore
 #RUN /opt/keycloak/bin/kc.sh build
 
-FROM quay.io/keycloak/keycloak:latest
+FROM quay.io/keycloak/keycloak:24.0.4
 COPY --from=builder /opt/keycloak/ /opt/keycloak/
 
 # change these values to point to a running postgres instance

docker-compose.yaml

@@ -121,7 +121,7 @@ services:
         condition: service_healthy
 
   sqlserver:
-    image: mysql
+    image: mysql:8.3.0
     container_name: sqlserver
     env_file: .env
     environment:
@@ -137,7 +137,7 @@ services:
       retries: 30
 
   keycloak:
-    image: quay.io/keycloak/keycloak
+    image: quay.io/keycloak/keycloak:24.0.4
     container_name: keycloak
     env_file: .env
     environment:
@@ -157,7 +157,7 @@ services:
       --import-realm
 
   nginx:
-    image: nginx
+    image: nginx:1.25.5
     container_name: nginx
     restart: unless-stopped
     volumes:

docker-description.md

@@ -0,0 +1,11 @@
+# AIOD Metadata Catalogue
+
+Image for AI on Demand's (AIOD) metadata catalogue REST API, developed on [Github](https://github.com/aiondemand/AIOD-rest-api/).
+This image requires a properly configured database setup to function. Additionally, to have all features working, authentication (via Keycloak) and search (through Elasticsearch) must also be configured. Please refer to the documentation available in the README of the ["AIOD-rest-api"](https://github.com/aiondemand/AIOD-rest-api) repository.
+
+The following tags are available:
+
+ - `latest`: the latest official release
+ - `develop`: the head of the development branch
+ - `v*`, e.g. `v1.3.20240308`: that specific release
+ - a number of custom tags may be introduced for testing, but these are not intended for general use.

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "aiod_metadata_catalogue"
 description = "A Metadata Catalogue for AI on Demand "
-version = "1.3.20240308"
+version = "1.3.20240619"
 requires-python = ">=3.11"
 authors = [
     { name = "Adrián Alcolea" },
@@ -47,7 +47,7 @@ dev = [
     "pytest-asyncio==0.23.2",
     "pytest-dotenv==0.5.2",
     "pytest-xdist==3.5.0",
-    "pre-commit==3.6.0",
+    "pre-commit==3.7.0",
     "responses==0.24.1",
     "freezegun==1.4.0",
 ]

scripts/realm_export.sh

@@ -9,7 +9,7 @@ source .env
 DATA_PATH=$(realpath "$DATA_PATH")
 LOCAL_BACKUP_PATH="$DATA_PATH"/keycloak_realm
 
-docker exec -i keycloak /bin/bash -c "/opt/keycloak/bin/kc.sh export --file /tmp/aiod.json --realm aiod"
+docker exec -i keycloak /bin/bash -c "/opt/keycloak/bin/kc.sh export --file /tmp/aiod.json --realm aiod --users realm_file"
 
 if [ ! -d "$LOCAL_BACKUP_PATH" ]; then
     mkdir "$LOCAL_BACKUP_PATH"

src/authentication.py

@@ -17,6 +17,7 @@
 performs a separate authorization request. The only downside is the overhead of the additional
 keycloak requests - if that becomes prohibitive in the future, we should reevaluate this design.
 """
+
 import logging
 import os
 
@@ -55,15 +56,15 @@ def has_any_role(self, *roles: str) -> bool:
         return bool(set(roles) & self.roles)
 
 
-async def get_current_user(token=Security(oidc)) -> User:
+async def _get_user(token) -> User:
     """
-    Use this function in Depends() to force authentication. Check the roles of the user for
-    authorization.
+    Check the roles of the user for authorization.
 
     Raises:
-        HTTPException with status 401 on missing token (unauthorized message), also status 401 on
-            any problem with the token (we don't want to leak information), status 500 on any
-            request if Keycloak is configured incorrectly.
+        NoTokenError on missing token (unauthorized message) and InvalidUserError on inactive user.
+        Also HTTPException with status 401 on any problem with the token
+        (we don't want to leak information), and status 500 on any request
+        if Keycloak is configured incorrectly.
     """
     if not client_secret:
         raise HTTPException(
@@ -73,11 +74,7 @@ async def get_current_user(token=Security(oidc)) -> User:
             "from a Keycloak Administrator of AIoD.",
         )
     if not token:
-        raise HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED,
-            detail="This endpoint requires authorization. You need to be logged in.",
-            headers={"WWW-Authenticate": "Bearer"},
-        )
+        raise NoTokenError("No token found")
     try:
         token = token.replace("Bearer ", "")
         # query the authorization server to determine the active state of this token and to
@@ -86,14 +83,58 @@ async def get_current_user(token=Security(oidc)) -> User:
 
         if not userinfo.get("active", False):
             logging.error("Invalid userinfo or inactive user.")
-            raise RuntimeError("Invalid userinfo or inactive user.")  # caught below
+            raise InvalidUserError("Invalid userinfo or inactive user")  # caught below
         return User(
             name=userinfo["username"], roles=set(userinfo.get("realm_access", {}).get("roles", []))
         )
+    except InvalidUserError:
+        raise
     except Exception as e:
         logging.error(f"Error while checking the access token: '{e}'")
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
             detail="Invalid authentication token",
             headers={"WWW-Authenticate": "Bearer"},
         )
+
+
+async def get_user_or_none(token=Security(oidc)) -> User | None:
+    """
+    Use this function in Depends() to ask for authentication.
+    This method should be only used to get the current user
+    without raising exception when the token is not found,
+    or the user is not active, or the userinfo is invalid.
+    """
+    try:
+        return await _get_user(token)
+    except (NoTokenError, InvalidUserError):
+        return None
+
+
+async def get_user_or_raise(token=Security(oidc)) -> User:
+    """
+    Use this function in Depends() to force authentication. Check the roles of the user for
+    authorization.
+
+    Raises:
+        HTTPException with status 401 on missing token (unauthorized message), or invalid user.
+        It also raises a HTTPException with status 401 on
+        any problem with the token (we don't want to leak information),
+        status 500 on any request if Keycloak is configured incorrectly.
+    """
+    try:
+        return await _get_user(token)
+    except (InvalidUserError, NoTokenError) as err:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=f"{err} - This endpoint requires authorization. You need to be logged in.",
+            headers={"WWW-Authenticate": "Bearer"},
+        ) from err
+
+
+class InvalidUserError(Exception):
+    """Raise an error on invalid userinfo or inactive user."""
+
+
+class NoTokenError(Exception):
+    """Raise an error when no token is found."""

src/database/model/agent/organisation.py

@@ -50,7 +50,8 @@ class Organisation(OrganisationBase, Agent, table=True):  # type: ignore [call-a
 
     class RelationshipConfig(Agent.RelationshipConfig):
         contact_details: int | None = OneToOne(
-            description="The contact details by which this organisation can be reached",
+            description="The identifier of the contact details by which this organisation "
+            "can be reached.",
             deserializer=FindByIdentifierDeserializer(Contact),
             _serializer=AttributeSerializer("identifier"),
         )

src/database/model/agent/person.py

@@ -58,7 +58,7 @@ class Person(PersonBase, Agent, table=True):  # type: ignore [call-arg]
 
     class RelationshipConfig(Agent.RelationshipConfig):
         contact_details: int | None = OneToOne(
-            description="The contact details by which this person can be reached",
+            description="The identifier of the contact details by which this person can be reached",
             deserializer=FindByIdentifierDeserializer(Contact),
             _serializer=AttributeSerializer("identifier"),
         )

src/database/model/ai_resource/resource.py

@@ -180,14 +180,15 @@ class RelationshipConfig(AIoDConcept.RelationshipConfig):
         )
         # TODO(jos): documentedIn - KnowledgeAsset. This should probably be defined on ResourceTable
         contact: list[int] = ManyToMany(
-            description="Contact information of persons/organisations that can be contacted about "
-            "this resource.",
+            description="The identifiers of the contact information of the persons and/or "
+            "organisations that can be contacted about this resource.",
             _serializer=AttributeSerializer("identifier"),
             deserializer=FindByIdentifierDeserializerList(Contact),
             default_factory_pydantic=list,
         )
         creator: list[int] = ManyToMany(
-            description="Contact information of persons/organisations that created this resource.",
+            description="The identifiers of the contact information of the persons and/or "
+            "organisations that created this resource.",
             _serializer=AttributeSerializer("identifier"),
             deserializer=FindByIdentifierDeserializerList(Contact),
             default_factory_pydantic=list,

src/database/model/platform/platform_names.py

@@ -9,6 +9,7 @@ class PlatformName(str, enum.Enum):
     """
 
     aiod = "aiod"
+    ai4europe_cms = "ai4europe_cms"
     example = "example"
     openml = "openml"
     huggingface = "huggingface"