Feature/ai builder connector

.env

@@ -20,6 +20,9 @@ AIOD_KEYCLOAK_PORT=8080
 
 EGICHECKINALIAS=
 
+#AIBUILDER
+AIBUILDER_API_TOKEN=""
+
 #ELASTICSEARCH
 ES_USER=elastic
 ES_PASSWORD=changeme

connectors/aibuilder/Dockerfile

@@ -0,0 +1,14 @@
+FROM aiod_metadata_catalogue
+
+USER root
+RUN apt-get update
+RUN apt -y install cron
+
+COPY cron /etc/cron.d/aiod
+COPY mlmodels.sh /opt/connectors/script/mlmodels.sh
+COPY entry.sh /opt/connectors/script/entry.sh
+
+RUN chmod +x /etc/cron.d/aiod /opt/connectors/script/mlmodels.sh
+RUN crontab /etc/cron.d/aiod
+
+WORKDIR /app

connectors/aibuilder/cron

@@ -0,0 +1 @@
+40 * * * * bash /opt/connectors/script/mlmodels.sh >> /opt/connectors/data/aibuilder/mlmodel/cron.log 2>&1

connectors/aibuilder/entry.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# If this directory does not exist, the cron job cannot log (and cannot run)
+mkdir -p /opt/connectors/data/aibuilder/mlmodel
+
+# Run cron on the foreground with log level WARN
+/usr/sbin/cron -f -l 4

connectors/aibuilder/mlmodels.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+WORK_DIR=/opt/connectors/data/aibuilder/mlmodel
+CONNECTOR=connectors.aibuilder.aibuilder_mlmodel_connector.AIBuilderMLModelConnector
+
+another_instance()
+{
+    echo $(date -u) "This script is already running in a different thread."
+    exit 1
+}
+exec 9< "$0"
+flock -n -x 9 || another_instance
+
+echo $(date -u) "Starting synchronization..."
+PYTHONPATH=/app /usr/local/bin/python3 /app/connectors/synchronization.py \
+      -c $CONNECTOR \
+      -w $WORK_DIR \
+      --from-date "2020-06-21" \
+      --save-every 100 >> ${WORK_DIR}/connector.log 2>&1
+echo $(date -u) "Synchronization Done."

docker-compose.yaml

@@ -109,6 +109,25 @@ services:
       app:
         condition: service_healthy
 
+  aibuilder-connector:
+    profiles: ["aibuilder"]
+    build:
+      context: connectors/aibuilder
+      dockerfile: Dockerfile
+    image: aiod_aibuilder_connector
+    container_name: aibuilder-connector
+    environment:
+      - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
+      - AIBUILDER_API_TOKEN=$AIBUILDER_API_TOKEN
+    volumes:
+      - ./src:/app
+      - ${DATA_PATH}/connectors:/opt/connectors/data
+    command: >
+      /bin/bash -c "/opt/connectors/script/entry.sh"
+    depends_on:
+      app:
+        condition: service_healthy
+
   sqlserver:
     image: mysql:8.3.0
     container_name: sqlserver

docs/README.md

@@ -132,13 +132,18 @@ It follows the same structure as the default file, but you only need to specify
 You can specify different connectors using
 
 ```bash
-docker compose --profile examples --profile huggingface-datasets --profile openml --profile zenodo-datasets up -d
-docker compose --profile examples --profile huggingface-datasets --profile openml --profile zenodo-datasets down
+docker compose --profile aibuilder --profile examples --profile huggingface-datasets --profile openml --profile zenodo-datasets up -d
+docker compose --profile aibuilder --profile examples --profile huggingface-datasets --profile openml --profile zenodo-datasets down
 ```
 
 Make sure you use the same profile for `up` and `down`, or use `./scripts/down.sh` (see below),
 otherwise some containers might keep running.
 
+##### Configuring AIBuilder connector
+To access the AIBuilder API you need to provide a valid API token through the `AIBUILDER_API_TOKEN` variable. \
+Use the `override.env` file for that as explained above. \
+Please note that for using the url of the `same_as` field of the AIBuilder models, you will need to substitute `AIBUILDER_API_TOKEN` on the url for your actual API token value.
+
 ### Shorthands
 We provide two auxiliary scripts for launching docker containers and bringing them down.
 The first, `./scripts/up.sh` invokes `docker compose up -d` and takes any number of profiles to launch as parameters.

src/connectors/aibuilder/aibuilder_mappings.py

@@ -0,0 +1,9 @@
+mlmodel_mapping = {
+    "platform_resource_identifier": "fullId",
+    "name": "name",
+    "date_published": "created",
+    "contact": "authors",
+    "creator": "publisher",
+    "description": "description",
+    "keyword": "tags",
+}