LinkedDataHub v5 WIP

.github/workflows/http-tests.yml

@@ -7,7 +7,7 @@ jobs:
     name: Build Docker image and run HTTP test suite against it
     runs-on: ubuntu-latest
     env:
-      ASF_ARCHIVE: http://archive.apache.org/dist/
+      ASF_ARCHIVE: https://archive.apache.org/dist/
       JENA_VERSION: 4.7.0
       BASE_URI: https://localhost:4443/
       OWNER_CERT_PWD: changeit
@@ -30,14 +30,14 @@ jobs:
         shell: bash
         working-directory: http-tests
       - name: Build Docker image & Run Docker containers
-        run: docker-compose -f docker-compose.yml -f ./http-tests/docker-compose.http-tests.yml --env-file ./http-tests/.env up --build -d
+        run: docker compose -f docker-compose.yml -f ./http-tests/docker-compose.http-tests.yml --env-file ./http-tests/.env up --build -d
       - name: Wait for the server to start...
         run: while ! (status=$(curl -k -s -w "%{http_code}\n" https://localhost:4443 -o /dev/null) && echo "$status" && echo "$status" | grep "403") ; do sleep 1 ; done # wait for the webapp to start (returns 403 by default)
       - name: Run HTTP test scripts
         run: ./run.sh "$PWD/ssl/owner/cert.pem" "${{env.OWNER_CERT_PWD}}" "$PWD/ssl/secretary/cert.pem" "${{env.SECRETARY_CERT_PWD}}"
         shell: bash
         working-directory: http-tests
       - name: Stop Docker containers
-        run: docker-compose --env-file ./http-tests/.env down
+        run: docker compose --env-file ./http-tests/.env down
       - name: Remove Docker containers
-        run: docker-compose --env-file ./http-tests/.env rm -f
+        run: docker compose --env-file ./http-tests/.env rm -f

.github/workflows/image.yml

@@ -0,0 +1,44 @@
+name: ci
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract version parts
+        id: version
+        run: |
+          VERSION="${{ github.ref_name }}"
+          MAJOR="${VERSION%%.*}"
+          MINOR="${VERSION%.*}"
+          MINOR="${MINOR#*.}"
+          echo "MAJOR=$MAJOR" >> $GITHUB_ENV
+          echo "MINOR=$MAJOR.$MINOR" >> $GITHUB_ENV
+          echo "FULL_VERSION=$VERSION" >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            atomgraph/linkeddatahub:latest
+            atomgraph/linkeddatahub:${{ env.FULL_VERSION }}
+            atomgraph/linkeddatahub:${{ env.MINOR }}
+            atomgraph/linkeddatahub:${{ env.MAJOR }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM maven:3.8.4-openjdk-17 as maven
+FROM maven:3.8.4-openjdk-17 AS maven
 
 # download and extract Jena
 
@@ -22,7 +22,7 @@
 
 # ==============================
 
-FROM atomgraph/letsencrypt-tomcat:10.1.4
+FROM atomgraph/letsencrypt-tomcat:10.1.34
 
 LABEL maintainer="martynas@atomgraph.com"
 
@@ -38,10 +38,6 @@
 
 WORKDIR $CATALINA_HOME
 
-# add XSLT stylesheet that makes changes to ROOT.xml
-
-COPY platform/context.xsl conf/context.xsl
-
 ENV CACHE_MODEL_LOADS=true
 
 ENV STYLESHEET=static/com/atomgraph/linkeddatahub/xsl/bootstrap/2.3.2/layout.xsl
@@ -102,12 +98,18 @@
 
 ENV MAX_TOTAL_CONN=40
 
+ENV MAX_REQUEST_RETRIES=3
+
 ENV IMPORT_KEEPALIVE=
 
+ENV MAX_IMPORT_THREADS=10
+
 ENV GOOGLE_CLIENT_ID=
 
 ENV GOOGLE_CLIENT_SECRET=
 
+ENV SERVLET_NAME=
+
 ENV GENERATE_SITEMAP=true
 
 # remove default Tomcat webapps and install xmlstarlet (used for XPath queries) and envsubst (for variable substitution)
@@ -120,6 +122,14 @@
     rm -rf webapps/* && \
     rm -rf /var/lib/apt/lists/*
 
+# add XSLT stylesheet that makes changes to ROOT.xml
+
+COPY platform/context.xsl /var/linkeddatahub/xsl/context.xsl
+
+# add XSLT stylesheet that makes changes to web.xml
+
+COPY platform/web.xsl /var/linkeddatahub/xsl/web.xsl
+
 # copy entrypoint
 
 COPY platform/entrypoint.sh entrypoint.sh
@@ -180,7 +190,7 @@
 
 RUN ./import-letsencrypt-stg-roots.sh
 
-HEALTHCHECK --start-period=80s --interval=20s --timeout=10s \
+HEALTHCHECK --start-period=80s --retries=5 \
     CMD curl -f -I "http://localhost:${HTTP_PORT}/ns" -H "Accept: application/n-triples" || exit 1 # relies on public access to the namespace document
 
 USER ldh

docker-compose.debug.yml

@@ -1,4 +1,3 @@
-version: "2.3"
 services:
   linkeddatahub:
     ports:

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "2.3"
 services:
   nginx:
     image: nginx:1.23.3
@@ -29,6 +28,10 @@ services:
   linkeddatahub:
     user: root # otherwise the ldh user does not have permissions to the mounted folder which is owner by root
     build: .
+    mem_limit: 2048m
+    depends_on:
+      - fuseki-admin
+      - fuseki-end-user
     environment:
     #  - JPDA_ADDRESS=*:8000 # debugger host - performance hit when enabled
       - CATALINA_OPTS=-XX:+UseContainerSupport -XX:MaxRAMPercentage=75 --add-exports java.base/sun.security.tools.keytool=ALL-UNNAMED # heap will use up to 75% of container's RAM
@@ -70,7 +73,7 @@ services:
       - ./platform/datasets/admin.trig:/var/linkeddatahub/datasets/admin.trig:ro
       - ./platform/datasets/end-user.trig:/var/linkeddatahub/datasets/end-user.trig:ro
   fuseki-admin:
-    image: atomgraph/fuseki:4.6.1
+    image: atomgraph/fuseki:4.7.0
     user: root # otherwise fuseki user does not have permissions to the mounted folder which is owner by root
     expose:
       - 3030
@@ -79,7 +82,7 @@ services:
       - ./data/admin:/fuseki/databases
     command: [ "--config", "/fuseki/config.ttl" ]
   fuseki-end-user:
-    image: atomgraph/fuseki:4.6.1
+    image: atomgraph/fuseki:4.7.0
     user: root # otherwise the fuseki user does not have permissions to the mounted folder which is owner by root
     expose:
       - 3030
@@ -97,12 +100,12 @@ services:
       - VARNISH_HTTP_PORT=6060
       - BACKEND_HOST=linkeddatahub
       - BACKEND_PORT=7070
-      - CLIENT_HOST=linkeddatahub
-      - VARNISH_TTL=86400
+      - CLIENT_HOST=localhost
       - VARNISH_SIZE=1G
-    entrypoint: /bin/sh -c "cp /etc/varnish/default.vcl.template /etc/varnish/default.vcl && sed -i 's|$${BACKEND_HOST}|'"$$BACKEND_HOST"'|g' /etc/varnish/default.vcl && sed -i 's|$${BACKEND_PORT}|'"$$BACKEND_PORT"'|g' /etc/varnish/default.vcl && sed -i 's|$${CLIENT_HOST}|'"$$CLIENT_HOST"'|g' /etc/varnish/default.vcl && /usr/local/bin/docker-varnish-entrypoint"
+    entrypoint: /bin/sh -c "cp /etc/varnish/default.vcl.template /etc/varnish/default.vcl && sed -i 's|$${BACKEND_HOST}|'"$$BACKEND_HOST"'|g' /etc/varnish/default.vcl && sed -i 's|$${BACKEND_PORT}|'"$$BACKEND_PORT"'|g' /etc/varnish/default.vcl && sed -i 's|$${CLIENT_HOST}|'"$$CLIENT_HOST"'|g' /etc/varnish/default.vcl && /usr/local/bin/docker-varnish-entrypoint \"$$0\" \"$$@\""
+    command: [ "-t", "86400" ] # time to live
     volumes:
-      - ./platform/varnish.vcl.template:/etc/varnish/default.vcl.template:ro
+      - ./platform/varnish-frontend.vcl.template:/etc/varnish/default.vcl.template:ro
   varnish-admin:
     image: varnish:7.3.0
     user: root # otherwise the varnish user does not have permissions to the mounted folder which is owner by root
@@ -113,11 +116,11 @@ services:
       - BACKEND_HOST=fuseki-admin
       - BACKEND_PORT=3030
       - CLIENT_HOST=linkeddatahub
-      - VARNISH_TTL=86400
       - VARNISH_SIZE=1G
-    entrypoint: /bin/sh -c "cp /etc/varnish/default.vcl.template /etc/varnish/default.vcl && sed -i 's|$${BACKEND_HOST}|'"$$BACKEND_HOST"'|g' /etc/varnish/default.vcl && sed -i 's|$${BACKEND_PORT}|'"$$BACKEND_PORT"'|g' /etc/varnish/default.vcl && sed -i 's|$${CLIENT_HOST}|'"$$CLIENT_HOST"'|g' /etc/varnish/default.vcl && /usr/local/bin/docker-varnish-entrypoint"
+    entrypoint: /bin/sh -c "cp /etc/varnish/default.vcl.template /etc/varnish/default.vcl && sed -i 's|$${BACKEND_HOST}|'"$$BACKEND_HOST"'|g' /etc/varnish/default.vcl && sed -i 's|$${BACKEND_PORT}|'"$$BACKEND_PORT"'|g' /etc/varnish/default.vcl && sed -i 's|$${CLIENT_HOST}|'"$$CLIENT_HOST"'|g' /etc/varnish/default.vcl && /usr/local/bin/docker-varnish-entrypoint \"$$0\" \"$$@\""
+    command: [ "-t", "86400", "-p", "timeout_idle=60s" ] # time to live
     volumes:
-      - ./platform/varnish.vcl.template:/etc/varnish/default.vcl.template:ro
+      - ./platform/varnish-backend.vcl.template:/etc/varnish/default.vcl.template:ro
   varnish-end-user:
     image: varnish:7.3.0
     user: root # otherwise varnish user does not have permissions to the mounted folder which is owner by root
@@ -128,11 +131,11 @@ services:
       - BACKEND_HOST=fuseki-end-user
       - BACKEND_PORT=3030
       - CLIENT_HOST=linkeddatahub
-      - VARNISH_TTL=86400
       - VARNISH_SIZE=1G
-    entrypoint: /bin/sh -c "cp /etc/varnish/default.vcl.template /etc/varnish/default.vcl && sed -i 's|$${BACKEND_HOST}|'"$$BACKEND_HOST"'|g' /etc/varnish/default.vcl && sed -i 's|$${BACKEND_PORT}|'"$$BACKEND_PORT"'|g' /etc/varnish/default.vcl && sed -i 's|$${CLIENT_HOST}|'"$$CLIENT_HOST"'|g' /etc/varnish/default.vcl && /usr/local/bin/docker-varnish-entrypoint"
+    entrypoint: /bin/sh -c "cp /etc/varnish/default.vcl.template /etc/varnish/default.vcl && sed -i 's|$${BACKEND_HOST}|'"$$BACKEND_HOST"'|g' /etc/varnish/default.vcl && sed -i 's|$${BACKEND_PORT}|'"$$BACKEND_PORT"'|g' /etc/varnish/default.vcl && sed -i 's|$${CLIENT_HOST}|'"$$CLIENT_HOST"'|g' /etc/varnish/default.vcl && /usr/local/bin/docker-varnish-entrypoint \"$$0\" \"$$@\""
+    command: [ "-t", "86400", "-p", "timeout_idle=60s" ] # time to live
     volumes:
-      - ./platform/varnish.vcl.template:/etc/varnish/default.vcl.template:ro
+      - ./platform/varnish-backend.vcl.template:/etc/varnish/default.vcl.template:ro
   email-server:
     image: namshi/smtp
     environment:

http-tests/admin/GET-agent-mbox-html.sh

@@ -9,7 +9,7 @@ purge_cache "$ADMIN_VARNISH_SERVICE"
 # request agent's HTML description
 
 html=$(curl -k -f -s -N \
-    -H 'Accept: text/html' \
+    -H 'Accept: application/xhtml+xml' \
   "$AGENT_URI")
 
 # check that the description does *not* include foaf:mbox property

http-tests/admin/GET-request-access-html.sh

@@ -11,7 +11,7 @@ purge_cache "$ADMIN_VARNISH_SERVICE"
 curl -k -w "%{http_code}\n" -o /dev/null -f -s \
   -G \
   -E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
-  -H 'Accept: text/html' \
+  -H 'Accept: application/xhtml+xml' \
   --data-urlencode "access-to=${END_USER_BASE_URL}" \
   "${ADMIN_BASE_URL}request%20access" \
 | grep -q "$STATUS_OK"

http-tests/admin/POST-request-access.sh

@@ -36,4 +36,4 @@ curl -w "%{http_code}\n" -o /dev/null -k -s \
   --data-urlencode "pu=http://xmlns.com/foaf/0.1/primaryTopic" \
   --data-urlencode "ob=request" \
   "${ADMIN_BASE_URL}request%20access" \
-| grep -q "$STATUS_OK"
+| grep -q "$STATUS_CREATED"

http-tests/admin/acl/add-post-authorization.sh

@@ -39,11 +39,15 @@ popd > /dev/null
 
 # access is allowed after authorization is created
 
-curl -k -w "%{http_code}\n" -o /dev/null -f -s \
+(
+curl -k -w "%{http_code}\n" -o /dev/null -s \
   -E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
   -H "Content-Type: application/n-triples" \
   -H "Accept: application/n-triples" \
-  -H "Content-Length: 0" \
   -X POST \
-  "$END_USER_BASE_URL" \
+   --data-binary @- \
+  "$END_USER_BASE_URL" <<EOF
+<http://s> <http://p> <http://o> .
+EOF
+) \
 | grep -q "$STATUS_NO_CONTENT"

http-tests/admin/acl/add-post-class-authorization.sh

@@ -39,11 +39,15 @@ popd > /dev/null
 
 # access is allowed after authorization is created
 
-curl -k -w "%{http_code}\n" -o /dev/null -f -s \
+(
+curl -k -w "%{http_code}\n" -o /dev/null -s \
   -E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
   -H "Content-Type: application/n-triples" \
   -H "Accept: application/n-triples" \
-  -H "Content-Length: 0" \
   -X POST \
-  "$END_USER_BASE_URL" \
+   --data-binary @- \
+  "$END_USER_BASE_URL" <<EOF
+<http://s> <http://p> <http://o> .
+EOF
+) \
 | grep -q "$STATUS_NO_CONTENT"

http-tests/admin/acl/add-post-group-authorization.sh

@@ -55,11 +55,15 @@ popd > /dev/null
 
 # access is allowed after authorization is created
 
-curl -k -w "%{http_code}\n" -o /dev/null -f -s \
+(
+curl -k -w "%{http_code}\n" -o /dev/null -s \
   -E "$AGENT_CERT_FILE":"$AGENT_CERT_PWD" \
   -H "Content-Type: application/n-triples" \
   -H "Accept: application/n-triples" \
-  -H "Content-Length: 0" \
   -X POST \
-  "$END_USER_BASE_URL" \
+   --data-binary @- \
+  "$END_USER_BASE_URL" <<EOF
+<http://s> <http://p> <http://o> .
+EOF
+) \
 | grep -q "$STATUS_NO_CONTENT"

http-tests/admin/acl/add-put-authorization.sh

@@ -41,7 +41,7 @@ popd > /dev/null
 
 pushd . && cd "$SCRIPT_ROOT"
 
-root_ntriples=$(./get-document.sh \
+root_ntriples=$(./get.sh \
   -f "$OWNER_CERT_FILE" \
   -p "$OWNER_CERT_PWD" \
   --accept 'application/n-triples' \

http-tests/admin/acl/add-put-class-authorization.sh

@@ -41,7 +41,7 @@ popd > /dev/null
 
 pushd . && cd "$SCRIPT_ROOT"
 
-root_ntriples=$(./get-document.sh \
+root_ntriples=$(./get.sh \
   -f "$OWNER_CERT_FILE" \
   -p "$OWNER_CERT_PWD" \
   --accept 'application/n-triples' \

http-tests/admin/acl/add-put-group-authorization.sh

@@ -57,7 +57,7 @@ popd > /dev/null
 
 pushd . && cd "$SCRIPT_ROOT"
 
-root_ntriples=$(./get-document.sh \
+root_ntriples=$(./get.sh \
   -f "$OWNER_CERT_FILE" \
   -p "$OWNER_CERT_PWD" \
   --accept 'application/n-triples' \

http-tests/admin/add-ontology-import.sh

@@ -10,7 +10,7 @@ pushd . > /dev/null && cd "$SCRIPT_ROOT/admin"
 
 namespace_doc="${END_USER_BASE_URL}ns"
 namespace="${namespace_doc}#"
-ontology_doc="${ADMIN_BASE_URL}model/ontologies/namespace/"
+ontology_doc="${ADMIN_BASE_URL}ontologies/namespace/"
 import_uri="http://www.w3.org/ns/auth/acl"
 
 # add ontology import

http-tests/admin/model/create-class.sh → http-tests/admin/model/add-class.sh

@@ -13,16 +13,15 @@ pushd . > /dev/null && cd "$SCRIPT_ROOT/admin/model"
 
 namespace_doc="${END_USER_BASE_URL}ns"
 namespace="${namespace_doc}#"
-ontology_doc="${ADMIN_BASE_URL}model/ontologies/namespace/"
+ontology_doc="${ADMIN_BASE_URL}ontologies/namespace/"
 class="${namespace_doc}#NewClass"
 
-./create-class.sh \
+./add-class.sh \
   -f "$OWNER_CERT_FILE" \
   -p "$OWNER_CERT_PWD" \
   -b "$ADMIN_BASE_URL" \
   --uri "$class" \
   --label "New class" \
-  --slug new-class \
   --sub-class-of "https://www.w3.org/ns/ldt/document-hierarchy#Item" \
   "$ontology_doc"