Merge branch 'IQSS:develop' into 9557_dataset_update_api_performance

.github/workflows/container_app_pr.yml

@@ -0,0 +1,83 @@
+---
+name: Preview Application Container Image
+
+on:
+    # We only run the push commands if we are asked to by an issue comment with the correct command.
+    # This workflow is always taken from the default branch and runs in repo context with access to secrets.
+    repository_dispatch:
+        types: [ push-image-command ]
+
+env:
+    IMAGE_TAG: unstable
+    BASE_IMAGE_TAG: unstable
+    PLATFORMS: "linux/amd64,linux/arm64"
+
+jobs:
+    deploy:
+        name: "Package & Push"
+        runs-on: ubuntu-latest
+        # Only run in upstream repo - avoid unnecessary runs in forks
+        if: ${{ github.repository_owner == 'IQSS' }}
+        steps:
+            # Checkout the pull request code as when merged
+            - uses: actions/checkout@v3
+              with:
+                  ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
+            - uses: actions/setup-java@v3
+              with:
+                  java-version: "11"
+                  distribution: 'adopt'
+            - uses: actions/cache@v3
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            # Note: Accessing, pushing tags etc. to GHCR will only succeed in upstream because secrets.
+            - name: Login to Github Container Registry
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ secrets.GHCR_USERNAME }}
+                  password: ${{ secrets.GHCR_TOKEN }}
+
+            - name: Set up QEMU for multi-arch builds
+              uses: docker/setup-qemu-action@v2
+
+            # Get the image tag from either the command or default to branch name (Not used for now)
+            #-   name: Get the target tag name
+            #    id: vars
+            #    run: |
+            #        tag=${{ github.event.client_payload.slash_command.args.named.tag }}
+            #        if [[ -z "$tag" ]]; then tag=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-'); fi
+            #        echo "IMAGE_TAG=$tag" >> $GITHUB_ENV
+
+            # Set image tag to branch name of the PR
+            - name: Set image tag to branch name
+              run: |
+                  echo "IMAGE_TAG=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
+
+            - name: Deploy multi-arch application container image
+              run: mvn -Pct deploy -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
+
+            - uses: marocchino/sticky-pull-request-comment@v2
+              with:
+                  header: app-registry-push
+                  hide_and_recreate: true
+                  hide_classify: "OUTDATED"
+                  number: ${{ github.event.client_payload.pull_request.number }}
+                  message: |
+                      :package: Pushed preview application image as
+                      ```
+                      ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container/package/dataverse). Use by referencing with full name as printed above, mind the registry name.
+
+            # Leave a note when things have gone sideways
+            - uses: peter-evans/create-or-update-comment@v3
+              if: ${{ failure() }}
+              with:
+                  issue-number: ${{ github.event.client_payload.pull_request.number }}
+                  body: >
+                      :package: Could not push preview image :disappointed:.
+                      See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.

.github/workflows/container_app_push.yml

@@ -18,10 +18,11 @@ env:
     BASE_IMAGE_TAG: unstable
     REGISTRY: "" # Empty means default to Docker Hub
     PLATFORMS: "linux/amd64,linux/arm64"
+    MASTER_BRANCH_TAG: alpha
 
 jobs:
     build:
-        name: Build & deploy
+        name: "Build & Test"
         runs-on: ubuntu-latest
         permissions:
             contents: read
@@ -49,19 +50,64 @@ jobs:
             - name: Build app container image with local architecture
               run: mvn -Pct package
 
-            # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
+            # TODO: add smoke / integration testing here
 
-            # Run this when triggered via push or schedule as reused workflow from base / maven unit tests
-            - if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' }}
-              name: Push description to DockerHub
-              uses: peter-evans/dockerhub-description@v3
+    hub-description:
+        needs: build
+        name: Push image description to Docker Hub
+        # Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
+        # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
+        if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v3
+            - uses: peter-evans/dockerhub-description@v3
               with:
                   username: ${{ secrets.DOCKERHUB_USERNAME }}
                   password: ${{ secrets.DOCKERHUB_TOKEN }}
                   repository: gdcc/dataverse
                   short-description: "Dataverse Application Container Image providing the executable"
                   readme-filepath: ./src/main/docker/README.md
 
+    # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
+    # We check for them here and subsequent jobs can rely on this to decide if they shall run.
+    check-secrets:
+        needs: build
+        name: Check for Secrets Availability
+        runs-on: ubuntu-latest
+        outputs:
+            available: ${{ steps.secret-check.outputs.available }}
+        steps:
+            - id: secret-check
+              # perform secret check & put boolean result as an output
+              shell: bash
+              run: |
+                  if [ "${{ secrets.DOCKERHUB_TOKEN }}" != '' ]; then
+                      echo "available=true" >> $GITHUB_OUTPUT;
+                  else
+                      echo "available=false" >> $GITHUB_OUTPUT;
+                  fi
+
+    deploy:
+        needs: check-secrets
+        name: "Package & Publish"
+        runs-on: ubuntu-latest
+        # Only run this job if we have access to secrets. This is true for events like push/schedule which run in
+        # context of main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
+        if: needs.check-secrets.outputs.available == 'true'
+        steps:
+            - uses: actions/checkout@v3
+            - uses: actions/setup-java@v3
+              with:
+                  java-version: "11"
+                  distribution: 'adopt'
+            - uses: actions/cache@v3
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            # Depending on context, we push to different targets. Login accordingly.
             - if: ${{ github.event_name != 'pull_request' }}
               name: Log in to Docker Hub registry
               uses: docker/login-action@v2
@@ -82,8 +128,8 @@ jobs:
             - name: Re-set image tag based on branch (if master)
               if: ${{ github.ref_name == 'master' }}
               run: |
-                  echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
-                  echo "BASE_IMAGE_TAG=alpha" >> $GITHUB_ENV
+                  echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
+                  echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
             - name: Re-set image tag and container registry when on PR
               if: ${{ github.event_name == 'pull_request' }}
               run: |
@@ -97,6 +143,11 @@ jobs:
               if: ${{ github.event_name == 'pull_request' }}
               with:
                   header: app-registry-push
+                  hide_and_recreate: true
+                  hide_classify: "OUTDATED"
                   message: |
-                      Pushed preview application image as [`ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}`](https://github.com/orgs/gdcc/packages/container/package/dataverse).
-                      Use it by referencing it with its full name as printed above.
+                      :package: Pushed preview application image as
+                      ```
+                      ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container/package/dataverse). Use by referencing with full name as printed above, mind the registry name.

.github/workflows/pr_comment_commands.yml

@@ -0,0 +1,20 @@
+name: PR Comment Commands
+on:
+    issue_comment:
+        types: [created]
+jobs:
+    dispatch:
+        # Avoid being triggered by forks in upstream
+        if: ${{ github.repository_owner == 'IQSS' }}
+        runs-on: ubuntu-latest
+        steps:
+            - name: Dispatch
+              uses: peter-evans/slash-command-dispatch@v3
+              with:
+                  # This token belongs to @dataversebot and has sufficient scope.
+                  token: ${{ secrets.GHCR_TOKEN }}
+                  commands: |
+                      push-image
+                  repository: IQSS/dataverse
+                  # Commenter must have at least write permission to repo to trigger dispatch
+                  permission: write

.readthedocs.yml

@@ -0,0 +1,21 @@
+version: 2
+
+# HTML is always built, these are additional formats only
+formats:
+    - pdf
+
+build:
+    os: ubuntu-22.04
+    tools:
+        python: "3.10"
+    apt_packages:
+        - graphviz
+
+python:
+    install:
+        - requirements: doc/sphinx-guides/requirements.txt
+
+
+sphinx:
+    configuration: doc/sphinx-guides/source/conf.py
+    fail_on_warning: true

doc/release-notes/9100-schema.org-updates.md

@@ -0,0 +1,3 @@
+Changes made in v5.13 and v5.14 in multiple PRs to improve the embedded Schema.org metadata in dataset pages will only be propagated to the Schema.Org JSON-LD metadata export if a reExportAll() is done.
+
+The 5.14 release notes should include the standard instructions for doing a reExportAll after updating the code.

doc/release-notes/9185-contact-email-updates.md

@@ -0,0 +1,12 @@
+## Contact Email Improvements
+
+Email sent from the contact forms to the contact(s) for a collection, dataset, or datafile can now optionally be cc'd to a support email address. The support email address can be changed from the default :SystemEmail address to a separate :SupportEmail address. When multiple contacts are listed, the system will now send one email to all contacts (with the optional cc if configured) instead of separate emails to each contact. Contact names with a comma that refer to Organizations will no longer have the name parts reversed in the email greeting. A new protected feedback API has been added.
+
+## Backward Incompatibilities
+
+When there are multiple contacts, the system will now send one email with all of the contacts in the To: header instead of sending one email to each contact (with no indication that others have been notified).
+
+## New JVM/MicroProfile Settings
+
+dataverse.mail.support-email - allows a separate email, distinct from the :SystemEmail to be used as the to address in emails from the contact form/ feedback api.
+dataverse.mail.cc-support-on-contact-emails - include the support email address as a CC: entry when contact/feedback emails are sent to the contacts for a collection, dataset, or datafile.

doc/sphinx-guides/source/api/native-api.rst

@@ -1618,6 +1618,8 @@ The fully expanded example above (without environment variables) looks like this
 
 The people who need to review the dataset (often curators or journal editors) can check their notifications periodically via API to see if any new datasets have been submitted for review and need their attention. See the :ref:`Notifications` section for details. Alternatively, these curators can simply check their email or notifications to know when datasets have been submitted (or resubmitted) for review.
 
+.. _return-a-dataset:
+
 Return a Dataset to Author
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -1645,6 +1647,8 @@ The fully expanded example above (without environment variables) looks like this
 
 The review process can sometimes resemble a tennis match, with the authors submitting and resubmitting the dataset over and over until the curators are satisfied. Each time the curators send a "reason for return" via API, that reason is persisted into the database, stored at the dataset version level.
 
+The :ref:`send-feedback` API call may be useful as a way to move the conversation to email. However, note that these emails go to contacts (versus authors) and there is no database record of the email contents. (:ref:`dataverse.mail.cc-support-on-contact-email` will send a copy of these emails to the support email address which would provide a record.)
+
 Link a Dataset
 ~~~~~~~~~~~~~~
 
@@ -4497,3 +4501,29 @@ A curl example using allowing access to a dataset's metadata
 
 Please see :ref:`dataverse.api.signature-secret` for the configuration option to add a shared secret, enabling extra
 security.
+
+.. _send-feedback:
+
+Send Feedback To Contact(s)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API call allows sending an email to the contacts for a collection, dataset, or datafile or to the support email address when no object is specified.
+The call is protected by the normal /admin API protections (limited to localhost or requiring a separate key), but does not otherwise limit the sending of emails.
+Administrators should be sure only trusted applications have access to avoid the potential for spam.
+
+The call is a POST with a JSON object as input with four keys:
+- "targetId" - the id of the collection, dataset, or datafile. Persistent ids and collection aliases are not supported. (Optional)
+- "subject" - the email subject line
+- "body" - the email body to send
+- "fromEmail" - the email to list in the reply-to field. (Dataverse always sends mail from the system email, but does it "on behalf of" and with a reply-to for the specified user.)
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+  export SERVER_URL=http://localhost
+  export JSON='{"targetId":24, "subject":"Data Question", "body":"Please help me understand your data. Thank you!", "fromEmail":"dataverseSupport@mailinator.com"}'
+
+  curl -X POST -H 'Content-Type:application/json' -d "$JSON" $SERVER_URL/api/admin/feedback
+
+Note that this call could be useful in coordinating with dataset authors (assuming they are also contacts) as an alternative/addition to the functionality provided by :ref:`return-a-dataset`.

doc/sphinx-guides/source/installation/config.rst

@@ -2349,6 +2349,27 @@ See :ref:`discovery-sign-posting` for details.
 
 Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_SIGNPOSTING_LEVEL1_ITEM_LIMIT``.
 
+dataverse.mail.support-email
+++++++++++++++++++++++++++++
+
+This provides an email address distinct from the :ref:`systemEmail` that will be used as the email address for Contact Forms and Feedback API. This address is used as the To address when the Contact form is launched from the Support entry in the top navigation bar and, if configured via :ref:`dataverse.mail.cc-support-on-contact-email`, as a CC address when the form is launched from a Dataverse/Dataset Contact button.
+This allows configuration of a no-reply email address for :ref:`systemEmail` while allowing feedback to go to/be cc'd to the support email address, which would normally accept replies. If not set, the :ref:`systemEmail` is used for the feedback API/contact form email.
+
+Note that only the email address is required, which you can supply without the ``<`` and ``>`` signs, but if you include the text, it's the way to customize the name of your support team, which appears in the "from" address in emails as well as in help text in the UI. If you don't include the text, the installation name (see :ref:`Branding Your Installation`) will appear in the "from" address.
+
+Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_MAIL_SUPPORT_EMAIL``.
+
+.. _dataverse.mail.cc-support-on-contact-email:
+
+dataverse.mail.cc-support-on-contact-email
+++++++++++++++++++++++++++++++++++++++++++
+
+If this setting is true, the contact forms and feedback API will cc the system (:SupportEmail if set, :SystemEmail if not) when sending email to the collection, dataset, or datafile contacts.
+A CC line is added to the contact form when this setting is true so that users are aware that the cc will occur.
+The default is false.
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_MAIL_CC_SUPPORT_ON_CONTACT_EMAIL``.
+
 
 .. _feature-flags:
 
@@ -3831,3 +3852,4 @@ A true(default)/false option determining whether datafiles listed on the dataset
 A true/false (default) option determining whether the dataset datafile table display includes checkboxes enabling users to turn folder ordering and/or category ordering (if an order is defined by :CategoryOrder) on and off dynamically. 
 
 .. _supported MicroProfile Config API source: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Overview.html
+