End-to-end automation test for reindexing #1751

.github/workflows/reindex.yml

@@ -0,0 +1,47 @@
+name: Reindex - Integration Tests
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'demo/**'
+      - '.github/workflows/site.yml'
+      - '.github/workflows/release.yml'
+
+jobs:
+  reindex:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.pull_request.labels.*.name, 'ci-skip')"
+    strategy:
+      matrix:
+        datastore: [ 'db2', 'derby', 'postgres' ]
+      fail-fast: false
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v2.3.4
+    - name: Set up java
+      uses: joschi/setup-jdk@v2.3.0
+      with:
+        java-version: 11
+    - name: Setup prerequisites
+      env:
+        WORKSPACE: ${{ github.workspace }}
+      run: bash build/reindex/bin/setup-prerequisites.sh ${{matrix.datastore}}
+    - name: Integration Tests
+      env:
+        WORKSPACE: ${{ github.workspace }}
+      run: |
+        bash build/reindex/bin/pre-integration-test.sh ${{matrix.datastore}}
+        bash build/reindex/bin/integration-test.sh ${{matrix.datastore}}
+        bash build/reindex/bin/post-integration-test.sh ${{matrix.datastore}}
+    - name: Gather error logs
+      if: failure()
+      env:
+        WORKSPACE: ${{ github.workspace }}
+      run: bash build/common/gather-logs.sh reindex ${{matrix.datastore}}
+    - name: Upload logs
+      if: always()
+      uses: actions/upload-artifact@v2.2.0
+      with:
+        name: integration-test-results-${{ matrix.datastore }}
+        path: build/reindex/integration-test-results

build/common/gather-logs.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# (C) Copyright IBM Corp. 2021
+#
+# SPDX-License-Identifier: Apache-2.0
+###############################################################################
+
+set -ex
+
+# Gathers the logs
+package_logs(){
+    workflow="${1}"
+    job="${2}"
+    echo "Gathering logs for [${workflow}/${job}]"
+
+    it_results=${WORKSPACE}/build/${workflow}/integration-test-results
+    if [ ! -d ${it_results} ]
+    then
+        rm -fr ${it_results} 2>/dev/null
+    fi
+
+    mkdir -p ${it_results}
+    mkdir -p ${it_results}/server-logs
+    mkdir -p ${it_results}/fhir-server-test
+
+    # Runtime Date
+    echo $(date) > ${it_results}/runtime.txt
+
+    # Look for the FHIR Server Container
+    containerId=$(docker ps -a | grep ibm-fhir-server | cut -d ' ' -f 1)
+    if [ -z "${containerId}" ]
+    then
+        echo "Warning: Could not find fhir container!!!"
+    else
+        echo "fhir container id: $containerId"
+        # Grab the container's console log
+        docker logs $containerId  >& ${it_results}/docker-console.txt
+
+        echo "Gathering post-test server logs from docker container: $containerId"
+        docker cp -L $containerId:/logs ${it_results}/server-logs
+    fi
+
+    echo "Gathering integration test output"
+    if [ -d ${WORKSPACE}/fhir-server-test/target/surefire-reports ]
+    then
+        cp -pr ${WORKSPACE}/fhir-server-test/target/surefire-reports/* ${it_results}/fhir-server-test
+    fi
+
+    if [ -f ${WORKSPACE}/build/${workflow}/${job}/workarea/${job}-test1.log ]
+    then
+        echo "Move the '${job}' Elements to the output area'"
+        cp -pr build/${workflow}/${job}/workarea/${job}-test*.log ${it_results}
+    fi
+}
+
+###############################################################################
+# Check if the workspace is set.
+if [ -z "${WORKSPACE}" ]
+then 
+    echo "The WORKSPACE value is unset"
+    exit -1
+fi 
+
+# Store the current directory to reset to
+pushd $(pwd) > /dev/null
+
+package_logs "${1}" "${2}"
+
+# Reset to Original Directory
+popd > /dev/null
+
+# EOF
+###############################################################################

build/reindex/.gitignore

@@ -0,0 +1,2 @@
+**/integration-test-results
+**/workarea

build/reindex/README.md

@@ -0,0 +1,48 @@
+# Integration Test Framework for the $reindex Operation
+
+This document outlines the end-to-end reindex automation framework. 
+
+The automation runs with these steps: 
+
+- **Checkout source code** - Checks out the git code and populates the `github` environment variables.
+- **Set up java** - Downloads and setup for Java 11
+- **Setup prerequisites** - This step builds the required artifacts necessary to test the build with the reindex. 
+- **Integration Tests** - The step executes the pre-integration-docker, then integration-test-docker and runs the post-integration-docker scripts.
+- **Gather error logs** - This step only runs upon a failure condition. 
+- **Upload logs** - The step uploads the results of the integration tests and the operational logs are posted to the job. 
+
+The GitHub Action is parameterized with a matrix for each new `$reindex` tests. Each additional entry in the array ends up creating multiple automation steps which must complete successfully for the workflow.
+
+``` yaml
+strategy:
+  matrix:
+    datastore: [ 'db2', 'derby', 'postgres' ]
+```
+
+Each datastore layer that is tested as part of the framework uses the default build files and the files that match the `matrix.datastore` name added to the `reindex.yml`.
+
+|Filename|Purpose|
+|----------|----------------|
+|bin/gather-logs.sh|Gathers the logs from the build|
+|bin/integration-test.sh|Run after the tests complete to release resources and package tests results|
+|bin/setup-prerequisites.sh|Builds the fhir-server|
+|bin/pre-integration-test.sh|Call the pre-integration-test step for `<datastore>`|
+|bin/post-integration-test.sh|Call the post-integration-test step for `<datastore>`|
+|`<datastore>`/integration-test.sh|if exists, overrides bin/integration-test.sh, replacing the prior test behavior.|
+|`<datastore>`/pre-integration-test.sh|Run before integration-test.sh to startup image and services for the integration testing|
+|`<datastore>`/post-integration-test.sh|if exists, runs after integration-test.sh to stop image and services from the integration testing|
+|`<datastore>`/.gitignore|Ignores files related to the reindex layer's tests|
+|`README.md`|This file describing the reindex framework|
+
+Note, `<datastore>` is replaced with your reindex layer such as `db2`. 
+
+Transaction Timeout is 300 seconds.
+
+Consult the reference implementation (`db2`) to start a new reindex tests. The minimum that must be implemented are the `pre-integration-test.sh` and `.gitignore`.
+
+## Test the Automation
+
+To test the build, be sure to pre-set the environment variable `WORKSPACE` with `export WORKSPACE=$(pwd)`.
+You must also start Docker, so the image is built that supports the IBM FHIR Server.
+
+If you have any questions, please reach out on Zulip.

build/reindex/bin/integration-test.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# (C) Copyright IBM Corp. 2021
+#
+# SPDX-License-Identifier: Apache-2.0
+###############################################################################
+
+set -ex
+set -o pipefail
+
+# spins for a set time until the server is up
+wait_for_it(){
+    # Wait until the fhir server is up and running...
+    echo "Waiting for fhir-server to complete initialization..."
+    healthcheck_url='https://localhost:9443/fhir-server/api/v4/$healthcheck'
+    tries=0
+    status=0
+    while [ $status -ne 200 -a $tries -lt 30 ]; do
+        tries=$((tries + 1))
+
+        set +o errexit
+        cmd="curl -k -o ${WORKSPACE}/health.json --max-time 5 -I -w "%{http_code}" -u fhiruser:change-password $healthcheck_url"
+        echo "Executing[$tries]: $cmd"
+        status=$($cmd)
+        set -o errexit
+
+        echo "Status code: $status"
+        if [ $status -ne 200 ]
+        then
+            echo "Sleeping 30 secs..."
+            sleep 30
+        fi
+    done
+
+    if [ $status -ne 200 ]
+    then
+        echo "Could not establish a connection to the fhir-server within $tries REST API invocations!"
+        exit 1
+    fi
+
+    echo "The fhir-server appears to be running..."
+}
+
+run_tests(){
+    # The integration tests may be overriden completely, or fall through to the default. 
+    reindex="${1}"
+
+    if [ ! -z "${reindex}" ] && [ -f "build/reindex/${reindex}/integration-test.sh" ]
+    then 
+        echo "Running [${reindex}] specific integration tests"
+        bash build/reindex/${reindex}/integration-test.sh
+    else
+        # Go to the Default
+        echo "Executing the default integration tests"
+        sed -i -e 's/test.reindex.enabled = false/test.reindex.enabled = true/g' ${WORKSPACE}/fhir-server-test/src/test/resources/test.properties
+
+        # Test 1 - Basic Tests for Reindex
+        mvn -B test -f fhir-server-test -DskipWebSocketTest=true --no-transfer-progress \
+            -DskipTests=false -Dtest="com.ibm.fhir.server.test.operation.ReindexOperationTest" | tee build/reindex/${reindex}/workarea/${reindex}-test1.log
+
+        # Test 2 - Long Run Tests *895 Resources*
+        mvn -B test -f fhir-server-test -DskipWebSocketTest=true --no-transfer-progress \
+            -DskipTests=false -Dtest="com.ibm.fhir.server.test.operation.ReindexOperationLongRunTest" | tee build/reindex/${reindex}/workarea/${reindex}-test2.log
+
+        # Test 3 Phase 1
+        mvn -B test -f fhir-server-test -DskipWebSocketTest=true --no-transfer-progress \
+            -DskipTests=false -Dtest="com.ibm.fhir.server.test.operation.ReindexOperationPhase1Test" | tee build/reindex/${reindex}/workarea/${reindex}-test3.log
+
+        # Update SPs
+        cp -pr ${WORKSPACE}/fhir-server-test/src/test/resources/testdata/reindex-operation/extension-search-parameters-test1.json \
+            ${WORKSPACE}/build/reindex/${reindex}/workarea/volumes/dist/config/default/extension-search-parameters.json
+
+        # Restart
+        cd build/reindex/${reindex}
+        docker-compose restart --timeout 30 fhir
+        cd -
+        wait_for_it
+
+        # Test 3 Phase 2
+        mvn -B test -f fhir-server-test -DskipWebSocketTest=true --no-transfer-progress \
+            -DskipTests=false -Dtest="com.ibm.fhir.server.test.operation.ReindexOperationPhase2Test" | tee build/reindex/${reindex}/workarea/${reindex}-test4.log
+
+        # Update SPs
+        cp -pr ${WORKSPACE}/fhir-server-test/src/test/resources/testdata/reindex-operation/extension-search-parameters-test2.json \
+            ${WORKSPACE}/build/reindex/${reindex}/workarea/volumes/dist/config/default/extension-search-parameters.json
+
+        # Restart
+        cd build/reindex/${reindex}
+        docker-compose restart --timeout 30 fhir
+        cd -
+        wait_for_it
+
+        # Test 3 Phase 3
+        mvn -B test -f fhir-server-test -DskipWebSocketTest=true --no-transfer-progress \
+            -DskipTests=false -Dtest="com.ibm.fhir.server.test.operation.ReindexOperationPhase3Test" | tee build/reindex/${reindex}/workarea/${reindex}-test5.log
+    fi
+}
+
+###############################################################################
+# Check if the workspace is set.
+if [ -z "${WORKSPACE}" ]
+then 
+    echo "The WORKSPACE value is unset"
+    exit -1
+fi 
+
+# Store the current directory to reset to
+pushd $(pwd) > /dev/null
+
+# Change to the reindex/bin directory
+cd "${WORKSPACE}"
+
+run_tests "${1}"
+
+# Reset to Original Directory
+popd > /dev/null
+
+# EOF
+###############################################################################

build/reindex/bin/post-integration-test.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# (C) Copyright IBM Corp. 2021
+#
+# SPDX-License-Identifier: Apache-2.0
+###############################################################################
+
+set -ex
+
+# reindex_post - executes for each reindex post integration steps
+reindex_post(){
+    reindex="${1}"
+    if [ ! -z "${reindex}" ] && [ -f build/reindex/${reindex}/post-integration-test.sh ]
+    then 
+        echo "Running [${reindex}] post-integration-test"
+        bash build/reindex/${reindex}/post-integration-test.sh
+    else
+        cd build/reindex/${reindex}
+        docker-compose down --remove-orphans --rmi local -v --timeout 30
+    fi
+}
+
+###############################################################################
+# Check if the workspace is set.
+if [ -z "${WORKSPACE}" ]
+then 
+    echo "The WORKSPACE value is unset"
+    exit -1
+fi 
+
+# Store the current directory to reset to
+pushd $(pwd) > /dev/null
+
+cd "${WORKSPACE}"
+
+reindex_post ${1}
+
+# Reset to Original Directory
+popd > /dev/null
+
+# EOF 
+###############################################################################

build/reindex/bin/pre-integration-test.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# (C) Copyright IBM Corp. 2021
+#
+# SPDX-License-Identifier: Apache-2.0
+###############################################################################
+
+set -ex
+
+# reindex_pre - executes for each reindex pre integration steps
+reindex_pre(){
+    reindex="${1}"
+    if [ ! -z "${reindex}" ] && [ -f build/reindex/${reindex}/pre-integration-test.sh ]
+    then 
+        echo "Running [${reindex}] pre-integration-test"
+        bash build/reindex/${reindex}/pre-integration-test.sh
+    fi
+}
+
+###############################################################################
+# Check if the workspace is set.
+if [ -z "${WORKSPACE}" ]
+then 
+    echo "The WORKSPACE value is unset"
+    exit -1
+fi 
+
+# Store the current directory to reset to
+pushd $(pwd) > /dev/null
+
+# Change to the reindex_pre
+cd "${WORKSPACE}"
+
+reindex_pre "${1}"
+
+# Reset to Original Directory
+popd > /dev/null
+
+# EOF 
+###############################################################################