ci/qase: some fixes and improvements with qase/ginkgo integration

.github/workflows/master-e2e.yaml

@@ -227,7 +227,7 @@ jobs:
       QASE_PROJECT_CODE: ELEMENTAL
       QASE_REPORT: 1
       QASE_RUN_COMPLETE: 1
-      QASE_RUN_DESCRIPTION: ${{ inputs.test_description }}
+      QASE_RUN_DESCRIPTION: TO_BE_CHANGED
       QASE_RUN_ID: ${{ inputs.qase_run_id }}
       QASE_RUN_NAME: TO_BE_CHANGED
       # For Rancher Manager
@@ -255,8 +255,14 @@ jobs:
           # 'env:' because GITHUB_WORKFLOW is a shell variable
           echo "QASE_RUN_NAME=${GITHUB_WORKFLOW}" >> ${GITHUB_ENV}
 
-          # Export it also to be used locally
+          # Also export URL of GH test run in Qase run description
+          GH_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          QASE_DESC="${{ inputs.test_description }} (${GH_RUN_URL})"
+          echo "QASE_RUN_DESCRIPTION=${QASE_DESC}" >> ${GITHUB_ENV}
+
+          # Export them also to be used locally
           export QASE_RUN_NAME="${GITHUB_WORKFLOW}"
+          export QASE_RUN_DESCRIPTION="${QASE_DESC}"
 
           # We need to always unset the faking ID variable
           unset QASE_RUN_ID
@@ -268,7 +274,7 @@ jobs:
           fi
 
           # Just an info for debugging purposes
-          echo "Export QASE_RUN_ID=${ID} and QASE_RUN_NAME=${GITHUB_WORKFLOW}"
+          echo "Export QASE_RUN_ID=${QASE_RUN_ID}, QASE_RUN_DESCRIPTION=${QASE_RUN_DESCRIPTION} and QASE_RUN_NAME=${QASE_RUN_NAME}"
       - name: Install Rancher+Elemental components
         env:
           CA_TYPE: ${{ inputs.ca_type }}
@@ -692,6 +698,18 @@ jobs:
             echo "Channel: ${{ inputs.upgrade_os_channel }}" >> ${GITHUB_STEP_SUMMARY}
             echo "Upgrade image: ${{ inputs.upgrade_image }}" >> ${GITHUB_STEP_SUMMARY}
           fi
+      - name: Delete Qase Run if job has been cancelled
+        if: ${{ always() && job.status == 'cancelled' }}
+        run: |
+          if [[ -n "${QASE_RUN_ID}" ]]; then
+            cd tests && make delete-qase-run
+          fi
+      - name: Finalize Qase Run and publish Results
+        if: ${{ always() && job.status != 'cancelled' }}
+        run: |
+          if [[ -n "${QASE_RUN_ID}" ]]; then
+            cd tests && make publish-qase-run
+          fi
       - name: Send failed status to slack
         if: ${{ failure() && github.event_name == 'schedule' }}
         uses: slackapi/slack-github-action@v1.23.0

tests/Makefile

@@ -44,9 +44,13 @@ deps:
 generate-readme:
 	@./scripts/generate-readme > README.md
 
-# Qase deps
+# Qase commands
 create-qase-run: deps
-	@go run qase/create_qase_run.go
+	@go run qase/qase_cmd.go -create
+delete-qase-run: deps
+	@go run qase/qase_cmd.go -delete
+publish-qase-run: deps
+	@go run qase/qase_cmd.go -publish
 
 # E2E tests
 e2e-airgap-rancher: deps

tests/e2e/helpers/qase/README.md

@@ -24,7 +24,6 @@ Here the variables you can use/define:
 | Variable | Description | Needed? |
 |:---:|:---:|:---:|
 | `QASE_API_TOKEN` | API token to access your Qase instance | Mandatory |
-| `QASE_DELETE_RUN` | Delete the specified run (mainly useful for debugging purposes) | Optional |
 | `QASE_ENVIRONNMENT_ID` | Use a specific environnment in a project | Optional |
 | `QASE_LOG_DEVEL` | Define log level use, see [logrus](https://pkg.go.dev/github.com/sirupsen/logrus#readme-level-logging) for more infos | Optional |
 | `QASE_PROJECT_CODE` | Code of your Qase project | Mandatory |
@@ -75,6 +74,9 @@ var _ = ReportAfterEach(func(report SpecReport) {
 Please note that you can change the `testCaseID` with your own variable, just pay attention to be consistent!
 You can also add more code in the `Report*` Ginkgo functions, this configuration is just the minimal stuff you need to add for the `qse-ginkgo` libray to work!
 
+**NOTE:** the `FinalizeResults()` call un `ReportAfterSuite` can be done outside with another Golang binary, this is just a simple example and you can adapt all to your needs.
+You can check another example on how it is done by having a look at the [Elemental project](https://www.github.com/rancher/elemental).
+
 Then, in your test code you have to add the case id value (`testCaseID` in our example) you want to report in Qase, for example like this:
 ```go
 func testFunc() ([]byte, error) {

tests/e2e/helpers/qase/qase.go

@@ -59,7 +59,6 @@ var (
 	apiToken       = os.Getenv("QASE_API_TOKEN")
 	environmentID  int64
 	envStrID       = os.Getenv("QASE_ENVIRONMENT_ID")
-	isDeleteRun    = os.Getenv("QASE_DELETE_RUN")
 	logLevel       = strings.ToUpper(os.Getenv("QASE_LOG_LEVEL"))
 	projectCode    = os.Getenv("QASE_PROJECT_CODE")
 	report         = os.Getenv("QASE_REPORT")
@@ -120,9 +119,7 @@ func createRun(client *qase.APIClient, name, description string, ids []int64) *q
 }
 
 /*
-This function creates a run in a specific project.
-  - @param name Name of the run
-  - @param description Short description of the run
+This function creates a run in a specific project. This is the exported version of the function.
   - @return ID of the created run
 */
 func CreateRun() int32 {
@@ -141,7 +138,9 @@ func CreateRun() int32 {
 		logrus.Debugf("Run named '%s' with description '%s' created with id %d", runName, runDescription, createdID)
 
 		// Check that we can access the run
-		checkRun(client, createdID)
+		if itExists := checkRun(client, createdID); !itExists {
+			logrus.Fatalf("Run %d doesn't exist", createdID)
+		}
 
 		// Export runID for all functions to use it
 		os.Setenv("QASE_RUN_ID", fmt.Sprint(runID))
@@ -156,11 +155,16 @@ This function checks the availability of a specific run.
   - @param id ID of the run to check
   - @return Fatal on error
 */
-func checkRun(client *qase.APIClient, id int32) {
+func checkRun(client *qase.APIClient, id int32) bool {
 	runResponse, _, err := client.RunsApi.GetRun(context.TODO(), projectCode, id)
-	if err != nil || !runResponse.Status {
-		logrus.Fatalf("Error on checking run: %v", err)
+	if err != nil {
+		logrus.Debugf("Error on checking run: %v", err)
+		// Return false in case if error
+		// As we don't care about the reason why, the run is not available anyway
+		return false
 	}
+
+	return runResponse.Status
 }
 
 /*
@@ -189,17 +193,53 @@ func deleteRun(client *qase.APIClient, id int32) {
 	}
 }
 
+/*
+This function completely deletes a specific run. This is the exported version of the function.
+  - @return Fatal on error
+*/
+func DeleteRun() {
+	// Do something only if run id is valid
+	if runID <= 0 {
+		logrus.Debug("Nothing to delete!")
+		return
+	}
+
+	cfg := qase.NewConfiguration()
+	cfg.AddDefaultHeader("Token", apiToken)
+	client := qase.NewAPIClient(cfg)
+
+	if checkProject(client, projectCode) {
+		logrus.Debugf("Project %s is validated", projectCode)
+
+		// Delete test run
+		deleteRun(client, runID)
+		logrus.Debugf("Run id %d has been deleted", runID)
+
+		// Check that we can't access the run
+		if itExists := checkRun(client, runID); itExists {
+			logrus.Fatalf("Run %d still exists", runID)
+		}
+	}
+}
+
 /*
 This function finalises the results for a specific run.
   - @return Fatal on error
 */
 func FinalizeResults() {
+	// Do something only if run id is valid
+	if runID <= 0 {
+		logrus.Debug("Nothing to finalize!")
+		return
+	}
+
 	cfg := qase.NewConfiguration()
 	cfg.AddDefaultHeader("Token", apiToken)
 	client := qase.NewAPIClient(cfg)
 
-	// Do something only if runID is valid
-	if runID > 0 {
+	if checkProject(client, projectCode) {
+		logrus.Debugf("Project %s is validated", projectCode)
+
 		// Complete run if needed
 		if runComplete != "" {
 			completeRun(client, runID)
@@ -219,17 +259,6 @@ func FinalizeResults() {
 			// Log in Ginkgo
 			ginkgo.GinkgoWriter.Printf("Report for run ID %d available: %s\n", runID, runPublicResponse.Result.Url)
 		}
-
-		// Delete Run if needed - mainly used for debugging purposes!
-		if isDeleteRun != "" {
-			deleteRun(client, runID)
-
-			// Also clean the variable and the env as well
-			os.Unsetenv("QASE_RUN_ID")
-			runID = 0
-		}
-	} else {
-		logrus.Debug("Nothing to finalize!")
 	}
 }
 
@@ -264,7 +293,7 @@ This function creates/updates run/cases for specific Ginkgo tests.
 */
 func Qase(id int64, testReport ginkgo.SpecReport) {
 	// A negative or zero run/case ID means that we won't log anything
-	if runID == 0 || id <= 0 {
+	if runID <= 0 || id <= 0 {
 		return
 	}
 

tests/e2e/helpers/qase/qase_example_test.go

@@ -17,7 +17,6 @@ Example code to use the Qase reporter helper in Ginkgo tests.
 
 This test can be run with this command:
 $ QASE_LOG_LEVEL=debug                                          \
-  QASE_DELETE_RUN=1                                             \
   QASE_REPORT=1                                                 \
   QASE_RUN_COMPLETE=1                                           \
   QASE_RUN_NAME="Qase/Gingo Integration Run"                    \

tests/e2e/suite_test.go

@@ -529,11 +529,6 @@ var _ = BeforeSuite(func() {
 	tools.HTTPShare("../..", ":8000")
 })
 
-var _ = ReportAfterSuite("Qase Report", func(report Report) {
-	// Finalize Qase report
-	FinalizeResults()
-})
-
 var _ = ReportBeforeEach(func(report SpecReport) {
 	// Reset case ID
 	testCaseID = -1

tests/qase/qase_cmd.go

@@ -0,0 +1,68 @@
+/*
+Copyright © 2022 - 2024 SUSE LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/rancher/elemental/tests/e2e/helpers/qase"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	runID    int32
+	runStrID = os.Getenv("QASE_RUN_ID")
+)
+
+func init() {
+	if runStrID != "" {
+		i, err := strconv.ParseInt(runStrID, 10, 32)
+		if err != nil {
+			logrus.Fatalf("Error on converting string to int64: %v", err)
+		}
+		runID = int32(i)
+	}
+}
+
+func main() {
+	// Define the allowed options
+	createRun := flag.Bool("create", false, "create a new Qase run")
+	deleteRun := flag.Bool("delete", false, "delete a Qase run, QASE_RUN_ID should be set")
+	publishRun := flag.Bool("publish", false, "publish a Qase report, QASE_RUN_ID should be set, it also depends on QASE_REPORT and QASE_RUN_COMPLETE")
+
+	// Parse the arguments
+	flag.Parse()
+
+	// Only one option at a time is allowed
+	if *createRun {
+		id := qase.CreateRun()
+		if id <= 0 {
+			logrus.Fatalln("Error on creating Qase run")
+		}
+		logrus.Debugf("Qase run id %d created", id)
+		fmt.Printf("%d", id)
+	} else if *deleteRun {
+		qase.DeleteRun()
+		logrus.Debugf("Qase run id %d deleted", runID)
+	} else if *publishRun {
+		qase.FinalizeResults()
+		logrus.Debugf("Qase finalization for run id %d has been done", runID)
+	} else {
+		logrus.Debugln("Nothing to do!")
+	}
+}