Add actions and workflows for QIT tests

.github/workflows/run-qit-all.yml

@@ -0,0 +1,70 @@
+name: Run QIT for all extensions
+
+# **What it does**: Runs a suite of tests for all Grow extensions.
+# **Why we have it**: To be able to run tests for all extensions at once. For example when we want to compatibility test a new version of the platform (WP/WC).
+
+on:
+  workflow_dispatch:
+    inputs:
+      # Basic params.
+      version:
+        description: 'Version to be tested'
+        required: true
+        type: choice
+        options:
+        - latest
+        - dev
+
+      # Configure which tests to run.
+      test-activation:
+        description: 'Should activation be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-security:
+        description: 'Should security be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-phpstan:
+        description: 'Should phpstan be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-api:
+        description: 'Should API be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-e2e:
+        description: 'Should E2E be tested? (takes a lot of time)'
+        required: true
+        default: false
+        type: boolean
+
+      # Advanced customization.
+      options:
+        description: 'Additional options for `qit` command, like `--optional_features=hpos`.'
+        required: false
+
+jobs:
+    qit-tests:
+        name: Run QIT Tests
+        uses: ./.github/workflows/run-qit-extension.yml
+        secrets: inherit
+        strategy:
+          # Allow to test extensions even if one of them fails.
+          fail-fast: false
+          matrix:
+            # List of extensions to be tested.
+            extension: [automatewoo, automatewoo-birthdays, automatewoo-referrals, google-listings-and-ads, woocommerce-google-analytics-integration]
+        with:
+          # Conditional statements are here to allow testing on push triggers, without manual input. To be removed before merging.
+          version: ${{ inputs.version }}
+          test-activation: ${{ inputs.test-activation }}
+          test-security: ${{ inputs.test-security }}
+          test-phpstan: ${{ inputs.test-phpstan }}
+          test-api: ${{ inputs.test-api }}
+          test-e2e: ${{ inputs.test-e2e }}
+          extension: ${{ matrix.extension }}
+          options: ${{ inputs.options }}

.github/workflows/run-qit-extension.yml

@@ -0,0 +1,154 @@
+name: Run QIT for a given extensions
+
+# **What it does**: Runs a set of QIT tests for a given extension.
+# **Why we have it**: To reuse across other repos, to make a full test of a single extension.
+
+on:
+  workflow_call:
+    inputs:
+      # Basic params.
+      extension:
+        description: Extension to test
+        required: true
+        type: string
+      version:
+        description: |
+          Version to be tested: `latest`, `dev`, or `local`.
+          `latest` will test latest released version,
+          `dev` will make the action look up for an `{extension}.zip` file in the root
+          of the repository under the `gha-dev-build` tag,
+          `local` will look up for an `{extension}.zip` artifact in the current workflow.
+        required: true
+        default: 'latest'
+        type: string
+
+      # Customize which types to run.
+      test-activation:
+        description: 'Should activation be tested?'
+        default: true
+        type: boolean
+      test-security:
+        description: 'Should security be tested?'
+        default: true
+        type: boolean
+      test-phpstan:
+        description: 'Should PHPStan be tested?'
+        default: true
+        type: boolean
+      test-api:
+        description: 'Should API be tested?'
+        default: true
+        type: boolean
+      test-e2e:
+        description: 'Should E2E be tested?'
+        default: true
+        type: boolean
+
+      # Advanced customization.
+      options:
+        description: 'Additional options for `qit` command, like `--optional_features=hpos`'
+        type: string
+
+    outputs:
+        statuses:
+            description: "Statuses of all tests. Array of integers."
+            value: ${{ jobs.qit-tests.outputs.statuses }}
+
+
+jobs:
+    qit-tests:
+        name: Run QIT Tests
+        runs-on: ubuntu-20.04
+        env:
+            QIT_DISABLE_ONBOARDING: yes
+            dev_build: ${{ inputs.version != 'latest' && format('{0}.zip', inputs.extension) || '' }}
+        outputs:
+            statuses: ${{ toJSON( steps.*.outputs.status ) }}
+        steps:
+
+            - name: Download `gha-dev-build`
+              if: ${{ inputs.version == 'dev' }}
+              uses: robinraju/release-downloader@v1.8
+              with:
+                repository: "woocommerce/${{ inputs.extension }}"
+                tag: 'gha-dev-build'
+                fileName: ${{ env.dev_build }}
+                token: ${{ secrets.BOT_GH_TOKEN }}
+
+            - name: Download artifact
+              if: ${{ inputs.version == 'local' }}
+              uses: actions/download-artifact@v3
+              with:
+                name: ${{ env.dev_build }}
+
+            - name: Install QIT via composer
+              run: composer require woocommerce/qit-cli
+            - name: Add Partner
+              run: |
+                ./vendor/bin/qit partner:add \
+                  --user='${{ secrets.QIT_PARTNER_USER }}' \
+                  --application_password='${{ secrets.QIT_PARTNER_SECRET }}'
+
+            - name: Create results dir
+              run: mkdir -p ./qit-results/${{ inputs.extension }}
+
+            - name: Activation test
+              id: activation-test
+              if: ${{ inputs.test-activation == true }}
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+              timeout-minutes: 5
+              with:
+                type: activation
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: Security test
+              id: security-test
+              if: ${{ inputs.test-security == true }}
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+              timeout-minutes: 5
+              with:
+                type: security
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: PHPStan test
+              id: phpstan-test
+              if: ${{ inputs.test-phpstan == true }}
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+              timeout-minutes: 5
+              with:
+                type: phpstan
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: API test
+              id: api-test
+              if: ${{ inputs.test-api == true }}
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+              timeout-minutes: 5
+              with:
+                type: api
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: E2E test
+              id: e2e-test
+              if: ${{ inputs.test-e2e == true }}
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+              timeout-minutes: 30
+              with:
+                type: e2e
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: Upload results
+              uses: actions/upload-artifact@v1
+              with:
+                  name: "qit-results-${{ inputs.extension }}"
+                  path: ./qit-results/${{ inputs.extension }}

packages/github-actions/README.md

@@ -20,6 +20,7 @@ Custom GitHub actions that help to composite GitHub workflows across the repos m
 - [`prepare-node`](actions/prepare-node) - Set up Node.js with a specific version, load npm cache, install Node dependencies
 - [`prepare-php`](actions/prepare-php) - Set up PHP with a specific version and tools, load Composer cache, install Composer dependencies
 - [`publish-extension-dev-build`](actions/publish-extension-dev-build) - Publish extension development build
+- [`run-qit-annotate`](actions/run-qit-annotate) - Runs QIT test and annotates the results
 - [`stylelint-annotation`](actions/stylelint-annotation) - Annotate stylelint results via stylelint formatter
 - [`update-version-tags`](actions/update-version-tags) - Update version tags
 

packages/github-actions/actions/run-qit-annotate/README.md

@@ -0,0 +1,55 @@
+# Run QIT test
+
+This action provides the following functionality for GitHub Actions users:
+
+- Run `qit` test of given type for a given extension
+- Annotate the results
+- Forward status code, so the consumer can decide how to conclude the results
+
+
+## Usage
+
+See [action.yml](action.yml)
+
+### Prerequisites
+
+- QIT needs to be [installed](https://woocommerce.github.io/qit-documentation/#/cli/getting-started?id=installing-qit) and [authenticated](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli)
+   - To setup QIT, you need to set `QIT_DISABLE_ONBOARDING` env to `yes`.
+- The action assumes there is `qit-results` directory. You can change it's name using `results-folder` input.
+
+
+### Basic:
+
+```yaml
+jobs:
+  qit-test:
+    name: Run QIT Tests
+    runs-on: ubuntu-20.04
+    env:
+      QIT_DISABLE_ONBOARDING: yes
+    steps:
+      - name: Install QIT via composer
+        run: composer require woocommerce/qit-cli
+
+      - name: Add Partner
+        run: |
+          ./vendor/bin/qit partner:add \
+            --user='${{ secrets.QIT_PARTNER_USER }}' \
+            --application_password='${{ secrets.QIT_PARTNER_SECRET }}'
+
+      - name: Create results dir
+        run: mkdir -p ./qit-results/automatewoo
+
+      - name: Security test
+        id: security-test
+        uses: woocommerce/grow/run-qit-annotate@actions-v1
+        timeout-minutes: 5
+        with:
+          type: security
+          extension: automatewoo
+          options: '--optional_features=hpos'
+
+      - name: Echo status
+        shell: bash
+        run: echo ${{ jobs.security-test.outputs.status }}
+```

packages/github-actions/actions/run-qit-annotate/action.yml

@@ -0,0 +1,94 @@
+name: Run QIT test
+description: Runs QIT test and annotates the results.
+
+# **What it does**: Runs a single QIT test and annotates the results.
+# **Why we have it**: To reuse across other workflows to test extensions.
+
+inputs:
+  # Basic params.
+  extension:
+    description: Extension to test
+    required: true
+  extension-file:
+    description: Custom build of the extension to test. If not given, the latest release will be used.
+    required: false
+  type:
+    description: Type of test to run
+    required: false
+    default: activation
+  # Advanced customization.
+  options:
+    description: Options to pass to the qit command
+    required: false
+    default: ''
+  results-filename:
+    description: Custom file name for results
+    required: false
+  results-folder:
+    description: Folder to store results
+    required: false
+    default: qit-results
+
+outputs:
+  status:
+    description: "Exit code of the test. May be used, for example, to fail the workflow if the test fails."
+    value: ${{ steps.run-test.outputs.status }}
+  summary:
+    description: "Short summary of the test"
+    value: ${{ steps.read-summary.outputs.summary }}
+  resultsURL:
+    description: "URL to the results of the test"
+    value: ${{ steps.read-summary.outputs.resultURL }}
+
+branding:
+  icon: 'award'
+  color: 'purple'
+
+runs:
+  using: composite
+  steps:
+    - name: Run test
+      id: run-test
+      # Do not fail when the `qit` fails, so we can annotate the results.
+      shell: bash --noprofile --norc {0}
+      continue-on-error: true
+      env:
+        # If the custom build is provided, pass it to the `zip` param.
+        zip: ${{ inputs.extension-file &&  format('--zip={0}', inputs.extension-file) || '' }}
+        report_file: ${{ inputs.results-filename || format( '{0}/{1}/{2}/{3}.txt', github.workspace, inputs.results-folder, inputs.extension, inputs.type ) }}
+        status: 0
+      run: |
+        ./vendor/bin/qit run:${{ inputs.type }} ${{ inputs.extension }} \
+          $zip \
+          ${{ inputs.options }} \
+          --wait \
+          > $report_file || status=$?
+        echo "status=$status" >> "$GITHUB_OUTPUT"
+        echo "report_file=$report_file" >> "$GITHUB_OUTPUT"
+        cat $report_file
+
+    # Parse the report file, to fetch the essential information.
+    - name: Read summary
+      id: read-summary
+      shell: bash
+      if: '!cancelled()'
+      run: |
+        summary=`grep -Po "(?<=Test Summary)\s+(.*)" ${{ steps.run-test.outputs.report_file }} --color=never`
+        resultURL=`grep -Po "(?<=Result Url)\s+(.*)" ${{ steps.run-test.outputs.report_file }} --color=never`
+        echo "summary=$summary" >> $GITHUB_OUTPUT
+        echo "resultURL=$resultURL" >> $GITHUB_OUTPUT
+
+    # Annotate the results according to the exit code.
+    - name: Annotate
+      if: '!cancelled()'
+      shell: bash
+      run: |
+        summary="${{ inputs.type }}: ${{ steps.read-summary.outputs.summary }} - ${{ steps.read-summary.outputs.resultURL }}";
+        case ${{ steps.run-test.outputs.status }} in
+          0) echo "::notice ::$summary"
+          ;;
+          2) echo "::warning ::$summary"
+          ;;
+          *) echo "::error ::$summary"
+          ;;
+        esac