Add Github Actions workflows and composite actions for packs to use

.circle/Makefile

@@ -9,12 +9,14 @@ ST2_REPO_PATH ?= /tmp/st2
 ST2_REPO_BRANCH ?= master
 FORCE_CHECK_ALL_FILES ?= false
 FORCE_CHECK_PACK ?= false
+LINT_CONFIGS_PATH ?= $(CI_DIR)/lint-configs
+REQUIREMENTS_DIR ?= $(CI_DIR)/.circle
 
 export ST2_REPO_PATH ROOT_DIR FORCE_CHECK_ALL_FILES FORCE_CHECK_PACK
 
 # All components are prefixed by st2
-COMPONENTS := $(wildcard /tmp/st2/st2*)
-COMPONENTS_RUNNERS := $(wildcard /tmp/st2/contrib/runners/*)
+COMPONENTS := $(wildcard $(ST2_REPO_PATH)/st2*)
+COMPONENTS_RUNNERS := $(wildcard $(ST2_REPO_PATH)/contrib/runners/*)
 
 .PHONY: all
 all: requirements lint packs-resource-register packs-tests
@@ -79,14 +81,14 @@ compile:
 	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
 		echo "Force flake8 checks on all files"; \
 		find $(ROOT_DIR)/* -name "*.py" | while read py_file; do \
-			flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \
+			flake8 --config=$(LINT_CONFIGS_PATH)/python/.flake8 $$py_file || exit 1; \
 		done; \
 	elif [ ${NUM_CHANGED_PY} -gt 0 ]; then \
 		echo "Checking ${NUM_CHANGED_PY} Python files"; \
 		$(CI_DIR)/utils/git-changes py >~/.git-changes-py; \
 		while read -r file; do \
 			if [ -n "$$file" ]; then \
-				flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \
+				flake8 --config=$(LINT_CONFIGS_PATH)/python/.flake8 $$file || exit 1; \
 			fi; \
 		done < ~/.git-changes-py; \
 		if [ -e ~/.git-changes-py ]; then rm ~/.git-changes-py; fi; \
@@ -102,11 +104,11 @@ compile:
 	. $(VIRTUALENV_DIR)/bin/activate; \
 	if [ "${COMMON_LIBS}" = "true" ]; then \
 		echo "Common libs PATH selected"; \
-		export PYTHONPATH=/home/circleci/repo/lib:${PYTHONPATH}; \
+		export PYTHONPATH=$(ROOT_DIR)/lib:${PYTHONPATH}; \
 	fi; \
 	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_PY} -gt 0 ]; then \
-		REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \
-		CONFIG_DIR=$(CI_DIR)/lint-configs/ \
+		REQUIREMENTS_DIR=$(REQUIREMENTS_DIR)/ \
+		CONFIG_DIR=$(LINT_CONFIGS_PATH)/ \
 		st2-check-pylint-pack $(ROOT_DIR) || exit 1; \
 	else \
 		echo "No files have changed, skipping run..."; \
@@ -237,8 +239,8 @@ compile:
 			echo "Missing LICENSE file in $(ROOT_DIR)"; \
 			exit 2;\
 		fi;\
-		if [ "$$(python -c 'import yaml; f = open("/home/circleci/repo/pack.yaml"); print(yaml.safe_load(f.read())["name"]); f.close();')" = "napalm_logs" ] || \
-		   [ "$$(python -c 'import yaml; f = open("/home/circleci/repo/pack.yaml"); print(yaml.safe_load(f.read())["name"]); f.close();')" = "netbox" ]; then \
+		if [ "$$(python -c 'import yaml; f = open("$(ROOT_DIR)/pack.yaml"); print(yaml.safe_load(f.read())["name"]); f.close();')" = "napalm_logs" ] || \
+		   [ "$$(python -c 'import yaml; f = open("$(ROOT_DIR)/pack.yaml"); print(yaml.safe_load(f.read())["name"]); f.close();')" = "netbox" ]; then \
 			cat $(ROOT_DIR)/LICENSE | grep -q "MIT License"  || (echo "LICENSE file doesn't contain MIT license text" ; exit 2); \
 			cat $(ROOT_DIR)/LICENSE | grep -q "Copyright (c) 2017 John Anderson"  || (echo "LICENSE file doesn't include John Anderson's name" ; exit 2); \
 			cat $(ROOT_DIR)/LICENSE | grep -q "Permission is hereby granted, free of charge, to any person obtaining a copy"  || (echo "LICENSE file doesn't contain MIT license text" ; exit 2); \
@@ -252,13 +254,12 @@ compile:
 	fi
 
 .PHONY: .clone_st2_repo
-.clone_st2_repo: /tmp/st2
-/tmp/st2:
+.clone_st2_repo:
 	@echo
 	@echo "==================== cloning st2 repo ===================="
 	@echo
-	@rm -rf /tmp/st2
-	@git clone https://github.com/StackStorm/st2.git --depth 1 --single-branch --branch $(ST2_REPO_BRANCH) /tmp/st2
+	@rm -rf $(ST2_REPO_PATH)
+	@git clone https://github.com/StackStorm/st2.git --depth 1 --single-branch --branch $(ST2_REPO_BRANCH) $(ST2_REPO_PATH)
 
 .PHONY: .install-runners
 .install-runners:
@@ -284,17 +285,17 @@ requirements: virtualenv .clone_st2_repo .install-runners
 	@echo "==================== requirements ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==20.3.3"
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-dev.txt
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-pack-tests.txt
+	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(REQUIREMENTS_DIR)/requirements-dev.txt
+	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(REQUIREMENTS_DIR)/requirements-pack-tests.txt
 
 .PHONY: requirements-ci
 requirements-ci:
 	@echo
 	@echo "==================== requirements-ci ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==20.3.3"
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-dev.txt
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-pack-tests.txt
+	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(REQUIREMENTS_DIR)/requirements-dev.txt
+	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(REQUIREMENTS_DIR)/requirements-pack-tests.txt
 
 .PHONY: virtualenv
 virtualenv: $(VIRTUALENV_DIR)/bin/activate

.circle/index.py

@@ -17,11 +17,23 @@
 
 from st2common.util.pack import get_pack_ref_from_metadata
 
-EXCHANGE_NAME = "StackStorm-Exchange"
-EXCHANGE_PREFIX = "stackstorm"
+EXCHANGE_NAME = os.environ.get("PACKS_ORG", "StackStorm-Exchange")
+EXCHANGE_PREFIX = os.environ.get("PACKS_PREFIX", "stackstorm")
 
+if os.environ.get("CIRCLECI"):
+    CI = "CircleCI"
+elif os.environ.get("GITHUB_ACTIONS"):
+    CI = "GHA"
+else:
+    CI = "unknown"
+
+
+# TODO: drop GITHUB_USERNAME once we drop support for CircleCI
 GITHUB_USERNAME = os.environ.get('MACHINE_USER')
+# TODO: drop MACHINE_PASSWORD once we drop support for CircleCI. Keep GH_TOKEN.
 GITHUB_PASSWORD = os.environ.get("MACHINE_PASSWORD", os.environ.get("GH_TOKEN"))
+# TODO: drop ACTIVE_PACK_NAME once we drop support for CircleCI.
+#       Only used for CircleCI-specific error message.
 ACTIVE_PACK_NAME = os.environ.get('PACK_NAME', "unknown")
 
 SESSION = requests.Session()
@@ -63,7 +75,7 @@ def build_index(path_glob, output_path):
     counter = 0
     failed_count = 0
     for filename in generator:
-        with open(filename, 'r') as pack:
+        with open(filename, 'r', encoding="utf8") as pack:
             pack_meta = yaml.safe_load(pack)
 
         pack_name = pack_meta['name']
@@ -102,12 +114,13 @@ def build_index(path_glob, output_path):
     result['metadata']['hash'] = data_hash.hexdigest()
 
     output_path = os.path.expanduser(os.path.join(output_path, 'index.json'))
-    with open(output_path, 'w') as outfile:
+    with open(output_path, 'w', encoding="utf8") as outfile:
         json.dump(result, outfile, indent=4, sort_keys=True,
                   separators=(',', ': '))
 
     failed_message = ''
-    if failed_count > 0:
+    # TODO: drop CircleCI error message once we drop support for CircleCI
+    if failed_count > 0 and CI == "CircleCI":
         failed_message = (
             ', {failed_count} packs failed to update.\n'
             'The GitHub Personal Access Tokens for CircleCI for the pack may '
@@ -120,6 +133,14 @@ def build_index(path_glob, output_path):
             'will need to ask a member of the StackStorm TSC to update the Personal\n'
             'Access Token on your behalf.'
         ).format(failed_count=failed_count, exchange_name=EXCHANGE_NAME)
+    elif failed_count > 0:
+        # If an issue is reported on GitHub Actions, update this error message
+        # to explain common causes and how to fix them.
+        failed_message = (
+            f', {failed_count} packs failed to update.\n'
+            'Please investigate why this failed and report an issue on:\n'
+            f'  https://github.com/{EXCHANGE_NAME}/ci\n'
+        )
 
     print('')
     print('Processed %s packs%s.' % (counter, failed_message))
@@ -152,13 +173,20 @@ def get_available_versions():
         proc.kill()
         outs, _ = proc.communicate()
     result = outs.decode().strip()
-    if proc.returncode != 0:
+    # TODO: drop CircleCI error message once we drop support for CircleCI
+    if proc.returncode != 0 and CI == "CircleCI":
         sys.exit(
             "Error retrieving data with github graphql API.\n"
             "The GitHub PAT might need to be regenerated:\n"
             "https://github.com/settings/tokens/new?scopes=public_repo"
             "&description=CircleCI%3A%20stackstorm-" + ACTIVE_PACK_NAME
         )
+    elif proc.returncode != 0:
+        # If an issue is reported on GitHub Actions, update this error message
+        # to explain common causes and how to fix them.
+        sys.exit(
+            "Error retrieving data with github graphql API.\n"
+        )
 
     # https://stackoverflow.com/a/43807246/1134951
     decoder = json.JSONDecoder()
@@ -194,7 +222,12 @@ def get_available_versions_for_pack(pack_ref):
 
     NOTE: This function uses Github API.
     """
+    # TODO: remove this if block once we discontinue CircleCI support. Keep the else block.
     if pack_ref not in PACK_VERSIONS:
+        # This will fail in GitHub Actions because the GITHUB_TOKEN there must be
+        # provided as bearer auth instead of basic auth. But using graphql is better
+        # anyway, so this is only needed as a backup on CircleCI if graphql doesn't work.
+        # graphql should always work on GHA.
         url = ('https://api.github.com/repos/%s/%s-%s/tags' %
                (EXCHANGE_NAME, EXCHANGE_PREFIX, pack_ref))
         resp = SESSION.get(url)

.circle/semver.py

@@ -16,8 +16,7 @@
 
 import sys
 import re
-
-import validate
+import yaml
 
 SEMVER_REGEX = re.compile(r"""^(?:0|[1-9]\d*)
                               \.
@@ -31,6 +30,13 @@
 DOUBLE_VERSION_REGEX = re.compile(r"^\d+\.\d+$")
 
 
+def load_yaml_file(path):
+    with open(path, 'r', encoding="utf8") as stream:
+        text = yaml.safe_load(stream)
+
+    return text
+
+
 def get_semver_string(version):
     if SINGLE_VERSION_REGEX.match(str(version)):
         semver = "%s.0.0" % version
@@ -44,5 +50,5 @@ def get_semver_string(version):
 
 
 if __name__ == '__main__':
-    pack = validate.load_yaml_file(sys.argv[1])
+    pack = load_yaml_file(sys.argv[1])
     print(get_semver_string(pack['version']))

.circle/validate.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 from __future__ import print_function
 
+import os
 import sys
 
 import yaml
@@ -22,18 +23,21 @@
 from st2common.util import schema as util_schema
 from st2common.util.pack import get_pack_ref_from_metadata
 
-PREFIX = 'stackstorm'
+PREFIX = os.environ.get("PACKS_PREFIX", "stackstorm")
 PACK_SCHEMA = PackAPI.schema
 
 
 def load_yaml_file(path):
-    with open(path, 'r') as stream:
+    with open(path, 'r', encoding="utf8") as stream:
         text = yaml.safe_load(stream)
 
     return text
 
 
 def validate_schema(instance, schema):
+    # validate() returns a cleaned instance with default values assigned.
+    # and it calls jsonschema.validate(instance, schema) so this will
+    # raise ValidationError if instance is not valid according to schema
     return util_schema.validate(instance=instance, schema=schema,
                                 cls=util_schema.CustomValidator,
                                 use_default=True,
@@ -51,12 +55,22 @@ def validate_repo_name(instance, repo_name):
 
 
 if __name__ == '__main__':
+    # If an exception is raised, python basically does sys.exit(1)
+    # Without an exception, the return code is 0.
+
     repo_name = sys.argv[1]
+    # raises if yaml is invalid
     pack_meta = load_yaml_file(sys.argv[2])
 
     # TODO: Figure out why this wasn't previously executed, and execute it
+    #       stackstorm-test-content-version repo is test_content_version pack
+    #       stackstorm-test2 repo is test pack
     # validate_repo_name(pack_meta, repo_name)
-    validate_schema(pack_meta, PACK_SCHEMA)
+
+    # raises ValidationError if pack_meta doesn't validate against PACK_SCHEMA
+    cleaned_pack_meta = validate_schema(pack_meta, PACK_SCHEMA)
+
+    # raises ValueError if pack ref not defined and pack name is not a valid ref
     pack_ref = validate_pack_contains_valid_ref_or_name(pack_meta)
 
     print(pack_ref)

.github/actions/apt-dependencies/action.yaml

@@ -0,0 +1,72 @@
+---
+name: Install APT Dependencies
+description: |
+  Install debian dependencies required for StackStorm-Exchange pack tests.
+  Before using this, make sure to run
+  StackStorm-Exchange/ci/.github/actions/checkout.
+author: StackStorm
+
+inputs:
+  cache-version:
+    required: false
+    default: "v0"
+  extra-apt-packages-file:
+    required: false
+    default: pack/.github/apt-packages.txt
+
+runs:
+  using: "composite"
+  steps:
+
+    # TODO: not working on GHA. Need to revisit caching.
+    #- name: Create a directory for debian packages so we can cache it
+    #  # this is what we did on CircleCI. Not sure if it'll work on GHA
+    #  shell: bash
+    #  run: >
+    #    sudo rm -rf /var/cache/apt/archives /var/lib/apt/lists
+    #    && sudo ln -s ~/apt_cache/archives /var/cache/apt/archives
+    #    && sudo ln -s ~/apt_cache/lists /var/lib/apt/lists
+    #    && mkdir -p ~/apt_cache/archives/partial ~/apt_cache/lists
+
+    # hashFiles only reads files relative to GITHUB_WORKSPACE
+    - name: Construct apt-packages.txt
+      shell: bash
+      run: |
+        cp ${{ github.action_path }}/apt-packages.txt ${GITHUB_WORKSPACE}/
+        if [[ -f ${{ inputs.extra-apt-packages-file }} ]]; then
+          cat ${{ inputs.extra-apt-packages-file }} >> ${GITHUB_WORKSPACE}/apt-packages.txt
+        fi
+
+    #- name: Cache APT Dependencies
+    #  id: cache-apt-deps
+    #  uses: actions/cache@v2
+    #  with:
+    #    path: |
+    #      ~/apt_cache/*.deb
+    #    key: ${{ inputs.cache-version }}-apt-archives-and-lists-${{ hashFiles('apt-packages.txt') }}
+    #    restore-keys: |
+    #      ${{ inputs.cache-version }}-apt-archives-and-lists-
+
+    - name: Install APT Dependencies
+      shell: bash
+      env:
+    #    CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
+        CACHE_HIT: "false"
+        APT_PACKAGES_FILE_PATH: apt-packages.txt
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        echo "::group::Install APT Dependencies"
+        if [[ "${CACHE_HIT}" != 'true' ]]; then
+          sudo apt-get -o=Dpkg::Use-Pty=0 -yq update
+        fi
+        APT_PACKAGES=$(grep -v '^#' "${APT_PACKAGES_FILE_PATH}" | xargs echo -n)
+        sudo apt-get -o=Dpkg::Use-Pty=0 -yq install ${APT_PACKAGES}
+        echo "::endgroup::"
+
+    - name: Print versions
+      shell: bash
+      run: |
+        echo "::group::Print Versions"
+        jq --version
+        gh --version
+        echo "::endgroup::"

.github/actions/apt-dependencies/apt-packages.txt

@@ -0,0 +1,10 @@
+# pre-installed on GHA
+#jq
+#gh # github cli
+#imagemagick
+
+# for st2 pip build
+libldap2-dev
+libsasl2-dev
+# st2 also installs
+#libssl-dev libyaml-dev ldap-utils

.github/actions/apt-dependencies/index-apt-packages.txt

@@ -0,0 +1,5 @@
+# In CircleCI, we couldn't directly install imagemagick, so
+# we used a workaround from https://discuss.circleci.com/t/error-installing-imagemagick/2963
+# TODO: determine if we should go back to using imagemagick instead of gmic and optipng
+gmic
+optipng