From b926795973906d77918843bf5f750f5c92315204 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 24 Sep 2024 18:04:18 +0100 Subject: [PATCH 1/5] ci: Move container registry authentication to an Azure Devops step This aims to - Reduce duplication - Seperate CI specific setup from test setup - Prepare for migration from Azure DevOps to GitHub Actions --- .ci/ansible_install.py | 11 ----------- .ci/azure-pipelines-steps.yml | 17 +++++++++++++---- .ci/debops_common_install.py | 3 --- .ci/localhost_ansible_install.py | 8 -------- .ci/mitogen_install.py | 14 -------------- .ci/mitogen_py24_install.py | 3 --- tox.ini | 6 ------ 7 files changed, 13 insertions(+), 49 deletions(-) delete mode 100755 .ci/ansible_install.py delete mode 100755 .ci/localhost_ansible_install.py delete mode 100755 .ci/mitogen_install.py diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py deleted file mode 100755 index 3b217ff2e..000000000 --- a/.ci/ansible_install.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ] -] - -ci_lib.run_batches(batches) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 919b992b1..791372af9 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -14,6 +14,19 @@ steps: versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') +- script: | + set -o errexit + set -o nounset + set -o pipefail + + aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws + displayName: Authenticate to container registry + condition: eq(variables['Agent.OS'], 'Linux') + env: + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) + - script: | set -o errexit set -o nounset @@ -90,7 +103,3 @@ steps: "$PYTHON" -m tox -e "$(tox.env)" displayName: "Run tests" - env: - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 565f94888..13217133e 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -9,9 +9,6 @@ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', ], - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ], ]) ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2') diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py deleted file mode 100755 index d08ddafc8..000000000 --- a/.ci/localhost_ansible_install.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ -] - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py deleted file mode 100755 index 23ff384b5..000000000 --- a/.ci/mitogen_install.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ -] - -if ci_lib.have_docker(): - batches.append([ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ]) - - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index 8af904059..85ea013c2 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -3,9 +3,6 @@ import ci_lib batches = [ - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ], [ 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', ] diff --git a/tox.ini b/tox.ini index 9bb82defb..9820a79ff 100644 --- a/tox.ini +++ b/tox.ini @@ -89,10 +89,7 @@ deps = install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = - mode_ansible: {toxinidir}/.ci/ansible_install.py mode_debops_common: {toxinidir}/.ci/debops_common_install.py - mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py - mode_mitogen: {toxinidir}/.ci/mitogen_install.py commands = mode_ansible: {toxinidir}/.ci/ansible_tests.py mode_debops_common: {toxinidir}/.ci/debops_common_tests.py @@ -100,9 +97,6 @@ commands = mode_mitogen: {toxinidir}/.ci/mitogen_tests.py passenv = ANSIBLE_* - AWS_ACCESS_KEY_ID - AWS_DEFAULT_REGION - AWS_SECRET_ACCESS_KEY HOME # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables From 8b92e09655e31f0404b302e001c1f4f024616c57 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 25 Sep 2024 12:19:39 +0100 Subject: [PATCH 2/5] ci: Extract container registry location into variables Preperation for migrating from Azure DevOps with Amazon Elastic Container Registry (AWS ECR), to GitHub Actions with GitHub Container Registry (GHCR). DebOps tests are not currently being run, the updates to .ci/debops*.py are best effort only. --- .ci/README.md | 1 - .ci/ansible_tests.py | 2 +- .ci/ci_lib.py | 48 ++++++++++++++---------- .ci/debops_common_install.py | 3 -- .ci/debops_common_tests.py | 9 +++-- tests/image_prep/_container_finalize.yml | 2 +- tests/image_prep/group_vars/all.yml | 3 ++ tests/testlib.py | 30 +++++++++++---- tox.ini | 1 + 9 files changed, 62 insertions(+), 37 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 67a3805b8..9248ac588 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` * `DISTROS`: the `ansible_` tests can run against multiple targets diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 102eda9c8..3ec48dfde 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ def pause_if_interactive(): with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers() + containers = ci_lib.container_specs(ci_lib.DISTROS) ci_lib.start_containers(containers) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 3e716385d..dfe49b97f 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -27,6 +27,13 @@ ) ) + +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + + _print = print def print(*args, **kwargs): file = kwargs.get('file', sys.stdout) @@ -193,8 +200,6 @@ def __exit__(self, _1, _2, _3): pass DISTRO = os.environ.get('DISTRO', 'debian9') # Used only when MODE=ansible DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() -TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) -BASE_PORT = 2200 TMP = TempDir().path @@ -217,6 +222,7 @@ def __exit__(self, _1, _2, _3): pass def get_docker_hostname(): """Return the hostname where the docker daemon is running. """ + # Duplicated in testlib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -225,27 +231,34 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] -def make_containers(name_prefix='', port_offset=0): +def container_specs( + distros, + base_port=2200, + image_template=IMAGE_TEMPLATE, + name_template='target-%(distro)s-%(index)d', +): """ >>> import pprint - >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6'] - >>> pprint.pprint(make_containers()) + >>> pprint.pprint(container_specs(['debian11-py3', 'centos6'])) [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'index': 1, 'name': 'target-debian11-1', 'port': 2201, - 'python_path': '/usr/bin/python'}, + 'python_path': '/usr/bin/python3'}, {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'index': 2, 'name': 'target-centos6-2', 'port': 2202, 'python_path': '/usr/bin/python'}] """ docker_hostname = get_docker_hostname() + # Code duplicated in testlib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) (?:-(?Ppy3))? @@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0): i = 1 lst = [] - for distro in DISTROS: + for distro in distros: + # Code duplicated in testlib.py, both should be updated together d = distro_pattern.match(distro).groupdict(default=None) - distro = d['distro'] - family = d['family'] - image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - if d['py'] == 'py3': + if d.pop('py') == 'py3': python_path = '/usr/bin/python3' else: python_path = '/usr/bin/python' - if d['count']: - count = int(count) - else: - count = 1 + count = int(d.pop('count') or '1', 10) for x in range(count): - lst.append({ - "distro": distro, "family": family, "image": image, - "name": name_prefix + ("target-%s-%s" % (distro, i)), + d['index'] = i + d.update({ + 'image': image_template % d, + 'name': name_template % d, "hostname": docker_hostname, - "port": BASE_PORT + i + port_offset, + 'port': base_port + i, "python_path": python_path, }) + lst.append(d) i += 1 return lst diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 13217133e..825126c7d 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -2,9 +2,6 @@ import ci_lib -# Naturally DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] - ci_lib.run_batches([ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 7db8a7976..b065486f8 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -6,9 +6,6 @@ import ci_lib -# DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT - project_dir = os.path.join(ci_lib.TMP, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' @@ -16,7 +13,11 @@ with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-') + containers = ci_lib.container_specs( + ['debian*2'], + base_port=2700, + name_template='debops-target-%(distro)s-%(index)d', + ) ci_lib.start_containers(containers) diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml index d61d9b3bf..5329fefa2 100644 --- a/tests/image_prep/_container_finalize.yml +++ b/tests/image_prep/_container_finalize.yml @@ -9,7 +9,7 @@ --change 'EXPOSE 22' --change 'CMD ["/usr/sbin/sshd", "-D"]' {{ inventory_hostname }} - public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test + {{ container_image_name }} delegate_to: localhost - name: Stop containers diff --git a/tests/image_prep/group_vars/all.yml b/tests/image_prep/group_vars/all.yml index 5f182f861..91ff934df 100644 --- a/tests/image_prep/group_vars/all.yml +++ b/tests/image_prep/group_vars/all.yml @@ -4,6 +4,9 @@ common_packages: - strace - sudo +container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test" +container_registry: public.ecr.aws/n5z0e8q9 + sudo_group: MacOSX: admin Debian: sudo diff --git a/tests/testlib.py b/tests/testlib.py index a52292ce6..76743e823 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,6 +51,12 @@ LOG = logging.getLogger(__name__) +DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + TESTS_DIR = os.path.join(os.path.dirname(__file__)) ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') @@ -509,6 +515,7 @@ def assertRaises(self, exc, func, *args, **kwargs): def get_docker_host(): + # Duplicated in ci_lib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -549,19 +556,23 @@ def start_container(self): ] subprocess.check_output(args) - def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): - if '-' in mitogen_test_distro: - distro, _py3 = mitogen_test_distro.split('-') - else: - distro = mitogen_test_distro - _py3 = None + def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + # Code duplicated in ci_lib.py, both should be updated together + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) + d = distro_pattern.match(distro).groupdict(default=None) - if _py3 == 'py3': + if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' else: self.python_path = '/usr/bin/python' - self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) + self.image = image_template % d self.start_container() self.host = self.get_host() self.port = self.get_port(self.container_name) @@ -601,6 +612,9 @@ def close(self): class BrokerMixin(object): broker_class = mitogen.master.Broker + + # Flag for tests that shutdown the broker themself + # e.g. unix_test.ListenerTest broker_shutdown = False def setUp(self): diff --git a/tox.ini b/tox.ini index 9820a79ff..9fb31bdcb 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,7 @@ commands = passenv = ANSIBLE_* HOME + MITOGEN_* # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables TF_BUILD From c6c8bfb690b8423b140917c9f5e9dcca72f17d30 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 25 Sep 2024 13:30:22 +0100 Subject: [PATCH 3/5] tests: Skip vanilla Ansible on Linux unpriviliged -> unprivileged become CI containers lack the necessary `setfacl` command. This has not previously been noticed because no vanilla Ansible jobs were being run on Linux, only on macOS. refs #1118 --- .../integration/become/su_password.yml | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index 52d420dbf..207980c49 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -53,20 +53,22 @@ vars: ansible_become_pass: user1_password when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - name: Ensure password su without chdir succeeds shell: whoami @@ -76,20 +78,22 @@ vars: ansible_become_pass: user1_password when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) tags: - su From 27214517a724431fb5dc73255b608bc4fe078479 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 26 Sep 2024 02:08:17 +0100 Subject: [PATCH 4/5] tests: Use a subprocess to check discovered python == running This replaces the use of `os.path.realpath()` which gave incorrect results on macOS - depending on the exact Python build, Python version, macOS version, installation method, and phase of the moon. realpath information kept around to aid debugging. --- .../ansible_2_8_tests.yml | 2 +- tests/ansible/lib/modules/test_echo_module.py | 112 +++++++++++++++++- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 0c7d30c9d..fc0d3cf78 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -99,7 +99,7 @@ that: - auto_out.ansible_facts.discovered_interpreter_python is defined - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen - - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved + - echoout.discovered_python.sys.executable.as_seen == echoout.running_python.sys.executable.as_seen fail_msg: - "auto_out: {{ auto_out }}" - "echoout: {{ echoout }}" diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index d6a5fb9e1..fe8ed69a9 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -10,11 +10,97 @@ __metaclass__ = type import os +import stat import platform +import subprocess import sys + from ansible.module_utils.basic import AnsibleModule +# trace_realpath() and _join_tracepath() adapated from stdlib posixpath.py +# https://github.com/python/cpython/blob/v3.12.6/Lib/posixpath.py#L423-L492 +# Copyright (c) 2001 - 2023 Python Software Foundation +# Copyright (c) 2024 Alex Willmer +# License: Python Software Foundation License Version 2 + +def trace_realpath(filename, strict=False): + """ + Return the canonical path of the specified filename, and a trace of + the route taken, eliminating any symbolic links encountered in the path. + """ + path, trace, ok = _join_tracepath(filename[:0], filename, strict, seen={}, trace=[]) + return os.path.abspath(path), trace + + +def _join_tracepath(path, rest, strict, seen, trace): + """ + Join two paths, normalizing and eliminating any symbolic links encountered + in the second path. + """ + trace.append(rest) + if isinstance(path, bytes): + sep = b'/' + curdir = b'.' + pardir = b'..' + else: + sep = '/' + curdir = '.' + pardir = '..' + + if os.path.isabs(rest): + rest = rest[1:] + path = sep + + while rest: + name, _, rest = rest.partition(sep) + if not name or name == curdir: + # current dir + continue + if name == pardir: + # parent dir + if path: + path, name = os.path.split(path) + if name == pardir: + path = os.path.join(path, pardir, pardir) + else: + path = pardir + continue + newpath = os.path.join(path, name) + try: + st = os.lstat(newpath) + except OSError: + if strict: + raise + is_link = False + else: + is_link = stat.S_ISLNK(st.st_mode) + if not is_link: + path = newpath + continue + # Resolve the symbolic link + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + else: + # Return already resolved part + rest of the path unchanged. + return os.path.join(newpath, rest), trace, False + seen[newpath] = None # not resolved symlink + path, trace, ok = _join_tracepath(path, os.readlink(newpath), strict, seen, trace) + if not ok: + return os.path.join(path, rest), False + seen[newpath] = path # resolved symlink + + return path, trace, True + + def main(): module = AnsibleModule(argument_spec=dict( facts_copy=dict(type=dict, default={}), @@ -33,7 +119,18 @@ def main(): sys.executable = "/usr/bin/python" facts_copy = module.params['facts_copy'] + discovered_interpreter_python = facts_copy['discovered_interpreter_python'] + d_i_p_realpath, d_i_p_trace = trace_realpath(discovered_interpreter_python) + d_i_p_proc = subprocess.Popen( + [discovered_interpreter_python, '-c', 'import sys; print(sys.executable)'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + + ) + d_i_p_stdout, d_i_p_stderr = d_i_p_proc.communicate() + + sys_exec_realpath, sys_exec_trace = trace_realpath(sys.executable) + result = { 'changed': False, 'ansible_facts': module.params['facts_to_override'], @@ -43,7 +140,17 @@ def main(): ), 'discovered_python': { 'as_seen': discovered_interpreter_python, - 'resolved': os.path.realpath(discovered_interpreter_python), + 'resolved': d_i_p_realpath, + 'trace': [os.path.abspath(p) for p in d_i_p_trace], + 'sys': { + 'executable': { + 'as_seen': d_i_p_stdout.decode('ascii').rstrip('\n'), + 'proc': { + 'stderr': d_i_p_stderr.decode('ascii'), + 'returncode': d_i_p_proc.returncode, + }, + }, + }, }, 'running_python': { 'platform': { @@ -54,7 +161,8 @@ def main(): 'sys': { 'executable': { 'as_seen': sys.executable, - 'resolved': os.path.realpath(sys.executable), + 'resolved': sys_exec_realpath, + 'trace': [os.path.abspath(p) for p in sys_exec_trace], }, 'platform': sys.platform, 'version_info': { From 4f60d01f09f2ae31c8f58cddafb17766e1054fa4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 26 Sep 2024 02:15:39 +0100 Subject: [PATCH 5/5] CI: Enable GitHub Actions testing workflow This replicate the existing Azure DevOps workflow, and adds a couple of new jobs (Python 2.7 on macOS, Python + vanilla Ansible on Linux). The GitHub Actions use container images hosted on GitHub Container Registry (GHCR, ghcr.io/mitogen-hq). These images have been copied straight from the existing Amazon Elastic Cloud Registry (AWS ECR, public.ecr.aws/n5z0e8q9). A short period of parallel running is planned. Then a second PR will remove the Azure DevOps workflow. --- .github/workflows/tests.yml | 326 ++++++++++++++++++++++++++++++++++++ docs/changelog.rst | 1 + 2 files changed, 327 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..4520c3cfb --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,326 @@ +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions + +name: Tests + +on: + pull_request: + push: + branches-ignore: + - docs-master + +env: + #ANSIBLE_VERBOSITY: 3 + #MITOGEN_LOG_LEVEL: DEBUG + MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test" + +# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners +# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support +jobs: + linux: + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + include: + - name: Ans_27_210 + tox_env: py27-mode_ansible-ansible2.10 + - name: Ans_27_4 + tox_env: py27-mode_ansible-ansible4 + + - name: Ans_36_210 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible2.10 + - name: Ans_36_4 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible4 + + - name: Ans_311_210 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible2.10 + - name: Ans_311_3 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible3 + - name: Ans_311_4 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible4 + - name: Ans_311_5 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible5 + - name: Ans_312_6 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible6 + - name: Ans_312_7 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible7 + - name: Ans_312_8 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible8 + - name: Ans_312_9 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible9 + - name: Ans_312_10 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible10 + - name: Van_312_10 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible10-strategy_linear + + - name: Mito_27_centos6 + tox_env: py27-mode_mitogen-distro_centos6 + - name: Mito_27_centos7 + tox_env: py27-mode_mitogen-distro_centos7 + - name: Mito_27_centos8 + tox_env: py27-mode_mitogen-distro_centos8 + - name: Mito_27_debian9 + tox_env: py27-mode_mitogen-distro_debian9 + - name: Mito_27_debian10 + tox_env: py27-mode_mitogen-distro_debian10 + - name: Mito_27_debian11 + tox_env: py27-mode_mitogen-distro_debian11 + - name: Mito_27_ubuntu1604 + tox_env: py27-mode_mitogen-distro_ubuntu1604 + - name: Mito_27_ubuntu1804 + tox_env: py27-mode_mitogen-distro_ubuntu1804 + - name: Mito_27_ubuntu2004 + tox_env: py27-mode_mitogen-distro_ubuntu2004 + + - name: Mito_36_centos6 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos6 + - name: Mito_36_centos7 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos7 + - name: Mito_36_centos8 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos8 + - name: Mito_36_debian9 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian9 + - name: Mito_36_debian10 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian10 + - name: Mito_36_debian11 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian11 + - name: Mito_36_ubuntu1604 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu1604 + - name: Mito_36_ubuntu1804 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu1804 + - name: Mito_36_ubuntu2004 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu2004 + + - name: Mito_312_centos6 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos6 + - name: Mito_312_centos7 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos7 + - name: Mito_312_centos8 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos8 + - name: Mito_312_debian9 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian9 + - name: Mito_312_debian10 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian10 + - name: Mito_312_debian11 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian11 + - name: Mito_312_ubuntu1604 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu1604 + - name: Mito_312_ubuntu1804 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu1804 + - name: Mito_312_ubuntu2004 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu2004 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install build deps + run: | + set -o errexit -o nounset -o pipefail + + sudo apt-get update + sudo apt-get install -y python2-dev python3-pip virtualenv + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + + macos: + # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + runs-on: macos-12 + timeout-minutes: 120 + + strategy: + fail-fast: false + matrix: + include: + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_312 + tox_env: py312-mode_mitogen + + - name: Loc_27_210 + tox_env: py27-mode_localhost-ansible2.10 + - name: Loc_312_10 + tox_env: py312-mode_localhost-ansible10 + + - name: Van_27_210 + tox_env: py27-mode_localhost-ansible2.10-strategy_linear + - name: Van_312_10 + tox_env: py312-mode_localhost-ansible10-strategy_linear + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 ]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + echo "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7: sys.executable: $(/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" diff --git a/docs/changelog.rst b/docs/changelog.rst index 66a8077e5..3adaec58c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,7 @@ Unreleased releases before 2.10 * :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3 compatibility +* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions v0.3.10 (2024-09-20)