diff --git a/.ci/README.md b/.ci/README.md index 9248ac588..9a5e8898e 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -2,7 +2,7 @@ # `.ci` This directory contains scripts for Continuous Integration platforms. Currently -Azure Pipelines, but they will also happily run on any Debian-like machine. +GitHub Actions, but ideally they will also run on any Debian-like machine. The scripts are usually split into `_install` and `_test` steps. The `_install` step will damage your machine, the `_test` step will just run the tests the way diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml deleted file mode 100644 index 791372af9..000000000 --- a/.ci/azure-pipelines-steps.yml +++ /dev/null @@ -1,105 +0,0 @@ -# Each step entry runs a task (Azure Pipelines analog of an Ansible module). -# https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool - -# `{script: ...}` is shorthand for `{task: CmdLine@, inputs: {script: ...}}`. -# The shell is bash. -# https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines -# https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines - -steps: -- task: UsePythonVersion@0 - displayName: Install python - inputs: - githubToken: '$(GITHUB_PYVER_TOKEN)' - versionSpec: '$(python.version)' - condition: ne(variables['python.version'], '') - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws - displayName: Authenticate to container registry - condition: eq(variables['Agent.OS'], 'Linux') - env: - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - sudo apt-get update - sudo apt-get install -y python2-dev python3-pip virtualenv - displayName: Install build deps - condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - # macOS builders lack a realpath command - type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version - type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version - type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version - echo - - if [ -e /usr/bin/python ]; then - echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" - fi - - if [ -e /usr/bin/python2 ]; then - echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" - fi - - if [ -e /usr/bin/python2.7 ]; then - echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" - fi - displayName: Show python versions - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) - PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') - - if [[ -z $PYTHON ]]; then - echo 1>&2 "Python interpreter could not be determined" - exit 1 - fi - - if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then - "$PYTHON" -m ensurepip --user --altinstall --no-default-pip - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - elif [[ $PYTHON == "python2.7" ]]; then - curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" - "$PYTHON" get-pip.py --user --no-python-version-warning - # Avoid Python 2.x pip masking system pip - rm -f ~/.local/bin/{easy_install,pip,wheel} - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - else - "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - fi - displayName: Install tooling - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) - PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') - - if [[ -z $PYTHON ]]; then - echo 1>&2 "Python interpreter could not be determined" - exit 1 - fi - - "$PYTHON" -m tox -e "$(tox.env)" - displayName: "Run tests" diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml deleted file mode 100644 index 0bf4556bf..000000000 --- a/.ci/azure-pipelines.yml +++ /dev/null @@ -1,157 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -# User defined variables are also injected as environment variables -# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#environment-variables -#variables: - #ANSIBLE_VERBOSITY: 3 - -trigger: - branches: - include: - - "*" - exclude: - - docs-master - -jobs: -- job: mac12 - # vanilla Ansible is really slow - timeoutInMinutes: 120 - steps: - - template: azure-pipelines-steps.yml - pool: - # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md - vmImage: macOS-12 - strategy: - matrix: - Mito_312: - tox.env: py312-mode_mitogen - Loc_312_10: - tox.env: py312-mode_localhost-ansible10 - Van_312_10: - tox.env: py312-mode_localhost-ansible10-strategy_linear - -- job: Linux - pool: - # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md - vmImage: ubuntu-20.04 - steps: - - template: azure-pipelines-steps.yml - strategy: - matrix: - Mito_27_centos6: - tox.env: py27-mode_mitogen-distro_centos6 - Mito_27_centos7: - tox.env: py27-mode_mitogen-distro_centos7 - Mito_27_centos8: - tox.env: py27-mode_mitogen-distro_centos8 - Mito_27_debian9: - tox.env: py27-mode_mitogen-distro_debian9 - Mito_27_debian10: - tox.env: py27-mode_mitogen-distro_debian10 - Mito_27_debian11: - tox.env: py27-mode_mitogen-distro_debian11 - Mito_27_ubuntu1604: - tox.env: py27-mode_mitogen-distro_ubuntu1604 - Mito_27_ubuntu1804: - tox.env: py27-mode_mitogen-distro_ubuntu1804 - Mito_27_ubuntu2004: - tox.env: py27-mode_mitogen-distro_ubuntu2004 - - Mito_36_centos6: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_centos6 - Mito_36_centos7: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_centos7 - Mito_36_centos8: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_centos8 - Mito_36_debian9: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_debian9 - Mito_36_debian10: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_debian10 - Mito_36_debian11: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_debian11 - Mito_36_ubuntu1604: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_ubuntu1604 - Mito_36_ubuntu1804: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_ubuntu1804 - Mito_36_ubuntu2004: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_ubuntu2004 - - Mito_312_centos6: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_centos6 - Mito_312_centos7: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_centos7 - Mito_312_centos8: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_centos8 - Mito_312_debian9: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_debian9 - Mito_312_debian10: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_debian10 - Mito_312_debian11: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_debian11 - Mito_312_ubuntu1604: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_ubuntu1604 - Mito_312_ubuntu1804: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_ubuntu1804 - Mito_312_ubuntu2004: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_ubuntu2004 - - Ans_27_210: - tox.env: py27-mode_ansible-ansible2.10 - Ans_27_4: - tox.env: py27-mode_ansible-ansible4 - - Ans_36_210: - python.version: '3.6' - tox.env: py36-mode_ansible-ansible2.10 - Ans_36_4: - python.version: '3.6' - tox.env: py36-mode_ansible-ansible4 - - Ans_311_210: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible2.10 - Ans_311_3: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible3 - Ans_311_4: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible4 - Ans_311_5: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible5 - Ans_312_6: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible6 - Ans_312_7: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible7 - Ans_312_8: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible8 - Ans_312_9: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible9 - Ans_312_10: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible10 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2c819e7d9..f3f31d82f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,24 +48,24 @@ jobs: - name: Ans_311_5 python_version: '3.11' tox_env: py311-mode_ansible-ansible5 - - name: Ans_312_6 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible6 - - name: Ans_312_7 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible7 - - name: Ans_312_8 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible8 - - name: Ans_312_9 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible9 - - name: Ans_312_10 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible10 - - name: Van_312_10 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible10-strategy_linear + - name: Ans_313_6 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible6 + - name: Ans_313_7 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible7 + - name: Ans_313_8 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible8 + - name: Ans_313_9 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible9 + - name: Ans_313_10 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible10 + - name: Van_313_10 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible10-strategy_linear - name: Mito_27_centos6 tox_env: py27-mode_mitogen-distro_centos6 @@ -114,33 +114,33 @@ jobs: python_version: '3.6' tox_env: py36-mode_mitogen-distro_ubuntu2004 - - name: Mito_312_centos6 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos6 - - name: Mito_312_centos7 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos7 - - name: Mito_312_centos8 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos8 - - name: Mito_312_debian9 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian9 - - name: Mito_312_debian10 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian10 - - name: Mito_312_debian11 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian11 - - name: Mito_312_ubuntu1604 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu1604 - - name: Mito_312_ubuntu1804 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu1804 - - name: Mito_312_ubuntu2004 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu2004 + - name: Mito_313_centos6 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos6 + - name: Mito_313_centos7 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos7 + - name: Mito_313_centos8 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos8 + - name: Mito_313_debian9 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian9 + - name: Mito_313_debian10 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian10 + - name: Mito_313_debian11 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian11 + - name: Mito_313_ubuntu1604 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu1604 + - name: Mito_313_ubuntu1804 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu1804 + - name: Mito_313_ubuntu2004 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu2004 steps: - uses: actions/checkout@v4 @@ -232,18 +232,21 @@ jobs: include: - name: Mito_27 tox_env: py27-mode_mitogen - - name: Mito_312 - tox_env: py312-mode_mitogen + - name: Mito_313 + python_version: '3.13' + tox_env: py313-mode_mitogen - name: Loc_27_210 tox_env: py27-mode_localhost-ansible2.10 - - name: Loc_312_10 - tox_env: py312-mode_localhost-ansible10 + - name: Loc_313_10 + python_version: '3.13' + tox_env: py313-mode_localhost-ansible10 - name: Van_27_210 tox_env: py27-mode_localhost-ansible2.10-strategy_linear - - name: Van_312_10 - tox_env: py312-mode_localhost-ansible10-strategy_linear + - name: Van_313_10 + python_version: '3.13' + tox_env: py313-mode_localhost-ansible10-strategy_linear steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index 0d4d1b30b..c3cd9a87d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # Mitogen +[![PyPI - Version](https://img.shields.io/pypi/v/mitogen)](https://pypi.org/project/mitogen/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mitogen)](https://pypi.org/project/mitogen/) +[![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster) + Please see the documentation. ![](https://i.imgur.com/eBM6LhJ.gif) [![Total alerts](https://img.shields.io/lgtm/alerts/g/mitogen-hq/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/mitogen-hq/mitogen/alerts/) - -[![Build Status](https://dev.azure.com/mitogen-hq/mitogen/_apis/build/status/mitogen-hq.mitogen?branchName=master)](https://dev.azure.com/mitogen-hq/mitogen/_build/latest?definitionId=1&branchName=master) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a3f66eacc..a715b2b0b 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -158,6 +158,7 @@ def _connect_ssh(spec): } } + def _connect_buildah(spec): """ Return ContextService arguments for a Buildah connection. @@ -173,6 +174,7 @@ def _connect_buildah(spec): } } + def _connect_docker(spec): """ Return ContextService arguments for a Docker connection. @@ -276,6 +278,7 @@ def _connect_podman(spec): } } + def _connect_setns(spec, kind=None): """ Return ContextService arguments for a mitogen_setns connection. diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 38f351eda..1b6512e85 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -36,8 +36,6 @@ import traceback import ansible -import ansible.constants -import ansible.plugins import ansible.plugins.action import ansible.utils.unsafe_proxy import ansible.vars.clean diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index a33363655..a9f672099 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -64,6 +64,7 @@ import abc import logging import os + import ansible.utils.shlex import ansible.constants as C import ansible.executor.interpreter_discovery @@ -88,12 +89,12 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth # keep trying different interpreters until we don't error if action._finding_python_interpreter: return action._possible_python_interpreter - + if s in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']: # python is the only supported interpreter_name as of Ansible 2.8.8 interpreter_name = 'python' discovered_interpreter_config = u'discovered_interpreter_%s' % interpreter_name - + if task_vars.get('ansible_facts') is None: task_vars['ansible_facts'] = {} @@ -134,7 +135,7 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth def parse_python_path(s, task_vars, action, rediscover_python): """ Given the string set for ansible_python_interpeter, parse it using shell - syntax and return an appropriate argument vector. If the value detected is + syntax and return an appropriate argument vector. If the value detected is one of interpreter discovery then run that first. Caches python interpreter discovery value in `facts_from_task_vars` like how Ansible handles this. """ @@ -433,7 +434,7 @@ def remote_addr(self): return self._play_context.remote_addr def remote_user(self): - return self._play_context.remote_user + return self._connection_option('remote_user') def become(self): return self._play_context.become @@ -463,7 +464,7 @@ def password(self): return optional_secret(self._connection_option('password')) def port(self): - return self._play_context.port + return self._connection_option('port') def python_path(self, rediscover_python=False): s = self._connection.get_task_var('ansible_python_interpreter') diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index d818edd7c..73ebba4dd 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -132,13 +132,13 @@ Noteworthy Differences | 5 | 3.8 - 3.11 | +-----------------+-----------------+ | 6 | | - +-----------------+ 3.8 - 3.12 | + +-----------------+ 3.8 - 3.13 | | 7 | | +-----------------+-----------------+ - | 8 | 3.9 - 3.12 | + | 8 | 3.9 - 3.13 | +-----------------+-----------------+ | 9 | | - +-----------------+ 3.10 - 3.12 | + +-----------------+ 3.10 - 3.13 | | 10 | | +-----------------+-----------------+ @@ -1275,7 +1275,7 @@ on each process whose name begins with ``mitogen:``:: [pid 29858] futex(0x55ea9be52f60, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, 0xffffffff ^C - $ + $ This shows one thread waiting on IO (``poll``) and two more waiting on the same lock. It is taken from a real example of a deadlock due to a forking bug. diff --git a/docs/changelog.rst b/docs/changelog.rst index dabf5f7dd..ea45a7350 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,7 +18,18 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -v0.3.11 (2024-10-07) +v0.3.13 (2024-10-09) +-------------------- + +* :gh:issue:`1138` CI: Complete migration from Azure DevOps Pipelines to + GitHub Actions +* :gh:issue:`1116` :mod:`ansible_mitogen`: Support for templated variable + `ansible_ssh_user`. +* :gh:issue:`978` :mod:`ansible_mitogen`: Support templated Ansible SSH port. +* :gh:issue:`1073` Python 3.13 support + + +v0.3.12 (2024-10-07) -------------------- * :gh:issue:`1106` :mod:`ansible_mitogen`: Support for `ansible_ssh_password` @@ -112,7 +123,7 @@ v0.3.4 (2023-07-02) * :gh:issue:`929` Support Ansible 6 and ansible-core 2.13 * :gh:issue:`832` Fix runtime error when using the ansible.builtin.dnf module multiple times -* :gh:issue:`925` :class:`ansible_mitogen.connection.Connection` no longer tries to close the +* :gh:issue:`925` :class:`ansible_mitogen.connection.Connection` no longer tries to close the connection on destruction. This is expected to reduce cases of `mitogen.core.Error: An attempt was made to enqueue a message with a Broker that has already exitted`. However it may result in resource leaks. diff --git a/docs/contributors.rst b/docs/contributors.rst index 4e9e58bde..69dc1e76e 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -127,6 +127,7 @@ sponsorship and outstanding future-thinking of its early adopters.
  • jgadling
  • John F Wall — Making Ansible Great with Massive Parallelism
  • Jonathan Rosser
    • +
  • Joshua M. Keyes
  • KennethC
  • Luca Berruti
  • Lewis Bellwood — Happy to be apart of a great project.
    • diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 27b109feb..d7606b11b 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -1038,7 +1038,7 @@ receive items in the order they are requested, as they become available. Mitogen enables SSH compression by default, there are circumstances where disabling SSH compression is desirable, and many scenarios for future connection methods where transport-layer compression is not supported at - all. + all. .. [#f2] Compression may seem redundant, however it is basically free and reducing IO is always a good idea. The 33% / 200 byte saving may mean the presence or diff --git a/examples/mitogen-fuse.py b/examples/mitogen-fuse.py index 55b272d91..73101fb87 100644 --- a/examples/mitogen-fuse.py +++ b/examples/mitogen-fuse.py @@ -119,7 +119,7 @@ def _chroot(path): os.chroot(path) -class Operations(fuse.Operations): # fuse.LoggingMixIn, +class Operations(fuse.Operations): # fuse.LoggingMixIn, def __init__(self, host, path='.'): self.host = host self.root = path diff --git a/examples/mitop.py b/examples/mitop.py index 8749e12a9..72a60bf3a 100644 --- a/examples/mitop.py +++ b/examples/mitop.py @@ -61,7 +61,7 @@ def child_main(sender, delay): Executed on the main thread of the Python interpreter running on each target machine, Context.call() from the master. It simply sends the output of the UNIX 'ps' command at regular intervals toward a Receiver on master. - + :param mitogen.core.Sender sender: The Sender to use for delivering our result. This could target anywhere, but the sender supplied by the master simply causes results diff --git a/examples/the_basics.py b/examples/the_basics.py index 0dcd4049d..af9903b00 100644 --- a/examples/the_basics.py +++ b/examples/the_basics.py @@ -10,7 +10,6 @@ import hashlib import io import os -import spwd import mitogen.core import mitogen.master @@ -57,21 +56,6 @@ def streamy_download_file(context, path): } -def get_password_hash(username): - """ - Fetch a user's password hash. - """ - try: - h = spwd.getspnam(username) - except KeyError: - return None - - # mitogen.core.Secret() is a Unicode subclass with a repr() that hides the - # secret data. This keeps secret stuff out of logs. Like blobs, secrets can - # also be serialized. - return mitogen.core.Secret(h) - - def md5sum(path): """ Return the MD5 checksum for a file. diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 003ea701b..48baa7c8b 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ #: Library version as a tuple. -__version__ = (0, 3, 12) +__version__ = (0, 3, 13) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/core.py b/mitogen/core.py index 9b225ed7a..49f92cae2 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -3290,7 +3290,7 @@ def stream_by_id(self, dst_id): This can be used from any thread, but its output is only meaningful from the context of the :class:`Broker` thread, as disconnection or replacement could happen in parallel on the broker thread at any - moment. + moment. """ return ( self._stream_by_id.get(dst_id) or diff --git a/mitogen/master.py b/mitogen/master.py index 51b29b824..865c9dc14 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -652,7 +652,7 @@ class ParentImpEnumerationMethod(FinderMethod): insane) parent package, and if no insane parents exist, simply use :mod:`sys.path` to search for it from scratch on the filesystem using the normal Python lookup mechanism. - + This is required for older versions of :mod:`ansible.compat.six`, :mod:`plumbum.colors`, Ansible 2.8 :mod:`ansible.module_utils.distro` and its submodule :mod:`ansible.module_utils.distro._distro`. diff --git a/mitogen/parent.py b/mitogen/parent.py index dd51b697f..fa3092c19 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -631,7 +631,7 @@ def __init__(self): def get_timeout(self): """ Return the floating point seconds until the next event is due. - + :returns: Floating point delay, or 0.0, or :data:`None` if no events are scheduled. diff --git a/setup.py b/setup.py index b17dab9dd..9cb304272 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ def long_description(): 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/README.md b/tests/README.md index 65226e87e..35a8775c8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -7,7 +7,7 @@ started in September 2017. Pull requests in this area are very welcome! ## Running The Tests -[![Build Status](https://dev.azure.com/mitogen-hq/mitogen/_apis/build/status/mitogen-hq.mitogen?branchName=master)](https://dev.azure.com/mitogen-hq/mitogen/_build/latest?definitionId=1&branchName=master) +[![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster) Your computer should have an Internet connection, and the ``docker`` command line tool should be able to connect to a working Docker daemon (localhost or diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 8ed807879..609cd9f88 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -25,11 +25,11 @@ tt-bare [tt_targets_bare:vars] ansible_host=localhost -ansible_user=mitogen__has_sudo_nopw [tt_targets_inventory] -tt-password ansible_password="{{ 'has_sudo_nopw_password' | trim }}" +tt-password ansible_password="{{ 'has_sudo_nopw_password' | trim }}" ansible_user=mitogen__has_sudo_nopw +tt-port ansible_password=has_sudo_nopw_password ansible_port="{{ 22 | int }}" ansible_user=mitogen__has_sudo_nopw +tt-remote-user ansible_password=has_sudo_nopw_password ansible_user="{{ 'mitogen__has_sudo_nopw' | trim }}" [tt_targets_inventory:vars] ansible_host=localhost -ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/integration/_expected_ssh_port.yml b/tests/ansible/integration/_expected_ssh_port.yml new file mode 100644 index 000000000..442659a52 --- /dev/null +++ b/tests/ansible/integration/_expected_ssh_port.yml @@ -0,0 +1,18 @@ +# Ansible removed its default SSH port in May 2021, defering to the SSH +# implementation. +# https://github.com/ansible/ansible/commit/45618a6f3856f7332df8afe4adc40d85649a70da + +# Careful templating is needed to preseve the type(s) of expected_ssh_port, +# particularly in combination with the assert_equal action plugin. +# Do: {{ expected_ssh_port }} +# Don't: {{ expected_ssh_port | any_filter }} +# Don't: {% if ...%}{{ expected_ssh_port }}{% else %}...{% endif %} +# https://stackoverflow.com/questions/66102524/ansible-set-fact-type-cast/66104814#66104814 + +- set_fact: + expected_ssh_port: null + when: ansible_version.full is version('2.11.1', '>=', strict=True) + +- set_fact: + expected_ssh_port: 22 + when: ansible_version.full is version('2.11.1', '<', strict=True) diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index f9ad9e0bd..60a67b82f 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -11,11 +11,11 @@ - name: integration/connection_delegation/delegate_to_template.yml vars: physical_host: "cd-normal-alias" - physical_hosts: ["cd-normal-alias", "cd-normal-normal"] hosts: test-targets gather_facts: no tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - meta: end_play when: @@ -67,7 +67,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', 'python_path': ['python3000'], 'remote_name': null, 'ssh_args': [ diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 8cf064bb4..b0475275f 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -55,6 +55,7 @@ - hosts: cd-normal tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: delegate_to: cd-alias register: out @@ -72,7 +73,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -98,6 +99,7 @@ - hosts: cd-alias tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -114,7 +116,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -140,6 +142,7 @@ - hosts: cd-normal-normal tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -167,7 +170,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -193,6 +196,7 @@ - hosts: cd-normal-alias tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -237,7 +241,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -262,6 +266,7 @@ - hosts: cd-newuser-normal-normal tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -289,7 +294,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -315,6 +320,7 @@ - hosts: cd-newuser-normal-normal tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: delegate_to: cd-alias register: out @@ -332,7 +338,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index af2b5e46b..f9770876c 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -52,7 +52,7 @@ {% if "1" == "1" %} {{ special_python }} {% else %} - python + python {% endif %} tags: - complex_args diff --git a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml index bc4ef1d83..fd4bc848e 100644 --- a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml +++ b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml @@ -3,6 +3,8 @@ gather_facts: false vars: ansible_password: "{{ 'has_sudo_nopw_password' | trim }}" + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ansible_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" tasks: - meta: reset_connection diff --git a/tests/ansible/integration/transport_config/port.yml b/tests/ansible/integration/transport_config/port.yml index 1b8e04a04..abc680587 100644 --- a/tests/ansible/integration/transport_config/port.yml +++ b/tests/ansible/integration/transport_config/port.yml @@ -6,13 +6,15 @@ hosts: tc-port-unset tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 1 - out.result[0].method == "ssh" - - out.result[0].kwargs.port == None + - out.result[0].kwargs.port == expected_ssh_port fail_msg: | + expected_ssh_port={{ expected_ssh_port }} out={{ out }} tags: - mitogen_only @@ -54,6 +56,7 @@ vars: {mitogen_via: tc-port-explicit-ssh} tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -61,8 +64,9 @@ - out.result[0].method == "ssh" - out.result[0].kwargs.port == 4321 - out.result[1].method == "ssh" - - out.result[1].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port fail_msg: | + expected_ssh_port={{ expected_ssh_port }} out={{ out }} tags: - mitogen_only @@ -86,6 +90,7 @@ vars: {mitogen_via: tc-port-explicit-port} tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -93,8 +98,9 @@ - out.result[0].method == "ssh" - out.result[0].kwargs.port == 1234 - out.result[1].method == "ssh" - - out.result[1].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port fail_msg: | + expected_ssh_port={{ expected_ssh_port }} out={{ out }} tags: - mitogen_only @@ -119,6 +125,7 @@ vars: {mitogen_via: tc-port-both} tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -126,8 +133,9 @@ - out.result[0].method == "ssh" - out.result[0].kwargs.port == 1532 - out.result[1].method == "ssh" - - out.result[1].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port fail_msg: | + expected_ssh_port={{ expected_ssh_port }} out={{ out }} tags: - mitogen_only diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index 37a0725a2..0fdef20b0 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -47,12 +47,12 @@ tt-bare ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} -ansible_user=mitogen__has_sudo_nopw [tt_targets_inventory] -tt-password ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} +tt-password ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} ansible_user=mitogen__has_sudo_nopw +tt-port ansible_password=has_sudo_nopw_password ansible_port="{{ '{{' }} {{ tt.port }} | int {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw +tt-remote-user ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_user="{{ '{{' }} 'mitogen__has_sudo_nopw' | trim {{ '}}' }}" [tt_targets_inventory:vars] ansible_host={{ tt.hostname }} ansible_python_interpreter={{ tt.python_path }} -ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/tests/utils_unsafe_test.py b/tests/ansible/tests/utils_unsafe_test.py index a020f55bf..9aa461c58 100644 --- a/tests/ansible/tests/utils_unsafe_test.py +++ b/tests/ansible/tests/utils_unsafe_test.py @@ -48,7 +48,7 @@ def test_builtins_roundtrip(self): self.assertCasts(wrap_var([]), []) self.assertCasts(wrap_var(u''), u'') self.assertCasts(wrap_var(()), []) - + def test_subtypes_roundtrip(self): self.assertCasts(wrap_var(Bytes()), b'') self.assertCasts(wrap_var(Dict()), {}) diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json index dcbcc7852..614a7f9c3 100644 --- a/tests/data/importer/webproject/modules_expected_py3x-new.json +++ b/tests/data/importer/webproject/modules_expected_py3x-new.json @@ -205,4 +205,3 @@ ] } } - \ No newline at end of file diff --git a/tests/image_prep/py24-build.sh b/tests/image_prep/py24-build.sh index b30cc24b9..b99e36a04 100755 --- a/tests/image_prep/py24-build.sh +++ b/tests/image_prep/py24-build.sh @@ -15,7 +15,7 @@ tar xzvf cpython-2.4.6.tar.gz ( cd cpython-2.4.6 - ./configure --prefix=/usr/local/python2.4.6 --with-pydebug --enable-debug CFLAGS="-g -O0" # --enable-debug + ./configure --prefix=/usr/local/python2.4.6 --with-pydebug --enable-debug CFLAGS="-g -O0" # --enable-debug echo 'zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz' >> Modules/Setup.config make -j 8 sudo make install diff --git a/tests/requirements.txt b/tests/requirements.txt index d64d8b873..c5671b37e 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,5 @@ cffi==1.15.1; python_version < '3.8' -cffi==1.16; python_version >= '3.8' +cffi==1.17.1; python_version >= '3.8' coverage==5.5; python_version == '2.7' coverage==6.2; python_version == '3.6' diff --git a/tox.ini b/tox.ini index 9fb31bdcb..6a0eb1806 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,7 @@ # I use this locally on Ubuntu 22.04, with the following additions # # sudo add-apt-repository ppa:deadsnakes/ppa -# sudo apt update -# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python{2,2.7,3} python3.{6..13}{,-venv} python-is-python3 sshpass tox +# sudo apt install lib{ldap2,sasl2,ssl}-dev python{2,2.7,3}{,-dev} python3.{7..13}{,-dev,-venv} python-is-python3 sshpass tox # Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== @@ -56,10 +55,10 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, - py{312}-mode_ansible-ansible{6,7,8,9,10}, - py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{313}-mode_ansible-ansible{6,7,8,9,10}, + py{27,36,313}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,313}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,313}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, [testenv] @@ -74,6 +73,7 @@ basepython = py310: python3.10 py311: python3.11 py312: python3.12 + py313: python3.13 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt @@ -99,9 +99,6 @@ passenv = ANSIBLE_* HOME MITOGEN_* - # Azure DevOps, TF_BUILD is set to 'True' when running in a build task - # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables - TF_BUILD setenv = # See also azure-pipelines.yml ANSIBLE_STRATEGY = mitogen_linear