From aea296c24d8610f12a3e2ff2b60d639c8c099590 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Fri, 19 Mar 2021 13:04:02 +0000 Subject: [PATCH] Make molecule reuse prerun from ansible-lint As ansible-lint is able to detect repository layout and run prepare ansible from running it, we rely on its logic in order to assure ansible is able to run properly. This means that dependencies will be installed into .cache folder and that the role import path will be updated to include it. --- .config/molecule/config.yml | 1 + .pre-commit-config.yaml | 5 +++-- .readthedocs.yml | 1 + docs/configuration.rst | 21 +++++++++++++++++++ mypy.ini | 3 +++ setup.cfg | 4 +++- src/molecule/command/base.py | 6 ++++++ src/molecule/config.py | 1 + src/molecule/data/validate-dockerfile.yml | 2 ++ src/molecule/provisioner/ansible.py | 1 + .../test/unit/provisioner/test_ansible.py | 10 ++++++--- tox.ini | 8 +++++-- 12 files changed, 55 insertions(+), 8 deletions(-) diff --git a/.config/molecule/config.yml b/.config/molecule/config.yml index 2c392aae3d..602c52a04e 100644 --- a/.config/molecule/config.yml +++ b/.config/molecule/config.yml @@ -1 +1,2 @@ # This is loaded by default +prerun: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5e026e524d..bf6dcd07f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,15 +50,16 @@ repos: entry: mypy src/ pass_filenames: false additional_dependencies: + - ansible-lint>=5.0.5a1 - packaging - enrich>=1.2.5 - subprocess-tee>=0.2.0 - repo: https://github.com/pre-commit/mirrors-pylint - rev: v2.6.0 + rev: v2.7.2 hooks: - id: pylint additional_dependencies: - - ansible-base + - ansible-lint>=5.0.5a1 - enrich>=1.2.5 - subprocess-tee>=0.2.0 - testinfra diff --git a/.readthedocs.yml b/.readthedocs.yml index 62b6635ea1..804662309f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,4 +12,5 @@ python: - method: pip path: . extra_requirements: + - ansible-base - docs diff --git a/docs/configuration.rst b/docs/configuration.rst index 00da2a92c2..d22aedf4bd 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -6,6 +6,27 @@ Configuration .. _variable substitution: + +Prerun +------ + +In order to help Ansible find used modules and roles, molecule will perform +a prerun set of actions. These involve installing dependencies from +``requiresments.yml`` specified at project level, install a standalone role +or a collection. The destination is ``project_dir/.cache`` and the code itself +is reused from ansible-lint, which has to do the same actions. + +This assures that when you include a role inside molecule playbooks, Ansible +will be able to find that role, and that the include is exactly the same as +the one you are expecting to use in production. + +If for some reason the prerun action does not suits your needs, you can still +disabled it by adding `prerun: false` inside the configuraiton file. + +Keep in mind that you can add this value to ``.config/molecule/config.yml`` +file in order to avoid adding it to each scenario. + + Variable Substitution --------------------- diff --git a/mypy.ini b/mypy.ini index 3f2d25be96..cca70f73fb 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,6 +7,9 @@ warn_redundant_casts = True no_implicit_optional = True # 3rd party ignores, to remove once they add hints +[mypy-ansiblelint.*] +ignore_missing_imports = True + [mypy-cerberus.*] ignore_missing_imports = True diff --git a/setup.cfg b/setup.cfg index a039797e49..ad8e12bb06 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,6 +66,7 @@ setup_requires = # These are required in actual runtime: install_requires = + ansible-lint >= 5.0.5a1 # only for the prerun functionality cerberus >= 1.3.1 click >= 7.0, < 8.0 # https://github.com/click-contrib/click-help-colors/issues/12 click-completion >= 0.5.1 @@ -120,9 +121,10 @@ test = pytest-xdist >= 2.1.0 pytest >= 6.1.2 lint = - ansible-lint[core,yamllint] >= 5.0.2, < 6 + # ansible-lint is not a core dependency, duplicating it here would confuse pip flake8 >= 3.8.4 pre-commit >= 2.10.1 + yamllint [options.entry_points] console_scripts = diff --git a/src/molecule/command/base.py b/src/molecule/command/base.py index 02ea3885d1..9b04d4ea74 100644 --- a/src/molecule/command/base.py +++ b/src/molecule/command/base.py @@ -28,6 +28,7 @@ from typing import Any, Callable import click +from ansiblelint.prerun import prepare_environment from click_help_colors import HelpColorsCommand, HelpColorsGroup import molecule.scenarios @@ -105,6 +106,11 @@ def execute_cmdline_scenarios(scenario_name, args, command_args, ansible_args=() ) for scenario in scenarios: + + if scenario.config.config["prerun"]: + LOG.info("Performing prerun...") + prepare_environment() + if command_args.get("subcommand") == "reset": LOG.info("Removing %s" % scenario.ephemeral_directory) shutil.rmtree(scenario.ephemeral_directory) diff --git a/src/molecule/config.py b/src/molecule/config.py index 2dbd49dcae..a4a341d261 100644 --- a/src/molecule/config.py +++ b/src/molecule/config.py @@ -348,6 +348,7 @@ def _get_defaults(self) -> MutableMapping: "safe_files": [], }, "platforms": [], + "prerun": True, "provisioner": { "name": "ansible", "config_options": {}, diff --git a/src/molecule/data/validate-dockerfile.yml b/src/molecule/data/validate-dockerfile.yml index 42e0d5ad16..3cd5569750 100644 --- a/src/molecule/data/validate-dockerfile.yml +++ b/src/molecule/data/validate-dockerfile.yml @@ -1,6 +1,8 @@ #!/usr/bin/env ansible-playbook --- - hosts: localhost + collections: + - community.docker vars: platforms: # platforms supported as being managed by molecule/ansible, this does diff --git a/src/molecule/provisioner/ansible.py b/src/molecule/provisioner/ansible.py index e3ffb5cd0b..99c859ee1c 100644 --- a/src/molecule/provisioner/ansible.py +++ b/src/molecule/provisioner/ansible.py @@ -453,6 +453,7 @@ def default_env(self): ), "/usr/share/ansible/roles", "/etc/ansible/roles", + *os.environ.get("ANSIBLE_ROLES_PATH", "").split(":"), ] ), self._config.ansible_collections_path: ":".join(collections_path_list), diff --git a/src/molecule/test/unit/provisioner/test_ansible.py b/src/molecule/test/unit/provisioner/test_ansible.py index 2ce332dc99..11b7e98178 100644 --- a/src/molecule/test/unit/provisioner/test_ansible.py +++ b/src/molecule/test/unit/provisioner/test_ansible.py @@ -202,7 +202,11 @@ def test_env_property(_instance): "config_instance", ["_provisioner_section_data"], indirect=True ) def test_env_appends_env_property(_instance): - x = [ + + # molecule could decide to add extra paths, so we only want to check + # that those that we need are kept inside the list + roles_path_list = _instance.env["ANSIBLE_ROLES_PATH"].split(":") + for x in [ util.abs_path( os.path.join(_instance._config.scenario.ephemeral_directory, "roles") ), @@ -213,8 +217,8 @@ def test_env_appends_env_property(_instance): "/usr/share/ansible/roles", "/etc/ansible/roles", util.abs_path(os.path.join(_instance._config.scenario.directory, "foo", "bar")), - ] - assert x == _instance.env["ANSIBLE_ROLES_PATH"].split(":") + ]: + assert x in roles_path_list x = _instance._get_modules_directories() x.append( diff --git a/tox.ini b/tox.ini index 1be3b5b895..27491fd558 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,7 @@ passenv = setenv = ANSIBLE_CONFIG={toxinidir}/.ansible.cfg ANSIBLE_DISPLAY_FAILED_STDERR=1 + ANSIBLE_NOCOWS=1 ANSIBLE_VERBOSITY=1 MOLECULE_NO_LOG=0 PYTHONDONTWRITEBYTECODE=1 @@ -44,9 +45,10 @@ deps = devel: ansible>=2.10.0a2,<2.11 # pytest-molecule not used but we want to check that it does not conflict devel: git+https://github.com/ansible-community/pytest-molecule#egg=pytest-molecule - dockerfile: ansible>=2.9.12 + dockerfile: ansible>=2.10 selinux py{36,37}: importlib-metadata<2,>=0.12 + py: ansible-base extras = docker lint @@ -58,10 +60,11 @@ extras = ; sh -c 'find {homedir}/.cache -type d -path "*/molecule_*" -exec rm -rfv \{\} +;' commands = ansibledevel: ansible-galaxy install git+https://github.com/ansible-collections/community.general.git + dockerfile: ansible-galaxy install community.docker # failsafe as pip may install incompatible dependencies pip check # failsafe for preventing changes that may break pytest collection - sh -c "PYTEST_ADDOPTS= python -m pytest -p no:cov --collect-only 2>&1 >{envlogdir}/collect.log" + sh -c "PYTEST_ADDOPTS= python -m pytest -p no:cov --collect-only" # -n auto used only on unit as is not supported by functional yet # html report is used by Zuul CI to display reports python -m pytest src/molecule/test/unit/ {env:_EXTRAS} {env:PYTEST_ADDOPTS:} {posargs} @@ -102,6 +105,7 @@ commands = 'import pathlib; '\ 'docs_dir = pathlib.Path(r"{toxinidir}") / "docs/docstree/html"; index_file = docs_dir / "index.html"; print(f"\nDocumentation available under `file://\{index_file\}`\n\nTo serve docs, use `python3 -m http.server --directory \{docs_dir\} 0`\n")' extras = + ansible-base docs [testenv:docs-livereload]