Embed ansible_compat in linter (#2994)

.config/requirements-lock.txt

@@ -4,7 +4,6 @@
 #
 #    pip-compile --no-annotate --output-file=.config/requirements-lock.txt --resolver=backtracking --strip-extras --unsafe-package=ruamel-yaml-clib pyproject.toml
 #
-ansible-compat==3.0.1
 ansible-core==2.14.1 ; python_version >= "3.9"
 attrs==22.2.0
 black==22.12.0

.config/requirements.in

@@ -2,7 +2,6 @@
 will-not-work-on-windows-try-from-wsl-instead; platform_system=='Windows'
 ansible-core>=2.12.0,<2.14.0; python_version<'3.9' # GPLv3
 ansible-core>=2.12.0; python_version>='3.9' # GPLv3
-ansible-compat>=3.0.1 # GPLv3
 # alphabetically sorted:
 black>=22.8.0 # MIT
 filelock>=3.3.0 # The Unlicense
@@ -11,5 +10,6 @@ packaging>=21.3 # Apache-2.0,BSD-2-Clause
 pyyaml>=5.4.1 # MIT (centos 9 has 5.3.1)
 rich>=12.0.0 # MIT
 ruamel.yaml>=0.17.21,<0.18 # MIT, next version is planned to have breaking changes
+subprocess-tee>=0.4.1 # MIT, used by ansible-compat
 yamllint >= 1.26.3 # GPLv3
 wcmatch>=8.1.2 # MIT

.config/requirements.txt

@@ -4,7 +4,6 @@
 #
 #    pip-compile --extra=docs --extra=test --no-annotate --output-file=.config/requirements.txt --resolver=backtracking --strip-extras --unsafe-package=ansible-core --unsafe-package=ruamel-yaml-clib pyproject.toml
 #
-ansible-compat==3.0.1
 astroid==2.14.1
 attrs==22.2.0
 black==22.12.0

.flake8

@@ -25,6 +25,8 @@ extend-exclude =
   env,
   # Metadata of `pip wheel` cmd is autogenerated
   pip-wheel-metadata,
+  # vendored code
+  src/ansiblelint/_vendor/*,
 
 # Let's not over-complicate the code:
 max-complexity = 10

.github/workflows/release.yml

@@ -28,6 +28,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0 # needed by setuptools-scm
+          submodules: true
       - name: Build dists
         run: python -m tox
       - name: Publish to pypi.org

.github/workflows/tox.yml

@@ -86,6 +86,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0 # needed by setuptools-scm
+          submodules: true
 
       - name: Set pre-commit cache
         uses: actions/cache@v3

.gitmodules

@@ -0,0 +1,3 @@
+[submodule ".projects/ansible-compat"]
+	path = .projects/ansible-compat
+	url = https://github.com/ansible/ansible-compat

.packit.yaml

@@ -9,6 +9,8 @@ actions:
   get-current-version:
     - ./tools/get-version.sh
   post-upstream-clone:
+    - "git submodule init"
+    - "git submodule update"
     - ./tools/update-version.sh
 srpm_build_deps:
   - python3-build

.pre-commit-config.yaml

@@ -11,6 +11,7 @@ ci:
     # https://github.com/pre-commit-ci/issues/issues/55
     - pip-compile
     - schemas
+  submodules: true
 exclude: >
   (?x)^(
     .config/requirements.*|
@@ -159,14 +160,15 @@ repos:
         # empty args needed in order to match mypy cli behavior
         args: [--strict]
         additional_dependencies:
-          - ansible-compat>=3.0.1
+          - ansible-compat
           - black>=22.10.0
           - cryptography>=39.0.1
           - filelock
           - jinja2
           - pytest>=7.2.0
           - rich>=13.2.0
           - ruamel.yaml
+          - subprocess-tee
           - types-PyYAML
           - types-jsonschema>=4.4.2
           - types-pkg_resources
@@ -184,7 +186,7 @@ repos:
         args:
           - --output-format=colorized
         additional_dependencies:
-          - ansible-compat>=3.0.1
+          - ansible-compat
           - ansible-core>=2.14.0
           - black>=22.10.0
           - docutils

.readthedocs.yml

@@ -21,3 +21,6 @@ python:
       path: .
       extra_requirements:
         - docs
+submodules:
+  include: all
+  recursive: true

pyproject.toml

@@ -72,7 +72,7 @@ source = ["src", ".tox/*/site-packages"]
 
 [tool.coverage.report]
 exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]
-omit = ["*/src/ansible_compat/*", "test/*"]
+omit = ["*/src/*/_vendor/*", "test/*"]
 # Increase it just so it would pass on any single-python run
 fail_under = 88
 skip_covered = true
@@ -123,7 +123,7 @@ preferred-modules = ["py:pathlib", "unittest:pytest"]
 # pylint defaults + f,fh,v,id
 good-names = ["i", "j", "k", "ex", "Run", "_", "f", "fh", "v", "id", "T"]
 # Ignore as being generated:
-ignore-paths = "^src/ansiblelint/_version.*$"
+ignore-paths = "^src/ansiblelint/(_version|_vendor).*$"
 
 [tool.pylint."MESSAGES CONTROL"]
 # increase from default is 50 which is too aggressive
@@ -190,13 +190,15 @@ norecursedirs = [
   "collections",
   "dist",
   "docs",
+  "src/ansiblelint/_vendor",
   "src/ansible_lint.egg-info",
   ".cache",
   ".eggs",
   ".git",
   ".github",
   ".tox",
   "*.egg",
+  ".projects",
 ]
 python_files = [
   "test_*.py",

src/ansiblelint/__init__.py

@@ -20,6 +20,10 @@
 """Main ansible-lint package."""
 from __future__ import annotations
 
+import ansiblelint._vendor
 from ansiblelint.version import __version__
 
+# make vendored top-level modules accessible EARLY
+
+
 __all__ = ("__version__",)

src/ansiblelint/_vendor/__init__.py

@@ -0,0 +1,48 @@
+import os
+import pkgutil
+import sys
+import warnings
+
+# This package exists to host vendored top-level Python packages for downstream packaging. Any Python packages
+# installed beneath this one will be masked from the Ansible loader, and available from the front of sys.path.
+# It is expected that the vendored packages will be loaded very early, so a warning will be fired on import of
+# the top-level ansible package if any packages beneath this are already loaded at that point.
+#
+# Python packages may be installed here during downstream packaging using something like:
+# pip install --upgrade -t (path to this dir) cryptography pyyaml packaging jinja2
+
+# mask vendored content below this package from being accessed as a subpackage
+__path__ = []
+
+
+def _ensure_vendored_path_entry() -> None:
+    """
+    Ensure that any downstream-bundled content beneath this package is available at the top of sys.path
+    """
+    # patch our vendored dir onto sys.path
+    vendored_path_entry = os.path.dirname(__file__)
+    vendored_module_names = {
+        m[1] for m in pkgutil.iter_modules([vendored_path_entry], "")
+    }  # m[1] == m.name
+
+    if vendored_module_names:
+        # patch us early to load vendored deps transparently
+        if vendored_path_entry in sys.path:
+            # handle reload case by removing the existing entry, wherever it might be
+            sys.path.remove(vendored_path_entry)
+        sys.path.insert(0, vendored_path_entry)
+
+        already_loaded_vendored_modules = set(sys.modules.keys()).intersection(
+            vendored_module_names
+        )
+
+        if already_loaded_vendored_modules:
+            warnings.warn(
+                "One or more Python packages bundled by this ansible-lint distribution were already "
+                "loaded ({}). This may result in undefined behavior.".format(
+                    ", ".join(sorted(already_loaded_vendored_modules))
+                )
+            )
+
+
+_ensure_vendored_path_entry()

src/ansiblelint/_vendor/ansible_compat

@@ -0,0 +1 @@
+../../../.projects/ansible-compat/src/ansible_compat