Improve the test suite setup time by caching the python package repos…

…itory created from tests/data/modules_v2 (Issue #4618, PR #4621)

Pull request opened by the merge tool on behalf of #4621

.gitignore

@@ -82,3 +82,6 @@ _build
 logs
 
 docs/reference/openapi.json
+
+# Ignore modules_v2 python package repo cache
+tests/data/modules_v2.cache

changelogs/unreleased/4618-cache-python-package-index.yml

@@ -0,0 +1,6 @@
+---
+description: Improve the test suite setup time by caching the python package repository created from tests/data/modules_v2
+issue-nr: 4618
+issue-repo: inmanta-core
+change-type: patch
+destination-branches: [master, iso5]

tests/conftest.py

@@ -1507,12 +1507,43 @@ def local_module_package_index(modules_v2_dir: str) -> Iterator[str]:
     Creates a local pip index for all v2 modules in the modules v2 dir. The modules are built and published to the index.
     :return: The path to the index
     """
-    with tempfile.TemporaryDirectory() as artifact_dir:
+    cache_dir = os.path.abspath(os.path.join(os.path.dirname(modules_v2_dir), f"{os.path.basename(modules_v2_dir)}.cache"))
+    build_dir = os.path.join(cache_dir, "build")
+    index_dir = os.path.join(build_dir, "simple")
+    timestamp_file = os.path.join(cache_dir, "cache_creation_timestamp")
+
+    def _should_rebuild_cache() -> bool:
+        if any(not os.path.exists(f) for f in [build_dir, index_dir, timestamp_file]):
+            # Cache doesn't exist
+            return True
+        if len(os.listdir(index_dir)) != len(os.listdir(modules_v2_dir)) + 1:  # #modules + index.html
+            # Modules were added/removed from the build_dir
+            return True
+        # Cache is dirty
+        return any(
+            os.path.getmtime(os.path.join(root, f)) > os.path.getmtime(timestamp_file)
+            for root, _, files in os.walk(modules_v2_dir)
+            for f in files
+        )
+
+    if _should_rebuild_cache():
+        logger.info(f"Cache {cache_dir} is dirty. Rebuilding cache.")
+        # Remove cache
+        if os.path.exists(cache_dir):
+            shutil.rmtree(cache_dir)
+        os.makedirs(build_dir)
+        # Build modules
         for module_dir in os.listdir(modules_v2_dir):
             path: str = os.path.join(modules_v2_dir, module_dir)
-            ModuleTool().build(path=path, output_dir=artifact_dir)
-        dir2pi(argv=["dir2pi", artifact_dir])
-        yield os.path.join(artifact_dir, "simple")
+            ModuleTool().build(path=path, output_dir=build_dir)
+        # Build python package repository
+        dir2pi(argv=["dir2pi", build_dir])
+        # Update timestamp file
+        open(timestamp_file, "w").close()
+    else:
+        logger.info(f"Using cache {cache_dir}")
+
+    yield index_dir
 
 
 @pytest.fixture