sc-12420 walk directories to find and parse configs (#6)

* sc-12420 initial directory walk code

* fix subdirectory walking and add test for multiple types

---------

Co-authored-by: Matthew Warren <matt.warren+github-commits@cloudtruth.com>

Dockerfile

@@ -8,4 +8,4 @@ WORKDIR /app
 COPY . /app
 RUN pip install -e .
 
-ENTRYPOINT [ "cloudtruth-dynamic-importer" ]
+ENTRYPOINT [ "cloudtruth-dynamic-importer" ]

src/dynamic_importer/main.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+from collections import defaultdict
 from time import time
 
 import click
@@ -13,6 +14,26 @@
 from dynamic_importer.util import validate_env_values
 
 CREATE_DATA_MSG_INTERVAL = 10
+DIRS_TO_IGNORE = [
+    ".git",
+    ".github",
+    ".vscode",
+    "__pycache__",
+    "venv",
+    "node_modules",
+    "dist",
+    "build",
+    "target",
+]
+# mime types think .env and tf files are plain text
+EXTENSIONS_TO_FILE_TYPES = {
+    ".json": "json",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".env": "dotenv",
+    ".tf": "tf",
+    ".tfvars": "tfvars",
+}
 
 
 @click.group()
@@ -63,7 +84,6 @@ def process_configs(file_type, default_values, env_values, output_dir):
         click.echo(f"Using {env}-specific values from: {file_path}")
         input_files[env] = file_path
     click.echo(f"Processing {file_type} files from: {', '.join(input_files)}")
-
     processing_class = get_processor_class(file_type)
     processor: BaseProcessor = processing_class(input_files)
     template, config_data = processor.process()
@@ -206,5 +226,103 @@ def create_data(data_file, template_file, project, k, c, u):
     click.echo("Data upload to CloudTruth complete!")
 
 
+@import_config.command()
+@click.option(
+    "-c",
+    "--config-dir",
+    help="Full path to directory to walk and locate configs",
+    required=True,
+)
+@click.option(
+    "-t",
+    "--file-types",
+    type=click.Choice(get_supported_formats(), case_sensitive=False),
+    help=f"Type of file to process. Must be one of: {get_supported_formats()}",
+    required=True,
+    multiple=True,
+)
+@click.option(
+    "-o",
+    "--output-dir",
+    help="Directory to write processed output to. Default is current directory",
+    default=".",
+    required=False,
+)
+def walk_directories(config_dir, file_types, output_dir):
+    walked_files = {}
+    output_dir = output_dir.rstrip("/")
+    for root, dirs, files in os.walk(config_dir):
+        root = root.rstrip("/")
+        last_project = None
+
+        # skip over known non-config directories
+        for dir in DIRS_TO_IGNORE:
+            if dir in dirs:
+                dirs.remove(dir)
+
+        for file in files:
+            file_path = f"{root}/{file}"
+            name, file_extension = os.path.splitext(file)
+            if name.startswith(".env"):
+                file_extension = ".env"
+            if data_type := EXTENSIONS_TO_FILE_TYPES.get(file_extension):
+                confirmed_type = click.prompt(
+                    f"File type {data_type} detected for {file_path}. Is this correct?",
+                    type=click.Choice(get_supported_formats(), case_sensitive=False),
+                    default=data_type,
+                )
+                if confirmed_type not in file_types:
+                    click.echo(
+                        f"Skipping {confirmed_type} file {file_path} as "
+                        f"it is not included in the supplied file types: {', '.join(file_types)}"
+                    )
+                    continue
+                project = click.prompt(
+                    f"Enter the CloudTruth project to import {file_path} into",
+                    default=last_project,
+                )
+                if project:
+                    last_project = project
+                env = click.prompt(
+                    f"Enter the CloudTruth environment to import {file_path} into",
+                )
+                if not env:
+                    env = click.prompt(
+                        "Environment cannot be empty. Please enter a CloudTruth environment"
+                    )
+                walked_files[file_path] = {
+                    "type": confirmed_type,
+                    "path": file_path,
+                    "project": project,
+                    "environment": env,
+                }
+
+    project_files = defaultdict(lambda: defaultdict(list))
+    for v in walked_files.values():
+        project_files[v["project"]][v["type"]].append(
+            {"path": v["path"], "environment": v["environment"]}
+        )
+    for project, type_info in project_files.items():
+        for file_type, file_meta in type_info.items():
+            env_paths = {d["environment"]: d["path"] for d in file_meta}
+            input_filename = ".".join(
+                list(env_paths.values())[0].split("/")[-1].split(".")[:-1]
+            )
+            click.echo(f"Processing {project} files: {', '.join(env_paths.values())}")
+            processing_class = get_processor_class(file_type)
+            processor: BaseProcessor = processing_class(env_paths)
+            template, config_data = processor.process()
+            template_out_file = f"{output_dir}/{input_filename}.cttemplate"
+            config_out_file = f"{output_dir}/{input_filename}.ctconfig"
+            click.echo(f"Writing template to: {template_out_file}")
+            with open(template_out_file, "w+") as fp:
+                template_body = processor.generate_template()
+                fp.write(template_body)
+
+            click.echo(f"Writing config data to: {config_out_file}")
+            with open(config_out_file, "w+") as fp:
+                json.dump(config_data, fp, indent=4)
+
+
 if __name__ == "__main__":
     import_config()

src/dynamic_importer/processors/__init__.py

@@ -35,17 +35,6 @@ def get_supported_formats() -> List[str]:
 
 class BaseProcessor:
     default_values = None
-    dirs_to_ignore = [
-        ".git",
-        ".github",
-        ".vscode",
-        "__pycache__",
-        "venv",
-        "node_modules",
-        "dist",
-        "build",
-        "target",
-    ]
     parameters_and_values: Dict = {}
     parameters = None
     raw_data: Dict = {}

src/dynamic_importer/processors/dotenv.py

@@ -12,6 +12,9 @@
 
 class DotEnvProcessor(BaseProcessor):
     def __init__(self, env_values: Dict) -> None:
+        # Due to an unknown bug, self.parameters_and_values can persist between
+        # Processor instances. Therefore, we reset it here.
+        self.parameters_and_values: Dict = {}
         for env, file_path in env_values.items():
             if not os.path.isfile(file_path):
                 raise ValueError(

src/dynamic_importer/processors/json.py

@@ -10,8 +10,8 @@
 
 class JSONProcessor(BaseProcessor):
     def __init__(self, env_values: Dict) -> None:
-        # the click test library seems to reuse Processor classes somehow
-        # so we reset self.parameters_and_values to avoid test pollution
+        # Due to an unknown bug, self.parameters_and_values can persist between
+        # Processor instances. Therefore, we reset it here.
         self.parameters_and_values: Dict = {}
         for env, file_path in env_values.items():
             with open(file_path, "r") as fp:

src/dynamic_importer/processors/tf.py

@@ -17,6 +17,9 @@ class TFProcessor(BaseProcessor):
     data_keys = {"type", "default"}
 
     def __init__(self, env_values: Dict) -> None:
+        # Due to an unknown bug, self.parameters_and_values can persist between
+        # Processor instances. Therefore, we reset it here.
+        self.parameters_and_values: Dict = {}
         for env, file_path in env_values.items():
             if not os.path.isfile(file_path):
                 raise ValueError(

src/dynamic_importer/processors/tfvars.py

@@ -10,6 +10,9 @@
 
 class TFVarsProcessor(BaseProcessor):
     def __init__(self, env_values: Dict) -> None:
+        # Due to an unknown bug, self.parameters_and_values can persist between
+        # Processor instances. Therefore, we reset it here.
+        self.parameters_and_values: Dict = {}
         for env, file_path in env_values.items():
             try:
                 with open(file_path, "r") as fp:

src/dynamic_importer/processors/yaml.py

@@ -10,6 +10,9 @@
 
 class YAMLProcessor(BaseProcessor):
     def __init__(self, env_values: Dict) -> None:
+        # Due to an unknown bug, self.parameters_and_values can persist between
+        # Processor instances. Therefore, we reset it here.
+        self.parameters_and_values: Dict = {}
         for env, file_path in env_values.items():
             try:
                 with open(file_path, "r") as fp:

src/tests/test_directory_walking.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import os
+import pathlib
+
+import pytest
+from click.testing import CliRunner
+from dynamic_importer.main import import_config
+
+"""
+Hey-o! Warren here. walk-directories prompts the user for information
+for every file in the supplied directory to walk. Therefore, the tests
+MUST supply input for the prompts. Otherwise, your tests will just
+hang indefinitely.
+"""
+
+
+@pytest.mark.usefixtures("tmp_path")
+def test_walk_directories_one_file_type(tmp_path):
+    runner = CliRunner()
+    current_dir = pathlib.Path(__file__).parent.resolve()
+
+    prompt_responses = [
+        "",
+        "myproj",
+        "default",
+        "",
+        "",
+        "development",
+        "",
+        "",
+        "production",
+        "",
+        "",
+        "staging",
+    ]
+    with runner.isolated_filesystem(temp_dir=tmp_path) as td:
+        result = runner.invoke(
+            import_config,
+            [
+                "walk-directories",
+                "-t",
+                "dotenv",
+                "-c",
+                f"{current_dir}/../samples/dotenvs",
+                "--output-dir",
+                td,
+            ],
+            input="\n".join(prompt_responses),
+            catch_exceptions=False,
+        )
+        assert result.exit_code == 0
+
+        assert pathlib.Path(f"{td}/.env.default.ctconfig").exists()
+        assert pathlib.Path(f"{td}/.env.default.cttemplate").exists()
+        assert os.path.getsize(f"{td}/.env.default.ctconfig") > 0
+        assert os.path.getsize(f"{td}/.env.default.cttemplate") > 0
+
+
+@pytest.mark.usefixtures("tmp_path")
+def test_walk_directories_multiple_file_types(tmp_path):
+    runner = CliRunner()
+    current_dir = pathlib.Path(__file__).parent.resolve()
+
+    prompt_responses = [
+        "",
+        "myproj",
+        "default",
+        "",
+        "",
+        "",
+        "default",
+        "",
+        "",
+        "",
+        "default",
+        "",
+        "dotty",
+        "default",
+        "",
+        "",
+        "development",
+        "",
+        "",
+        "production",
+        "",
+        "",
+        "staging",
+    ]
+    with runner.isolated_filesystem(temp_dir=tmp_path) as td:
+        result = runner.invoke(
+            import_config,
+            [
+                "walk-directories",
+                "-t",
+                "dotenv",
+                "-t",
+                "json",
+                "-c",
+                f"{current_dir}/../samples",
+                "--output-dir",
+                td,
+            ],
+            input="\n".join(prompt_responses),
+            catch_exceptions=False,
+        )
+        assert result.exit_code == 0
+
+        assert pathlib.Path(f"{td}/.env.default.ctconfig").exists()
+        assert pathlib.Path(f"{td}/.env.default.cttemplate").exists()
+        assert os.path.getsize(f"{td}/.env.default.ctconfig") > 0
+        assert os.path.getsize(f"{td}/.env.default.cttemplate") > 0
+
+        assert pathlib.Path(f"{td}/.env.ctconfig").exists()
+        assert pathlib.Path(f"{td}/.env.cttemplate").exists()
+        assert os.path.getsize(f"{td}/.env.ctconfig") > 0
+        assert os.path.getsize(f"{td}/.env.cttemplate") > 0
+
+        assert pathlib.Path(f"{td}/short.ctconfig").exists()
+        assert pathlib.Path(f"{td}/short.cttemplate").exists()
+        assert os.path.getsize(f"{td}/short.ctconfig") > 0
+        assert os.path.getsize(f"{td}/short.cttemplate") > 0
+
+        assert not pathlib.Path(f"{td}/variables.ctconfig").exists()
+        assert not pathlib.Path(f"{td}/variables.cttemplate").exists()