diff --git a/Dockerfile b/Dockerfile index 34e9659..2a6f821 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,4 +8,4 @@ WORKDIR /app COPY . /app RUN pip install -e . -ENTRYPOINT [ "cloudtruth-dynamic-importer" ] \ No newline at end of file +ENTRYPOINT [ "cloudtruth-dynamic-importer" ] diff --git a/src/dynamic_importer/main.py b/src/dynamic_importer/main.py index ce5b66e..ddf9b9f 100644 --- a/src/dynamic_importer/main.py +++ b/src/dynamic_importer/main.py @@ -2,6 +2,7 @@ import json import os +from collections import defaultdict from time import time import click @@ -13,6 +14,26 @@ from dynamic_importer.util import validate_env_values CREATE_DATA_MSG_INTERVAL = 10 +DIRS_TO_IGNORE = [ + ".git", + ".github", + ".vscode", + "__pycache__", + "venv", + "node_modules", + "dist", + "build", + "target", +] +# mime types think .env and tf files are plain text +EXTENSIONS_TO_FILE_TYPES = { + ".json": "json", + ".yaml": "yaml", + ".yml": "yaml", + ".env": "dotenv", + ".tf": "tf", + ".tfvars": "tfvars", +} @click.group() @@ -63,7 +84,6 @@ def process_configs(file_type, default_values, env_values, output_dir): click.echo(f"Using {env}-specific values from: {file_path}") input_files[env] = file_path click.echo(f"Processing {file_type} files from: {', '.join(input_files)}") - processing_class = get_processor_class(file_type) processor: BaseProcessor = processing_class(input_files) template, config_data = processor.process() @@ -206,5 +226,103 @@ def create_data(data_file, template_file, project, k, c, u): click.echo("Data upload to CloudTruth complete!") +@import_config.command() +@click.option( + "-c", + "--config-dir", + help="Full path to directory to walk and locate configs", + required=True, +) +@click.option( + "-t", + "--file-types", + type=click.Choice(get_supported_formats(), case_sensitive=False), + help=f"Type of file to process. Must be one of: {get_supported_formats()}", + required=True, + multiple=True, +) +@click.option( + "-o", + "--output-dir", + help="Directory to write processed output to. Default is current directory", + default=".", + required=False, +) +def walk_directories(config_dir, file_types, output_dir): + walked_files = {} + output_dir = output_dir.rstrip("/") + for root, dirs, files in os.walk(config_dir): + root = root.rstrip("/") + last_project = None + + # skip over known non-config directories + for dir in DIRS_TO_IGNORE: + if dir in dirs: + dirs.remove(dir) + + for file in files: + file_path = f"{root}/{file}" + name, file_extension = os.path.splitext(file) + if name.startswith(".env"): + file_extension = ".env" + if data_type := EXTENSIONS_TO_FILE_TYPES.get(file_extension): + confirmed_type = click.prompt( + f"File type {data_type} detected for {file_path}. Is this correct?", + type=click.Choice(get_supported_formats(), case_sensitive=False), + default=data_type, + ) + if confirmed_type not in file_types: + click.echo( + f"Skipping {confirmed_type} file {file_path} as " + f"it is not included in the supplied file types: {', '.join(file_types)}" + ) + continue + project = click.prompt( + f"Enter the CloudTruth project to import {file_path} into", + default=last_project, + ) + if project: + last_project = project + env = click.prompt( + f"Enter the CloudTruth environment to import {file_path} into", + ) + if not env: + env = click.prompt( + "Environment cannot be empty. Please enter a CloudTruth environment" + ) + walked_files[file_path] = { + "type": confirmed_type, + "path": file_path, + "project": project, + "environment": env, + } + + project_files = defaultdict(lambda: defaultdict(list)) + for v in walked_files.values(): + project_files[v["project"]][v["type"]].append( + {"path": v["path"], "environment": v["environment"]} + ) + for project, type_info in project_files.items(): + for file_type, file_meta in type_info.items(): + env_paths = {d["environment"]: d["path"] for d in file_meta} + input_filename = ".".join( + list(env_paths.values())[0].split("/")[-1].split(".")[:-1] + ) + click.echo(f"Processing {project} files: {', '.join(env_paths.values())}") + processing_class = get_processor_class(file_type) + processor: BaseProcessor = processing_class(env_paths) + template, config_data = processor.process() + template_out_file = f"{output_dir}/{input_filename}.cttemplate" + config_out_file = f"{output_dir}/{input_filename}.ctconfig" + click.echo(f"Writing template to: {template_out_file}") + with open(template_out_file, "w+") as fp: + template_body = processor.generate_template() + fp.write(template_body) + + click.echo(f"Writing config data to: {config_out_file}") + with open(config_out_file, "w+") as fp: + json.dump(config_data, fp, indent=4) + + if __name__ == "__main__": import_config() diff --git a/src/dynamic_importer/processors/__init__.py b/src/dynamic_importer/processors/__init__.py index aea227e..88cb897 100644 --- a/src/dynamic_importer/processors/__init__.py +++ b/src/dynamic_importer/processors/__init__.py @@ -35,17 +35,6 @@ def get_supported_formats() -> List[str]: class BaseProcessor: default_values = None - dirs_to_ignore = [ - ".git", - ".github", - ".vscode", - "__pycache__", - "venv", - "node_modules", - "dist", - "build", - "target", - ] parameters_and_values: Dict = {} parameters = None raw_data: Dict = {} diff --git a/src/dynamic_importer/processors/dotenv.py b/src/dynamic_importer/processors/dotenv.py index 8c75314..be853ee 100644 --- a/src/dynamic_importer/processors/dotenv.py +++ b/src/dynamic_importer/processors/dotenv.py @@ -12,6 +12,9 @@ class DotEnvProcessor(BaseProcessor): def __init__(self, env_values: Dict) -> None: + # Due to an unknown bug, self.parameters_and_values can persist between + # Processor instances. Therefore, we reset it here. + self.parameters_and_values: Dict = {} for env, file_path in env_values.items(): if not os.path.isfile(file_path): raise ValueError( diff --git a/src/dynamic_importer/processors/json.py b/src/dynamic_importer/processors/json.py index 5a53349..12f36b3 100644 --- a/src/dynamic_importer/processors/json.py +++ b/src/dynamic_importer/processors/json.py @@ -10,8 +10,8 @@ class JSONProcessor(BaseProcessor): def __init__(self, env_values: Dict) -> None: - # the click test library seems to reuse Processor classes somehow - # so we reset self.parameters_and_values to avoid test pollution + # Due to an unknown bug, self.parameters_and_values can persist between + # Processor instances. Therefore, we reset it here. self.parameters_and_values: Dict = {} for env, file_path in env_values.items(): with open(file_path, "r") as fp: diff --git a/src/dynamic_importer/processors/tf.py b/src/dynamic_importer/processors/tf.py index ee70745..f370b70 100644 --- a/src/dynamic_importer/processors/tf.py +++ b/src/dynamic_importer/processors/tf.py @@ -17,6 +17,9 @@ class TFProcessor(BaseProcessor): data_keys = {"type", "default"} def __init__(self, env_values: Dict) -> None: + # Due to an unknown bug, self.parameters_and_values can persist between + # Processor instances. Therefore, we reset it here. + self.parameters_and_values: Dict = {} for env, file_path in env_values.items(): if not os.path.isfile(file_path): raise ValueError( diff --git a/src/dynamic_importer/processors/tfvars.py b/src/dynamic_importer/processors/tfvars.py index 825a231..2bac1b5 100644 --- a/src/dynamic_importer/processors/tfvars.py +++ b/src/dynamic_importer/processors/tfvars.py @@ -10,6 +10,9 @@ class TFVarsProcessor(BaseProcessor): def __init__(self, env_values: Dict) -> None: + # Due to an unknown bug, self.parameters_and_values can persist between + # Processor instances. Therefore, we reset it here. + self.parameters_and_values: Dict = {} for env, file_path in env_values.items(): try: with open(file_path, "r") as fp: diff --git a/src/dynamic_importer/processors/yaml.py b/src/dynamic_importer/processors/yaml.py index dd8bf57..22e05f8 100644 --- a/src/dynamic_importer/processors/yaml.py +++ b/src/dynamic_importer/processors/yaml.py @@ -10,6 +10,9 @@ class YAMLProcessor(BaseProcessor): def __init__(self, env_values: Dict) -> None: + # Due to an unknown bug, self.parameters_and_values can persist between + # Processor instances. Therefore, we reset it here. + self.parameters_and_values: Dict = {} for env, file_path in env_values.items(): try: with open(file_path, "r") as fp: diff --git a/src/tests/test_directory_walking.py b/src/tests/test_directory_walking.py new file mode 100644 index 0000000..0ca0058 --- /dev/null +++ b/src/tests/test_directory_walking.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import os +import pathlib + +import pytest +from click.testing import CliRunner +from dynamic_importer.main import import_config + +""" +Hey-o! Warren here. walk-directories prompts the user for information +for every file in the supplied directory to walk. Therefore, the tests +MUST supply input for the prompts. Otherwise, your tests will just +hang indefinitely. +""" + + +@pytest.mark.usefixtures("tmp_path") +def test_walk_directories_one_file_type(tmp_path): + runner = CliRunner() + current_dir = pathlib.Path(__file__).parent.resolve() + + prompt_responses = [ + "", + "myproj", + "default", + "", + "", + "development", + "", + "", + "production", + "", + "", + "staging", + ] + with runner.isolated_filesystem(temp_dir=tmp_path) as td: + result = runner.invoke( + import_config, + [ + "walk-directories", + "-t", + "dotenv", + "-c", + f"{current_dir}/../samples/dotenvs", + "--output-dir", + td, + ], + input="\n".join(prompt_responses), + catch_exceptions=False, + ) + assert result.exit_code == 0 + + assert pathlib.Path(f"{td}/.env.default.ctconfig").exists() + assert pathlib.Path(f"{td}/.env.default.cttemplate").exists() + assert os.path.getsize(f"{td}/.env.default.ctconfig") > 0 + assert os.path.getsize(f"{td}/.env.default.cttemplate") > 0 + + +@pytest.mark.usefixtures("tmp_path") +def test_walk_directories_multiple_file_types(tmp_path): + runner = CliRunner() + current_dir = pathlib.Path(__file__).parent.resolve() + + prompt_responses = [ + "", + "myproj", + "default", + "", + "", + "", + "default", + "", + "", + "", + "default", + "", + "dotty", + "default", + "", + "", + "development", + "", + "", + "production", + "", + "", + "staging", + ] + with runner.isolated_filesystem(temp_dir=tmp_path) as td: + result = runner.invoke( + import_config, + [ + "walk-directories", + "-t", + "dotenv", + "-t", + "json", + "-c", + f"{current_dir}/../samples", + "--output-dir", + td, + ], + input="\n".join(prompt_responses), + catch_exceptions=False, + ) + assert result.exit_code == 0 + + assert pathlib.Path(f"{td}/.env.default.ctconfig").exists() + assert pathlib.Path(f"{td}/.env.default.cttemplate").exists() + assert os.path.getsize(f"{td}/.env.default.ctconfig") > 0 + assert os.path.getsize(f"{td}/.env.default.cttemplate") > 0 + + assert pathlib.Path(f"{td}/.env.ctconfig").exists() + assert pathlib.Path(f"{td}/.env.cttemplate").exists() + assert os.path.getsize(f"{td}/.env.ctconfig") > 0 + assert os.path.getsize(f"{td}/.env.cttemplate") > 0 + + assert pathlib.Path(f"{td}/short.ctconfig").exists() + assert pathlib.Path(f"{td}/short.cttemplate").exists() + assert os.path.getsize(f"{td}/short.ctconfig") > 0 + assert os.path.getsize(f"{td}/short.cttemplate") > 0 + + assert not pathlib.Path(f"{td}/variables.ctconfig").exists() + assert not pathlib.Path(f"{td}/variables.cttemplate").exists()