sc-12541 allow for multiple dir trees to walk (#20)

Co-authored-by: Matthew Warren <matt.warren+github-commits@cloudtruth.com>
cloudtruth · May 2, 2024 · c84a58d · c84a58d
1 parent 6231b2f
commit c84a58d
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ This utility is distributed as a Docker container and can be pulled from cloudtr
 You can feed a directory of files into the `walk-directories` command, which will find all files matching the supplied types and parse them into CloudTruth config formats. If you supply your CLOUDTRUTH_API_KEY via docker, the data will be uploaded to your CloudTruth account.
 
 ```
-docker run --rm -e CLOUDTRUTH_API_KEY="myverysecureS3CR3T!!" -v ${PWD}/files:/app/files cloudtruth/dynamic-importer walk-directories --config-dir /app/samples/ -t dotenv -t json -t tf
+docker run --rm -e CLOUDTRUTH_API_KEY="myverysecureS3CR3T!!" -v ${PWD}/files:/app/files cloudtruth/dynamic-importer walk-directories --config-dirs /app/samples/ -t dotenv -t json -t tf
 ```
 
 ## Processing a single file

diff --git a/src/dynamic_importer/main.py b/src/dynamic_importer/main.py
@@ -262,9 +262,10 @@ def _create_data(
 
 @import_config.command()
 @click.option(
-    "--config-dir",
+    "--config-dirs",
     help="Full path to directory to walk and locate configs",
     required=True,
+    multiple=True,
 )
 @click.option(
     "-t",
@@ -293,30 +294,33 @@ def _create_data(
 @click.option("-c", help="Create missing projects and enviroments", is_flag=True)
 @click.option("-u", help="Upsert values", is_flag=True)
 def walk_directories(
-    config_dir, file_types, exclude_dirs, create_hierarchy, parse_descriptions, k, c, u
+    config_dirs, file_types, exclude_dirs, create_hierarchy, parse_descriptions, k, c, u
 ):
     """
     Walks a directory, constructs templates and config data, and uploads to CloudTruth.
     This is an interactive version of the process_configs and create_data commands. The
     user will be prompted for project and environment names as files are walked.
     """
     walked_files = {}
-    for root, dirs, files in os.walk(config_dir):
-        root = root.rstrip("/")
-
-        # skip over known non-config directories
-        for dir in DIRS_TO_IGNORE:
-            if dir in dirs:
-                dirs.remove(dir)
-
-        # skip over user-specified non-config directories
-        for dir in exclude_dirs:
-            dir = dir.rstrip("/")
-            if os.path.abspath(dir) in [f"{os.path.abspath(root)}/{d}" for d in dirs]:
-                click.echo(f"Excluding directory: {os.path.abspath(dir)}")
-                dirs.remove(os.path.basename(dir))
-
-        walked_files.update(walk_files(root, files, file_types, create_hierarchy))
+    for config_dir in config_dirs:
+        for root, dirs, files in os.walk(config_dir):
+            root = root.rstrip("/")
+
+            # skip over known non-config directories
+            for dir in DIRS_TO_IGNORE:
+                if dir in dirs:
+                    dirs.remove(dir)
+
+            # skip over user-specified non-config directories
+            for dir in exclude_dirs:
+                dir = dir.rstrip("/")
+                if os.path.abspath(dir) in [
+                    f"{os.path.abspath(root)}/{d}" for d in dirs
+                ]:
+                    click.echo(f"Excluding directory: {os.path.abspath(dir)}")
+                    dirs.remove(os.path.basename(dir))
+
+            walked_files.update(walk_files(root, files, file_types, create_hierarchy))
 
     project_files = defaultdict(lambda: defaultdict(list))
     for v in walked_files.values():

diff --git a/src/tests/test_cli.py b/src/tests/test_cli.py
@@ -96,7 +96,7 @@ def test_walk_directories_parse_descriptions(self, mock_client):
                 "walk-directories",
                 "-t",
                 "yaml",
-                "--config-dir",
+                "--config-dirs",
                 f"{current_dir}/../../samples/advanced",
                 "-c",
                 "-u",

diff --git a/src/tests/test_directory_walking.py b/src/tests/test_directory_walking.py
@@ -68,7 +68,7 @@ def test_walk_directories_one_file_type(mock_client):
             "walk-directories",
             "-t",
             "dotenv",
-            "--config-dir",
+            "--config-dirs",
             f"{current_dir}/../../samples/dotenvs",
             "-c",
             "-u",
@@ -147,7 +147,7 @@ def test_walk_directories_multiple_file_types(mock_client):
             "walk-directories",
             "-t",
             "dotenv",
-            "--config-dir",
+            "--config-dirs",
             f"{current_dir}/../../samples",
         ],
         input="\n".join(
@@ -199,7 +199,7 @@ def test_walk_directories_with_exclusion(mock_client):
             "dotenv",
             "-t",
             "yaml",
-            "--config-dir",
+            "--config-dirs",
             f"{current_dir}/../../samples",
             "--exclude-dirs",
             f"{current_dir}/../../samples/dotenvs",
@@ -261,7 +261,7 @@ def test_walk_directories_with_inheritance(mock_client):
             "yaml",
             "--create-hierarchy",
             "-c",
-            "--config-dir",
+            "--config-dirs",
             f"{current_dir}/../../samples",
         ],
         input="\n".join(prompt_responses),