From 06efba2808420634c3616735f3f3adc81d673ec8 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 11 Oct 2022 15:25:08 +0200 Subject: [PATCH 01/11] Added utility functions to load bundlenames Ticket: CFE-3990 Changelog: None Signed-off-by: Lars Erik Wik --- cfbs/utils.py | 14 ++++++++++++++ tests/test_utils.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cfbs/utils.py b/cfbs/utils.py index 11edddf0..6c37c302 100644 --- a/cfbs/utils.py +++ b/cfbs/utils.py @@ -288,3 +288,17 @@ def wrapper(*args, **kwargs): def canonify(s: str): s = "".join([c if c.isalnum() else "_" for c in s]) return s + + +def load_bundlenames(file: str): + with open(file, "r") as f: + policy = f.read() + return loads_bundlenames(policy) + + +def loads_bundlenames(policy: str): + # The lookbehind only supports fixed length strings + policy = re.sub(r"[ \t]+", " ", policy) + + regex = r"(?<=^bundle agent )[a-zA-Z0-9_\200-\377]+" + return re.findall(regex, policy, re.MULTILINE) diff --git a/tests/test_utils.py b/tests/test_utils.py index 391ce664..39df7006 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,4 @@ -from cfbs.utils import canonify, merge_json +from cfbs.utils import canonify, merge_json, loads_bundlenames def test_canonify(): @@ -33,3 +33,33 @@ def test_merge_json(): merged["variables"]["cfbs:create_single_file.filename"]["comment"] == "Added by 'cfbs input'" ) + + +def test_loads_bundlenames_single_bundle(): + policy = """bundle agent bogus +{ + reports: + "Hello World"; +} +""" + bundles = loads_bundlenames(policy) + assert len(bundles) == 1 + assert bundles[0] == "bogus" + + +def test_loads_bundlenames_multiple_bundles(): + policy = """bundle\tagent bogus { + reports: + "Bogus!"; +} + +bundle agent doofus +{ + reports: + "Doofus!"; +} +""" + bundles = loads_bundlenames(policy) + assert len(bundles) == 2 + assert bundles[0] == "bogus" + assert bundles[1] == "doofus" From 4a606becc27410667100a6b0330b3d62c2c2d73c Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 11 Oct 2022 15:27:25 +0200 Subject: [PATCH 02/11] Added message to assert in `prompt_user` Added message to assert in `prompt_user` in order to make it easier to debug when `default` is neither `None` nor in `choices`. Ticket: None Changelog: None Signed-off-by: Lars Erik Wik --- cfbs/prompts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cfbs/prompts.py b/cfbs/prompts.py index 3af1fc64..c5cfe904 100644 --- a/cfbs/prompts.py +++ b/cfbs/prompts.py @@ -12,7 +12,9 @@ def prompt_user(non_interactive, prompt, choices=None, default=None): prompt_separator = " " if prompt.endswith("?") else ": " if choices: - assert default is None or str(default) in choices + assert ( + default is None or str(default) in choices + ), "'%s' not 'None' and '%s' not in '%s'" % (default, default, choices) choices_str = "/".join( choice.upper() if choice == str(default) else choice for choice in choices ) From 7c807ffc27e44e72b73efedcf5931979ed6bbbae Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 11 Oct 2022 15:31:48 +0200 Subject: [PATCH 03/11] Prompt user which bundle to run cfbs now prompts user which bundle to run when adding a local module or directory. Ticket: CFE-3990 Changelog: Body Signed-off-by: Lars Erik Wik --- cfbs/cfbs_config.py | 76 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/cfbs/cfbs_config.py b/cfbs/cfbs_config.py index 2fa1e7aa..01c450a6 100644 --- a/cfbs/cfbs_config.py +++ b/cfbs/cfbs_config.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os import copy +import glob import logging as log from collections import OrderedDict @@ -10,6 +11,7 @@ read_file, find, write_json, + load_bundlenames, ) from cfbs.internal_file_management import ( clone_url_repo, @@ -225,6 +227,79 @@ def _find_dependencies(self, modules, exclude): dependencies += self._find_dependencies(dependencies, exclude) return dependencies + def _add_to_inputs(self, module): + name = module["name"] + step = "policy_files %s" % name + module["steps"].append(step) + log.debug("Added build step '%s' for module '%s'" % (step, name)) + + def _add_to_bundleseqence(self, module, policy_files): + name = module["name"] + choices = [] + first = True + prompt_str = "Which bundle should be evaluated (added to bundle sequence)?" + + for file in policy_files: + log.debug("Looking for bundles in policy file '%s'" % file) + for bundle in load_bundlenames(file): + log.debug("Found bundle '%s'" % bundle) + choices.append(bundle) + prompt_str += "\n%2d. %s:%s" % (len(choices), file, bundle) + if first: + prompt_str += " (default)" + first = False + + if not choices: + log.warning("Did not find any bundles to add to bundlesequence") + return + + choices.append(None) + prompt_str += "\n%2d. (None)\n" % (len(choices)) + + response = prompt_user( + self.non_interactive, + prompt_str, + [str(i + 1) for i in range(len(choices))], + 1, + ) + bundle = choices[int(response) - 1] + if bundle is None: + log.debug("User chose not to add any bundles to the bundlesequence") + return + log.debug("User chose to add '%s' to the bundlesequence" % bundle) + + step = "bundles %s" % bundle + module["steps"].append(step) + log.debug("Added build step '%s' for module '%s'" % (step, name)) + + def _handle_local_module(self, module): + name = module["name"] + if not ( + name.startswith("./") + and name.endswith((".cf", "/")) + and "local" in module["tags"] + ): + log.debug("Module '%s' does not appear to be a local module" % name) + return + + if name.endswith(".cf"): + policy_files = [name] + else: + pattern = "%s/**/*.cf" % name + policy_files = glob.glob(pattern, recursive=True) + + for file in policy_files: + if _has_autorun_tag(file): + log.warning( + "Found bundle tagged with autorun in local policy file '%s': " + % file + + "Note that the autorun tag is ignored when adding local policy files or subdirectories." + ) + # TODO: Support adding local modules with autorun tag + + self._add_to_inputs(module) + self._add_to_bundleseqence(module, policy_files) + def _add_without_dependencies(self, modules): assert modules assert len(modules) > 0 @@ -237,6 +312,7 @@ def _add_without_dependencies(self, modules): module["index"] = self.index.custom_index self["build"].append(module) self.validate_added_module(module) + self._handle_local_module(module) added_by = module["added_by"] if added_by == "cfbs add": From 3f939c80820ee5f6cd39f2a932baa4981c8632f1 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Mon, 17 Oct 2022 09:45:04 +0200 Subject: [PATCH 04/11] Added unit test for utility function `merge_json` Added unit test for utility function `merge_json` during some debugging in order to make sure the function worked as I expected. Might as well keep it. Ticket: None Changelog: None Signed-off-by: Lars Erik Wik --- tests/test_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 39df7006..7a016bf1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -34,6 +34,26 @@ def test_merge_json(): == "Added by 'cfbs input'" ) + original = { + "classes": {"services_autorun": ["any"]}, + "inputs": ["services/cfbs/bogus.cf"], + "vars": {"control_common_bundlesequence_end": ["bogus"]}, + } + extras = { + "inputs": ["services/cfbs/doofus/doofus.cf", "services/cfbs/doofus/foo/foo.cf"] + } + merged = merge_json(original, extras) + expected = { + "classes": {"services_autorun": ["any"]}, + "inputs": [ + "services/cfbs/bogus.cf", + "services/cfbs/doofus/doofus.cf", + "services/cfbs/doofus/foo/foo.cf", + ], + "vars": {"control_common_bundlesequence_end": ["bogus"]}, + } + assert merged == expected + def test_loads_bundlenames_single_bundle(): policy = """bundle agent bogus From 2f4bede5984ceb051e6ab3b383ff07686adfceed Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Mon, 17 Oct 2022 09:50:41 +0200 Subject: [PATCH 05/11] Remove autorun when creating module from local file Removed autorun dependency added when creating module from local policy file, and changed the destion from `/services/autorun/` to `/services/cfbs/`. Ticket: CFE-3990 Changelog: Body Signed-off-by: Lars Erik Wik --- cfbs/index.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cfbs/index.py b/cfbs/index.py index 02bb5f03..1706b646 100644 --- a/cfbs/index.py +++ b/cfbs/index.py @@ -14,12 +14,11 @@ def _local_module_data_cf_file(module): - target = os.path.basename(module) + dst = os.path.join("services", "cfbs", module[2:]) return { "description": "Local policy file added using cfbs command line", "tags": ["local"], - "dependencies": ["autorun"], - "steps": ["copy %s services/autorun/%s" % (module, target)], + "steps": ["copy %s %s" % (module, dst)], "added_by": "cfbs add", } From dbf9dc09f917d8a625a59e8f43138d89783782f3 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Mon, 17 Oct 2022 12:02:23 +0200 Subject: [PATCH 06/11] Remove autorun logic from directory build step Remove logic that adds policy files to the inputs attribute in def.json. Also the copy logic now keeps the directory hierachy of the module. Ticket: CFE-3990 Changelog: Body Signed-off-by: Lars Erik Wik --- cfbs/build.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/cfbs/build.py b/cfbs/build.py index 5064e882..ae2b9fbf 100644 --- a/cfbs/build.py +++ b/cfbs/build.py @@ -120,26 +120,17 @@ def _perform_build_step(module, step, max_length): merged = read_json(defjson) if not merged: merged = {} - if "classes" not in merged: - merged["classes"] = {} - if "services_autorun_bundles" not in merged["classes"]: - merged["classes"]["services_autorun_bundles"] = ["any"] - inputs = [] - for root, dirs, files in os.walk(src): + for root, _, files in os.walk(src): for f in files: - if f.endswith(".cf"): - inputs.append(os.path.join(dstarg, f)) - cp(os.path.join(root, f), os.path.join(destination, dstarg, f)) - elif f == "def.json": + if f == "def.json": extra = read_json(os.path.join(root, f)) if extra: merged = merge_json(merged, extra) else: - cp(os.path.join(root, f), os.path.join(destination, dstarg, f)) - if "inputs" in merged: - merged["inputs"].extend(inputs) - else: - merged["inputs"] = inputs + s = os.path.join(root, f) + d = os.path.join(destination, dstarg, root[len(src) :], f) + log.debug("Copying '%s' to '%s'" % (s, d)) + cp(s, d) write_json(defjson, merged) elif operation == "input": src, dst = args From 785930188564c4277fceccd1713a149d0fca0a67 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 18 Oct 2022 11:07:25 +0200 Subject: [PATCH 07/11] No warning adding files without autorun Removed the warning printed when adding files not tagged with autorun. Ticket: CFE-3990 Changelog: Body Signed-off-by: Lars Erik Wik --- cfbs/cfbs_config.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/cfbs/cfbs_config.py b/cfbs/cfbs_config.py index 01c450a6..e0143c17 100644 --- a/cfbs/cfbs_config.py +++ b/cfbs/cfbs_config.py @@ -51,27 +51,6 @@ class CFBSConfig(CFBSJson): def exists(path="./cfbs.json"): return os.path.exists(path) - @staticmethod - def validate_added_module(module): - """Try to help the user with warnings in appropriate cases""" - - name = module["name"] - if name.startswith("./") and name.endswith(".cf"): - assert os.path.isfile(name) - if not _has_autorun_tag(name): - log.warning("No autorun tag found in policy file: '%s'" % name) - log.warning("Tag the bundle(s) you want evaluated:") - log.warning(' meta: "tags" slist => { "autorun" };') - return - if name.startswith("./") and name.endswith("/"): - assert os.path.isdir(name) - policy_files = list(find(name, extension=".cf")) - with_autorun = (x for x in policy_files if _has_autorun_tag(x)) - if any(policy_files) and not any(with_autorun): - log.warning("No bundles tagged with autorun found in: '%s'" % name) - log.warning("Tag the bundle(s) you want evaluated in .cf policy files:") - log.warning(' meta: "tags" slist => { "autorun" };') - @classmethod def get_instance(cls, index=None, non_interactive=False): if cls.instance is not None: @@ -128,7 +107,6 @@ def add_with_dependencies(self, module, remote_config=None, dependent=None): print("Added module: %s (Dependency of %s)" % (module["name"], dependent)) else: print("Added module: %s" % module["name"]) - self.validate_added_module(module) def _add_using_url( self, @@ -311,7 +289,6 @@ def _add_without_dependencies(self, modules): if self.index.custom_index != None: module["index"] = self.index.custom_index self["build"].append(module) - self.validate_added_module(module) self._handle_local_module(module) added_by = module["added_by"] From a37067cb19787db3231e17e0d397f2cb5df1ab99 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 12 Oct 2022 13:24:55 +0200 Subject: [PATCH 08/11] Added build step `policy_files` Added build step `policy_files` that adds policy files to inputs. Ticket: CFE-3990 Changelog: Body Signed-off-by: Lars Erik Wik --- cfbs/build.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cfbs/build.py b/cfbs/build.py index ae2b9fbf..a2c09eb1 100644 --- a/cfbs/build.py +++ b/cfbs/build.py @@ -1,4 +1,5 @@ import os +import glob import logging as log from cfbs.utils import ( canonify, @@ -161,6 +162,32 @@ def _perform_build_step(module, step, max_length): merged = extras log.debug("Merged def.json: %s", pretty(merged)) write_json(dst, merged) + elif operation == "policy_files": + files = [] + for file in args: + if file.startswith("./"): + file = file[2:] + if file.endswith(".cf"): + files.append(file) + elif file.endswith("/"): + pattern = "%s**/*.cf" % file + files += glob.glob(pattern, recursive=True) + else: + user_error( + "Unsupported filetype '%s' for build step '%s': " + % (file, operation) + + "Expected directory (*/) of policy file (*.cf)" + ) + files = [os.path.join("services", "cfbs", file) for file in files] + print("%s policy_files '%s'" % (prefix, "' '".join(files) if files else "")) + augment = {"inputs": files} + log.debug("Generated augment: %s" % pretty(augment)) + path = os.path.join(destination, "def.json") + original = read_json(path) + log.debug("Original def.json: %s" % pretty(original)) + merged = merge_json(original, augment) if original else augment + log.debug("Merged def.json: %s", pretty(merged)) + write_json(path, merged) else: user_error("Unknown build step operation: %s" % operation) From b49df849d55b54212fb242b84a178c39333fc477 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 12 Oct 2022 13:28:51 +0200 Subject: [PATCH 09/11] Added build step `bundles` Added build step `bundles` that adds bundles to the bundlesequence. Ticket: CFE-3990 Changelog: Body Signed-off-by: Lars Erik Wik --- cfbs/build.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cfbs/build.py b/cfbs/build.py index a2c09eb1..b6553536 100644 --- a/cfbs/build.py +++ b/cfbs/build.py @@ -188,6 +188,17 @@ def _perform_build_step(module, step, max_length): merged = merge_json(original, augment) if original else augment log.debug("Merged def.json: %s", pretty(merged)) write_json(path, merged) + elif operation == "bundles": + bundles = args + print("%s bundles '%s'" % (prefix, "' '".join(bundles) if bundles else "")) + augment = {"vars": {"control_common_bundlesequence_end": bundles}} + log.debug("Generated augment: %s" % pretty(augment)) + path = os.path.join(destination, "def.json") + original = read_json(path) + log.debug("Original def.json: %s" % pretty(original)) + merged = merge_json(original, augment) if original else augment + log.debug("Merged def.json: %s", pretty(merged)) + write_json(path, merged) else: user_error("Unknown build step operation: %s" % operation) From 1222b51bb329bea7786ccb4b336ac1b916706151 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Mon, 17 Oct 2022 17:25:57 +0200 Subject: [PATCH 10/11] Modify 010_local_add shell test Modify 010_local_add shell test to work with changes from CFE-3990. Ticket: CFE-3990 Changelog: None Signed-off-by: Lars Erik Wik --- tests/shell/010_local_add.sh | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/shell/010_local_add.sh b/tests/shell/010_local_add.sh index a6d0b43f..f93a9d97 100644 --- a/tests/shell/010_local_add.sh +++ b/tests/shell/010_local_add.sh @@ -8,18 +8,27 @@ rm -rf .git cfbs --non-interactive init cfbs status -echo ' -bundle agent test_bundle -{ - meta: - "tags" slist => { "autorun" }; + +echo 'bundle agent bogus { reports: - "test"; + "This is $(this.promise_filename):$(this.bundle)!"; } -' > test_policy.cf -cfbs --non-interactive add ./test_policy.cf -grep '"name": "autorun"' cfbs.json -grep '"name": "./test_policy.cf"' cfbs.json +' > bogus.cf + + +cfbs --non-interactive add ./bogus.cf + +grep '"name": "./bogus.cf"' cfbs.json +grep '"policy_files ./bogus.cf"' cfbs.json +grep '"bundles bogus"' cfbs.json + cfbs status cfbs build -ls out/masterfiles/services/autorun/test_policy.cf + +grep '"inputs"' out/masterfiles/def.json +grep '"services/cfbs/bogus.cf"' out/masterfiles/def.json + +grep '"control_common_bundlesequence_end"' out/masterfiles/def.json +grep '"bogus"' out/masterfiles/def.json + +ls out/masterfiles/services/cfbs/bogus.cf From 3ec21875318247e2d834c312f5bbc57fd8c7a782 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 18 Oct 2022 10:57:30 +0200 Subject: [PATCH 11/11] Modify 016_add_folders shell test Modify 016_add_folders shell test to work with changes from CFE-3990. Ticket: CFE-3990 Changelog: None Signed-off-by: Lars Erik Wik --- tests/shell/016_add_folders.sh | 65 +++++++++++++++------------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/tests/shell/016_add_folders.sh b/tests/shell/016_add_folders.sh index 01d6f2b5..6a6b1fb6 100644 --- a/tests/shell/016_add_folders.sh +++ b/tests/shell/016_add_folders.sh @@ -5,63 +5,54 @@ mkdir -p ./tmp/ cd ./tmp/ touch cfbs.json && rm cfbs.json rm -rf .git -rm -rf one -rm -rf two -mkdir one -echo ' -bundle agent bundle_one -{ - meta: - "tags" slist => { "autorun" }; +mkdir -p doofus +echo 'bundle agent doofus { reports: - "one"; + "This is $(this.promise_filename):$(this.bundle)!"; } -' > one/policy.cf -echo '{} -' > one/data.json +' > doofus/doofus.cf -mkdir two -mkdir two/three -echo ' -bundle agent bundle_two -{ - meta: - "tags" slist => { "autorun" }; +mkdir -p doofus/foo +echo 'bundle agent foo { reports: - "two"; + "This is $(this.promise_filename):$(this.bundle)!"; } -' > two/three/policy.cf +' > doofus/foo/foo.cf + +echo '{} +' > doofus/data.json + echo '{ "vars": { "foo_thing": "awesome" } } -' > two/three/def.json -echo 'Hello -' > two/three/file.txt +' > doofus/foo/def.json cfbs --non-interactive init cfbs status -cfbs --non-interactive add ./one -cfbs --non-interactive add ./two/ +cfbs --non-interactive add ./doofus/ cfbs status -cfbs status | grep "./one/" -cfbs status | grep "./two/" -cat cfbs.json | grep "directory ./ services/cfbs/one/" -cat cfbs.json | grep "directory ./ services/cfbs/two/" +cfbs status | grep "./doofus/" +grep '"name": "./doofus/"' cfbs.json +grep '"directory ./ services/cfbs/doofus/"' cfbs.json +grep '"policy_files ./doofus/"' cfbs.json +grep '"bundles doofus"' cfbs.json cfbs build -ls out/masterfiles/services/cfbs/one -grep "bundle_one" out/masterfiles/services/cfbs/one/policy.cf -ls out/masterfiles/services/cfbs/one/data.json +grep '"inputs"' out/masterfiles/def.json +grep '"services/cfbs/doofus/doofus.cf"' out/masterfiles/def.json +grep '"services/cfbs/doofus/foo/foo.cf"' out/masterfiles/def.json + +grep '"control_common_bundlesequence_end"' out/masterfiles/def.json +grep '"doofus"' out/masterfiles/def.json -ls out/masterfiles/services/cfbs/two -grep "bundle_two" out/masterfiles/services/cfbs/two/policy.cf -grep "Hello" out/masterfiles/services/cfbs/two/file.txt +grep '"foo_thing": "awesome"' out/masterfiles/def.json -grep "awesome" out/masterfiles/def.json +ls out/masterfiles/services/cfbs/doofus/doofus.cf +ls out/masterfiles/services/cfbs/doofus/foo/foo.cf