diff --git a/cfbs/build.py b/cfbs/build.py index 5064e882..b6553536 100644 --- a/cfbs/build.py +++ b/cfbs/build.py @@ -1,4 +1,5 @@ import os +import glob import logging as log from cfbs.utils import ( canonify, @@ -120,26 +121,17 @@ def _perform_build_step(module, step, max_length): merged = read_json(defjson) if not merged: merged = {} - if "classes" not in merged: - merged["classes"] = {} - if "services_autorun_bundles" not in merged["classes"]: - merged["classes"]["services_autorun_bundles"] = ["any"] - inputs = [] - for root, dirs, files in os.walk(src): + for root, _, files in os.walk(src): for f in files: - if f.endswith(".cf"): - inputs.append(os.path.join(dstarg, f)) - cp(os.path.join(root, f), os.path.join(destination, dstarg, f)) - elif f == "def.json": + if f == "def.json": extra = read_json(os.path.join(root, f)) if extra: merged = merge_json(merged, extra) else: - cp(os.path.join(root, f), os.path.join(destination, dstarg, f)) - if "inputs" in merged: - merged["inputs"].extend(inputs) - else: - merged["inputs"] = inputs + s = os.path.join(root, f) + d = os.path.join(destination, dstarg, root[len(src) :], f) + log.debug("Copying '%s' to '%s'" % (s, d)) + cp(s, d) write_json(defjson, merged) elif operation == "input": src, dst = args @@ -170,6 +162,43 @@ def _perform_build_step(module, step, max_length): merged = extras log.debug("Merged def.json: %s", pretty(merged)) write_json(dst, merged) + elif operation == "policy_files": + files = [] + for file in args: + if file.startswith("./"): + file = file[2:] + if file.endswith(".cf"): + files.append(file) + elif file.endswith("/"): + pattern = "%s**/*.cf" % file + files += glob.glob(pattern, recursive=True) + else: + user_error( + "Unsupported filetype '%s' for build step '%s': " + % (file, operation) + + "Expected directory (*/) of policy file (*.cf)" + ) + files = [os.path.join("services", "cfbs", file) for file in files] + print("%s policy_files '%s'" % (prefix, "' '".join(files) if files else "")) + augment = {"inputs": files} + log.debug("Generated augment: %s" % pretty(augment)) + path = os.path.join(destination, "def.json") + original = read_json(path) + log.debug("Original def.json: %s" % pretty(original)) + merged = merge_json(original, augment) if original else augment + log.debug("Merged def.json: %s", pretty(merged)) + write_json(path, merged) + elif operation == "bundles": + bundles = args + print("%s bundles '%s'" % (prefix, "' '".join(bundles) if bundles else "")) + augment = {"vars": {"control_common_bundlesequence_end": bundles}} + log.debug("Generated augment: %s" % pretty(augment)) + path = os.path.join(destination, "def.json") + original = read_json(path) + log.debug("Original def.json: %s" % pretty(original)) + merged = merge_json(original, augment) if original else augment + log.debug("Merged def.json: %s", pretty(merged)) + write_json(path, merged) else: user_error("Unknown build step operation: %s" % operation) diff --git a/cfbs/cfbs_config.py b/cfbs/cfbs_config.py index 2fa1e7aa..e0143c17 100644 --- a/cfbs/cfbs_config.py +++ b/cfbs/cfbs_config.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os import copy +import glob import logging as log from collections import OrderedDict @@ -10,6 +11,7 @@ read_file, find, write_json, + load_bundlenames, ) from cfbs.internal_file_management import ( clone_url_repo, @@ -49,27 +51,6 @@ class CFBSConfig(CFBSJson): def exists(path="./cfbs.json"): return os.path.exists(path) - @staticmethod - def validate_added_module(module): - """Try to help the user with warnings in appropriate cases""" - - name = module["name"] - if name.startswith("./") and name.endswith(".cf"): - assert os.path.isfile(name) - if not _has_autorun_tag(name): - log.warning("No autorun tag found in policy file: '%s'" % name) - log.warning("Tag the bundle(s) you want evaluated:") - log.warning(' meta: "tags" slist => { "autorun" };') - return - if name.startswith("./") and name.endswith("/"): - assert os.path.isdir(name) - policy_files = list(find(name, extension=".cf")) - with_autorun = (x for x in policy_files if _has_autorun_tag(x)) - if any(policy_files) and not any(with_autorun): - log.warning("No bundles tagged with autorun found in: '%s'" % name) - log.warning("Tag the bundle(s) you want evaluated in .cf policy files:") - log.warning(' meta: "tags" slist => { "autorun" };') - @classmethod def get_instance(cls, index=None, non_interactive=False): if cls.instance is not None: @@ -126,7 +107,6 @@ def add_with_dependencies(self, module, remote_config=None, dependent=None): print("Added module: %s (Dependency of %s)" % (module["name"], dependent)) else: print("Added module: %s" % module["name"]) - self.validate_added_module(module) def _add_using_url( self, @@ -225,6 +205,79 @@ def _find_dependencies(self, modules, exclude): dependencies += self._find_dependencies(dependencies, exclude) return dependencies + def _add_to_inputs(self, module): + name = module["name"] + step = "policy_files %s" % name + module["steps"].append(step) + log.debug("Added build step '%s' for module '%s'" % (step, name)) + + def _add_to_bundleseqence(self, module, policy_files): + name = module["name"] + choices = [] + first = True + prompt_str = "Which bundle should be evaluated (added to bundle sequence)?" + + for file in policy_files: + log.debug("Looking for bundles in policy file '%s'" % file) + for bundle in load_bundlenames(file): + log.debug("Found bundle '%s'" % bundle) + choices.append(bundle) + prompt_str += "\n%2d. %s:%s" % (len(choices), file, bundle) + if first: + prompt_str += " (default)" + first = False + + if not choices: + log.warning("Did not find any bundles to add to bundlesequence") + return + + choices.append(None) + prompt_str += "\n%2d. (None)\n" % (len(choices)) + + response = prompt_user( + self.non_interactive, + prompt_str, + [str(i + 1) for i in range(len(choices))], + 1, + ) + bundle = choices[int(response) - 1] + if bundle is None: + log.debug("User chose not to add any bundles to the bundlesequence") + return + log.debug("User chose to add '%s' to the bundlesequence" % bundle) + + step = "bundles %s" % bundle + module["steps"].append(step) + log.debug("Added build step '%s' for module '%s'" % (step, name)) + + def _handle_local_module(self, module): + name = module["name"] + if not ( + name.startswith("./") + and name.endswith((".cf", "/")) + and "local" in module["tags"] + ): + log.debug("Module '%s' does not appear to be a local module" % name) + return + + if name.endswith(".cf"): + policy_files = [name] + else: + pattern = "%s/**/*.cf" % name + policy_files = glob.glob(pattern, recursive=True) + + for file in policy_files: + if _has_autorun_tag(file): + log.warning( + "Found bundle tagged with autorun in local policy file '%s': " + % file + + "Note that the autorun tag is ignored when adding local policy files or subdirectories." + ) + # TODO: Support adding local modules with autorun tag + + self._add_to_inputs(module) + self._add_to_bundleseqence(module, policy_files) + def _add_without_dependencies(self, modules): assert modules assert len(modules) > 0 @@ -236,7 +289,7 @@ def _add_without_dependencies(self, modules): if self.index.custom_index != None: module["index"] = self.index.custom_index self["build"].append(module) - self.validate_added_module(module) + self._handle_local_module(module) added_by = module["added_by"] if added_by == "cfbs add": diff --git a/cfbs/index.py b/cfbs/index.py index 02bb5f03..1706b646 100644 --- a/cfbs/index.py +++ b/cfbs/index.py @@ -14,12 +14,11 @@ def _local_module_data_cf_file(module): - target = os.path.basename(module) + dst = os.path.join("services", "cfbs", module[2:]) return { "description": "Local policy file added using cfbs command line", "tags": ["local"], - "dependencies": ["autorun"], - "steps": ["copy %s services/autorun/%s" % (module, target)], + "steps": ["copy %s %s" % (module, dst)], "added_by": "cfbs add", } diff --git a/cfbs/prompts.py b/cfbs/prompts.py index 3af1fc64..c5cfe904 100644 --- a/cfbs/prompts.py +++ b/cfbs/prompts.py @@ -12,7 +12,9 @@ def prompt_user(non_interactive, prompt, choices=None, default=None): prompt_separator = " " if prompt.endswith("?") else ": " if choices: - assert default is None or str(default) in choices + assert ( + default is None or str(default) in choices + ), "'%s' not 'None' and '%s' not in '%s'" % (default, default, choices) choices_str = "/".join( choice.upper() if choice == str(default) else choice for choice in choices ) diff --git a/cfbs/utils.py b/cfbs/utils.py index 11edddf0..6c37c302 100644 --- a/cfbs/utils.py +++ b/cfbs/utils.py @@ -288,3 +288,17 @@ def wrapper(*args, **kwargs): def canonify(s: str): s = "".join([c if c.isalnum() else "_" for c in s]) return s + + +def load_bundlenames(file: str): + with open(file, "r") as f: + policy = f.read() + return loads_bundlenames(policy) + + +def loads_bundlenames(policy: str): + # The lookbehind only supports fixed length strings + policy = re.sub(r"[ \t]+", " ", policy) + + regex = r"(?<=^bundle agent )[a-zA-Z0-9_\200-\377]+" + return re.findall(regex, policy, re.MULTILINE) diff --git a/tests/shell/010_local_add.sh b/tests/shell/010_local_add.sh index a6d0b43f..f93a9d97 100644 --- a/tests/shell/010_local_add.sh +++ b/tests/shell/010_local_add.sh @@ -8,18 +8,27 @@ rm -rf .git cfbs --non-interactive init cfbs status -echo ' -bundle agent test_bundle -{ - meta: - "tags" slist => { "autorun" }; + +echo 'bundle agent bogus { reports: - "test"; + "This is $(this.promise_filename):$(this.bundle)!"; } -' > test_policy.cf -cfbs --non-interactive add ./test_policy.cf -grep '"name": "autorun"' cfbs.json -grep '"name": "./test_policy.cf"' cfbs.json +' > bogus.cf + + +cfbs --non-interactive add ./bogus.cf + +grep '"name": "./bogus.cf"' cfbs.json +grep '"policy_files ./bogus.cf"' cfbs.json +grep '"bundles bogus"' cfbs.json + cfbs status cfbs build -ls out/masterfiles/services/autorun/test_policy.cf + +grep '"inputs"' out/masterfiles/def.json +grep '"services/cfbs/bogus.cf"' out/masterfiles/def.json + +grep '"control_common_bundlesequence_end"' out/masterfiles/def.json +grep '"bogus"' out/masterfiles/def.json + +ls out/masterfiles/services/cfbs/bogus.cf diff --git a/tests/shell/016_add_folders.sh b/tests/shell/016_add_folders.sh index 01d6f2b5..6a6b1fb6 100644 --- a/tests/shell/016_add_folders.sh +++ b/tests/shell/016_add_folders.sh @@ -5,63 +5,54 @@ mkdir -p ./tmp/ cd ./tmp/ touch cfbs.json && rm cfbs.json rm -rf .git -rm -rf one -rm -rf two -mkdir one -echo ' -bundle agent bundle_one -{ - meta: - "tags" slist => { "autorun" }; +mkdir -p doofus +echo 'bundle agent doofus { reports: - "one"; + "This is $(this.promise_filename):$(this.bundle)!"; } -' > one/policy.cf -echo '{} -' > one/data.json +' > doofus/doofus.cf -mkdir two -mkdir two/three -echo ' -bundle agent bundle_two -{ - meta: - "tags" slist => { "autorun" }; +mkdir -p doofus/foo +echo 'bundle agent foo { reports: - "two"; + "This is $(this.promise_filename):$(this.bundle)!"; } -' > two/three/policy.cf +' > doofus/foo/foo.cf + +echo '{} +' > doofus/data.json + echo '{ "vars": { "foo_thing": "awesome" } } -' > two/three/def.json -echo 'Hello -' > two/three/file.txt +' > doofus/foo/def.json cfbs --non-interactive init cfbs status -cfbs --non-interactive add ./one -cfbs --non-interactive add ./two/ +cfbs --non-interactive add ./doofus/ cfbs status -cfbs status | grep "./one/" -cfbs status | grep "./two/" -cat cfbs.json | grep "directory ./ services/cfbs/one/" -cat cfbs.json | grep "directory ./ services/cfbs/two/" +cfbs status | grep "./doofus/" +grep '"name": "./doofus/"' cfbs.json +grep '"directory ./ services/cfbs/doofus/"' cfbs.json +grep '"policy_files ./doofus/"' cfbs.json +grep '"bundles doofus"' cfbs.json cfbs build -ls out/masterfiles/services/cfbs/one -grep "bundle_one" out/masterfiles/services/cfbs/one/policy.cf -ls out/masterfiles/services/cfbs/one/data.json +grep '"inputs"' out/masterfiles/def.json +grep '"services/cfbs/doofus/doofus.cf"' out/masterfiles/def.json +grep '"services/cfbs/doofus/foo/foo.cf"' out/masterfiles/def.json + +grep '"control_common_bundlesequence_end"' out/masterfiles/def.json +grep '"doofus"' out/masterfiles/def.json -ls out/masterfiles/services/cfbs/two -grep "bundle_two" out/masterfiles/services/cfbs/two/policy.cf -grep "Hello" out/masterfiles/services/cfbs/two/file.txt +grep '"foo_thing": "awesome"' out/masterfiles/def.json -grep "awesome" out/masterfiles/def.json +ls out/masterfiles/services/cfbs/doofus/doofus.cf +ls out/masterfiles/services/cfbs/doofus/foo/foo.cf diff --git a/tests/test_utils.py b/tests/test_utils.py index 391ce664..7a016bf1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,4 @@ -from cfbs.utils import canonify, merge_json +from cfbs.utils import canonify, merge_json, loads_bundlenames def test_canonify(): @@ -33,3 +33,53 @@ def test_merge_json(): merged["variables"]["cfbs:create_single_file.filename"]["comment"] == "Added by 'cfbs input'" ) + + original = { + "classes": {"services_autorun": ["any"]}, + "inputs": ["services/cfbs/bogus.cf"], + "vars": {"control_common_bundlesequence_end": ["bogus"]}, + } + extras = { + "inputs": ["services/cfbs/doofus/doofus.cf", "services/cfbs/doofus/foo/foo.cf"] + } + merged = merge_json(original, extras) + expected = { + "classes": {"services_autorun": ["any"]}, + "inputs": [ + "services/cfbs/bogus.cf", + "services/cfbs/doofus/doofus.cf", + "services/cfbs/doofus/foo/foo.cf", + ], + "vars": {"control_common_bundlesequence_end": ["bogus"]}, + } + assert merged == expected + + +def test_loads_bundlenames_single_bundle(): + policy = """bundle agent bogus +{ + reports: + "Hello World"; +} +""" + bundles = loads_bundlenames(policy) + assert len(bundles) == 1 + assert bundles[0] == "bogus" + + +def test_loads_bundlenames_multiple_bundles(): + policy = """bundle\tagent bogus { + reports: + "Bogus!"; +} + +bundle agent doofus +{ + reports: + "Doofus!"; +} +""" + bundles = loads_bundlenames(policy) + assert len(bundles) == 2 + assert bundles[0] == "bogus" + assert bundles[1] == "doofus"