From c563e46e6f3b604792e210eadb735dceba1623f5 Mon Sep 17 00:00:00 2001 From: Matthew Warren Date: Thu, 25 Apr 2024 16:35:53 -0400 Subject: [PATCH] sc-12669 detect secret parameters and mark them as such for CT encryption --- samples/advanced/values.yaml | 2 +- src/dynamic_importer/processors/__init__.py | 25 +++++++++++++----- src/tests/processors/test_yaml.py | 29 ++++++++++++++++++--- src/tests/test_cli.py | 17 ++++++++++++ 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/samples/advanced/values.yaml b/samples/advanced/values.yaml index 246ce6a..16d15b4 100644 --- a/samples/advanced/values.yaml +++ b/samples/advanced/values.yaml @@ -81,7 +81,7 @@ tolerations: [] affinity: {} appSettings: - apiKey: + apiKey: "abc123-sodapop48fizzybusiness0202" apiUrl: pollingInterval: debug: false diff --git a/src/dynamic_importer/processors/__init__.py b/src/dynamic_importer/processors/__init__.py index 29e1e02..a170d51 100644 --- a/src/dynamic_importer/processors/__init__.py +++ b/src/dynamic_importer/processors/__init__.py @@ -8,6 +8,7 @@ import importlib import os import pkgutil +import re from copy import deepcopy from typing import Any from typing import Dict @@ -16,15 +17,22 @@ from typing import Tuple from typing import Union +RE_WORDS = "(pas+wo?r?d|pass(phrase)?|pwd|token|secrete?|api(\\W|_)?key)" +RE_CANDIDATES = re.compile("(^{0}$|_{0}_|^{0}_|_{0}$)".format(RE_WORDS), re.IGNORECASE) + def get_processor_class(file_type: str) -> BaseProcessor: ft_lower = file_type.lower() - if processor_module := importlib.import_module( - f"dynamic_importer.processors.{ft_lower}" - ): - for subclass in BaseProcessor.__subclasses__(): - if subclass.__name__.lower() == f"{ft_lower}processor": - return getattr(processor_module, subclass.__name__) + try: + processor_module = importlib.import_module( + f"dynamic_importer.processors.{ft_lower}" + ) + except ModuleNotFoundError: + raise ValueError(f"No processor found for file type: {file_type}") + + for subclass in BaseProcessor.__subclasses__(): + if subclass.__name__.lower() == f"{ft_lower}processor": + return getattr(processor_module, subclass.__name__) raise ValueError(f"No processor found for file type: {file_type}") @@ -49,6 +57,9 @@ class BaseProcessor: def __init__(self, env_values: Dict) -> None: raise NotImplementedError("Subclasses must implement the __init__ method") + def is_param_secret(self, param_name: str) -> bool: + return bool(RE_CANDIDATES.search(param_name)) + def guess_type(self, value): """ Guess the type of the value and return it as a string. @@ -138,7 +149,7 @@ def _traverse_data( "values": {env: obj}, "param_name": param_name, "type": obj_type, - "secret": False, + "secret": self.is_param_secret(param_name), } } diff --git a/src/tests/processors/test_yaml.py b/src/tests/processors/test_yaml.py index 953aae0..f5e8df9 100644 --- a/src/tests/processors/test_yaml.py +++ b/src/tests/processors/test_yaml.py @@ -12,10 +12,13 @@ class YamlTestCase(TestCase): + def setUp(self) -> None: + self.current_dir = pathlib.Path(__file__).parent.resolve() + return super().setUp() + def test_yaml_with_embedded_templates(self): - current_dir = pathlib.Path(__file__).parent.resolve() processor = YAMLProcessor( - {"default": f"{current_dir}/../../../samples/advanced/values.yaml"} + {"default": f"{self.current_dir}/../../../samples/advanced/values.yaml"} ) processed_template, processed_data = processor.process() @@ -50,9 +53,10 @@ def test_yaml_with_embedded_templates(self): ) def test_yaml_double_quoting(self): - current_dir = pathlib.Path(__file__).parent.resolve() processor = YAMLProcessor( - {"default": f"{current_dir}/../../../samples/advanced/app-config.yaml.hbs"} + { + "default": f"{self.current_dir}/../../../samples/advanced/app-config.yaml.hbs" + } ) _, processed_data = processor.process() template_str = processor.generate_template(processed_data) @@ -66,3 +70,20 @@ def test_yaml_double_quoting(self): template_str[begin_idx:end_idx], ) self.assertEqual(template_str.count('"'), 2) + + def test_yaml_secret_masking(self): + processor = YAMLProcessor( + {"default": f"{self.current_dir}/../../../samples/advanced/values.yaml"} + ) + processed_template, processed_data = processor.process() + + self.assertTrue(processed_data["[appSettings][apiKey]"]["secret"]) + self.assertTrue( + processed_data["[projectMappings][root][resource_templates][secret]"][ + "secret" + ] + ) + # These shouldn't be true but "secret" in the name makes them marked as secrets + # This is a limitation of the current implementation but users can manually override + self.assertTrue(processed_data["[secret][create]"]["secret"]) + self.assertTrue(processed_data["[secret][name]"]["secret"]) diff --git a/src/tests/test_cli.py b/src/tests/test_cli.py index 2f2aace..e6e044c 100644 --- a/src/tests/test_cli.py +++ b/src/tests/test_cli.py @@ -17,6 +17,7 @@ import pytest from click.testing import CliRunner from dynamic_importer.main import import_config +from dynamic_importer.processors import get_processor_class from tests.fixtures.requests import mocked_requests_localhost_get from tests.fixtures.requests import mocked_requests_localhost_post @@ -46,6 +47,22 @@ def test_process_configs_no_args(self): result.output, ) + def test_process_configs_invalid_type(self): + runner = CliRunner() + result = runner.invoke( + import_config, ["process-configs", "-t", "spam", "-p", "testproj"] + ) + self.assertEqual(result.exit_code, 2) + self.assertIn( + "Error: Invalid value for '-t' / '--file-type': " + "'spam' is not one of 'yaml', 'dotenv', 'json', 'tf', 'tfvars'", + result.output, + ) + + def test_get_processor_class_invalid_type(self): + with pytest.raises(ValueError): + get_processor_class("spam") + def test_regenerate_template_no_args(self): runner = CliRunner() result = runner.invoke(