From 6f1ee32e9fc013d6a1ec9cba55894fcb3e78359e Mon Sep 17 00:00:00 2001 From: Aaron Loo Date: Fri, 21 Dec 2018 12:06:02 -0800 Subject: [PATCH 1/4] nicer AWS type --- detect_secrets/plugins/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detect_secrets/plugins/aws.py b/detect_secrets/plugins/aws.py index f545b550f..55f49c949 100644 --- a/detect_secrets/plugins/aws.py +++ b/detect_secrets/plugins/aws.py @@ -9,7 +9,7 @@ class AWSKeyDetector(RegexBasedDetector): - secret_type = 'AWS key' + secret_type = 'AWS Access Key' blacklist = ( re.compile(r'AKIA[0-9A-Z]{16}'), ) From 278290f4426f2a5551e06e809f40b48c77d6d633 Mon Sep 17 00:00:00 2001 From: Aaron Loo Date: Fri, 21 Dec 2018 12:06:59 -0800 Subject: [PATCH 2/4] other cleanup --- README.md | 2 +- test_data/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e42310301..311bc7882 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ committing secrets. ### Things that won't be prevented * Multi-line secrets -* Default passwords that do not trigger the `KeywordDetector` (e.g. `paaassword = "paaassword"`) +* Default passwords that do not trigger the `KeywordDetector` (e.g. `login = "hunter2"`) ### Plugin Configuration diff --git a/test_data/config.yaml b/test_data/config.yaml index 329a7c21e..336615948 100644 --- a/test_data/config.yaml +++ b/test_data/config.yaml @@ -3,7 +3,7 @@ credentials: other_value_here: 1234567890a nested: value: AKIAabcdefghijklmnop - value: abcdefghijklmnop + other_value: abcdefghijklmnop list_of_keys: - 123 - 456 From 1c899da1e5d9bd2c757bc44ff77a1c651c2aebc2 Mon Sep 17 00:00:00 2001 From: Aaron Loo Date: Fri, 21 Dec 2018 14:29:56 -0800 Subject: [PATCH 3/4] more support for env variables in files --- .secrets.baseline | 40 ++++++++++----- .../plugins/core/ini_file_parser.py | 11 ++++- .../plugins/high_entropy_strings.py | 37 ++++++++------ test_data/config.env | 1 + tests/plugins/high_entropy_strings_test.py | 49 ++++++++++++------- 5 files changed, 93 insertions(+), 45 deletions(-) create mode 100644 test_data/config.env diff --git a/.secrets.baseline b/.secrets.baseline index 4362363e4..396228a0d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,11 +1,17 @@ { - "exclude_regex": "test_data/.*|tests/.*", - "generated_at": "2018-07-12T23:20:29Z", + "exclude_regex": "test_data/.*|tests/.*|^.secrets.baseline$", + "generated_at": "2018-12-21T22:29:02Z", "plugins_used": [ + { + "name": "AWSKeyDetector" + }, { "base64_limit": 4.5, "name": "Base64HighEntropyString" }, + { + "name": "BasicAuthDetector" + }, { "hex_limit": 3, "name": "HexHighEntropyString" @@ -15,11 +21,18 @@ } ], "results": { + "README.md": [ + { + "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", + "line_number": 153, + "type": "Basic Auth Credentials" + } + ], "detect_secrets/plugins/high_entropy_strings.py": [ { "hashed_secret": "88a7b59d2e9172960b72b65f7839b9da2453f3e9", "is_secret": false, - "line_number": 215, + "line_number": 261, "type": "Hex High Entropy String" } ], @@ -27,46 +40,51 @@ { "hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b", "is_secret": false, - "line_number": 34, + "line_number": 43, "type": "Private Key" }, { "hashed_secret": "daefe0b4345a654580dcad25c7c11ff4c944a8c0", "is_secret": false, - "line_number": 35, + "line_number": 44, "type": "Private Key" }, { "hashed_secret": "f0778f3e140a61d5bbbed5430773e52af2f5fba4", "is_secret": false, - "line_number": 36, + "line_number": 45, "type": "Private Key" }, { "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd", "is_secret": false, - "line_number": 37, + "line_number": 46, "type": "Private Key" }, { "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_secret": false, - "line_number": 38, + "line_number": 47, "type": "Private Key" }, { "hashed_secret": "11200d1bf5e1eb358b5d823c443347d97e982a85", "is_secret": false, - "line_number": 39, + "line_number": 48, "type": "Private Key" }, { "hashed_secret": "9279619d0c9a9529b0b223e3b809f4df24b8ba8b", "is_secret": false, - "line_number": 40, + "line_number": 49, + "type": "Private Key" + }, + { + "hashed_secret": "4ada9713ec27066b2ffe0b7bd9c9c8d635dc4ab2", + "line_number": 50, "type": "Private Key" } ] }, - "version": "0.9.1" + "version": "0.11.0" } diff --git a/detect_secrets/plugins/core/ini_file_parser.py b/detect_secrets/plugins/core/ini_file_parser.py index 1bfd8db6f..41ade35fa 100644 --- a/detect_secrets/plugins/core/ini_file_parser.py +++ b/detect_secrets/plugins/core/ini_file_parser.py @@ -4,10 +4,17 @@ class IniFileParser(object): - def __init__(self, file): + def __init__(self, file, add_header=False): self.parser = configparser.ConfigParser() self.parser.optionxform = str - self.parser.read_file(file) + + if not add_header: + self.parser.read_file(file) + else: + # This supports environment variables, or other files that look + # like config files, without a section header. + content = file.read() + self.parser.read_string('[global]\n' + content) # Hacky way to keep track of line location file.seek(0) diff --git a/detect_secrets/plugins/high_entropy_strings.py b/detect_secrets/plugins/high_entropy_strings.py index 54d9110d2..ec4ecacb6 100644 --- a/detect_secrets/plugins/high_entropy_strings.py +++ b/detect_secrets/plugins/high_entropy_strings.py @@ -52,17 +52,23 @@ def __init__(self, charset, limit, *args): def analyze(self, file, filename): file_type_analyzers = ( - (self._analyze_ini_file, configparser.Error,), + (self._analyze_ini_file(), configparser.Error,), (self._analyze_yaml_file, yaml.YAMLError,), + (super(HighEntropyStringsPlugin, self).analyze, Exception,), + (self._analyze_ini_file(True), configparser.Error,), ) for analyze_function, exception_class in file_type_analyzers: try: - return analyze_function(file, filename) + output = analyze_function(file, filename) + if output: + return output except exception_class: - file.seek(0) + pass - return super(HighEntropyStringsPlugin, self).analyze(file, filename) + file.seek(0) + + return {} def calculate_shannon_entropy(self, data): """Returns the entropy of a given string. @@ -158,21 +164,24 @@ def non_quoted_string_regex(self, strict=True): finally: self.regex = old_regex - def _analyze_ini_file(self, file, filename): + def _analyze_ini_file(self, add_header=False): """ :returns: same format as super().analyze() """ - potential_secrets = {} + def wrapped(file, filename): + potential_secrets = {} - with self.non_quoted_string_regex(): - for value, lineno in IniFileParser(file).iterator(): - potential_secrets.update(self.analyze_string( - value, - lineno, - filename, - )) + with self.non_quoted_string_regex(): + for value, lineno in IniFileParser(file, add_header).iterator(): + potential_secrets.update(self.analyze_string( + value, + lineno, + filename, + )) - return potential_secrets + return potential_secrets + + return wrapped def _analyze_yaml_file(self, file, filename): """ diff --git a/test_data/config.env b/test_data/config.env new file mode 100644 index 000000000..342541295 --- /dev/null +++ b/test_data/config.env @@ -0,0 +1 @@ +mimi=gX69YO4CvBsVjzAwYxdGyDd30t5+9ez31gKATtj4 diff --git a/tests/plugins/high_entropy_strings_test.py b/tests/plugins/high_entropy_strings_test.py index b9b6f374d..d6bdebbb2 100644 --- a/tests/plugins/high_entropy_strings_test.py +++ b/tests/plugins/high_entropy_strings_test.py @@ -108,6 +108,25 @@ def test_ignored_lines(self, content_to_format): assert len(results) == 0 + def test_entropy_lower_limit(self): + with pytest.raises(ValueError): + Base64HighEntropyString(-1) + + def test_entropy_upper_limit(self): + with pytest.raises(ValueError): + Base64HighEntropyString(15) + + +class TestBase64HighEntropyStrings(HighEntropyStringsTest): + + def setup(self): + super(TestBase64HighEntropyStrings, self).setup( + # Testing default limit, as suggested by truffleHog. + Base64HighEntropyString(4.5), + 'c3VwZXIgc2VjcmV0IHZhbHVl', # too short for high entropy + 'c3VwZXIgbG9uZyBzdHJpbmcgc2hvdWxkIGNhdXNlIGVub3VnaCBlbnRyb3B5', + ) + def test_ini_file(self): # We're testing two files here, because we want to make sure that # the HighEntropyStrings regex is reset back to normal after @@ -148,31 +167,25 @@ def test_yaml_file(self): with open('test_data/config.yaml') as f: secrets = plugin.analyze(f, 'test_data/config.yaml') - assert len(secrets.values()) == 1 + assert len(secrets.values()) == 2 for secret in secrets.values(): location = str(secret).splitlines()[1] assert location in ( 'Location: test_data/config.yaml:3', + 'Location: test_data/config.yaml:5', ) - def test_entropy_lower_limit(self): - with pytest.raises(ValueError): - Base64HighEntropyString(-1) - - def test_entropy_upper_limit(self): - with pytest.raises(ValueError): - Base64HighEntropyString(15) + def test_env_file(self): + plugin = Base64HighEntropyString(4.5) + with open('test_data/config.env') as f: + secrets = plugin.analyze(f, 'test_data/config.env') - -class TestBase64HighEntropyStrings(HighEntropyStringsTest): - - def setup(self): - super(TestBase64HighEntropyStrings, self).setup( - # Testing default limit, as suggested by truffleHog. - Base64HighEntropyString(4.5), - 'c3VwZXIgc2VjcmV0IHZhbHVl', # too short for high entropy - 'c3VwZXIgbG9uZyBzdHJpbmcgc2hvdWxkIGNhdXNlIGVub3VnaCBlbnRyb3B5', - ) + assert len(secrets.values()) == 1 + for secret in secrets.values(): + location = str(secret).splitlines()[1] + assert location in ( + 'Location: test_data/config.env:1', + ) class TestHexHighEntropyStrings(HighEntropyStringsTest): From 0c7588084006149c35b001f327132ca5df46baf4 Mon Sep 17 00:00:00 2001 From: Aaron Loo Date: Fri, 21 Dec 2018 15:05:35 -0800 Subject: [PATCH 4/4] python2.7 compat --- detect_secrets/main.py | 2 +- .../plugins/core/ini_file_parser.py | 10 ++- .../plugins/high_entropy_strings.py | 2 +- testing/mocks.py | 2 +- tests/main_test.py | 62 +++++++++++-------- tests/plugins/base_test.py | 2 +- 6 files changed, 48 insertions(+), 32 deletions(-) diff --git a/detect_secrets/main.py b/detect_secrets/main.py index 718955e6d..e29c6b6c0 100644 --- a/detect_secrets/main.py +++ b/detect_secrets/main.py @@ -133,7 +133,7 @@ def _get_existing_baseline(import_filename): return json.loads(stdin) -def _read_from_file(filename): +def _read_from_file(filename): # pragma: no cover """Used for mocking.""" with open(filename) as f: return json.loads(f.read()) diff --git a/detect_secrets/plugins/core/ini_file_parser.py b/detect_secrets/plugins/core/ini_file_parser.py index 41ade35fa..feb037daa 100644 --- a/detect_secrets/plugins/core/ini_file_parser.py +++ b/detect_secrets/plugins/core/ini_file_parser.py @@ -13,8 +13,14 @@ def __init__(self, file, add_header=False): else: # This supports environment variables, or other files that look # like config files, without a section header. - content = file.read() - self.parser.read_string('[global]\n' + content) + content = '[global]\n' + file.read() + + try: + # python2.7 compatible + self.parser.read_string(unicode(content)) + except NameError: + # python3 compatible + self.parser.read_string(content) # Hacky way to keep track of line location file.seek(0) diff --git a/detect_secrets/plugins/high_entropy_strings.py b/detect_secrets/plugins/high_entropy_strings.py index ec4ecacb6..7e97b6f67 100644 --- a/detect_secrets/plugins/high_entropy_strings.py +++ b/detect_secrets/plugins/high_entropy_strings.py @@ -55,7 +55,7 @@ def analyze(self, file, filename): (self._analyze_ini_file(), configparser.Error,), (self._analyze_yaml_file, yaml.YAMLError,), (super(HighEntropyStringsPlugin, self).analyze, Exception,), - (self._analyze_ini_file(True), configparser.Error,), + (self._analyze_ini_file(add_header=True), configparser.Error,), ) for analyze_function, exception_class in file_type_analyzers: diff --git a/testing/mocks.py b/testing/mocks.py index c1e468958..18e1d23f6 100644 --- a/testing/mocks.py +++ b/testing/mocks.py @@ -135,7 +135,7 @@ class PrinterShim(object): def __init__(self): self.clear() - def add(self, message): + def add(self, message, *args, **kwargs): self.message += str(message) + '\n' def clear(self): diff --git a/tests/main_test.py b/tests/main_test.py index f9091a8c8..3d5156dd9 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -15,32 +15,6 @@ from testing.mocks import mock_printer -@pytest.fixture -def mock_baseline_initialize(): - def mock_initialize_function(plugins, exclude_regex, *args, **kwargs): - return secrets_collection_factory( - plugins=plugins, - exclude_regex=exclude_regex, - ) - - with mock.patch( - 'detect_secrets.main.baseline.initialize', - side_effect=mock_initialize_function, - ) as mock_initialize: - yield mock_initialize - - -@pytest.fixture -def mock_merge_baseline(): - with mock.patch( - 'detect_secrets.main.baseline.merge_baseline', - ) as m: - # This return value needs to have the `results` key, so that it can - # formatted appropriately for output. - m.return_value = {'results': {}} - yield m - - class TestMain(object): """These are smoke tests for the console usage of detect_secrets. Most of the functional test cases should be within their own module tests. @@ -269,6 +243,16 @@ def test_audit_short_file(self, filename, expected_output): BashColor.enable_color() + def test_audit_diff_not_enough_files(self): + assert main('audit --diff fileA'.split()) == 1 + + def test_audit_same_file(self): + with mock_printer(main_module) as printer_shim: + assert main('audit --diff .secrets.baseline .secrets.baseline'.split()) == 0 + assert printer_shim.message.strip() == ( + 'No difference, because it\'s the same file!' + ) + @contextmanager def mock_stdin(response=None): @@ -282,3 +266,29 @@ def mock_stdin(response=None): m.stdin.isatty.return_value = False m.stdin.read.return_value = response yield + + +@pytest.fixture +def mock_baseline_initialize(): + def mock_initialize_function(plugins, exclude_regex, *args, **kwargs): + return secrets_collection_factory( + plugins=plugins, + exclude_regex=exclude_regex, + ) + + with mock.patch( + 'detect_secrets.main.baseline.initialize', + side_effect=mock_initialize_function, + ) as mock_initialize: + yield mock_initialize + + +@pytest.fixture +def mock_merge_baseline(): + with mock.patch( + 'detect_secrets.main.baseline.merge_baseline', + ) as m: + # This return value needs to have the `results` key, so that it can + # formatted appropriately for output. + m.return_value = {'results': {}} + yield m diff --git a/tests/plugins/base_test.py b/tests/plugins/base_test.py index 98ee81faf..3d2da92ba 100644 --- a/tests/plugins/base_test.py +++ b/tests/plugins/base_test.py @@ -6,7 +6,7 @@ def test_fails_if_no_secret_type_defined(): - class MockPlugin(BasePlugin): + class MockPlugin(BasePlugin): # pragma: no cover def analyze_string(self, *args, **kwargs): pass