From 68e01321b04212419ce7cc46e1a149525560cf2d Mon Sep 17 00:00:00 2001 From: "Justin D. Eyster" Date: Mon, 8 Apr 2019 14:56:39 -0400 Subject: [PATCH 1/6] Adds artifactory credential detector to plugins --- README.md | 6 ++ detect_secrets/core/usage.py | 6 ++ detect_secrets/plugins/artifactory.py | 17 +++++ detect_secrets/plugins/common/initialize.py | 1 + tests/core/usage_test.py | 1 + tests/main_test.py | 71 +++++++++++++++++++++ tests/plugins/artifactory_test.py | 37 +++++++++++ tests/pre_commit_hook_test.py | 3 + 8 files changed, 142 insertions(+) create mode 100644 detect_secrets/plugins/artifactory.py create mode 100644 tests/plugins/artifactory_test.py diff --git a/README.md b/README.md index 0c5ca0f2b..8ff30834b 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ The current heuristic searches we implement out of the box include: * **KeywordDetector**: checks to see if certain keywords are being used e.g. `password` or `secret` +<<<<<<< HEAD See [detect_secrets/ plugins](https://github.com/Yelp/detect-secrets/tree/master/detect_secrets/plugins) for more details. @@ -178,6 +179,11 @@ This preset amount can be adjusted in several ways: * Specifying it within the config file, for server scanning. * Specifying it with command line flags (e.g. `--base64-limit`) +======= +* **ArtifactoryDetector**: checks to see if Artifactory credentials are present. + +## IBM versioning and rebase guide +>>>>>>> 5ac3771... Adds artifactory credential detector to plugins (#136) Lowering these limits will identify more potential secrets, but also create more false positives. Adjust these limits to suit your needs. diff --git a/detect_secrets/core/usage.py b/detect_secrets/core/usage.py index 84b571e60..496b5ef79 100644 --- a/detect_secrets/core/usage.py +++ b/detect_secrets/core/usage.py @@ -290,6 +290,12 @@ class PluginOptions(object): disable_flag_text='--no-slack-scan', disable_help_text='Disables scanning for Slack tokens.', ), + PluginDescriptor( + classname='ArtifactoryDetector', + disable_flag_text='--no-artifactory-scan', + disable_help_text='Disable scanning for Artifactory credentials', + is_default=True, + ), ] def __init__(self, parser): diff --git a/detect_secrets/plugins/artifactory.py b/detect_secrets/plugins/artifactory.py new file mode 100644 index 000000000..512939725 --- /dev/null +++ b/detect_secrets/plugins/artifactory.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +import re + +from .base import RegexBasedDetector + + +class ArtifactoryDetector(RegexBasedDetector): + + secret_type = 'Artifactory Credentials' + + blacklist = [ + # artifactory tokens begin with AKC + re.compile(r'(\s|=|"|^)AKC\w{10,}'), # api token + # artifactory encrypted passwords begin with AP6 + re.compile(r'(\s|=|"|^)AP6\w{10,}'), # password + ] diff --git a/detect_secrets/plugins/common/initialize.py b/detect_secrets/plugins/common/initialize.py index 0e2dfb96d..5d5089c05 100644 --- a/detect_secrets/plugins/common/initialize.py +++ b/detect_secrets/plugins/common/initialize.py @@ -12,6 +12,7 @@ from ..keyword import KeywordDetector # noqa: F401 from ..private_key import PrivateKeyDetector # noqa: F401 from ..slack import SlackDetector # noqa: F401 +from ..artifactory import ArtifactoryDetector # noqa: F401 from detect_secrets.core.log import log from detect_secrets.core.usage import PluginOptions diff --git a/tests/core/usage_test.py b/tests/core/usage_test.py index 4282be33a..5905d5ba2 100644 --- a/tests/core/usage_test.py +++ b/tests/core/usage_test.py @@ -39,6 +39,7 @@ def test_consolidates_output_basic(self): 'PrivateKeyDetector': {}, 'AWSKeyDetector': {}, 'SlackDetector': {}, + 'ArtifactoryDetector': {}, } assert not hasattr(args, 'no_private_key_scan') diff --git a/tests/main_test.py b/tests/main_test.py index 94b6f9d3c..b6cfb7ddf 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -89,6 +89,7 @@ def test_scan_string_basic( assert main('scan --string'.split()) == 0 assert uncolor(printer_shim.message) == textwrap.dedent(""" AWSKeyDetector : False + ArtifactoryDetector : False Base64HighEntropyString: {} BasicAuthDetector : False HexHighEntropyString : {} @@ -102,6 +103,29 @@ def test_scan_string_basic( mock_baseline_initialize.assert_not_called() +<<<<<<< HEAD +======= + def test_scan_string_basic_default( + self, + mock_baseline_initialize, + ): + with mock_stdin( + '012345678ab', + ), mock_printer( + main_module, + ) as printer_shim: + assert main('scan --string'.split()) == 0 + assert uncolor(printer_shim.message) == textwrap.dedent(""" + AWSKeyDetector : False + ArtifactoryDetector: False + BasicAuthDetector : False + PrivateKeyDetector : False + SlackDetector : False + """)[1:] + + mock_baseline_initialize.assert_not_called() + +>>>>>>> 5ac3771... Adds artifactory credential detector to plugins (#136) def test_scan_string_cli_overrides_stdin(self): with mock_stdin( '012345678ab', @@ -111,6 +135,7 @@ def test_scan_string_cli_overrides_stdin(self): assert main('scan --string 012345'.split()) == 0 assert uncolor(printer_shim.message) == textwrap.dedent(""" AWSKeyDetector : False + ArtifactoryDetector : False Base64HighEntropyString: False (2.585) BasicAuthDetector : False HexHighEntropyString : False (2.121) @@ -232,6 +257,9 @@ def test_old_baseline_ignored_with_update_flag( { "name": "AWSKeyDetector", }, + { + "name": "ArtifactoryDetector", + }, { "base64_limit": 1.5, "name": "Base64HighEntropyString", @@ -267,6 +295,9 @@ def test_old_baseline_ignored_with_update_flag( { "name": "AWSKeyDetector", }, + { + "name": "ArtifactoryDetector", + }, { "name": "BasicAuthDetector", }, @@ -351,6 +382,9 @@ def test_old_baseline_ignored_with_update_flag( { "name": "AWSKeyDetector", }, + { + "name": "ArtifactoryDetector", + }, { "base64_limit": 5.5, "name": "Base64HighEntropyString", @@ -381,6 +415,9 @@ def test_old_baseline_ignored_with_update_flag( { "name": "AWSKeyDetector", }, + { + "name": "ArtifactoryDetector", + }, { "base64_limit": 2.5, "name": "Base64HighEntropyString", @@ -506,6 +543,40 @@ def test_audit_short_file(self, filename, expected_output): expected_output, ) +<<<<<<< HEAD +======= + def test_scan_with_default_plugin(self): + filename = 'test_data/short_files/last_line.ini' + plugins_used = [ + { + "name": "AWSKeyDetector", + }, + { + "name": "ArtifactoryDetector", + }, + { + "name": "BasicAuthDetector", + }, + { + "name": "PrivateKeyDetector", + }, + { + "name": "SlackDetector", + }, + ] + + with mock_stdin(), mock_printer( + # To extract the baseline output + main_module, + ) as printer_shim: + main(['scan', filename]) + baseline = printer_shim.message + + baseline_dict = json.loads(baseline) + assert baseline_dict['results'] == {} + assert baseline_dict['plugins_used'] == plugins_used + +>>>>>>> 5ac3771... Adds artifactory credential detector to plugins (#136) def test_audit_diff_not_enough_files(self): assert main('audit --diff fileA'.split()) == 1 diff --git a/tests/plugins/artifactory_test.py b/tests/plugins/artifactory_test.py new file mode 100644 index 000000000..41f6458a1 --- /dev/null +++ b/tests/plugins/artifactory_test.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import + +import pytest + +from detect_secrets.plugins.artifactory import ArtifactoryDetector + + +class TestArtifactoryDetector(object): + + @pytest.mark.parametrize( + 'payload, should_flag', + [ + ('AP6xxxxxxxxxx', True), + ('AKCxxxxxxxxxx', True), + (' AP6xxxxxxxxxx', True), + (' AKCxxxxxxxxxx', True), + ('=AP6xxxxxxxxxx', True), + ('=AKCxxxxxxxxxx', True), + ('\"AP6xxxxxxxxxx\"', True), + ('\"AKCxxxxxxxxxx\"', True), + ('X-JFrog-Art-Api: AKCxxxxxxxxxx', True), + ('X-JFrog-Art-Api: AP6xxxxxxxxxx', True), + ('artifactoryx:_password=AKCxxxxxxxxxx', True), + ('artifactoryx:_password=AP6xxxxxxxxxx', True), + ('testAKCwithinsomeirrelevantstring', False), + ('testAP6withinsomeirrelevantstring', False), + ('X-JFrog-Art-Api: $API_KEY', False), + ('X-JFrog-Art-Api: $PASSWORD', False), + ('artifactory:_password=AP6xxxxxxxx', False), + ('artifactory:_password=AKCxxxxxxxx', False), + ], + ) + def test_analyze_string(self, payload, should_flag): + logic = ArtifactoryDetector() + + output = logic.analyze_string(payload, 1, 'mock_filename') + assert len(output) == int(should_flag) diff --git a/tests/pre_commit_hook_test.py b/tests/pre_commit_hook_test.py index 28bf8d66d..40212bd44 100644 --- a/tests/pre_commit_hook_test.py +++ b/tests/pre_commit_hook_test.py @@ -171,6 +171,9 @@ def test_that_baseline_gets_updated( { 'name': 'AWSKeyDetector', }, + { + 'name': 'ArtifactoryDetector', + }, { 'base64_limit': 4.5, 'name': 'Base64HighEntropyString', From 272cffdf42a54a63709a1b3668a4c8fd7c4bb85c Mon Sep 17 00:00:00 2001 From: Justin Eyster Date: Mon, 8 Apr 2019 15:14:54 -0400 Subject: [PATCH 2/6] Removed cherry-pick markers --- README.md | 10 +--------- tests/main_test.py | 6 ------ 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/README.md b/README.md index 8ff30834b..86c07ccd4 100644 --- a/README.md +++ b/README.md @@ -150,10 +150,7 @@ The current heuristic searches we implement out of the box include: * **KeywordDetector**: checks to see if certain keywords are being used e.g. `password` or `secret` -<<<<<<< HEAD -See [detect_secrets/ -plugins](https://github.com/Yelp/detect-secrets/tree/master/detect_secrets/plugins) -for more details. +* **ArtifactoryDetector**: checks to see if Artifactory credentials are present. ## Caveats @@ -179,11 +176,6 @@ This preset amount can be adjusted in several ways: * Specifying it within the config file, for server scanning. * Specifying it with command line flags (e.g. `--base64-limit`) -======= -* **ArtifactoryDetector**: checks to see if Artifactory credentials are present. - -## IBM versioning and rebase guide ->>>>>>> 5ac3771... Adds artifactory credential detector to plugins (#136) Lowering these limits will identify more potential secrets, but also create more false positives. Adjust these limits to suit your needs. diff --git a/tests/main_test.py b/tests/main_test.py index b6cfb7ddf..69bf6b23a 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -103,8 +103,6 @@ def test_scan_string_basic( mock_baseline_initialize.assert_not_called() -<<<<<<< HEAD -======= def test_scan_string_basic_default( self, mock_baseline_initialize, @@ -125,7 +123,6 @@ def test_scan_string_basic_default( mock_baseline_initialize.assert_not_called() ->>>>>>> 5ac3771... Adds artifactory credential detector to plugins (#136) def test_scan_string_cli_overrides_stdin(self): with mock_stdin( '012345678ab', @@ -543,8 +540,6 @@ def test_audit_short_file(self, filename, expected_output): expected_output, ) -<<<<<<< HEAD -======= def test_scan_with_default_plugin(self): filename = 'test_data/short_files/last_line.ini' plugins_used = [ @@ -576,7 +571,6 @@ def test_scan_with_default_plugin(self): assert baseline_dict['results'] == {} assert baseline_dict['plugins_used'] == plugins_used ->>>>>>> 5ac3771... Adds artifactory credential detector to plugins (#136) def test_audit_diff_not_enough_files(self): assert main('audit --diff fileA'.split()) == 1 From 5b8ff0b66a073eca47184fb9d0e579ef8b6c57a7 Mon Sep 17 00:00:00 2001 From: Justin Eyster Date: Mon, 8 Apr 2019 15:16:57 -0400 Subject: [PATCH 3/6] Accidentally deleted something from README. Restores. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 86c07ccd4..eecd3a1d2 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,10 @@ The current heuristic searches we implement out of the box include: * **ArtifactoryDetector**: checks to see if Artifactory credentials are present. +See [detect_secrets/ +plugins](https://github.com/Yelp/detect-secrets/tree/master/detect_secrets/plugins) +for more details. + ## Caveats This is not meant to be a sure-fire solution to prevent secrets from entering From 51045a789eb7fd4b8768e7a5cdc52b95f380a80a Mon Sep 17 00:00:00 2001 From: Justin Eyster Date: Mon, 8 Apr 2019 15:28:03 -0400 Subject: [PATCH 4/6] Removed is_default option from IBM fork --- detect_secrets/core/usage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/detect_secrets/core/usage.py b/detect_secrets/core/usage.py index 496b5ef79..507c7c3f9 100644 --- a/detect_secrets/core/usage.py +++ b/detect_secrets/core/usage.py @@ -294,7 +294,6 @@ class PluginOptions(object): classname='ArtifactoryDetector', disable_flag_text='--no-artifactory-scan', disable_help_text='Disable scanning for Artifactory credentials', - is_default=True, ), ] From 98c6eb0e4aa7550107f299d762f0000f9ed70f38 Mon Sep 17 00:00:00 2001 From: Justin Eyster Date: Mon, 8 Apr 2019 15:41:07 -0400 Subject: [PATCH 5/6] Removed additional tests pulled from IBM fork in main_test.py --- tests/main_test.py | 51 ---------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index 69bf6b23a..f4eceab24 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -103,26 +103,6 @@ def test_scan_string_basic( mock_baseline_initialize.assert_not_called() - def test_scan_string_basic_default( - self, - mock_baseline_initialize, - ): - with mock_stdin( - '012345678ab', - ), mock_printer( - main_module, - ) as printer_shim: - assert main('scan --string'.split()) == 0 - assert uncolor(printer_shim.message) == textwrap.dedent(""" - AWSKeyDetector : False - ArtifactoryDetector: False - BasicAuthDetector : False - PrivateKeyDetector : False - SlackDetector : False - """)[1:] - - mock_baseline_initialize.assert_not_called() - def test_scan_string_cli_overrides_stdin(self): with mock_stdin( '012345678ab', @@ -540,37 +520,6 @@ def test_audit_short_file(self, filename, expected_output): expected_output, ) - def test_scan_with_default_plugin(self): - filename = 'test_data/short_files/last_line.ini' - plugins_used = [ - { - "name": "AWSKeyDetector", - }, - { - "name": "ArtifactoryDetector", - }, - { - "name": "BasicAuthDetector", - }, - { - "name": "PrivateKeyDetector", - }, - { - "name": "SlackDetector", - }, - ] - - with mock_stdin(), mock_printer( - # To extract the baseline output - main_module, - ) as printer_shim: - main(['scan', filename]) - baseline = printer_shim.message - - baseline_dict = json.loads(baseline) - assert baseline_dict['results'] == {} - assert baseline_dict['plugins_used'] == plugins_used - def test_audit_diff_not_enough_files(self): assert main('audit --diff fileA'.split()) == 1 From 3e17ccd240a2248beb9ae8cd72d892ae3ad528be Mon Sep 17 00:00:00 2001 From: Justin Eyster Date: Tue, 9 Apr 2019 11:21:09 -0400 Subject: [PATCH 6/6] Moved import to be alphabetical --- detect_secrets/plugins/common/initialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detect_secrets/plugins/common/initialize.py b/detect_secrets/plugins/common/initialize.py index 5d5089c05..f9ad91b32 100644 --- a/detect_secrets/plugins/common/initialize.py +++ b/detect_secrets/plugins/common/initialize.py @@ -4,6 +4,7 @@ except ImportError: # pragma: no cover from functools32 import lru_cache +from ..artifactory import ArtifactoryDetector # noqa: F401 from ..aws import AWSKeyDetector # noqa: F401 from ..base import BasePlugin from ..basic_auth import BasicAuthDetector # noqa: F401 @@ -12,7 +13,6 @@ from ..keyword import KeywordDetector # noqa: F401 from ..private_key import PrivateKeyDetector # noqa: F401 from ..slack import SlackDetector # noqa: F401 -from ..artifactory import ArtifactoryDetector # noqa: F401 from detect_secrets.core.log import log from detect_secrets.core.usage import PluginOptions