Add per line exclude regex via --exclude-line

detect_secrets/core/baseline.py

@@ -10,8 +10,8 @@
 
 def initialize(
     plugins,
-    exclude_files_re=None,
-    exclude_lines_re=None,
+    exclude_files_regex=None,
+    exclude_lines_regex=None,
     path='.',
     scan_all_files=False,
 ):
@@ -21,14 +21,18 @@ def initialize(
     :type plugins: tuple of detect_secrets.plugins.base.BasePlugin
     :param plugins: rules to initialize the SecretsCollection with.
 
-    :type exclude_files_re: str|None
-    :type exclude_lines_re: str|None
+    :type exclude_files_regex: str|None
+    :type exclude_lines_regex: str|None
     :type path: str
     :type scan_all_files: bool
 
     :rtype: SecretsCollection
     """
-    output = SecretsCollection(plugins, exclude_files_re)
+    output = SecretsCollection(
+        plugins,
+        exclude_files=exclude_files_regex,
+        exclude_lines=exclude_lines_regex,
+    )
 
     if os.path.isfile(path):
         # This option allows for much easier adhoc usage.
@@ -41,11 +45,11 @@ def initialize(
     if not files_to_scan:
         return output
 
-    if exclude_files_re:
-        exclude_files_re = re.compile(exclude_files_re, re.IGNORECASE)
+    if exclude_files_regex:
+        exclude_files_regex = re.compile(exclude_files_regex, re.IGNORECASE)
         files_to_scan = filter(
             lambda file: (
-                not exclude_files_re.search(file)
+                not exclude_files_regex.search(file)
             ),
             files_to_scan,
         )
@@ -68,13 +72,13 @@ def get_secrets_not_in_baseline(results, baseline):
     :rtype: SecretsCollection
     :returns: SecretsCollection of new results (filtering out baseline)
     """
-    exclude_files_re = None
+    exclude_files_regex = None
     if baseline.exclude_files:
-        exclude_files_re = re.compile(baseline.exclude_files, re.IGNORECASE)
+        exclude_files_regex = re.compile(baseline.exclude_files, re.IGNORECASE)
 
     new_secrets = SecretsCollection()
     for filename in results.data:
-        if exclude_files_re and exclude_files_re.search(filename):
+        if exclude_files_regex and exclude_files_regex.search(filename):
             continue
 
         if filename not in baseline.data:

detect_secrets/core/secrets_collection.py

@@ -93,7 +93,7 @@ def load_baseline_from_dict(cls, data):
             plugin_classname = plugin.pop('name')
             plugins.append(initialize.from_plugin_classname(
                 plugin_classname,
-                exclude_lines_re=result.exclude_lines,
+                exclude_lines_regex=result.exclude_lines,
                 **plugin
             ))
         result.plugins = tuple(plugins)

detect_secrets/main.py

@@ -33,7 +33,7 @@ def main(argv=None):
         # we want to get the latest updates.
         plugins = initialize.from_parser_builder(
             args.plugins,
-            exclude_lines_re=args.exclude_lines,
+            exclude_lines_regex=args.exclude_lines,
         )
         if args.string:
             line = args.string
@@ -140,8 +140,8 @@ def _perform_scan(args, plugins):
 
     new_baseline = baseline.initialize(
         plugins=plugins,
-        exclude_files_re=args.exclude_files,
-        exclude_lines_re=args.exclude_lines,
+        exclude_files_regex=args.exclude_files,
+        exclude_lines_regex=args.exclude_lines,
         path=args.path,
         scan_all_files=args.all_files,
     ).format_for_baseline_output()
@@ -188,12 +188,12 @@ def _add_baseline_to_exclude_files(args):
     """
     Modifies args.exclude_files in-place.
     """
-    baseline_name_re = r'^{}$'.format(args.import_filename[0])
+    baseline_name_regex = r'^{}$'.format(args.import_filename[0])
 
     if not args.exclude_files:
-        args.exclude_files = baseline_name_re
-    elif baseline_name_re not in args.exclude_files:
-        args.exclude_files += r'|{}'.format(baseline_name_re)
+        args.exclude_files = baseline_name_regex
+    elif baseline_name_regex not in args.exclude_files:
+        args.exclude_files += r'|{}'.format(baseline_name_regex)
 
 
 if __name__ == '__main__':

detect_secrets/plugins/aws.py

@@ -9,7 +9,9 @@
 
 
 class AWSKeyDetector(RegexBasedDetector):
+
     secret_type = 'AWS Access Key'
+
     blacklist = (
         re.compile(r'AKIA[0-9A-Z]{16}'),
     )

detect_secrets/plugins/base.py

@@ -11,16 +11,21 @@ class BasePlugin(object):
     """This is an abstract class to define Plugins API"""
 
     __metaclass__ = ABCMeta
+
     secret_type = None
 
-    def __init__(self, exclude_lines_re=None, **kwargs):
+    def __init__(self, exclude_lines_regex=None, **kwargs):
+        """
+        :type exclude_lines_regex: str|None
+        :param exclude_lines_regex: optional regex for ignored lines.
+        """
         if not self.secret_type:
             raise ValueError('Plugins need to declare a secret_type.')
 
-        self.exclude_lines_re = None
-        if exclude_lines_re:
-            self.exclude_lines_re = re.compile(
-                exclude_lines_re,
+        self.exclude_lines_regex = None
+        if exclude_lines_regex:
+            self.exclude_lines_regex = re.compile(
+                exclude_lines_regex,
             )
 
     def analyze(self, file, filename):
@@ -34,26 +39,46 @@ def analyze(self, file, filename):
         """
         potential_secrets = {}
         for line_num, line in enumerate(file.readlines(), start=1):
-            if any(regex.search(line) for regex in WHITELIST_REGEXES):
-                continue
-            if (
-                self.exclude_lines_re
-                and self.exclude_lines_re.search(line)
-            ):
-                continue
             secrets = self.analyze_string(line, line_num, filename)
             potential_secrets.update(secrets)
 
         return potential_secrets
 
-    @abstractmethod
     def analyze_string(self, string, line_num, filename):
         """
         :param string:    string; the line to analyze
         :param line_num:  integer; line number that is currently being analyzed
         :param filename:  string; name of file being analyzed
         :returns:         dictionary
 
+        NOTE: line_num and filename are used for PotentialSecret creation only.
+        """
+        if (
+            any(
+                whitelist_regex.search(string) for whitelist_regex in WHITELIST_REGEXES
+            )
+
+            or (
+                self.exclude_lines_regex and
+                self.exclude_lines_regex.search(string)
+            )
+        ):
+            return {}
+
+        return self._analyze_string(
+            string,
+            line_num,
+            filename,
+        )
+
+    @abstractmethod
+    def _analyze_string(self, string, line_num, filename):
+        """
+        :param string:    string; the line to analyze
+        :param line_num:  integer; line number that is currently being analyzed
+        :param filename:  string; name of file being analyzed
+        :returns:         dictionary
+
         NOTE: line_num and filename are used for PotentialSecret creation only.
         """
         raise NotImplementedError
@@ -103,14 +128,16 @@ def __dict__(self):
 
 
 class RegexBasedDetector(BasePlugin):
-    """Base class for regular-expressed based detectors.
+    """Base class for regular-expression based detectors.
 
     To create a new regex-based detector, subclass this and set
     `secret_type` with a description and `blacklist`
     with a sequence of regular expressions, like:
 
     class FooDetector(RegexBasedDetector):
+
         secret_type = "foo"
+
         blacklist = (
             re.compile(r'foo'),
         )
@@ -125,7 +152,7 @@ def secret_type(self):
     def blacklist(self):
         raise NotImplementedError
 
-    def analyze_string(self, string, line_num, filename):
+    def _analyze_string(self, string, line_num, filename):
         output = {}
 
         for identifier in self.secret_generator(string):

detect_secrets/plugins/basic_auth.py

@@ -12,6 +12,7 @@
 class BasicAuthDetector(RegexBasedDetector):
 
     secret_type = 'Basic Auth Credentials'
+
     blacklist = [
         re.compile(
             r'://[^{}\s]+:([^{}\s]+)@'.format(

detect_secrets/plugins/common/ini_file_parser.py

@@ -6,11 +6,20 @@ class IniFileParser(object):
 
     _comment_regex = re.compile(r'\s*[;#]')
 
-    def __init__(self, file, add_header=False, exclude_lines_re=None):
+    def __init__(self, file, add_header=False, exclude_lines_regex=None):
+        """
+        :type file: file object
+
+        :type add_header: bool
+        :param add_header: whether or not to add a top-level [global] header.
+
+        :type exclude_lines_regex: regex object
+        :param exclude_lines_regex: optional regex for ignored lines.
+        """
         self.parser = configparser.ConfigParser()
         self.parser.optionxform = str
 
-        self.exclude_lines_re = exclude_lines_re
+        self.exclude_lines_regex = exclude_lines_regex
 
         if not add_header:
             self.parser.read_file(file)
@@ -80,8 +89,8 @@ def _get_value_and_line_offset(self, key, values):
                 continue
 
             if (
-                self.exclude_lines_re
-                and self.exclude_lines_re.search(line)
+                self.exclude_lines_regex
+                and self.exclude_lines_regex.search(line)
             ):
                 continue
 

detect_secrets/plugins/common/initialize.py

@@ -16,17 +16,21 @@
 from detect_secrets.core.usage import PluginOptions
 
 
-def from_parser_builder(plugins_dict, exclude_lines_re):
+def from_parser_builder(plugins_dict, exclude_lines_regex):
     """
     :param plugins_dict: plugins dictionary received from ParserBuilder.
         See example in tests.core.usage_test.
+
+    :type exclude_lines_regex: str|None
+    :param exclude_lines_regex: optional regex for ignored lines.
+
     :returns: tuple of initialized plugins
     """
     output = []
     for plugin_name in plugins_dict:
         output.append(from_plugin_classname(
             plugin_name,
-            exclude_lines_re=exclude_lines_re,
+            exclude_lines_regex=exclude_lines_regex,
             **plugins_dict[plugin_name]
         ))
 
@@ -95,7 +99,7 @@ def _remove_key(d, key):
 
         return from_parser_builder(
             plugins_dict,
-            exclude_lines_re=args.exclude_lines,
+            exclude_lines_regex=args.exclude_lines,
         )
 
     # Use baseline plugin as starting point
@@ -123,18 +127,18 @@ def _remove_key(d, key):
 
     return from_parser_builder(
         plugins_dict,
-        exclude_lines_re=args.exclude_lines,
+        exclude_lines_regex=args.exclude_lines,
     )
 
 
-def from_plugin_classname(plugin_classname, exclude_lines_re=None, **kwargs):
+def from_plugin_classname(plugin_classname, exclude_lines_regex=None, **kwargs):
     """Initializes a plugin class, given a classname and kwargs.
 
     :type plugin_classname: str
-    :param plugin_classname: subclass of BasePlugin
+    :param plugin_classname: subclass of BasePlugin.
 
-    :type exclude_lines_re: str|None
-    :param exclude_lines_re: subclass of BasePlugin
+    :type exclude_lines_regex: str|None
+    :param exclude_lines_regex: optional regex for ignored lines.
     """
     klass = globals()[plugin_classname]
 
@@ -143,7 +147,7 @@ def from_plugin_classname(plugin_classname, exclude_lines_re=None, **kwargs):
         raise TypeError
 
     try:
-        instance = klass(exclude_lines_re=exclude_lines_re, **kwargs)
+        instance = klass(exclude_lines_regex=exclude_lines_regex, **kwargs)
     except TypeError:
         log.warning(
             'Unable to initialize plugin!',

detect_secrets/plugins/common/yaml_file_parser.py

@@ -38,9 +38,15 @@ class YamlFileParser(object):
     This parsing method is inspired by https://stackoverflow.com/a/13319530.
     """
 
-    def __init__(self, file, exclude_lines_re=None):
+    def __init__(self, file, exclude_lines_regex=None):
+        """
+        :type file: file object
+
+        :type exclude_lines_regex: regex object
+        :param exclude_lines_regex: optional regex for ignored lines.
+        """
         self.content = file.read()
-        self.exclude_lines_re = exclude_lines_re
+        self.exclude_lines_regex = exclude_lines_regex
 
         self.loader = yaml.SafeLoader(self.content)
         self.loader.compose_node = self._compose_node_shim
@@ -131,8 +137,8 @@ def get_ignored_lines(self):
                 WHITELIST_REGEX['yaml'].search(line)
 
                 or (
-                    self.exclude_lines_re and
-                    self.exclude_lines_re.search(line)
+                    self.exclude_lines_regex and
+                    self.exclude_lines_regex.search(line)
                 )
             ):
                 ignored_lines.add(line_number)