From 8595dbb8cac8608bb039d15ff58a2daf55b54031 Mon Sep 17 00:00:00 2001
From: Al Finkelstein <al.finkelstein@ibm.com>
Date: Fri, 20 Nov 2020 16:20:54 -0600
Subject: [PATCH] Add selective bulk fetcher execution (#85)

---
 CHANGES.md                           |  4 ++++
 compliance/__init__.py               |  2 +-
 compliance/runners.py                | 14 +++++++++++++-
 compliance/scripts/compliance_cli.py | 16 ++++++++++++++++
 doc-source/design-principles.rst     | 17 ++++++++++++++---
 5 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index cc19e0b5..08de4e78 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,7 @@
+# [1.8.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.8.0)
+
+- [ADDED] Selective fetcher bulk `--include` and `--exclude` execution is now possible.
+
 # [1.7.2](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.7.2)
 
 - [FIXED] LazyLoader namedtuple defaults removed; Framework compatible with Python 3.6 again.
diff --git a/compliance/__init__.py b/compliance/__init__.py
index 1e0fafd6..beecc2de 100644
--- a/compliance/__init__.py
+++ b/compliance/__init__.py
@@ -14,4 +14,4 @@
 # limitations under the License.
 """Compliance automation package."""
 
-__version__ = '1.7.2'
+__version__ = '1.8.0'
diff --git a/compliance/runners.py b/compliance/runners.py
index 0f6d6a23..56e3480e 100644
--- a/compliance/runners.py
+++ b/compliance/runners.py
@@ -15,6 +15,7 @@
 """Compliance automation flow management module."""
 
 import inspect
+import json
 import os
 import re
 import sys
@@ -180,7 +181,18 @@ def get_fetchers(self):
                         self.load_errors.add(load_err)
                 except AttributeError:
                     pass
-        return fetchers
+        if not (self.opts.include or self.opts.exclude):
+            return fetchers
+        include = {f'{f.__module__}.{f.__name__}' for f in fetchers}
+        if self.opts.include:
+            include = set(json.loads(open(self.opts.include).read()))
+        exclude = set()
+        if self.opts.exclude:
+            exclude = set(json.loads(open(self.opts.exclude).read()))
+        return filter(
+            lambda f: f'{f.__module__}.{f.__name__}' in include - exclude,
+            fetchers
+        )
 
     def run_fetchers(self, reruns=None):
         """
diff --git a/compliance/scripts/compliance_cli.py b/compliance/scripts/compliance_cli.py
index 111c4776..bf102044 100644
--- a/compliance/scripts/compliance_cli.py
+++ b/compliance/scripts/compliance_cli.py
@@ -107,6 +107,18 @@ def _init_arguments(self):
             action='append',
             default=[]
         )
+        self.add_argument(
+            '--include',
+            help='Specifies the path/name of the fetcher include JSON file.',
+            metavar='fetchers.json',
+            default=None
+        )
+        self.add_argument(
+            '--exclude',
+            help='Specifies the path/name of the fetcher exclude JSON file.',
+            metavar='fetchers.json',
+            default=None
+        )
 
     def _validate_arguments(self, args):
         if 'stdout' not in args.notify:
@@ -117,6 +129,10 @@ def _validate_arguments(self, args):
             )
         if not args.fetch and not args.check:
             self.parser.error('--fetch or --check option is expected.')
+        if not args.fetch and (args.include or args.exclude):
+            self.parser.error(
+                '--include/--exclude options only valid with --fetch.'
+            )
         if not args.check and args.fix != 'off':
             self.parser.error('--fix option only valid with --check.')
 
diff --git a/doc-source/design-principles.rst b/doc-source/design-principles.rst
index 65fccf41..c1fd8903 100644
--- a/doc-source/design-principles.rst
+++ b/doc-source/design-principles.rst
@@ -235,9 +235,20 @@ due to an unavailable evidence dependency.
 Fetcher Execution
 =================
 
-The Auditree framework will run all fetchers (tests prefixed by ``fetch_``)
-that it can find.
-
+By default the Auditree framework will run all fetchers (tests prefixed by
+``fetch_``) that it can find.  However, it is possible to limit fetcher
+execution in bulk by using the ``--include`` and/or ``exclude`` CLI options
+while providing a file path/name to a JSON config file containing a list of
+fetchers to include/exclude.  The format of the JSON config file is a list of
+fetcher classes.  Where a fetcher class is represented as a string dot notation
+path to the fetcher class.
+
+Fetcher include/exclude JSON config file example::
+
+  [
+    "fetcher_pkg.path_to_my_checks.checks.fetch_module_foo.FooFetcherClass",
+    "fetcher_pkg.path_to_my_checks.checks.fetch_module_bar.BarFetcherClass"
+  ]
 
 Compliance Checks
 ~~~~~~~~~~~~~~~~~