diff --git a/CHANGES.md b/CHANGES.md index d08c3261..f044b387 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ -# [1.24.1](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.24.1) +# [1.25.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.25.0) +- [ADDED] Documentation on how to use it with 1Password CLI. +- [CHANGED] "--creds-path" does not default to "~/.credentials". - [FIXED] Number of errors/warnings shown correctly for single checks. # [1.24.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.24.0) diff --git a/compliance/__init__.py b/compliance/__init__.py index cf2d7688..81f8e1f2 100644 --- a/compliance/__init__.py +++ b/compliance/__init__.py @@ -13,4 +13,4 @@ # limitations under the License. """Compliance automation package.""" -__version__ = "1.24.1" +__version__ = "1.25.0" diff --git a/compliance/config.py b/compliance/config.py index 6103db39..916c0095 100644 --- a/compliance/config.py +++ b/compliance/config.py @@ -68,11 +68,9 @@ def __init__(self): @property def creds(self): """Credentials used for locker management and running fetchers.""" - if self.creds_path is None: - raise ValueError("Path to credentials file not provided") - - if self._creds is None: - self._creds = Config(self.creds_path) + if not self._creds: + path = None if self.creds_path is None else str(self.creds_path) + self._creds = Config(path) return self._creds @property diff --git a/compliance/runners.py b/compliance/runners.py index e6b6fb37..87b37832 100644 --- a/compliance/runners.py +++ b/compliance/runners.py @@ -91,11 +91,15 @@ def get_test_candidates(self, suite): yield test def _load_compliance_config(self): - creds_path = Path(self.opts.creds_path).expanduser() - if not creds_path.is_file(): - raise ValueError(f"{creds_path} file does not exist.") self.config = get_config() - self.config.creds_path = str(creds_path) + creds_path = None + if self.opts.creds_path is not None: + creds_path = Path(self.opts.creds_path).expanduser() + if not creds_path.is_file(): + raise ValueError( + f"Invalid path to credentials file '{str(creds_path)}'" + ) + self.config.creds_path = creds_path self.config.load(self.opts.compliance_config) def _init_dirs(self): diff --git a/compliance/scripts/compliance_cli.py b/compliance/scripts/compliance_cli.py index ea2da712..95a19f52 100644 --- a/compliance/scripts/compliance_cli.py +++ b/compliance/scripts/compliance_cli.py @@ -83,7 +83,7 @@ def _init_arguments(self): "Defaults to %(default)s." ), metavar="/path/to/creds.ini", - default="~/.credentials", + default=None, ) notify_options = [k for k in get_notifiers().keys() if k != "stdout"] self.add_argument( diff --git a/compliance/utils/credentials.py b/compliance/utils/credentials.py index 76b649c5..bb141f4a 100644 --- a/compliance/utils/credentials.py +++ b/compliance/utils/credentials.py @@ -36,8 +36,9 @@ def __init__(self, cfg_file="~/.credentials"): :param cfg_file: The path to the RawConfigParser compatible config file """ self._cfg = RawConfigParser() - self._cfg.read(str(Path(cfg_file).expanduser())) self._cfg_file = cfg_file + if cfg_file is not None: + self._cfg.read(str(Path(cfg_file).expanduser())) def __getitem__(self, section): """ @@ -60,13 +61,16 @@ def _getattr_wrapper(t, attr): try: return t.__getattribute__(attr) except AttributeError as exc: - exc.args = ( - ( + if self._cfg_file: + msg = ( f'Unable to locate attribute "{attr}" ' f'in section "{type(t).__name__}" ' f'at config file "{self._cfg_file}"' - ), - ) + ) + else: + env_var_name = f"{type(t).__name__}_{attr}".upper() + msg = f"Unable to find the env var: {env_var_name}" + exc.args = (msg,) raise exc env_vars = [k for k in environ.keys() if k.startswith(f"{section.upper()}_")] diff --git a/doc-source/design-principles.rst b/doc-source/design-principles.rst index 0f8cffd2..d2566bb9 100644 --- a/doc-source/design-principles.rst +++ b/doc-source/design-principles.rst @@ -750,9 +750,51 @@ levels: Credentials ~~~~~~~~~~~ -If you want to configure your credentials locally, the framework -will look for a credentials file at ``~/.credentials`` by default. This -file should be similar to this: +There are 2 ways for providing credentials: -.. include:: credentials-example.cfg - :literal: +1. *Local file*: if you want to configure your credentials in a local file, + you will have to provide the the framework using ``--creds-path`` option. + This file should be similar to this: + + .. include:: credentials-example.cfg + :literal: + +1. *Environment variables*: each section and field of the local file can be + rendered as an environment variable. + For instance, suppose your code requires ``creds['github'].token`` or ``creds['slack'].webhook``. + You just need to export: + + * ``GITHUB_TOKEN = XXX`` + + * ``MY_SERVICE_API_KEY = YYY`` + + This is equivalent to the credentials file:: + + [github] + token=XXX + + [my_service] + api_key=YYY + +Creds with ``.env`` files and 1Password ++++++++++++++++++++++++++++++++++++++++ + +Combining the method based on passing env vars to Auditree and `1Password CLI `_, +it is possible to grab the secrets from 1Password and inject them into Auditree. +Here it is how to do it: + +1. Create the following alias:: + + alias compliance="op run --env-file .env -- compliance" + +1. In your fetchers/checks project, create an ``.env`` file with the following schema:: + +
_="op:////" + + For example:: + + GITHUB_TOKEN="op://Private/github/token" + MY_SERVICE_ORG="the-org-id" + MY_SERVICE_API_KEY="op://Shared/my_service/api_key" + +1. Now running ``compliance`` will pull credentials from 1Password vaults. diff --git a/doc-source/notifiers.rst b/doc-source/notifiers.rst index 28bc8711..11fbee8f 100644 --- a/doc-source/notifiers.rst +++ b/doc-source/notifiers.rst @@ -134,7 +134,9 @@ You can also use a Slack app token (recommended if you need to post messages to private channels):: [slack] - slack=XXX + token=XXX + +Note that you can do the same thing using env vars ``SLACK_WEBHOOK`` and ``SLACK_TOKEN``. In case you need private channels as part of the list, you have to specify the channel ID:: diff --git a/doc-source/running-on-travis.rst b/doc-source/running-on-travis.rst index 0c72a865..d35463c3 100644 --- a/doc-source/running-on-travis.rst +++ b/doc-source/running-on-travis.rst @@ -57,104 +57,22 @@ This is a typical `.travis.yml` file: - "3.7" install: - pip install -r requirements.txt - - ./travis/gen-credentials.py > ~/.credentials script: - make clean - ./travis/run.sh - after_script: - - rm ~/.credentials Basically, this will firstly install the dependencies through ``pip install -r requirements.txt`` and then generate the credentials file from using Travis environment variables. -Credentials generation -~~~~~~~~~~~~~~~~~~~~~~ +Credentials +~~~~~~~~~~~ -This is an implementation you might want to use for your project of -``gen-credentials.py``: +The recommended way to use credentials in a CI job is to export them as environment variables. +Auditree will automatically parsed the environment variables available to the process and make them available to the fetchers if they follow a specific structure. -.. code-block:: python +For more information on how to do this, have a look to the :ref:`credentials` section. - #!/usr/bin/env python - # -*- coding:utf-8; mode:python -*- - - '''This script generates a config file suitable to be used by - `utilitarian.credentials.Config` from envvars. This is useful for - Travis CI that allows to deploy credentials safely using envvars. - - Any new supported credential must be added to SUPPORTED_SECTIONS which - includes a list of sections of `Config` supported by the script. For - example, adding 'github' will make the script to generate - `github.username` from GITHUB_USERNAME and `github.password` from - GITHUB_PASSWORD, if both envvars are defined. - ''' - - import os - import sys - import ConfigParser - - - SUPPORTED_SECTIONS = ['github', 'slack'] - - - def main(): - matched_keys = filter( - lambda k: any([k.lower().startswith(x) for x in SUPPORTED_SECTIONS]), - os.environ.keys() - ) - if not matched_keys: - return 0 - - cfg_parser = ConfigParser.ConfigParser() - for k in matched_keys: - # split the section name and option from this env var (max() - # to ensure the longest match) - section = max( - [s for s in SUPPORTED_SECTIONS if k.lower().startswith(s)], - key=len - ) - option = k.split(section.upper())[1][1:].lower() - - # add to the config - if not cfg_parser.has_section(section): - cfg_parser.add_section(section) - cfg_parser.set(section, option, os.environ[k]) - - cfg_parser.write(sys.stdout) - - return 0 - - - if __name__ == '__main__': - exit(main()) - -So, for instance, using the previous script you will be able to create -the credentials required for ``github`` and ``slack`` by -defining the following environment variables in Travis: - -* ``GITHUB_TOKEN = XXX`` - -* ``SLACK_WEBHOOK = YYY`` - -Using those variables, ``./travis/gen-credentials.py > -~/.credentials`` will generate:: - - [github] - token=XXX - - [slack] - webhook=YYY - -This method has a few limitation: - -* Do not use ``$`` as part of the value of any variable as they will - be evaluated by bash. - -* You will need to add a new service into the - ``SUPPORTED_SECTIONS``. This is actually good since a manual - addition requires a code change (so new credentials are - tracked). ``travis/run.sh`` ~~~~~~~~~~~~~~~~~