From 2a0bcc97625b2d830fcd30e24de1335b91e3e4d8 Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Mon, 28 Jun 2021 07:53:12 -0700 Subject: [PATCH 1/3] SEC-1493 Added the ability to group linters via linter packs. --- lib/functions/validation.sh | 64 +++++++++++++++---------------------- lib/linter.sh | 16 ++++++++++ 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/lib/functions/validation.sh b/lib/functions/validation.sh index d97597078d0..dbb2b3f015c 100755 --- a/lib/functions/validation.sh +++ b/lib/functions/validation.sh @@ -53,53 +53,39 @@ function GetValidationInfo() { eval "${VALIDATE_LANGUAGE}=${!VALIDATE_LANGUAGE,,}" done - ################################################ - # Determine if any linters were explicitly set # - ################################################ - ANY_SET="false" - ANY_TRUE="false" - ANY_FALSE="false" - # Loop through all languages - for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do - # build the variable - VALIDATE_LANGUAGE="VALIDATE_${LANGUAGE}" - # Check to see if the variable was set - if [ -n "${!VALIDATE_LANGUAGE}" ]; then - # It was set, need to set flag - ANY_SET="true" - if [ "${!VALIDATE_LANGUAGE}" == "true" ]; then - ANY_TRUE="true" - elif [ "${!VALIDATE_LANGUAGE}" == "false" ]; then - ANY_FALSE="true" + # Loop through linter packs to turn on any linters in those packs. + for LINTER_PACK in "${LINTER_PACKS[@]}"; do + LINTERS="${LINTER_PACK}[@]" + # Build the RUN_*_TOOLS variable + RUN_LINTER_PACK="RUN_${LINTER_PACK}" + + for LANGUAGE in "${!LINTERS}"; do + # Build the VALIDATE_* variable + VALIDATE_LANGUAGE="VALIDATE_${LANGUAGE}" + SHOULD_VALIDATE_LANGUAGE="SHOULD_${VALIDATE_LANGUAGE}" + if [[ -n "${!VALIDATE_LANGUAGE}" ]]; then + # If variable is explicitly set, that will override any linter pack settings. + continue fi - fi + # Should run current linter if *any* linter pack flips it to true. + if [[ "${!SHOULD_VALIDATE_LANGUAGE}" != "true" ]]; then + eval "${SHOULD_VALIDATE_LANGUAGE}='${!RUN_LINTER_PACK}'" + fi + done done - if [ $ANY_TRUE == "true" ] && [ $ANY_FALSE == "true" ]; then - fatal "Behavior not supported, please either only include (VALIDATE=true) or exclude (VALIDATE=false) linters, but not both" - fi - - ######################################################### - # Validate if we should check/omit individual languages # - ######################################################### - # Loop through all languages + # Loop through all languages. for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do # build the variable VALIDATE_LANGUAGE="VALIDATE_${LANGUAGE}" - # Check if ANY_SET was set - if [[ ${ANY_SET} == "true" ]]; then - # Check to see if the variable was set - if [ -z "${!VALIDATE_LANGUAGE}" ]; then - # Flag was not set, default to: - # if ANY_TRUE then set to false - # if ANY_FALSE then set to true - eval "${VALIDATE_LANGUAGE}='$ANY_FALSE'" - fi + SHOULD_VALIDATE_LANGUAGE="SHOULD_${VALIDATE_LANGUAGE}" + if [ -n "${!VALIDATE_LANGUAGE}" ]; then + # If variable was explicitly set, honor that. + continue else - # No linter flags were set - default all to true - eval "${VALIDATE_LANGUAGE}='true'" + # Else use linter pack settings to determine whether it should run. + eval "${VALIDATE_LANGUAGE}='${!SHOULD_VALIDATE_LANGUAGE:-false}'; export ${VALIDATE_LANGUAGE}" fi - eval "export ${VALIDATE_LANGUAGE}" done ####################################### diff --git a/lib/linter.sh b/lib/linter.sh index 1f739a785b7..9b2aff6f166 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -203,6 +203,14 @@ else JAVASCRIPT_STYLE='standard' fi +####################### +# Linter pack toggles # +####################### +# shellcheck disable=SC2034 # Variable is referenced indirectly +RUN_SECURITY_TOOLS="${RUN_SECURITY_TOOLS:-true}" +# shellcheck disable=SC2034 # Variable is referenced indirectly +RUN_CODE_QUALITY_TOOLS="${RUN_CODE_QUALITY_TOOLS:-false}" + ################## # Language array # ################## @@ -292,6 +300,14 @@ LINTER_NAMES_ARRAY['YAML']="yamllint" ############################################ LINTED_LANGUAGES_ARRAY=() # Will be filled at run time with all languages that were linted +################ +# Linter packs # +################ +LINTER_PACKS=('SECURITY_TOOLS' 'CODE_QUALITY_TOOLS') +SECURITY_TOOLS=('CLOUDFORMATION_CFN_NAG' 'DOCKERFILE_HADOLINT' 'GITLEAKS' 'PYTHON_BANDIT' 'TERRAFORM_TERRASCAN') +# CODE_QUALITY_TOOLS are all tools (i.e., items in LANGUAGE_ARRAY) that are not in SECURITY_TOOLS +CODE_QUALITY_TOOLS=($(printf '%s\n' "${LANGUAGE_ARRAY[@]}" "${SECURITY_TOOLS[@]}" | sort | uniq -u)) + ################### # GitHub ENV Vars # ################### From c161a2a362e2a442166c63143985e709c3191e5b Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Wed, 30 Jun 2021 05:20:27 -0700 Subject: [PATCH 2/3] SEC-1493 Addressed linter issues and updated README. --- README.md | 25 +++++++++++++++++++++++-- lib/functions/validation.sh | 6 +++--- lib/linter.sh | 5 ++++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0a13a26034f..8fd49233880 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,9 @@ Example usage: The super-linter allows you to pass the following `ENV` variables to be able to trigger different functionality. +
+ The behavior in this section has changed with 23andMe's custom logic (see next section). Expand to see original. + _Note:_ All the `VALIDATE_[LANGUAGE]` variables behave in a very specific way: - If none of them are passed, then they all default to true. @@ -274,6 +277,7 @@ _Note:_ All the `VALIDATE_[LANGUAGE]` variables behave in a very specific way: This means that if you run the linter "out of the box", all languages will be checked. But if you wish to select or exclude specific linters, we give you full control to choose which linters are run, and won't run anything unexpected. +
| **ENV VAR** | **Default Value** | **Notes** | | ---------------------------------- | ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -383,17 +387,34 @@ But if you wish to select or exclude specific linters, we give you full control | **VALIDATE_YAML** | `true` | Flag to enable or disable the linting process of the YAML language. | | **YAML_CONFIG_FILE** | `.yaml-lint.yml` | Filename for [Yamllint configuration](https://yamllint.readthedocs.io/en/stable/configuration.html) (ex: `.yaml-lint.yml`, `.yamllint.yml`) | -### 23andMe Custom Environment Variables +### 23andMe Custom Behavior and Environment Variables + +23andMe's custom super-linter introduced the ability to group linters into linter packs, which can then be toggled on or off as a group. +As a result, the `VALIDATE_[LANGUAGE]` variables behave differently from those in the base super-linter: + +- **All variables default to `false`, unless explicitly toggled to true.** +- A single linter/language can appear in multiple linter packs. Its `VALIDATE_[LANGUAGE]` variable will be set to `true` if *any* of the linter packs containing it are marked `true`, *except*... +- If there is an explicit `VALIDATE_[LANGUAGE]` variable for that specific linter/language, the variable will override all linter pack settings. + | **ENV VAR** | **Default Value** | **Notes** | | -------------------------------------- | ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **CLOUDFORMATION_CFN_NAG_CONFIG_FILE** | `.cfnnag` | Filename for [cfn-nag configuration](https://github.com/stelligent/cfn_nag#profiles) | | **GITLEAKS_CONFIG_FILE** | `none` | Filename for optional [gitleaks configuration](https://github.com/zricethezav/gitleaks#configuration) (ex: `simple_regex_config.toml`). Uses the [gitleaks default](https://github.com/zricethezav/gitleaks/blob/master/config/default.go) if unspecified | | **OUTPUT_MODE** | `none` | Additional means for outputting findings, other than writing to stdout. Supported modes: lintly | | **PYTHON_BANDIT_CONFIG_FILE** | `.bandit` | Filename for [bandit configuration](https://bandit.readthedocs.io/en/latest/config.html) (ex: `.bandit`, `tox.ini`) | -| **VALIDATE_CLOUDFORMATION_CFN_NAG** | `true` | Flag to enable or disable the security linting process of the AWS CloudFormation language. | +| **RUN_CODE_QUALITY_TOOLS** | `false ` | Flag to enable or disable the entire suite of code-quality-related linters. See below for a list of linters in this pack. | +| **RUN_SECURITY_TOOLS** | `true ` | Flag to enable or disable the entire suite of security tools. See below for a list of tools in this pack. | +| **VALIDATE_CLOUDFORMATION_CFN_NAG** | `true` | Flag to enable or disable the security linting process of the AWS CloudFormation language. | | **VALIDATE_GITLEAKS** | `true` | Flag to enable or disable the linting process of potentially leaked secrets across all files and languages. | | **VALIDATE_PYTHON_BANDIT** | `true` | Flag to enable or disable the linting process of the Python language. (Utilizing: bandit) | +The following is a list of supported linter packs. + +| **Linter pack** | **Included Tools** | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| **RUN_SECURITY_TOOLS** | `CLOUDFORMATION_CFN_NAG`
`DOCKERFILE_HADOLINT`
`GITLEAKS`
`PYTHON_BANDIT`
`TERRAFORM_TERRASCAN` | +| **RUN_CODE_QUALITY_TOOLS** | _(All other tools that are currently not security tools)_ | + ### Template rules files You can use the **GitHub** **Super-Linter** _with_ or _without_ your own personal rules sets. This allows for greater flexibility for each individual code base. The Template rules all try to follow the standards we believe should be enabled at the basic level. diff --git a/lib/functions/validation.sh b/lib/functions/validation.sh index dbb2b3f015c..2b147506efa 100755 --- a/lib/functions/validation.sh +++ b/lib/functions/validation.sh @@ -54,10 +54,10 @@ function GetValidationInfo() { done # Loop through linter packs to turn on any linters in those packs. - for LINTER_PACK in "${LINTER_PACKS[@]}"; do - LINTERS="${LINTER_PACK}[@]" + for PACK in "${LINTER_PACKS[@]}"; do + LINTERS="${PACK}[@]" # Build the RUN_*_TOOLS variable - RUN_LINTER_PACK="RUN_${LINTER_PACK}" + RUN_LINTER_PACK="RUN_${PACK}" for LANGUAGE in "${!LINTERS}"; do # Build the VALIDATE_* variable diff --git a/lib/linter.sh b/lib/linter.sh index 9b2aff6f166..b171bb2bd7c 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -304,9 +304,12 @@ LINTED_LANGUAGES_ARRAY=() # Will be filled at run time with all languages that w # Linter packs # ################ LINTER_PACKS=('SECURITY_TOOLS' 'CODE_QUALITY_TOOLS') +# shellcheck disable=SC2034 # Variable is referenced indirectly SECURITY_TOOLS=('CLOUDFORMATION_CFN_NAG' 'DOCKERFILE_HADOLINT' 'GITLEAKS' 'PYTHON_BANDIT' 'TERRAFORM_TERRASCAN') # CODE_QUALITY_TOOLS are all tools (i.e., items in LANGUAGE_ARRAY) that are not in SECURITY_TOOLS -CODE_QUALITY_TOOLS=($(printf '%s\n' "${LANGUAGE_ARRAY[@]}" "${SECURITY_TOOLS[@]}" | sort | uniq -u)) +# shellcheck disable=SC2034 # Variable is referenced indirectly +mapfile -t CODE_QUALITY_TOOLS < <(printf '%s\n' "${LANGUAGE_ARRAY[@]}" "${SECURITY_TOOLS[@]}" | sort | uniq -u) +export LINTER_PACKS ################### # GitHub ENV Vars # From c4ad61f211f0878966333698f917e0bb08913a02 Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Tue, 13 Jul 2021 13:27:45 -0700 Subject: [PATCH 3/3] SEC-1493 Renamed "linters" to "languages" for consistency. --- README.md | 10 +++++----- lib/functions/validation.sh | 18 +++++++++--------- lib/linter.sh | 16 ++++++++-------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8fd49233880..c5f3fde1f98 100644 --- a/README.md +++ b/README.md @@ -389,12 +389,12 @@ But if you wish to select or exclude specific linters, we give you full control ### 23andMe Custom Behavior and Environment Variables -23andMe's custom super-linter introduced the ability to group linters into linter packs, which can then be toggled on or off as a group. +23andMe's custom super-linter introduced the ability to group linters into language packs, which can then be toggled on or off as a group. As a result, the `VALIDATE_[LANGUAGE]` variables behave differently from those in the base super-linter: - **All variables default to `false`, unless explicitly toggled to true.** -- A single linter/language can appear in multiple linter packs. Its `VALIDATE_[LANGUAGE]` variable will be set to `true` if *any* of the linter packs containing it are marked `true`, *except*... -- If there is an explicit `VALIDATE_[LANGUAGE]` variable for that specific linter/language, the variable will override all linter pack settings. +- A single linter/language can appear in multiple linter packs. Its `VALIDATE_[LANGUAGE]` variable will be set to `true` if *any* of the language packs containing it are marked `true`, *except*... +- If there is an explicit `VALIDATE_[LANGUAGE]` variable for that specific linter/language, the variable will override all language pack settings. | **ENV VAR** | **Default Value** | **Notes** | | -------------------------------------- | ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -408,9 +408,9 @@ As a result, the `VALIDATE_[LANGUAGE]` variables behave differently from those i | **VALIDATE_GITLEAKS** | `true` | Flag to enable or disable the linting process of potentially leaked secrets across all files and languages. | | **VALIDATE_PYTHON_BANDIT** | `true` | Flag to enable or disable the linting process of the Python language. (Utilizing: bandit) | -The following is a list of supported linter packs. +The following is a list of supported language packs. -| **Linter pack** | **Included Tools** | +| **Language pack** | **Included Tools** | | -------------------------- | ------------------------------------------------------------------------------------------------------------------- | | **RUN_SECURITY_TOOLS** | `CLOUDFORMATION_CFN_NAG`
`DOCKERFILE_HADOLINT`
`GITLEAKS`
`PYTHON_BANDIT`
`TERRAFORM_TERRASCAN` | | **RUN_CODE_QUALITY_TOOLS** | _(All other tools that are currently not security tools)_ | diff --git a/lib/functions/validation.sh b/lib/functions/validation.sh index 2b147506efa..45c64f52595 100755 --- a/lib/functions/validation.sh +++ b/lib/functions/validation.sh @@ -53,23 +53,23 @@ function GetValidationInfo() { eval "${VALIDATE_LANGUAGE}=${!VALIDATE_LANGUAGE,,}" done - # Loop through linter packs to turn on any linters in those packs. - for PACK in "${LINTER_PACKS[@]}"; do - LINTERS="${PACK}[@]" + # Loop through language packs to turn on any linters in those packs. + for PACK in "${LANGUAGE_PACKS[@]}"; do + LANGUAGES="${PACK}[@]" # Build the RUN_*_TOOLS variable - RUN_LINTER_PACK="RUN_${PACK}" + RUN_LANGUAGE_PACK="RUN_${PACK}" - for LANGUAGE in "${!LINTERS}"; do + for LANGUAGE in "${!LANGUAGES}"; do # Build the VALIDATE_* variable VALIDATE_LANGUAGE="VALIDATE_${LANGUAGE}" SHOULD_VALIDATE_LANGUAGE="SHOULD_${VALIDATE_LANGUAGE}" if [[ -n "${!VALIDATE_LANGUAGE}" ]]; then - # If variable is explicitly set, that will override any linter pack settings. + # If variable is explicitly set, that will override any language pack settings. continue fi - # Should run current linter if *any* linter pack flips it to true. + # Should run current linter if *any* language pack flips it to true. if [[ "${!SHOULD_VALIDATE_LANGUAGE}" != "true" ]]; then - eval "${SHOULD_VALIDATE_LANGUAGE}='${!RUN_LINTER_PACK}'" + eval "${SHOULD_VALIDATE_LANGUAGE}='${!RUN_LANGUAGE_PACK}'" fi done done @@ -83,7 +83,7 @@ function GetValidationInfo() { # If variable was explicitly set, honor that. continue else - # Else use linter pack settings to determine whether it should run. + # Else use language pack settings to determine whether it should run. eval "${VALIDATE_LANGUAGE}='${!SHOULD_VALIDATE_LANGUAGE:-false}'; export ${VALIDATE_LANGUAGE}" fi done diff --git a/lib/linter.sh b/lib/linter.sh index b171bb2bd7c..2da79ab279f 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -203,9 +203,9 @@ else JAVASCRIPT_STYLE='standard' fi -####################### -# Linter pack toggles # -####################### +######################### +# Language pack toggles # +######################### # shellcheck disable=SC2034 # Variable is referenced indirectly RUN_SECURITY_TOOLS="${RUN_SECURITY_TOOLS:-true}" # shellcheck disable=SC2034 # Variable is referenced indirectly @@ -300,16 +300,16 @@ LINTER_NAMES_ARRAY['YAML']="yamllint" ############################################ LINTED_LANGUAGES_ARRAY=() # Will be filled at run time with all languages that were linted -################ -# Linter packs # -################ -LINTER_PACKS=('SECURITY_TOOLS' 'CODE_QUALITY_TOOLS') +################## +# Language packs # +################## +LANGUAGE_PACKS=('SECURITY_TOOLS' 'CODE_QUALITY_TOOLS') # shellcheck disable=SC2034 # Variable is referenced indirectly SECURITY_TOOLS=('CLOUDFORMATION_CFN_NAG' 'DOCKERFILE_HADOLINT' 'GITLEAKS' 'PYTHON_BANDIT' 'TERRAFORM_TERRASCAN') # CODE_QUALITY_TOOLS are all tools (i.e., items in LANGUAGE_ARRAY) that are not in SECURITY_TOOLS # shellcheck disable=SC2034 # Variable is referenced indirectly mapfile -t CODE_QUALITY_TOOLS < <(printf '%s\n' "${LANGUAGE_ARRAY[@]}" "${SECURITY_TOOLS[@]}" | sort | uniq -u) -export LINTER_PACKS +export LANGUAGE_PACKS ################### # GitHub ENV Vars #