Prepare release4.6.0 (#1169)

CHANGELOG.md

@@ -1,16 +1,30 @@
 # Changelog
 
 ## Unreleased
-* Fail builds when aqua scan detects remotely exploitable security vulnerabilities with solutions ([#1147](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1147))
 
 ### Added
 
 ### Changed
 * Enhance SSDS Document Generation Performance using New Atlassian APIs ([#1084](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1084))
+* Deprecation of vuln-type and scanners config in Trivy ([#1150](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1150))
 
 ### Fixed
 * Fix Tailor deployment drifts for D, Q envs ([#1055](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1055))
 
+## [4.6.0] - 2024-10-23
+
+### Changed
+* Fail builds when aqua scan detects remotely exploitable security vulnerabilities with solutions ([#1147](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1147))
+* Fail the release manager pipeline and create security vulnerability issues or move them to TODO state if already present ([#1151](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1151))
+* Aqua log readability update and whitelisting mechanism fix ([#1161](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1161))
+* Aqua remotely exploitable critical vulnerabilities improvements ([#1157](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1157))
+* In the release manager pipeline, use the default integration branch for component ([#1144](https://github.com/opendevstack/od-jenkins-shared-library/pull/1144))
+* Use riskPriority value as number instead of its value as text in RA ([#1156](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1156))
+
+### Fixed
+* Fix branch calculation for re-deploy ([#1162](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1162))
+* Preserve quickstarter files in case a repository already contain files with same names ([#1165](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1165))
+
 ## [4.5.6] - 2024-09-16
 
 ### Fixed

build.gradle

@@ -139,7 +139,7 @@ codenarc {
     toolVersion = '1.6'
     configFile = file('codenarc.groovy')
     maxPriority1Violations = 0
-    maxPriority2Violations = 1
+    maxPriority2Violations = 0
     maxPriority3Violations = 300
     reportFormat = 'html'
 }

docs/modules/jenkins-shared-library/partials/odsComponentStageScanWithAqua.adoc

@@ -5,6 +5,7 @@ The "Aqua Security Scan" stage scans an image that was previously built in that
 As a result, a Bitbucket Code Insight entry is added to the git commit (in Bitbucket) that basically
 contains a link to the scan result on the Aqua platform. The Bitbucket Code Insight entry can be seen in a pull request.
 The pull request in Bitbucket shows the Code Insight of the latest commit of the PR.
+In case the Aqua scan detects remotely exploitable cirtical vulnerabilities with solutions the build fails until the solution is implemented.
 
 To get started, make sure you have a `ConfigMap` in OpenDevStack project namespace (usually ods) in OpenShift that has these fields:
 ----

docs/modules/jenkins-shared-library/partials/odsComponentStageScanWithAqua.adoc.tmpl

@@ -3,6 +3,7 @@ The "Aqua Security Scan" stage scans an image that was previously built in that
 As a result, a Bitbucket Code Insight entry is added to the git commit (in Bitbucket) that basically
 contains a link to the scan result on the Aqua platform. The Bitbucket Code Insight entry can be seen in a pull request.
 The pull request in Bitbucket shows the Code Insight of the latest commit of the PR.
+In case the Aqua scan detects remotely exploitable cirtical vulnerabilities with solutions the build fails until the solution is implemented.
 
 To get started, make sure you have a `ConfigMap` in OpenDevStack project namespace (usually ods) in OpenShift that has these fields:
 ----

render-adoc.go

@@ -30,15 +30,19 @@ import (
 // optimization for later?
 
 const (
-	groovydocComponentPath     = "build/docs/groovydoc/org/ods/component"
-	antoraPartialsPath         = "docs/modules/jenkins-shared-library/partials"
+	pathSep                    = string(os.PathSeparator)
 	globalComponentStagePrefix = "odsComponentStage"
 	adocTemplateSuffix         = ".adoc.tmpl"
 )
 
+var (
+	groovydocComponentPath = filepath.FromSlash("build/docs/groovydoc/org/ods/component")
+	antoraPartialsPath     = filepath.FromSlash("docs/modules/jenkins-shared-library/partials")
+)
+
 func main() {
-	adocTemplatePrefix := fmt.Sprintf("%s/%s", antoraPartialsPath, globalComponentStagePrefix)
-	templateFiles, err := filepath.Glob(fmt.Sprintf("%s/*%s", antoraPartialsPath, adocTemplateSuffix))
+	adocTemplatePrefix := fmt.Sprintf(filepath.FromSlash("%s/%s"), antoraPartialsPath, globalComponentStagePrefix)
+	templateFiles, err := filepath.Glob(fmt.Sprintf(filepath.FromSlash("%s/*%s"), antoraPartialsPath, adocTemplateSuffix))
 	check(err)
 	for _, templateFile := range templateFiles {
 		stageName := strings.TrimPrefix(templateFile, adocTemplatePrefix)

src/org/ods/component/Context.groovy

@@ -152,8 +152,11 @@ class Context implements IContext {
 
         logger.debug 'Retrieving Git information ...'
         config.gitUrl = retrieveGitUrl()
+        logger.debug("Retrieved Git Url: ${config.gitUrl}")
         config.gitBranch = retrieveGitBranch()
+        logger.debug("Retrieved Git Branch: ${config.gitBranch}")
         config.gitCommit = retrieveGitCommit()
+        logger.debug("Retrieved Git Commit: ${config.gitCommit}")
         config.gitCommitAuthor = retrieveGitCommitAuthor()
         config.gitCommitMessage = retrieveGitCommitMessage()
         config.gitCommitRawMessage = retrieveGitCommitRawMessage()
@@ -712,7 +715,8 @@ class Context implements IContext {
                 label: 'getting GIT branch to build').trim()
             branch = script.sh(
                 returnStdout: true,
-                script: "git name-rev ${branch} | cut -d ' ' -f2  | sed -e 's|remotes/origin/||g'",
+                script: "git name-rev --name-only --exclude=tags/* ${branch} | cut -d ' ' -f2  |" +
+                    " sed -e 's|remotes/origin/||g'",
                 label: 'resolving to real GIT branch to build').trim()
         }
         logger.debug "resolved branch ${branch}"

src/org/ods/component/HelmDeploymentStrategy.groovy

@@ -192,7 +192,7 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
                         'helmValuesFiles': options.helmValuesFiles,
                         'helmValues': options.helmValues,
                         'helmDefaultFlags': options.helmDefaultFlags,
-                        'helmAdditionalFlags': options.helmAdditionalFlags
+                        'helmAdditionalFlags': options.helmAdditionalFlags,
                     ])
                 rolloutData["${resourceKind}/${resourceName}"] = podData
                 // TODO: Once the orchestration pipeline can deal with multiple replicas,

src/org/ods/component/Pipeline.groovy

@@ -267,7 +267,7 @@ class Pipeline implements Serializable {
                         script.stage('odsPipeline error') {
                             logger.warnClocked("${context.componentId}",
                                 "***** Finished ODS Pipeline for ${context.componentId} (with error) *****")
-                            logger.warn "Error: ${err}"
+                            logger.error "${err}"
                             updateBuildStatus('FAILURE')
                             setBitbucketBuildStatus('FAILED')
                             if (notifyNotGreen) {

src/org/ods/component/ScanWithAquaStage.groovy

@@ -2,6 +2,9 @@ package org.ods.component
 
 import groovy.transform.TypeChecked
 import groovy.transform.TypeCheckingMode
+import org.apache.commons.lang3.StringUtils
+import org.ods.orchestration.util.GitUtil
+import org.ods.services.AquaRemoteCriticalVulnerabilityWithSolutionException
 import org.ods.services.AquaService
 import org.ods.services.BitbucketService
 import org.ods.services.NexusService
@@ -11,6 +14,8 @@ import org.ods.util.ILogger
 @TypeChecked
 class ScanWithAquaStage extends Stage {
 
+    static final String REMOTE_EXPLOIT_TYPE = 'remote'
+    static final String CRITICAL_AQUA_SEVERITY = 'critical'
     static final String STAGE_NAME = 'Aqua Security Scan'
     static final String AQUA_CONFIG_MAP_NAME = "aqua"
     static final String BITBUCKET_AQUA_REPORT_KEY = "org.opendevstack.aquasec"
@@ -20,8 +25,8 @@ class ScanWithAquaStage extends Stage {
     private final OpenShiftService openShift
     private final NexusService nexus
     private final ScanWithAquaOptions options
-    private Map configurationAquaCluster
-    private Map configurationAquaProject
+    private final Map configurationAquaCluster
+    private final Map configurationAquaProject
 
     @SuppressWarnings('ParameterCount')
     @TypeChecked(TypeCheckingMode.SKIP)
@@ -101,10 +106,20 @@ class ScanWithAquaStage extends Stage {
         if (![AquaService.AQUA_SUCCESS, AquaService.AQUA_POLICIES_ERROR].contains(returnCode)) {
             errorMessages += "<li>Error executing Aqua CLI</li>"
         }
+        List actionableVulnerabilities = null
+        String nexusReportLink = null
         // If report exists
         if ([AquaService.AQUA_SUCCESS, AquaService.AQUA_POLICIES_ERROR].contains(returnCode)) {
             try {
                 def resultInfo = steps.readJSON(text: steps.readFile(file: jsonFile) as String) as Map
+
+                Set whitelistedRECVs = []
+                actionableVulnerabilities = filterRemoteCriticalWithSolutionVulnerabilities(resultInfo,
+                    whitelistedRECVs)
+                if (whitelistedRECVs.size() > 0) {
+                    logger.warn(buildWhiteListedRECVsMessage(whitelistedRECVs))
+                }
+
                 Map vulnerabilities = resultInfo.vulnerability_summary as Map
                 // returnCode is 0 --> Success or 4 --> Error policies
                 // with sum of errorCodes > 0 BitbucketCodeInsight is FAIL
@@ -113,8 +128,9 @@ class ScanWithAquaStage extends Stage {
                                   vulnerabilities.malware ?: 0]
 
                 URI reportUriNexus = archiveReportInNexus(reportFile, nexusRepository)
-                createBitbucketCodeInsightReport(url, nexusRepository ? reportUriNexus.toString() : null,
-                    registry, imageRef, errorCodes.sum() as int, errorMessages)
+                nexusReportLink = nexusRepository ? reportUriNexus.toString() : null
+                createBitbucketCodeInsightReport(url, nexusReportLink,
+                    registry, imageRef, errorCodes.sum() as int, errorMessages, actionableVulnerabilities)
                 archiveReportInJenkins(!context.triggeredByOrchestrationPipeline, reportFile)
             } catch (err) {
                 logger.warn("Error archiving the Aqua reports due to: ${err}")
@@ -125,9 +141,104 @@ class ScanWithAquaStage extends Stage {
         }
 
         notifyAquaProblem(alertEmails, errorMessages)
+
+        if (actionableVulnerabilities?.size() > 0) { // We need to mark the pipeline and delete the image
+            performActionsForRECVs(actionableVulnerabilities, nexusReportLink)
+        }
+
         return
     }
 
+    private void performActionsForRECVs(List actionableVulnerabilities, String nexusReportLink) {
+        def scannedBranch = computeScannedBranch()
+        logger.info("Aqua scanned branch: ${scannedBranch}")
+        addAquaVulnerabilityObjectsToContext(actionableVulnerabilities, nexusReportLink, scannedBranch)
+        String response = openShift.deleteImage(context.getComponentId() + ":" + context.getShortGitCommit())
+        logger.info("Delete image response: " + response)
+        throw new AquaRemoteCriticalVulnerabilityWithSolutionException(
+            buildActionableMessageForAquaVulnerabilities(actionableVulnerabilities: actionableVulnerabilities,
+                nexusReportLink: nexusReportLink, gitUrl: context.getGitUrl(), gitBranch: scannedBranch,
+                gitCommit: context.getGitCommit(), repoName: context.getRepoName()))
+    }
+
+    private void addAquaVulnerabilityObjectsToContext(List actionableVulnerabilities, String nexusReportLink,
+            String scannedBranch) {
+        context.addArtifactURI('aquaCriticalVulnerability', actionableVulnerabilities)
+        context.addArtifactURI('jiraComponentId', context.getComponentId())
+        context.addArtifactURI('gitUrl', context.getGitUrl())
+        context.addArtifactURI('gitBranch', scannedBranch)
+        context.addArtifactURI('repoName', context.getRepoName())
+        context.addArtifactURI('nexusReportLink', nexusReportLink)
+    }
+
+    private String buildWhiteListedRECVsMessage(Set whiteListedRECVs) {
+        StringBuilder message = new StringBuilder("The Aqua scan detected the following remotely " +
+            "exploitable critical vulnerabilities which were whitelisted in Aqua: ")
+        message.append(whiteListedRECVs.join(", "))
+        return message.toString()
+    }
+
+    private String buildActionableMessageForAquaVulnerabilities(Map args) {
+        StringBuilder message = new StringBuilder()
+        String gitBranchUrl = GitUtil.buildGitBranchUrl(args.gitUrl as String, context.getProjectId(),
+            args.repoName as String, args.gitBranch as String)
+        message.append("We detected remotely exploitable critical vulnerabilities in repository ${gitBranchUrl}. " +
+            "Due to their high severity, we must stop the delivery " +
+            "process until all vulnerabilities have been addressed. ")
+
+        message.append("\n\nThe following vulnerabilities were found:")
+        def count = 1
+        for (def vulnerability : args.actionableVulnerabilities) {
+            message.append("\n\n${count}.    Vulnerability name: " + (vulnerability as Map).name as String)
+            message.append("\n\n${count}.1.  Description: " + (vulnerability as Map).description as String)
+            message.append("\n\n${count}.2.  Solution: " + (vulnerability as Map).solution as String)
+            message.append("\n")
+            count++
+        }
+        def openPRs = getOpenPRsForCommit(args.gitCommit as String, args.repoName as String)
+        if (openPRs.size() > 0) {
+            message.append("\nThis commit exists in the following open pull requests: ")
+            def cnt = 1
+            for (def pr : openPRs) {
+                message.append("\n\n${cnt}.    Pull request: " + (pr as Map).title as String)
+                message.append("\n\n${cnt}.1.  Link: " + (pr as Map).link as String)
+                message.append("\n")
+                cnt++
+            }
+        }
+        if (args.nexusReportLink != null) {
+            message.append("\nYou can find the complete security scan report here: ${args.nexusReportLink}.\n")
+        }
+        return message.toString()
+    }
+
+    private List getOpenPRsForCommit(String gitCommit, String repoName) {
+        def apiResponse = bitbucket.getPullRequestsForCommit(repoName, gitCommit)
+        def prs = []
+        try {
+            def js = steps.readJSON(text: apiResponse) as Map
+            prs = js['values']
+            if (prs == null) {
+                throw new RuntimeException('Field "values" of JSON response must not be empty!')
+            }
+        } catch (Exception ex) {
+            logger.warn "Could not understand API response. Error was: ${ex}"
+            return []
+        }
+        def response = []
+        for (def i = 0; i < (prs as List).size(); i++) {
+            Map pr = (prs as List)[i] as Map
+            if (!(pr.open as Boolean)) { // We only consider Open PRs
+                continue
+            }
+            response.add([
+                title: pr.title,
+                link: (((pr.links as Map).self as List)[0] as Map).href,
+            ])
+        }
+        response
+    }
+
     private String getImageRef() {
         // take the image ref of the image that is being build in the image build stage
         Map<String, String> buildInfo =
@@ -166,7 +277,8 @@ class ScanWithAquaStage extends Stage {
 
     @SuppressWarnings('ParameterCount')
     private createBitbucketCodeInsightReport(String aquaUrl, String nexusUrlReport,
-                                             String registry, String imageRef, int returnCode, String messages) {
+                                             String registry, String imageRef, int returnCode, String messages,
+                                                List actionableVulnerabilities) {
         String aquaScanUrl = aquaUrl + "/#/images/" + registry + "/" + imageRef.replace("/", "%2F") + "/vulns"
         String title = "Aqua Security"
         String details = "Please visit the following links to review the Aqua Security scan report:"
@@ -199,13 +311,22 @@ class ScanWithAquaStage extends Stage {
                 [ title: "Messages", value: prepareMessageToBitbucket(messages), ]
             ])
         }
+        if (actionableVulnerabilities?.size() > 0) {
+            if (!data.messages) {
+                data.put("messages", [])
+            }
+            ((List) data.messages).add([
+                title: "Blocking",
+                value: "Yes",
+            ])
+        }
 
         bitbucket.createCodeInsightReport(data, context.repoName, context.gitCommit)
     }
 
     private createBitbucketCodeInsightReport(String messages) {
         String title = "Aqua Security"
-        String details = "There was some problems with Aqua:"
+        String details = "There were some problems with Aqua:"
 
         String result = "FAIL"
 
@@ -286,4 +407,42 @@ class ScanWithAquaStage extends Stage {
         }
     }
 
+    private List filterRemoteCriticalWithSolutionVulnerabilities(Map aquaJsonMap, Set whitelistedRECVs) {
+        List result = []
+        aquaJsonMap.resources.each { it ->
+            (it as Map).vulnerabilities.each { vul ->
+                Map vulnerability = vul as Map
+                if ((vulnerability?.exploit_type as String)?.equalsIgnoreCase(REMOTE_EXPLOIT_TYPE)
+                    && (vulnerability?.aqua_severity as String)?.equalsIgnoreCase(CRITICAL_AQUA_SEVERITY)
+                    && !StringUtils.isEmpty((vulnerability?.solution as String).trim())) {
+                    if (Boolean.parseBoolean(vulnerability?.already_acknowledged as String)) {
+                        whitelistedRECVs.add(vulnerability.name)
+                    } else {
+                        result.push(vulnerability)
+                    }
+                }
+            }
+        }
+        return result
+    }
+
+    private String computeScannedBranch() {
+        def scannedBranch = context.getGitBranch()
+        if (scannedBranch.toLowerCase().startsWith("release/")) {
+            // We need to check that the branch was created in BitBucket otherwise we scanned the default branch
+            Map branchesResponse = bitbucket.findRepoBranches(context.getRepoName(), scannedBranch)
+            List matchedBranches = branchesResponse['values'] as List
+            if (matchedBranches?.size() > 0) {
+                for (def i = 0; i < matchedBranches.size(); i++) {
+                    Map branch = matchedBranches[i]
+                    if (branch.displayId == scannedBranch) {
+                        return scannedBranch
+                    }
+                }
+            }
+            scannedBranch = bitbucket.getDefaultBranch(context.getRepoName())
+        }
+        return scannedBranch
+    }
+
 }