From bebdb6a70316043da9108fb13f9cbb0078d9c351 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 22 Nov 2019 14:42:54 -0500 Subject: [PATCH] Add support for Slack notifications We need to add more monitoring on the pipeline. Otherwise it's going to be very easy to go red for a few days without noticing. My initial goal was to hook up to IRC, but the reality is that I think we'll need both Slack and IRC because many folks mostly live in Slack nowadays. (That said, I still fully intend to add IRC support via fedmsgs as discussed in #41). Now, how this patch works is that we add two new Jenkins plugins: - configuration-as-code - slack The first one is used to configure the second one. More broadly, it's able to configure almost all of Jenkins and its plugins via YAML instead of dropping to XML, so we'll likely be leveraging it some more in the future. Of course, this is all still optional. The local developer workflow should still work fine. --- HACKING.md | 12 +++++++++++- Jenkinsfile | 33 ++++++++++++++++++++++++++++++++- jenkins/master/plugins.txt | 2 ++ manifests/jenkins.yaml | 20 ++++++++++++++++++++ manifests/pipeline.yaml | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 2 deletions(-) diff --git a/HACKING.md b/HACKING.md index b1c999325..5f422bfda 100644 --- a/HACKING.md +++ b/HACKING.md @@ -163,7 +163,7 @@ If you're planning to test changes, it would be best to fork this repo so that you do your work there. The workflow requires a remote repo to which to push changes. -### Creating AWS credentials configs +### [OPTIONAL] Creating AWS credentials configs If you are in production where we upload builds to S3 OR you want to test uploading to S3 as part of your pipeline development, you need to @@ -217,6 +217,16 @@ $ aws s3 mb my-fcos-bucket And provide it to `--bucket` below. +### [OPTIONAL] Slack integration + +If you want to be able to have build status messages appear in Slack, +create a `slack-api-token` secret: + +``` +$ echo -n "$TOKEN" > slack-token +$ oc create secret generic slack-api-token --from-file=token=slack-token +``` + ### Create a Jenkins instance with a persistent volume backing store ``` diff --git a/Jenkinsfile b/Jenkinsfile index cc17ab949..d164aedbc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -125,6 +125,11 @@ node { podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultContainer: 'jnlp') { node('coreos-assembler') { container('coreos-assembler') { + // declare this early so we can use it in Slack + def newBuildID + + try { + // this is defined IFF we *should* and we *can* upload to S3 def s3_stream_dir @@ -219,7 +224,7 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon """) } - def newBuildID = utils.shwrap_capture("readlink builds/latest") + newBuildID = utils.shwrap_capture("readlink builds/latest") if (prevBuildID == newBuildID) { currentBuild.result = 'SUCCESS' currentBuild.description = "[${params.STREAM}] 💤 (no new build)" @@ -423,5 +428,31 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon } } } + + } catch (e) { + currentBuild.result = 'FAILURE' + throw e + } finally { + def color + def message = "[${params.STREAM}] <${env.BUILD_URL}|${env.BUILD_NUMBER}>" + + if (build.result == 'SUCCESS') { + message = ":fcos: :sparkles: ${message} - SUCCESS" + color = 'good'; + } else { + message = ":fcos: :trashfire: ${message} - FAILURE" + color = 'danger'; + } + + if (newBuildID) { + message = "${message} (${newBuildID})" + } + + try { + slackSend(color: color, message: message) + } finally { + echo message + } + } }} } diff --git a/jenkins/master/plugins.txt b/jenkins/master/plugins.txt index 034c9afc0..527dd7789 100644 --- a/jenkins/master/plugins.txt +++ b/jenkins/master/plugins.txt @@ -1 +1,3 @@ github-oauth:0.33 +configuration-as-code:1.33 +slack:2.34 diff --git a/manifests/jenkins.yaml b/manifests/jenkins.yaml index 4f0d1a51f..ed28f5d31 100644 --- a/manifests/jenkins.yaml +++ b/manifests/jenkins.yaml @@ -74,6 +74,9 @@ objects: value: "true" - name: JNLP_SERVICE_NAME value: ${JNLP_SERVICE_NAME} + # DELTA: point c-as-c plugin to config map files; see below + - name: CASC_JENKINS_CONFIG + value: /var/lib/jenkins/jcasc image: ' ' imagePullPolicy: IfNotPresent livenessProbe: @@ -100,6 +103,14 @@ objects: volumeMounts: - mountPath: /var/lib/jenkins name: ${JENKINS_SERVICE_NAME}-data + # DELTA: mount c-as-c config map + - name: ${JENKINS_SERVICE_NAME}-cfg + mountPath: /var/lib/jenkins/jcasc + readOnly: true + # DELTA: mount Slack token; see below + - name: slack-token + mountPath: /var/run/secrets/slack-api-token + readOnly: true dnsPolicy: ClusterFirst restartPolicy: Always serviceAccountName: ${JENKINS_SERVICE_NAME} @@ -107,6 +118,15 @@ objects: - name: ${JENKINS_SERVICE_NAME}-data persistentVolumeClaim: claimName: ${JENKINS_SERVICE_NAME} + # DELTA: add a configmap -- it's defined in pipeline.yaml + - name: ${JENKINS_SERVICE_NAME}-cfg + configMap: + name: jenkins-cfg + # DELTA: add the Slack token + - name: slack-token + secret: + secretName: slack-api-token + optional: true triggers: - imageChangeParams: automatic: true diff --git a/manifests/pipeline.yaml b/manifests/pipeline.yaml index 8882f3191..2bcdf3ef1 100644 --- a/manifests/pipeline.yaml +++ b/manifests/pipeline.yaml @@ -44,6 +44,12 @@ parameters: - description: Whether to use KVM device plugin or legacy OCI KVM hook name: KVM_SELECTOR value: kvm-device-plugin + - description: Slack domain on which to post build status updates + name: SLACK_DOMAIN + value: coreos + - description: Slack channel on which to post build status updates + name: SLACK_CHANNEL + value: jenkins-coreos objects: @@ -123,6 +129,34 @@ objects: importPolicy: scheduled: true + ### JENKINS CONFIGURATION ### + + # this uses the configuration-as-code plugin: + # https://github.com/jenkinsci/configuration-as-code-plugin + - apiVersion: v1 + kind: ConfigMap + metadata: + name: jenkins-cfg + annotations: + coreos.com/deploy-default: "true" + data: + jenkins.yaml: | + credentials: + system: + domainCredentials: + - credentials: + - string: + scope: GLOBAL + id: slack-token + secret: /var/run/secrets/slack-api-token/token + description: Slack API token + unclassified: + slackNotifier: + teamDomain: ${SLACK_DOMAIN} + tokenCredentialId: slack-token + room: "#${SLACK_CHANNEL}" + + ### COREOS-ASSEMBLER ### # keep a local copy of coreos-assembler so we're not constantly pulling it