From 5db8ace2b52f01d242f4f0e235846aec4549d842 Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Fri, 21 Aug 2020 14:15:28 +0200 Subject: [PATCH 1/4] Implement Git.areChangesStagedForCommit() --- README.md | 1 + src/com/cloudogu/ces/cesbuildlib/Git.groovy | 17 +++++++++++++++++ .../com/cloudogu/ces/cesbuildlib/GitTest.groovy | 12 ++++++++++++ 3 files changed, 30 insertions(+) diff --git a/README.md b/README.md index 93e1704b..18f6e5aa 100644 --- a/README.md +++ b/README.md @@ -473,6 +473,7 @@ gitWithCreds 'https://your.repo' // Implicitly passed credentials * `git.commitMessage` - Last commit message e.g. `Implements new functionality...` * `git.commitHash` - e.g. `fb1c8820df462272011bca5fddbe6933e91d69ed` * `git.commitHashShort` - e.g. `fb1c882` +* `git.areChangesStagedForCommit()` - `true` if changes are staged for commit. If `false`, `git.commit()` will fail. * `git.repositoryUrl` - e.g. `https://github.com/orga/repo.git` * `git.gitHubRepositoryName` - e.g. `orga/repo` * Tags - Note that the git plugin might not fetch tags for all builds. Run `sh "git fetch --tags"` to make sure. diff --git a/src/com/cloudogu/ces/cesbuildlib/Git.groovy b/src/com/cloudogu/ces/cesbuildlib/Git.groovy index 10359667..05258679 100644 --- a/src/com/cloudogu/ces/cesbuildlib/Git.groovy +++ b/src/com/cloudogu/ces/cesbuildlib/Git.groovy @@ -197,6 +197,23 @@ class Git implements Serializable { } } + /** + * @return true when changes are staged for commit, i.e. "git add" detected changes. + * Note that this will not work on a branch which has no commits, e.g. newly initialized repositories. + */ + boolean areChangesStagedForCommit() { + // See https://stackoverflow.com/a/3879077/ + + // '--' at the end avoids matching a file called "HEAD" + String returnCode = script.sh(returnStatus: true, script: 'git update-index --refresh && git diff-index --exit-code HEAD --') + // --exit-code: exits with 1 if there were differences and 0 means no differences. + if (returnCode.equals("0")) { + return false + } else { + true + } + } + /** * Sets Tag with message using the name and email of the last committer as author and committer. * diff --git a/test/com/cloudogu/ces/cesbuildlib/GitTest.groovy b/test/com/cloudogu/ces/cesbuildlib/GitTest.groovy index 9af124c0..61964ff1 100644 --- a/test/com/cloudogu/ces/cesbuildlib/GitTest.groovy +++ b/test/com/cloudogu/ces/cesbuildlib/GitTest.groovy @@ -125,6 +125,18 @@ class GitTest { assertEquals(expectedReturnValue, git.commitHashShort) } + @Test + void changesStagedForCommit() { + scriptMock.expectedDefaultShRetValue = 1 + assertTrue git.areChangesStagedForCommit() + } + + @Test + void noChangesStagedForCommit() { + scriptMock.expectedDefaultShRetValue = 0 + assertFalse git.areChangesStagedForCommit() + } + @Test void getRepositoryUrl() { String expectedReturnValue = "https://github.com/orga/repo.git" From 23ff9d6497ca3250d31212348cc50fbc722b289b Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Fri, 21 Aug 2020 14:54:30 +0200 Subject: [PATCH 2/4] SonarQube: Fix "Malformed key for Project" when not using branch plugin. Probably introduced with SonarQube version: 7.9.3 --- .../cloudogu/ces/cesbuildlib/SonarQube.groovy | 12 +++++-- .../ces/cesbuildlib/SonarQubeTest.groovy | 32 +++++++++++++++---- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/com/cloudogu/ces/cesbuildlib/SonarQube.groovy b/src/com/cloudogu/ces/cesbuildlib/SonarQube.groovy index ca3a7298..82f3474b 100644 --- a/src/com/cloudogu/ces/cesbuildlib/SonarQube.groovy +++ b/src/com/cloudogu/ces/cesbuildlib/SonarQube.groovy @@ -136,11 +136,19 @@ class SonarQube implements Serializable { // CHANGE_TARGET=develop def artifactId = mvn.artifactId.trim() - mvn.additionalArgs += " -Dsonar.projectKey=${mvn.groupId}:${artifactId}:${script.env.BRANCH_NAME}" + - " -Dsonar.projectName=${artifactId}:${script.env.BRANCH_NAME} " + // Malformed key for Project: 'groupId:artifactId:feature/something'. + // Allowed characters are alphanumeric, '-', '_', '.' and ':', with at least one non-digit. + String projectKey = replaceCharactersNotAllowedInProjectKey(script.env.BRANCH_NAME) + String projectName = script.env.BRANCH_NAME + mvn.additionalArgs += " -Dsonar.projectKey=${mvn.groupId}:${artifactId}:${projectKey}" + + " -Dsonar.projectName=${artifactId}:${projectName} " } } + protected String replaceCharactersNotAllowedInProjectKey(String potentialProjectKey) { + return potentialProjectKey.replaceAll("[^a-zA-Z0-9-_.:]", "_"); + } + protected String determineIntegrationBranch() { if (config['integrationBranch']) { return config['integrationBranch'] diff --git a/test/com/cloudogu/ces/cesbuildlib/SonarQubeTest.groovy b/test/com/cloudogu/ces/cesbuildlib/SonarQubeTest.groovy index dd57fae3..d8754a14 100644 --- a/test/com/cloudogu/ces/cesbuildlib/SonarQubeTest.groovy +++ b/test/com/cloudogu/ces/cesbuildlib/SonarQubeTest.groovy @@ -37,16 +37,17 @@ class SonarQubeTest { void analyzeWithToken() throws Exception { def sonarQube = new SonarQube(scriptMock, [token: 'secretTextCred', sonarHostUrl: 'http://ces/sonar']) + def branchName = 'develop.Or:somehing-completely_.different' scriptMock.env = [ SONAR_AUTH_TOKEN: 'auth', - BRANCH_NAME : 'develop' + BRANCH_NAME : branchName ] sonarQube.analyzeWith(mavenMock) assert mavenMock.args == 'sonar:sonar -Dsonar.host.url=http://ces/sonar -Dsonar.login=auth ' - assertBranchName() + assertBranchName(branchName, branchName) assert scriptMock.actualStringArgs['credentialsId'] == 'secretTextCred' } @@ -74,7 +75,7 @@ class SonarQubeTest { assert mavenMock.args == 'sonar:sonar -Dsonar.host.url=http://ces/sonar -Dsonar.login=usr -Dsonar.password=pw ' - assertBranchName() + assertBranchName('develop', 'develop') assert scriptMock.actualUsernamePasswordArgs[0]['credentialsId'] == 'usrPwCred' } @@ -127,7 +128,7 @@ class SonarQubeTest { assert mavenMock.args == 'sonar:sonar -Dsonar.host.url=host -Dsonar.login=auth -DextraKey=extraValue' - assertBranchName() + assertBranchName('develop', 'develop') assert scriptMock.actualSonarQubeEnv == 'sqEnv' } @@ -152,6 +153,23 @@ class SonarQubeTest { assert mavenMock.additionalArgs == '' } + + @Test + void analyzeWithBranchContainCharsNotValidForProjectKey() throws Exception { + String branchName = 'feature/abc' + String projectKey = 'feature_abc' + String projectName = branchName + + def sonarQube = new SonarQube(scriptMock, [usernamePassword: 'usrPwCred', sonarHostUrl: 'http://ces/sonar']) + scriptMock.env = [ + SONAR_AUTH_TOKEN: 'auth', + BRANCH_NAME : branchName + ] + + sonarQube.analyzeWith(mavenMock) + + assertBranchName(projectKey, projectName) + } @Test void analyzeWithBranchPlugin() throws Exception { @@ -303,7 +321,7 @@ class SonarQubeTest { assert exception.message == "Missing required 'sonarHostUrl' parameter." } - void assertBranchName() { - assert mavenMock.additionalArgs.contains("-Dsonar.projectKey=com.cloudogu.ces:ces-build-lib:develop -Dsonar.projectName=ces-build-lib:develop ") + void assertBranchName(String projectKey, String projectName) { + assert mavenMock.additionalArgs.contains("-Dsonar.projectKey=com.cloudogu.ces:ces-build-lib:${projectKey} -Dsonar.projectName=ces-build-lib:${projectName}") } -} +} \ No newline at end of file From a86b7a971a0b44cf8458853ead09252c243ac0ca Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Mon, 24 Aug 2020 16:12:09 +0200 Subject: [PATCH 3/4] Implement Docker.Image.repoDigests() --- README.md | 5 +++ .../cloudogu/ces/cesbuildlib/Docker.groovy | 18 ++++++++++- .../ces/cesbuildlib/DockerTest.groovy | 31 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 18f6e5aa..e4880c21 100644 --- a/README.md +++ b/README.md @@ -599,6 +599,11 @@ Example from Jenkinsfile: ## Additional features provided by the `Docker.Image` class +* `repoDigests()`: Returns the repo digests, a content addressable unique digest of an image that was pushed + to or pulled from repositories. + If the image was built locally and not pushed, returns an empty list. + If the image was pulled from or pushed to a repo, returns a list containing one item. + If the image was pulled from or pushed to multiple repos, might also contain more than one digest. * `mountJenkinsUser()`: Setting this to `true` provides the user that executes the build within docker container's `/etc/passwd`. This is necessary for some commands such as npm, ansible, git, id, etc. Those might exit with errors withouta user present. diff --git a/src/com/cloudogu/ces/cesbuildlib/Docker.groovy b/src/com/cloudogu/ces/cesbuildlib/Docker.groovy index edf3f7fe..46d20b86 100644 --- a/src/com/cloudogu/ces/cesbuildlib/Docker.groovy +++ b/src/com/cloudogu/ces/cesbuildlib/Docker.groovy @@ -21,7 +21,7 @@ class Docker implements Serializable { String findIp(container) { sh.returnStdOut "docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${container.id}" } - + /** * @return the IP address in the current context: the docker host ip (when outside of a container) or the ip of the * container this is running in @@ -290,6 +290,22 @@ class Docker implements Serializable { return this } + /** + * Returns the repo digests, a content addressable unique digest of an image that was pushed to or pulled from + * a repository. + * + * @return If the image was built locally and not pushed, returns an empty list. + * If the image was pulled from or pushed to a repo, returns a list containing one item. + * If the image was pulled from or pushed to multiple repos, might also contain more than one digest. + */ + List repoDigests() { + def split = sh.returnStdOut( + "docker image inspect ${imageIdString} -f '{{range .RepoDigests}}{{printf \"%s\\n\" .}}{{end}}'") + .split('\n') + // Remove empty lines, e.g. the superflous last linebreak + return split - '' + } + private extendArgs(String args) { String extendedArgs = args if (mountJenkinsUser) { diff --git a/test/com/cloudogu/ces/cesbuildlib/DockerTest.groovy b/test/com/cloudogu/ces/cesbuildlib/DockerTest.groovy index 046f7e3d..b9cf35aa 100644 --- a/test/com/cloudogu/ces/cesbuildlib/DockerTest.groovy +++ b/test/com/cloudogu/ces/cesbuildlib/DockerTest.groovy @@ -16,6 +16,7 @@ class DockerTest { def actualDockerGroup = "docker:x:$actualDockerGroupId:jenkins" Map actualWriteFileArgs = [:] def actualShArgs = new LinkedList() + def actualRepoDigests = '' @Test void findIpOfContainer() { @@ -394,6 +395,35 @@ class DockerTest { assert actualShArgs[0].contains("${actualDockerServerVersion}.tgz") } + @Test + void "repo digest"() { + def expectedDigest = "hello-world@sha256:7f0a9f93b4aa3022c3a4c147a449bf11e0941a1fd0bf4a8e6c9408b2600777c5" + actualRepoDigests = expectedDigest + "\n\n" + def digests = createWithImage(mockedImageMethodInside()).image(expectedImage).repoDigests() + + assert digests.size() == 1 + assert digests[0] == expectedDigest + } + + @Test + void "repo digest empty"() { + actualRepoDigests = '\n' + def digests = createWithImage(mockedImageMethodInside()).image(expectedImage).repoDigests() + + assert digests.size() == 0 + } + + @Test + void "repo digest multiple"() { + actualRepoDigests = "a\nb\nc\n\n" + def digests = createWithImage(mockedImageMethodInside()).image(expectedImage).repoDigests() + + assert digests.size() == 3 + assert digests[0] == 'a' + assert digests[1] == 'b' + assert digests[2] == 'c' + } + private Docker create(Map mockedMethod) { Map> mockedScript = [ docker: mockedMethod @@ -429,6 +459,7 @@ class DockerTest { if (script.contains(actualDockerGroup)) return actualDockerGroupId if (script.contains('cat /etc/passwd | grep')) return actualPasswd if (script.contains('docker version --format \'{{.Server.Version}}\'')) return " ${actualDockerServerVersion} " + if (script.contains('RepoDigests')) return " ${actualRepoDigests} " else fail("Unexpected sh call. Script: " + script) }, pwd: { return expectedHome }, From 6b1c6441117d0d6bbd9c2ef940bdd402da4bb19d Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Mon, 24 Aug 2020 17:13:26 +0200 Subject: [PATCH 4/4] Docker.Image avoid exception due to inaccessible default param. "Scripts not permitted to use field org.jenkinsci.plugins.docker.workflow.ImageNameTokens tag" --- .../cloudogu/ces/cesbuildlib/Docker.groovy | 20 +++++++- .../ces/cesbuildlib/DockerTest.groovy | 50 +++++++++++++++++-- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/com/cloudogu/ces/cesbuildlib/Docker.groovy b/src/com/cloudogu/ces/cesbuildlib/Docker.groovy index 46d20b86..6afcf60a 100644 --- a/src/com/cloudogu/ces/cesbuildlib/Docker.groovy +++ b/src/com/cloudogu/ces/cesbuildlib/Docker.groovy @@ -235,17 +235,33 @@ class Docker implements Serializable { * Runs docker tag to record a tag of this image (defaulting to the tag it already has). Will rewrite an * existing tag if one exists. */ - void tag(String tagName = image().parsedId.tag, boolean force = true) { + void tag(String tagName, boolean force) { image().tag(tagName, force) } + + void tag(String tagName) { + image().tag(tagName) + } + + void tag() { + image().tag() + } /** * Pushes an image to the registry after tagging it as with the tag method. For example, you can use image().push * 'latest' to publish it as the latest version in its repository. */ - void push(String tagName = image().parsedId.tag, boolean force = true) { + void push(String tagName, boolean force) { image().push(tagName, force) } + + void push(String tagName) { + image().push(tagName) + } + + void push() { + image().push() + } /** * Provides the user that executes the build within docker container's /etc/passwd. diff --git a/test/com/cloudogu/ces/cesbuildlib/DockerTest.groovy b/test/com/cloudogu/ces/cesbuildlib/DockerTest.groovy index b9cf35aa..5fdce8db 100644 --- a/test/com/cloudogu/ces/cesbuildlib/DockerTest.groovy +++ b/test/com/cloudogu/ces/cesbuildlib/DockerTest.groovy @@ -17,6 +17,8 @@ class DockerTest { Map actualWriteFileArgs = [:] def actualShArgs = new LinkedList() def actualRepoDigests = '' + def actualPushParams + def actualTagParams @Test void findIpOfContainer() { @@ -399,7 +401,7 @@ class DockerTest { void "repo digest"() { def expectedDigest = "hello-world@sha256:7f0a9f93b4aa3022c3a4c147a449bf11e0941a1fd0bf4a8e6c9408b2600777c5" actualRepoDigests = expectedDigest + "\n\n" - def digests = createWithImage(mockedImageMethodInside()).image(expectedImage).repoDigests() + def digests = createWithImage().image(expectedImage).repoDigests() assert digests.size() == 1 assert digests[0] == expectedDigest @@ -408,7 +410,7 @@ class DockerTest { @Test void "repo digest empty"() { actualRepoDigests = '\n' - def digests = createWithImage(mockedImageMethodInside()).image(expectedImage).repoDigests() + def digests = createWithImage().image(expectedImage).repoDigests() assert digests.size() == 0 } @@ -416,7 +418,7 @@ class DockerTest { @Test void "repo digest multiple"() { actualRepoDigests = "a\nb\nc\n\n" - def digests = createWithImage(mockedImageMethodInside()).image(expectedImage).repoDigests() + def digests = createWithImage().image(expectedImage).repoDigests() assert digests.size() == 3 assert digests[0] == 'a' @@ -424,6 +426,42 @@ class DockerTest { assert digests[2] == 'c' } + @Test + void "push image"() { + createWithImage().image(expectedImage).push() + assert actualPushParams == ['', null] + } + + @Test + void "push image with name"() { + createWithImage().image(expectedImage).push('name') + assert actualPushParams == ['name', null] + } + + @Test + void "push image with name and force"() { + createWithImage().image(expectedImage).push('name', true) + assert actualPushParams == ['name', true] + } + + @Test + void "tag image"() { + createWithImage().image(expectedImage).tag() + assert actualTagParams == ['', null] + } + + @Test + void "tag image with name"() { + createWithImage().image(expectedImage).tag('name') + assert actualTagParams == ['name', null] + } + + @Test + void "tag image with name and force"() { + createWithImage().image(expectedImage).tag('name', true) + assert actualTagParams == ['name', true] + } + private Docker create(Map mockedMethod) { Map> mockedScript = [ docker: mockedMethod @@ -435,11 +473,13 @@ class DockerTest { * @return Mock Docker instance with mock image, that contains mocked methods. */ @SuppressWarnings("GroovyAssignabilityCheck") - private Docker createWithImage(Map mockedMethod) { + private Docker createWithImage(Map mockedMethod = [:]) { def mockedScript = [ docker: [image: { String id -> assert id == expectedImage mockedMethod.put('id', id) + mockedMethod.put('push', { String param1 = '', Boolean param2 = null -> actualPushParams = [param1, param2] }) + mockedMethod.put('tag', { String param1 = '', Boolean param2 = null -> actualTagParams = [param1, param2] }) return mockedMethod } ], @@ -483,7 +523,7 @@ class DockerTest { private def mockedImageMethodInside() { [inside: { String param1, Closure param2 -> return [param1, param2] }] } - + /** * @return a map that defines a withRun() method returning its params, to be used as param in createWithImage()}. */