added cleanDockerImages plugin

cleanup/cleanDockerImages/CleanDockerImagesTest.groovy

@@ -0,0 +1,100 @@
+import groovy.json.JsonSlurper
+import org.apache.http.client.HttpResponseException
+import org.jfrog.artifactory.client.ArtifactoryClientBuilder
+import org.jfrog.artifactory.client.model.repository.settings.impl.DockerRepositorySettingsImpl
+import spock.lang.Specification
+
+class CleanDockerImagesTest extends Specification {
+    def 'simple clean docker images plugin test'() {
+        setup:
+        def baseurl = 'http://localhost:8088/artifactory'
+        def artifactory = ArtifactoryClientBuilder.create().setUrl(baseurl)
+            .setUsername('admin').setPassword('password').build()
+
+        def builder = artifactory.repositories().builders()
+        def local = builder.localRepositoryBuilder().key('example-docker-local')
+            .repositorySettings(new DockerRepositorySettingsImpl()).build()
+        artifactory.repositories().create(0, local)
+        def repo = artifactory.repository('example-docker-local')
+        mkImage(repo, 'foo1/bar/manifest.json', 'text1', 1)
+        mkImage(repo, 'foo1/baz/manifest.json', 'text2', 1)
+        mkImage(repo, 'foo1/ban/manifest.json', 'text3', 1)
+        mkImage(repo, 'foo2/bar/manifest.json', 'text4', 2)
+        mkImage(repo, 'foo2/baz/manifest.json', 'text5', 2)
+        mkImage(repo, 'foo2/ban/manifest.json', 'text6', 2)
+        mkImage(repo, 'foo3/bar/manifest.json', 'text7', 3)
+        mkImage(repo, 'foo3/baz/manifest.json', 'text8', 3)
+        mkImage(repo, 'foo3/ban/manifest.json', 'text9', 3)
+
+        when:
+        def resp = artifactory.plugins().execute('cleanDockerImages').sync()
+
+        then:
+        new JsonSlurper().parseText(resp).status == 'okay'
+
+        when:
+        repo.file('foo1/bar/manifest.json').info()
+
+        then:
+        thrown(HttpResponseException)
+
+        when:
+        repo.file('foo1/baz/manifest.json').info()
+
+        then:
+        thrown(HttpResponseException)
+
+        when:
+        repo.file('foo1/ban/manifest.json').info()
+
+        then:
+        notThrown(HttpResponseException)
+
+        when:
+        repo.file('foo2/bar/manifest.json').info()
+
+        then:
+        thrown(HttpResponseException)
+
+        when:
+        repo.file('foo2/baz/manifest.json').info()
+
+        then:
+        notThrown(HttpResponseException)
+
+        when:
+        repo.file('foo2/ban/manifest.json').info()
+
+        then:
+        notThrown(HttpResponseException)
+
+        when:
+        repo.file('foo3/bar/manifest.json').info()
+
+        then:
+        notThrown(HttpResponseException)
+
+        when:
+        repo.file('foo3/baz/manifest.json').info()
+
+        then:
+        notThrown(HttpResponseException)
+
+        when:
+        repo.file('foo3/ban/manifest.json').info()
+
+        then:
+        notThrown(HttpResponseException)
+
+        cleanup:
+        repo.delete()
+    }
+
+    void mkImage(repo, path, content, ct) {
+        def prop = 'docker.label.com.jfrog.artifactory.retention.maxCount'
+        def stream = new ByteArrayInputStream(content.getBytes('utf-8'))
+        repo.upload(path, stream).doUpload()
+        repo.file(path).properties().addProperty(prop, "$ct").doSet()
+        sleep(100)
+    }
+}

cleanup/cleanDockerImages/README.md

@@ -0,0 +1,54 @@
+Artifactory Clean Docker Images User Plugin
+===========================================
+
+This plugin is used to clean Docker repositories based on configured cleanup
+policies.
+
+Configuration
+-------------
+
+The `cleanDockerImages.properties` file has the following field:
+
+- `dockerRepos`: A list of Docker repositories to clean. If a repo is not in
+  this list, it will not be cleaned.
+
+For example:
+
+``` json
+dockerRepos = ["example-docker-local", "example-docker-local-2"]
+```
+
+Usage
+-----
+
+Cleanup policies are specified as labels on the Docker image. Currently, this
+plugin supports the following policies:
+
+- `maxDays`: The maximum number of days a Docker image can exist in an
+  Artifactory repository. Any images older than this will be deleted.
+- `maxCount`: The maximum number of versions of a particular image which should
+  exist. For example, if there are 10 versions of a Docker image and `maxCount`
+  is set to 6, the oldest 4 versions of the image will be deleted.
+
+To set these labels for an image, add them to the Dockerfile before building:
+
+``` dockerfile
+LABEL com.jfrog.artifactory.retention.maxCount="10"
+LABEL com.jfrog.artifactory.retention.maxDays="7"
+```
+
+When a Docker image is deployed, Artifactory will automatically create
+properties reflecting each of its labels. These properties are read by the
+plugin in order to decide on the cleanup policy for the image.
+
+Cleanup can be triggered via a REST endpoint. For example:
+
+``` shell
+curl -XPOST -uadmin:password http://localhost:8081/artifactory/api/plugins/execute/cleanDockerImages
+```
+
+A dry run can also be triggered:
+
+``` shell
+curl -XPOST -uadmin:password "http://localhost:8081/artifactory/api/plugins/execute/cleanDockerImages?params=dryRun=true"
+```

cleanup/cleanDockerImages/cleanDockerImages.groovy

@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2017 JFrog Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Created by Madhu Reddy on 6/16/17.
+
+import groovy.json.JsonBuilder
+import java.util.concurrent.TimeUnit
+import org.artifactory.repo.RepoPathFactory
+
+// usage: curl -X POST http://localhost:8088/artifactory/api/plugins/execute/cleanDockerImages
+
+executions {
+    cleanDockerImages() { params ->
+        def deleted = []
+        def etcdir = ctx.artifactoryHome.etcDir
+        def propsfile = new File(etcdir, "plugins/cleanDockerImages.properties")
+        def repos = new ConfigSlurper().parse(propsfile.toURL()).dockerRepos
+        def dryRun = params['dryRun'] ? params['dryRun'][0] as boolean : false
+        repos.each {
+            log.debug("Cleaning Docker images in repo: $it")
+            def del = buildParentRepoPaths(RepoPathFactory.create(it), dryRun)
+            deleted.addAll(del)
+        }
+        def json = [status: 'okay', dryRun: dryRun, deleted: deleted]
+        message = new JsonBuilder(json).toPrettyString()
+        status = 200
+    }
+}
+
+def buildParentRepoPaths(path, dryRun) {
+    def deleted = [], oldSet = [], imagesPathMap = [:], imagesCount = [:]
+    def parentInfo = repositories.getItemInfo(path)
+    simpleTraverse(parentInfo, oldSet, imagesPathMap, imagesCount)
+    for (img in oldSet) {
+        deleted << img.id
+        if (!dryRun) repositories.delete(img)
+    }
+    for (key in imagesPathMap.keySet()) {
+        def repoList = imagesPathMap[key]
+        def maxImagesCount = imagesCount[key]
+        // If number of current docker images is more than maxcount, delete them
+        if (maxImagesCount <= 0 || repoList.size() <= maxImagesCount) continue
+        repoList = repoList.sort { it[1] }
+        def deleteCount = repoList.size() - maxImagesCount
+        for (i = 0; i < deleteCount; i += 1) {
+            deleted << repoList[i][0].id
+            if (!dryRun) repositories.delete(repoList[i][0])
+        }
+    }
+    return deleted
+}
+
+// Traverse through the docker repo (directories and sub-directories) and:
+// - delete the images immediately if the maxDays policy applies
+// - Aggregate the images that qualify for maxCount policy (to get deleted in
+//   the execution closure)
+def simpleTraverse(parentInfo, oldSet, imagesPathMap, imagesCount) {
+    def maxCount = null
+    def parentRepoPath = parentInfo.repoPath
+    for (childItem in repositories.getChildren(parentRepoPath)) {
+        def currentPath = childItem.repoPath
+        if (childItem.isFolder()) {
+            simpleTraverse(childItem, oldSet, imagesPathMap, imagesCount)
+            continue
+        }
+        log.debug("Scanning File: $currentPath.name")
+        if (currentPath.name != "manifest.json") continue
+        // get the properties here and delete based on policies:
+        // - implement daysPassed policy first and delete the images that
+        //   qualify
+        // - aggregate the image info to group by image and sort by create
+        //   date for maxCount policy
+        if (checkDaysPassedForDelete(childItem)) {
+            log.debug("Adding to OLD MAP: $parentRepoPath")
+            oldSet << parentRepoPath
+        } else if ((maxCount = getMaxCountForDelete(childItem)) > 0) {
+            log.debug("Adding to IMAGES MAP: $parentRepoPath")
+            def parentCreatedDate = parentInfo.created
+            def parentId = parentRepoPath.parent.id
+            def oldmax = maxCount
+            if (parentId in imagesCount) oldmax = imagesCount[parentId]
+            imagesCount[parentId] = maxCount > oldmax ? maxCount : oldmax
+            if (!imagesPathMap.containsKey(parentId)) {
+                imagesPathMap[parentId] = []
+            }
+            imagesPathMap[parentId] << [parentRepoPath, childItem.created]
+        }
+        break
+    }
+}
+
+// This method checks if the docker image's manifest has the property
+// "com.jfrog.artifactory.retention.maxDays" for purge
+def checkDaysPassedForDelete(item) {
+    def maxDaysProp = "docker.label.com.jfrog.artifactory.retention.maxDays"
+    def oneday = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)
+    def prop = repositories.getProperty(item.repoPath, maxDaysProp)
+    if (!prop) return false
+    log.debug("PROPERTY $maxDaysProp FOUND = $prop IN MANIFEST FILE")
+    prop = prop.isInteger() ? prop.toInteger() : null
+    if (prop == null) return false
+    return ((new Date().time - item.created) / oneday) >= prop
+}
+
+// This method checks if the docker image's manifest has the property
+// "com.jfrog.artifactory.retention.maxCount" for purge
+def getMaxCountForDelete(item) {
+    def maxCountProp = "docker.label.com.jfrog.artifactory.retention.maxCount"
+    def prop = repositories.getProperty(item.repoPath, maxCountProp)
+    if (!prop) return 0
+    log.debug "PROPERTY $maxCountProp FOUND = $prop IN MANIFEST FILE"
+    prop = prop.isInteger() ? prop.toInteger() : 0
+    return prop > 0 ? prop : 0
+}

cleanup/cleanDockerImages/cleanDockerImages.properties

@@ -0,0 +1 @@
+dockerRepos = ["example-docker-local"]