diff --git a/cleanup/artifactDeprecate/artifactDeprecate.groovy b/cleanup/artifactDeprecate/artifactDeprecate.groovy deleted file mode 100644 index efc6cfb6..00000000 --- a/cleanup/artifactDeprecate/artifactDeprecate.groovy +++ /dev/null @@ -1,92 +0,0 @@ -import org.apache.commons.lang3.StringUtils -import org.artifactory.api.repo.exception.ItemNotFoundRuntimeException - -import groovy.time.TimeCategory -import groovy.time.TimeDuration -import groovy.transform.Field -import org.artifactory.security.UserInfo - -// Usage example: -// curl -i -uadmin:password -X POST "http://localhost:8081/artifactory/api/plugins/execute/deprecate?params=months=0;repos=docker-local;directory=android;dryRun=true;numberArtifactsToKeep=3" -// curl -i -uadmin:password -X POST "http://localhost:8081/artifactory/api/plugins/execute/deprecate?params=months=3;repos=docker-local;directory=ansible;numberArtifactsToKeep=3" - -@Field final String PROPERTIES_FILE_PATH = "plugins/${this.class.name}.properties" -def config = new ConfigSlurper().parse(new File(ctx.artifactoryHome.haAwareEtcDir, PROPERTIES_FILE_PATH).toURL()) - -config.policies.each{ policySettings -> - def cron = policySettings[ 0 ] ? policySettings[ 0 ] as String : ["0 0 5 ? * 1"] - def repos = policySettings[ 1 ] ? policySettings[ 1 ] as String[] : ["__none__"] - def directory = policySettings[ 2 ] ? policySettings[ 2 ] as String : ["__none__"] - def months = policySettings[ 3 ] ? policySettings[ 3 ] as int : 6 - def dryRun = policySettings[ 4 ] ? policySettings[ 4 ] as Boolean : false - def numberArtifactsToKeep = policySettings[ 5 ] ? policySettings[ 5 ] as int : 3 - - log.info "Schedule job policy list: $config.policies" - - jobs { - "scheduledCleanup_$cron"(cron: cron) { - log.info "Policy settings for scheduled run at($cron): repo list($repos), directory($directory), months($months), dryrun($dryRun), numberArtifactsToKeep($numberArtifactsToKeep)" - artifactDeprecate( months, repos, directory, log, dryRun, numberArtifactsToKeep ) - } - } -} - -def pluginGroup = 'deprecates' - -executions { - deprecate(groups: [pluginGroup]) { params -> - def months = params['months'] ? params['months'][0] as int : 6 - def repos = params['repos'] as String[] - def directory = params['directory'][0] as String - def dryRun = params['dryRun'] ? params['dryRun'][0].toBoolean() : false - def numberArtifactsToKeep = params['numberArtifactsToKeep'] ? params['numberArtifactsToKeep'][0] as int : 3 - artifactDeprecate(months, repos, directory, log, dryRun, numberArtifactsToKeep) - } -} - - -private def artifactDeprecate(int months, String[] repos, String directory, log, dryRun = false, numberArtifactsToKeep = 3) { - - log.info "-----------[ Starting Deprecating Artifacts... ]-----------" - log.info "---> Variables: \n months: $months \n repos: $repos \n directory: ${directory} \n log: $log \n dryRun: $dryRun \n numberArtifactsToKeep: $numberArtifactsToKeep \n" - - def monthsUntil = Calendar.getInstance() - monthsUntil.add(Calendar.MONTH, -months) - - int cntFoundArtifacts = 0 - int cntNoDeletePermissions = 0 - def artifactsList = searches.artifactsCreatedOrModifiedInRange(null, monthsUntil, repos) - def artifactsCleanedUp = artifactsList.findAll { it =~ directory } - def sortedArtifactsCleanedUp = artifactsCleanedUp.sort { repositories.getItemInfo(it)?.lastUpdated } - int numberArtifactsToDelete = sortedArtifactsCleanedUp.size - numberArtifactsToKeep - def artifactsToDelete = sortedArtifactsCleanedUp.take(numberArtifactsToDelete) - - - log.info "\n ===> sortedArtifactsCleanedUp: $sortedArtifactsCleanedUp\n\n ===> artifactsToDelete: $artifactsToDelete" - - artifactsToDelete.find { - try { - if (!security.canDelete(it)) { - cntNoDeletePermissions++ - } - if (dryRun) { - log.info "Found $it!" - log.info "\t==> currentUser: ${security.currentUser().getUsername()}" - log.info "\t==> canDelete: ${security.canDelete(it)}" - } else { - if (security.canDelete(it)) { - log.info "Deleting $it!" - repositories.delete it - } else { - log.info "Can't delete $it (user ${security.currentUser().getUsername()} has no delete permissions), " + - "$cntFoundArtifacts/$artifactsCleanedUp.size" - cntNoDeletePermissions++ - } - } - } catch (ItemNotFoundRuntimeException ex) { - log.info "Failed to find $it, skipping" - } - - return false - } -} \ No newline at end of file diff --git a/cleanup/artifactDeprecate/artifactDeprecate.properties b/cleanup/artifactDeprecate/artifactDeprecate.properties deleted file mode 100644 index f05fb5eb..00000000 --- a/cleanup/artifactDeprecate/artifactDeprecate.properties +++ /dev/null @@ -1,6 +0,0 @@ -// Support different policies for different repos. Parameters between '|' are optional -// Policy syntax: [ , [ ], directory, months |, |, numberArtifactsToKeep | } -// Example: [ "0 0 5 ? * 1", [ "libs-releases-local" ], 3, true, 3 ], -policies = [ - [ "0 0 5 ? * 1", [ "docker-local" ], "ansible" 3, true, 3 ], - ] diff --git a/cleanup/cleanDockerImages/CleanDockerImagesTest.groovy b/cleanup/cleanDockerImages/CleanDockerImagesTest.groovy new file mode 100644 index 00000000..204f34f9 --- /dev/null +++ b/cleanup/cleanDockerImages/CleanDockerImagesTest.groovy @@ -0,0 +1,100 @@ +import groovy.json.JsonSlurper +import org.apache.http.client.HttpResponseException +import org.jfrog.artifactory.client.ArtifactoryClientBuilder +import org.jfrog.artifactory.client.model.repository.settings.impl.DockerRepositorySettingsImpl +import spock.lang.Specification + +class CleanDockerImagesTest extends Specification { + def 'simple clean docker images plugin test'() { + setup: + def baseurl = 'http://localhost:8088/artifactory' + def artifactory = ArtifactoryClientBuilder.create().setUrl(baseurl) + .setUsername('admin').setPassword('password').build() + + def builder = artifactory.repositories().builders() + def local = builder.localRepositoryBuilder().key('example-docker-local') + .repositorySettings(new DockerRepositorySettingsImpl()).build() + artifactory.repositories().create(0, local) + def repo = artifactory.repository('example-docker-local') + mkImage(repo, 'foo1/bar/manifest.json', 'text1', 1) + mkImage(repo, 'foo1/baz/manifest.json', 'text2', 1) + mkImage(repo, 'foo1/ban/manifest.json', 'text3', 1) + mkImage(repo, 'foo2/bar/manifest.json', 'text4', 2) + mkImage(repo, 'foo2/baz/manifest.json', 'text5', 2) + mkImage(repo, 'foo2/ban/manifest.json', 'text6', 2) + mkImage(repo, 'foo3/bar/manifest.json', 'text7', 3) + mkImage(repo, 'foo3/baz/manifest.json', 'text8', 3) + mkImage(repo, 'foo3/ban/manifest.json', 'text9', 3) + + when: + def resp = artifactory.plugins().execute('cleanDockerImages').sync() + + then: + new JsonSlurper().parseText(resp).status == 'okay' + + when: + repo.file('foo1/bar/manifest.json').info() + + then: + thrown(HttpResponseException) + + when: + repo.file('foo1/baz/manifest.json').info() + + then: + thrown(HttpResponseException) + + when: + repo.file('foo1/ban/manifest.json').info() + + then: + notThrown(HttpResponseException) + + when: + repo.file('foo2/bar/manifest.json').info() + + then: + thrown(HttpResponseException) + + when: + repo.file('foo2/baz/manifest.json').info() + + then: + notThrown(HttpResponseException) + + when: + repo.file('foo2/ban/manifest.json').info() + + then: + notThrown(HttpResponseException) + + when: + repo.file('foo3/bar/manifest.json').info() + + then: + notThrown(HttpResponseException) + + when: + repo.file('foo3/baz/manifest.json').info() + + then: + notThrown(HttpResponseException) + + when: + repo.file('foo3/ban/manifest.json').info() + + then: + notThrown(HttpResponseException) + + cleanup: + repo.delete() + } + + void mkImage(repo, path, content, ct) { + def prop = 'docker.label.com.jfrog.artifactory.retention.maxCount' + def stream = new ByteArrayInputStream(content.getBytes('utf-8')) + repo.upload(path, stream).doUpload() + repo.file(path).properties().addProperty(prop, "$ct").doSet() + sleep(100) + } +} diff --git a/cleanup/cleanDockerImages/README.md b/cleanup/cleanDockerImages/README.md new file mode 100644 index 00000000..48d93a48 --- /dev/null +++ b/cleanup/cleanDockerImages/README.md @@ -0,0 +1,54 @@ +Artifactory Clean Docker Images User Plugin +=========================================== + +This plugin is used to clean Docker repositories based on configured cleanup +policies. + +Configuration +------------- + +The `cleanDockerImages.properties` file has the following field: + +- `dockerRepos`: A list of Docker repositories to clean. If a repo is not in + this list, it will not be cleaned. + +For example: + +``` json +dockerRepos = ["example-docker-local", "example-docker-local-2"] +``` + +Usage +----- + +Cleanup policies are specified as labels on the Docker image. Currently, this +plugin supports the following policies: + +- `maxDays`: The maximum number of days a Docker image can exist in an + Artifactory repository. Any images older than this will be deleted. +- `maxCount`: The maximum number of versions of a particular image which should + exist. For example, if there are 10 versions of a Docker image and `maxCount` + is set to 6, the oldest 4 versions of the image will be deleted. + +To set these labels for an image, add them to the Dockerfile before building: + +``` dockerfile +LABEL com.jfrog.artifactory.retention.maxCount="10" +LABEL com.jfrog.artifactory.retention.maxDays="7" +``` + +When a Docker image is deployed, Artifactory will automatically create +properties reflecting each of its labels. These properties are read by the +plugin in order to decide on the cleanup policy for the image. + +Cleanup can be triggered via a REST endpoint. For example: + +``` shell +curl -XPOST -uadmin:password http://localhost:8081/artifactory/api/plugins/execute/cleanDockerImages +``` + +A dry run can also be triggered: + +``` shell +curl -XPOST -uadmin:password "http://localhost:8081/artifactory/api/plugins/execute/cleanDockerImages?params=dryRun=true" +``` diff --git a/cleanup/cleanDockerImages/cleanDockerImages.groovy b/cleanup/cleanDockerImages/cleanDockerImages.groovy new file mode 100644 index 00000000..500ae280 --- /dev/null +++ b/cleanup/cleanDockerImages/cleanDockerImages.groovy @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2017 JFrog Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Created by Madhu Reddy on 6/16/17. + +import groovy.json.JsonBuilder +import java.util.concurrent.TimeUnit +import org.artifactory.repo.RepoPathFactory + +// usage: curl -X POST http://localhost:8088/artifactory/api/plugins/execute/cleanDockerImages + +executions { + cleanDockerImages() { params -> + def deleted = [] + def etcdir = ctx.artifactoryHome.etcDir + def propsfile = new File(etcdir, "plugins/cleanDockerImages.properties") + def repos = new ConfigSlurper().parse(propsfile.toURL()).dockerRepos + def dryRun = params['dryRun'] ? params['dryRun'][0] as boolean : false + repos.each { + log.debug("Cleaning Docker images in repo: $it") + def del = buildParentRepoPaths(RepoPathFactory.create(it), dryRun) + deleted.addAll(del) + } + def json = [status: 'okay', dryRun: dryRun, deleted: deleted] + message = new JsonBuilder(json).toPrettyString() + status = 200 + } +} + +def buildParentRepoPaths(path, dryRun) { + def deleted = [], oldSet = [], imagesPathMap = [:], imagesCount = [:] + def parentInfo = repositories.getItemInfo(path) + simpleTraverse(parentInfo, oldSet, imagesPathMap, imagesCount) + for (img in oldSet) { + deleted << img.id + if (!dryRun) repositories.delete(img) + } + for (key in imagesPathMap.keySet()) { + def repoList = imagesPathMap[key] + def maxImagesCount = imagesCount[key] + // If number of current docker images is more than maxcount, delete them + if (maxImagesCount <= 0 || repoList.size() <= maxImagesCount) continue + repoList = repoList.sort { it[1] } + def deleteCount = repoList.size() - maxImagesCount + for (i = 0; i < deleteCount; i += 1) { + deleted << repoList[i][0].id + if (!dryRun) repositories.delete(repoList[i][0]) + } + } + return deleted +} + +// Traverse through the docker repo (directories and sub-directories) and: +// - delete the images immediately if the maxDays policy applies +// - Aggregate the images that qualify for maxCount policy (to get deleted in +// the execution closure) +def simpleTraverse(parentInfo, oldSet, imagesPathMap, imagesCount) { + def maxCount = null + def parentRepoPath = parentInfo.repoPath + for (childItem in repositories.getChildren(parentRepoPath)) { + def currentPath = childItem.repoPath + if (childItem.isFolder()) { + simpleTraverse(childItem, oldSet, imagesPathMap, imagesCount) + continue + } + log.debug("Scanning File: $currentPath.name") + if (currentPath.name != "manifest.json") continue + // get the properties here and delete based on policies: + // - implement daysPassed policy first and delete the images that + // qualify + // - aggregate the image info to group by image and sort by create + // date for maxCount policy + if (checkDaysPassedForDelete(childItem)) { + log.debug("Adding to OLD MAP: $parentRepoPath") + oldSet << parentRepoPath + } else if ((maxCount = getMaxCountForDelete(childItem)) > 0) { + log.debug("Adding to IMAGES MAP: $parentRepoPath") + def parentCreatedDate = parentInfo.created + def parentId = parentRepoPath.parent.id + def oldmax = maxCount + if (parentId in imagesCount) oldmax = imagesCount[parentId] + imagesCount[parentId] = maxCount > oldmax ? maxCount : oldmax + if (!imagesPathMap.containsKey(parentId)) { + imagesPathMap[parentId] = [] + } + imagesPathMap[parentId] << [parentRepoPath, childItem.created] + } + break + } +} + +// This method checks if the docker image's manifest has the property +// "com.jfrog.artifactory.retention.maxDays" for purge +def checkDaysPassedForDelete(item) { + def maxDaysProp = "docker.label.com.jfrog.artifactory.retention.maxDays" + def oneday = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS) + def prop = repositories.getProperty(item.repoPath, maxDaysProp) + if (!prop) return false + log.debug("PROPERTY $maxDaysProp FOUND = $prop IN MANIFEST FILE") + prop = prop.isInteger() ? prop.toInteger() : null + if (prop == null) return false + return ((new Date().time - item.created) / oneday) >= prop +} + +// This method checks if the docker image's manifest has the property +// "com.jfrog.artifactory.retention.maxCount" for purge +def getMaxCountForDelete(item) { + def maxCountProp = "docker.label.com.jfrog.artifactory.retention.maxCount" + def prop = repositories.getProperty(item.repoPath, maxCountProp) + if (!prop) return 0 + log.debug "PROPERTY $maxCountProp FOUND = $prop IN MANIFEST FILE" + prop = prop.isInteger() ? prop.toInteger() : 0 + return prop > 0 ? prop : 0 +} diff --git a/cleanup/cleanDockerImages/cleanDockerImages.properties b/cleanup/cleanDockerImages/cleanDockerImages.properties new file mode 100644 index 00000000..b2ac1a82 --- /dev/null +++ b/cleanup/cleanDockerImages/cleanDockerImages.properties @@ -0,0 +1 @@ +dockerRepos = ["example-docker-local"] diff --git a/cleanup/deleteDeprecated/DeleteDeprecatedTest.groovy b/cleanup/deleteDeprecated/DeleteDeprecatedTest.groovy index e0eabdea..7349aece 100644 --- a/cleanup/deleteDeprecated/DeleteDeprecatedTest.groovy +++ b/cleanup/deleteDeprecated/DeleteDeprecatedTest.groovy @@ -1,14 +1,15 @@ -import groovyx.net.http.HttpResponseException +import org.apache.http.client.HttpResponseException import spock.lang.Specification import org.jfrog.artifactory.client.model.repository.settings.impl.MavenRepositorySettingsImpl -import static org.jfrog.artifactory.client.ArtifactoryClient.create +import org.jfrog.artifactory.client.ArtifactoryClientBuilder class DeleteDeprecatedTest extends Specification { def 'delete deprecated plugin test'() { setup: def baseurl = 'http://localhost:8088/artifactory' - def artifactory = create(baseurl, 'admin', 'password') + def artifactory = ArtifactoryClientBuilder.create().setUrl(baseurl) + .setUsername('admin').setPassword('password').build() def builder = artifactory.repositories().builders() def local = builder.localRepositoryBuilder().key('maven-local')