From 012edeae89f62654087edb2e0ac11af1ecf3f138 Mon Sep 17 00:00:00 2001 From: Martin Migasiewicz Date: Wed, 21 Jul 2021 13:10:20 +0200 Subject: [PATCH] chore(artifactCleanup): document new keep artifacts feature. --- cleanup/artifactCleanup/README.md | 57 ++++++++++++++----- .../artifactCleanup/artifactCleanup.groovy | 14 ++--- cleanup/artifactCleanup/artifactCleanup.json | 26 ++++----- .../artifactCleanup.properties | 6 +- 4 files changed, 65 insertions(+), 38 deletions(-) diff --git a/cleanup/artifactCleanup/README.md b/cleanup/artifactCleanup/README.md index ac410652..c55364a7 100644 --- a/cleanup/artifactCleanup/README.md +++ b/cleanup/artifactCleanup/README.md @@ -7,6 +7,7 @@ which is by default 1 month. It can be run manually from the REST API, or automa Many delete operations can affect performance due to disk I/O occurring. A new parameter now allows a delay per delete operation. See below. To ensure logging for this plugin, edit ${ARTIFACTORY_HOME}/etc/logback.xml to add: + ```xml @@ -25,15 +26,10 @@ Parameters - `timeInterval`: The time interval to look back before deleting an artifact. Default *1*. - `repos`: A list of repositories to clean. This parameter is required. - `dryRun`: If this parameter is passed, artifacts will not actually be deleted. Default *false*. +- `disablePropertiesSupport`: Disable the support of Artifactory Properties (see below *Artifactory Properties support* section). Default - `paceTimeMS`: The number of milliseconds to delay between delete operations. Default *0*. -- `disablePropertiesSupport`: Disable the support of Artifactory Properties (see below *Artifactory Properties support* section). Default *false*. - -Properties ----------- - -- `monthsUntil`: The number of months back to look before deleting an artifact. -- `repos`: A list of repositories to clean. -- `paceTimeMS`: The number of milliseconds to delay between delete operations. +- `keepArtifacts`: Enable protection of artifacts that should not be deleted. Default *false*. +- `keepArtifactsRegex`: Regular expression for protecting artifacts to not be deleted. Default `/.*-[\.\d+]*[-+][\.\d+]*\..*/`. Artifactory Properties support ---------- @@ -42,22 +38,25 @@ Some Artifactory [Properties](https://www.jfrog.com/confluence/display/RTF/Prope - `cleanup.skip`: Skip the artifact deletion if property defined on artifact's path ; artifact itself or in a parent folder(s). - `artifactCleanup.json` ---------- The json contains the policies for scheduling cleanup jobs for different repositories. The following properties are supported by each policy descriptor object: + - `cron`: The cron time when the job is executed. Default `0 0 5 ? * 1` *should* be redefined. - `repos`: The mandatory list of repositories the policies will be applied to. - `timeUnit`: The unit of the time interval. *year*, *month*, *day*, *hour* or *minute* are allowed values. Default *month*. - `timeInterval`: The time interval to look back before deleting an artifact. Default *1*. -- `dryRun`: If this parameter is passed, artifacts will not actually be deleted. Default *false*. +- `dryRun`: If this parameter is passed, artifacts will not actually be deleted. Default *false*. - `paceTimeMS`: The number of milliseconds to delay between delete operations. Default *0*. - `disablePropertiesSupport`: Disable the support of Artifactory Properties (see above *Artifactory Properties support* section). +- `keepArtifacts`: Enable protection of artifacts that should not be deleted. Default *false*. +- `keepArtifactsRegex`: Regular expression for protecting artifacts to not be deleted. Default `/.*-[\.\d+]*[-+][\.\d+]*\..*/`. An example file could contain the following json: + ```json { "policies": [ @@ -70,7 +69,9 @@ An example file could contain the following json: "timeInterval": 3, "dryRun": true, "paceTimeMS": 500, - "disablePropertiesSupport": true + "disablePropertiesSupport": true, + "keepArtifacts": true, + "keepArtifactsRegex": "~/.\_-\d+\.\d+\.\d+\-\d{2}\.\d\.\*/" } ] } @@ -78,12 +79,36 @@ An example file could contain the following json: **Note**: If a deprecated `artifactCleanup.properties` is defined it will only be applied if no `artifactCleanup.json` is present. +`artifactCleanup.properties` (DEPRECATED) +---------- + +The properties file contains the policies for scheduling cleanup jobs for different repositories. + +The following properties are supported by each policy descriptor object: + +- `cron`: The cron time when the job is executed. Default `0 0 5 ? * 1` *should* be redefined. +- `repos`: The mandatory list of repositories the policies will be applied to. +- `months`: The number of months back to look before deleting an artifact. Default: *6*. +- `paceTimeMS`: The number of milliseconds to delay between delete operations. Default *0*. +- `dryRun`: If this parameter is passed, artifacts will not actually be deleted. Default *false*. +- `disablePropertiesSupport`: Disable the support of Artifactory Properties (see above *Artifactory Properties support* section). +- `keepArtifacts`: Enable protection of artifacts that should not be deleted. Default *false*. +- `keepArtifactsRegex`: Regular expression for protecting artifacts to not be deleted. Default `~/.*-[\.\d+]*[-+][\.\d+]*\..*/`. + +An example file could contain the following policy: + +``` +policies = [ + [ "0 0 5 ? * 1", [ "libs-release-local" ], 3, 500, true, true, true, ~/.*-\d+\.\d+\.\d+\.*/ ], + ] +``` + Executing --------- To execute the plugin: -For Artifactory 4.x: +For Artifactory 4.x: `curl -X POST -v -u admin:password "http://localhost:8080/artifactory/api/plugins/execute/cleanup?params=timeUnit=month|timeInterval=1|repos=libs-release-local|dryRun=true|paceTimeMS=2000|disablePropertiesSupport=true"` For Artifactory 5.x or higher: @@ -104,17 +129,19 @@ The plugin have 4 control options: - `stop`: When detected, the loop deleting artifacts is exited and the script ends. Example: `curl -X POST -v -u admin:password "http://localhost:8080/artifactory/api/plugins/execute/cleanupCtl?params=command=stop"` + - `pause`: Suspend operation. The thread continues to run with a 1 minute sleep for retest. Example: `curl -X POST -v -u admin:password "http://localhost:8080/artifactory/api/plugins/execute/cleanupCtl?params=command=pause"` + - `resume`: Resume normal execution. Example: `curl -X POST -v -u admin:password "http://localhost:8080/artifactory/api/plugins/execute/cleanupCtl?params=command=resume"` -- `adjustPaceTimeMS`: Modify the running delay factor by increasing/decreasing the delay value. Example: +- `adjustPaceTimeMS`: Modify the running delay factor by increasing/decreasing the delay value. Example: For Artifactory 4.x -`curl -X POST -v -u admin:password "http://localhost:8080/artifactory/api/plugins/execute/cleanupCtl?params=command=adjustPaceTimeMS|value=-1000"` +`curl -X POST -v -u admin:password "http://localhost:8080/artifactory/api/plugins/execute/cleanupCtl?params=command=adjustPaceTimeMS|value=-1000"` For Artifactory 5.x or higher: -`curl -X POST -v -u admin:password "http://localhost:8080/artifactory/api/plugins/execute/cleanupCtl?params=command=adjustPaceTimeMS;value=-1000"` +`curl -X POST -v -u admin:password "http://localhost:8080/artifactory/api/plugins/execute/cleanupCtl?params=command=adjustPaceTimeMS;value=-1000"` diff --git a/cleanup/artifactCleanup/artifactCleanup.groovy b/cleanup/artifactCleanup/artifactCleanup.groovy index 4a896a7c..67a1f081 100644 --- a/cleanup/artifactCleanup/artifactCleanup.groovy +++ b/cleanup/artifactCleanup/artifactCleanup.groovy @@ -62,7 +62,7 @@ executions { def paceTimeMS = params['paceTimeMS'] ? params['paceTimeMS'][0] as int : 0 def keepArtifacts = params['keepArtifacts'] ? new Boolean(params['keepArtifacts'][0]) : false def keepArtifactsRegex = params['keepArtifactsRegex'] ? params['keepArtifactsRegex'] as Pattern : regex - + // Enable fallback support for deprecated month parameter if ( params['months'] && !params['timeInterval'] ) { log.info('Deprecated month parameter is still in use, please use the new timeInterval parameter instead!', properties) @@ -136,7 +136,7 @@ if ( deprecatedConfigFile.exists() ) { } if ( configFile.exists() ) { - + def config = new JsonSlurper().parse(configFile.toURL()) log.info "Schedule job policy list: $config.policies" log.info "Schedule regex: $keepArtifactsRegex" @@ -160,7 +160,7 @@ if ( configFile.exists() ) { } } count++ - } + } } if ( deprecatedConfigFile.exists() && configFile.exists() ) { @@ -233,10 +233,10 @@ private def artifactCleanup(String timeUnit, int timeInterval, String[] repos, l } else { log.debug "Can't delete $it (user ${security.currentUser().getUsername()} has no delete permissions), " + "$cntFoundArtifacts/$artifactsCleanedUp.size total $bytesFound bytes" - if (security.canDelete(it)) { - // Because previous block was not processed, security.canDelete(it) AND/OR checkName is FALSE. + if (security.canDelete(it)) { + // Because previous block was not processed, security.canDelete(it) AND/OR checkName is FALSE. // I am checking if permissions to delete files are proper. If that's true, that means function checkName returned FALSE - // and artifact is protected by Regex. + // and artifact is protected by Regex. bytesFoundWithRegexProtection += repositories.getItemInfo(it)?.getSize() cntNoDeleteRegexPermissions++ } @@ -335,4 +335,4 @@ private def mapTimeUnitToCalendar (String timeUnit) { private def checkName(keepArtifacts, keepArtifactsRegex, artifactName) { return !keepArtifacts || !((artifactName =~ keepArtifactsRegex).count > 0) -} \ No newline at end of file +} diff --git a/cleanup/artifactCleanup/artifactCleanup.json b/cleanup/artifactCleanup/artifactCleanup.json index 5d073a8c..801836e8 100644 --- a/cleanup/artifactCleanup/artifactCleanup.json +++ b/cleanup/artifactCleanup/artifactCleanup.json @@ -1,15 +1,15 @@ { - "policies": [ - { - "cron": "0 0 1 ? * 1", - "repos": [ - "libs-release-local" - ], - "timeUnit": "day", - "timeInterval": 3, - "dryRun": true, - "paceTimeMS": 500, - "disablePropertiesSupport": true - } - ] + "policies": [{ + "cron": "0 0 1 ? * 1", + "repos": [ + "libs-release-local" + ], + "timeUnit": "day", + "timeInterval": 3, + "dryRun": true, + "paceTimeMS": 500, + "disablePropertiesSupport": true, + "keepArtifacts": true, + "keepArtifactsRegex": "~/.\\_-\\d+\\.\\d+\\.\\d+\\-\\d{2}\\.\\d\\.\\*/" + }] } diff --git a/cleanup/artifactCleanup/artifactCleanup.properties b/cleanup/artifactCleanup/artifactCleanup.properties index 9c99cd57..16767511 100644 --- a/cleanup/artifactCleanup/artifactCleanup.properties +++ b/cleanup/artifactCleanup/artifactCleanup.properties @@ -1,6 +1,6 @@ // Support different policies for different repos. Parameters between '|' are optional -// Policy syntax: [ , [ ], months |, , |, |, |, releaseRegex | } +// Policy syntax: [ , [ ], months |, , |, |, |, keepArtifactsRegex | } // Example: [ "0 0 5 ? * 1", [ "libs-releases-local" ], 3, 500, true, true, true, ~/.*-\d+\.\d+\.\d+\.*/ ], -policies = [ - [ "0 0 5 ? * 1", [ "docker-local" ], 3, 500, true, true, true ], +policies = [ + [ "0 0 5 ? * 1", [ "libs-release-local" ], 3, 500, true, true, true, ~/.*-\d+\.\d+\.\d+\.*/ ], ]