From 434163b50333f2fe671216db3e9ecae872a60c18 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 16 Nov 2020 19:20:08 +0100 Subject: [PATCH 1/8] cache ocis build artifcats for testing --- .drone.star | 110 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 13 deletions(-) diff --git a/.drone.star b/.drone.star index 30858856535..a27925485d9 100644 --- a/.drone.star +++ b/.drone.star @@ -117,12 +117,15 @@ def main(ctx): Returns: none """ + pipelines = [] before = \ + [ buildOcisBinaryForTesting(ctx) ] + \ testOcisModules(ctx) + \ testPipelines(ctx) + stages = [ docker(ctx, 'amd64'), docker(ctx, 'arm64'), @@ -200,7 +203,7 @@ def testPipelines(ctx): return pipelines def testOcisModule(ctx, module): - steps = generate(module) + [ + steps = makeGenerate(module) + [ { 'name': 'vet', 'image': 'webhippie/golang:1.14', @@ -307,6 +310,34 @@ def testOcisModule(ctx, module): ], } +def buildOcisBinaryForTesting(ctx): + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'build_ocis_binary_for_testing', + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': + makeGenerate('ocis') + + build() + + rebuildBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis'), + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/v*', + 'refs/pull/**', + ], + }, + 'volumes': [ + { + 'name': 'gopath', + 'temp': {}, + }, + ], + } + def uploadCoverage(ctx): return { 'kind': 'pipeline', @@ -385,8 +416,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo 'arch': 'amd64', }, 'steps': - generate('ocis') + - build() + + restoreBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + cloneCoreRepos(coreBranch, coreCommit) + [ { @@ -421,6 +451,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo 'temp': {}, }, ], + 'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]), 'trigger': { 'ref': [ 'refs/heads/master', @@ -440,8 +471,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n 'arch': 'amd64', }, 'steps': - generate('ocis') + - build() + + restoreBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + cloneCoreRepos(coreBranch, coreCommit) + [ { @@ -477,6 +507,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n 'temp': {}, }, ], + 'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]), 'trigger': { 'ref': [ 'refs/heads/master', @@ -509,8 +540,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '', 'arch': 'amd64', }, 'steps': - generate('ocis') + - build() + + restoreBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + [ { 'name': 'webUITests', @@ -562,6 +592,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '', 'temp': {} } ], + 'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]), 'trigger': { 'ref': [ 'refs/heads/master', @@ -581,8 +612,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc 'arch': 'amd64', }, 'steps': - generate('ocis') + - build() + + restoreBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + [ { 'name': 'WebUIAcceptanceTests', @@ -655,6 +685,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc 'temp': {} } ], + 'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]), 'trigger': { 'ref': [ 'refs/heads/master', @@ -674,7 +705,7 @@ def docker(ctx, arch): 'arch': arch, }, 'steps': - generate('ocis') + + makeGenerate('ocis') + build() + [ { 'name': 'dryrun', @@ -747,7 +778,7 @@ def dockerEos(ctx): 'arch': 'amd64', }, 'steps': - generate('ocis') + + makeGenerate('ocis') + build() + [ { 'name': 'dryrun-eos-ocis', @@ -858,7 +889,7 @@ def binary(ctx, name): 'arch': 'amd64', }, 'steps': - generate('ocis') + [ + makeGenerate('ocis') + [ { 'name': 'build', 'image': 'webhippie/golang:1.14', @@ -1312,7 +1343,7 @@ def docs(ctx): }, } -def generate(module): +def makeGenerate(module): return [ { 'name': 'generate', @@ -1541,3 +1572,56 @@ def build(): ], }, ] + +def genericCache(name, action, mounts, cache_key): + rebuild = 'false' + restore = 'false' + if action == 'rebuild': + rebuild = 'true' + action = 'rebuild' + else: + restore = 'true' + action = 'restore' + + cleaned_mounts = [] + for mount in mounts: + cleaned_mounts.append(mount.lstrip('/')) + + step = { + 'name': '%s_%s' %(action, name), + 'image': 'meltwater/drone-cache:v1', + 'pull': 'always', + 'environment': { + 'AWS_ACCESS_KEY_ID': { + 'from_secret': 'cache_s3_access_key', + }, + 'AWS_SECRET_ACCESS_KEY': { + 'from_secret': 'cache_s3_secret_key', + }, + }, + 'settings': { + 'endpoint': { + 'from_secret': 'cache_s3_endpoint' + }, + 'bucket': 'cache', + 'region': 'us-east-1', # not used at all, but failes if not given! + 'path_style': 'true', + 'cache_key': cache_key, + 'rebuild': rebuild, + 'restore': restore, + 'mount': cleaned_mounts, + }, + } + + return step + +def genericBuildArtifactCache(name, action, path): + name = '%s_build_artifact_cache' %(name) + cache_key = '{{ .Repo.Name }}/{{ .Commit.Sha }}-{{ .Build.Number }}/%s' %(name) + return genericCache(name, action, [path], cache_key) + +def restoreBuildArtifactCache(name, path): + return [genericBuildArtifactCache(name, 'restore', path)] + +def rebuildBuildArtifactCache(name, path): + return [genericBuildArtifactCache(name, 'rebuild', path)] From e2817731fcad52091d686afb424aebc3a5cfee08 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 27 Nov 2020 10:56:13 +0100 Subject: [PATCH 2/8] fix ci run when docs_only is somewhere in pr message, should only check title --- .drone.star | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index a27925485d9..16703c53211 100644 --- a/.drone.star +++ b/.drone.star @@ -153,7 +153,12 @@ def main(ctx): pipelines = before + [ notify_pipeline ] - elif '[docs-only]' in (ctx.build.title + ctx.build.message): + elif \ + (ctx.build.event == "pull" and '[docs-only]' in ctx.build.title) \ + or \ + (ctx.build.event != "pull" and '[docs-only]' in (ctx.build.title + ctx.build.message)): + # [docs-only] is not taken from PR messages, but from commit messages + docs_pipeline = docs(ctx) docs_pipeline['depends_on'] = [] docs_pipelines = [ docs_pipeline ] From 4439a3109e93bd766450c9a790374ce7d4015057 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 27 Nov 2020 13:22:36 +0100 Subject: [PATCH 3/8] purge build artifact cache after it is used anymore --- .drone.star | 77 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/.drone.star b/.drone.star index 16703c53211..5de941433ae 100644 --- a/.drone.star +++ b/.drone.star @@ -107,7 +107,6 @@ def getPipelineNames(pipelines=[]): names.append(pipeline['name']) return names - def main(ctx): """main is the entrypoint for drone @@ -125,7 +124,6 @@ def main(ctx): testOcisModules(ctx) + \ testPipelines(ctx) - stages = [ docker(ctx, 'amd64'), docker(ctx, 'arm64'), @@ -137,6 +135,9 @@ def main(ctx): releaseSubmodule(ctx), ] + purge = purgeBuildArtifactCache(ctx, 'ocis-binary-amd64') + purge['depends_on'] = getPipelineNames(testPipelines(ctx)) + after = [ manifest(ctx), changelog(ctx), @@ -144,6 +145,7 @@ def main(ctx): badges(ctx), docs(ctx), updateDeployment(ctx), + #purge, ] if ctx.build.event == "cron": @@ -190,7 +192,6 @@ def testOcisModules(ctx): return pipelines + [coverage_upload] - def testPipelines(ctx): pipelines = [ localApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], 'owncloud', 'apiOcisSpecific'), @@ -327,7 +328,7 @@ def buildOcisBinaryForTesting(ctx): 'steps': makeGenerate('ocis') + build() + - rebuildBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis'), + rebuildBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis'), 'trigger': { 'ref': [ 'refs/heads/master', @@ -421,7 +422,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo 'arch': 'amd64', }, 'steps': - restoreBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis') + + restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + cloneCoreRepos(coreBranch, coreCommit) + [ { @@ -476,7 +477,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n 'arch': 'amd64', }, 'steps': - restoreBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis') + + restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + cloneCoreRepos(coreBranch, coreCommit) + [ { @@ -545,7 +546,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '', 'arch': 'amd64', }, 'steps': - restoreBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis') + + restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + [ { 'name': 'webUITests', @@ -617,7 +618,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc 'arch': 'amd64', }, 'steps': - restoreBuildArtifactCache('ocis-binary-amd64', 'ocis/bin/ocis') + + restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + [ { 'name': 'WebUIAcceptanceTests', @@ -1588,10 +1589,6 @@ def genericCache(name, action, mounts, cache_key): restore = 'true' action = 'restore' - cleaned_mounts = [] - for mount in mounts: - cleaned_mounts.append(mount.lstrip('/')) - step = { 'name': '%s_%s' %(action, name), 'image': 'meltwater/drone-cache:v1', @@ -1614,19 +1611,57 @@ def genericCache(name, action, mounts, cache_key): 'cache_key': cache_key, 'rebuild': rebuild, 'restore': restore, - 'mount': cleaned_mounts, + 'mount': mounts, }, } - return step -def genericBuildArtifactCache(name, action, path): +def genericCachePurge(name, cache_key): + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'purge_%s_build_artifact_cache' %(name), + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': [ + { + 'name': 'purge-cache', + 'image': 'minio/mc', + 'environment': { + 'MC_HOST_cache': { + 'from_secret': 'cache_s3_connection_url' + } + }, + 'commands': [ + 'mc rm --force cache/%s' % (cache_key), + ] + }, + ], + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/v*', + 'refs/pull/**', + ], + }, + } + +def genericBuildArtifactCache(ctx, name, action, path): name = '%s_build_artifact_cache' %(name) - cache_key = '{{ .Repo.Name }}/{{ .Commit.Sha }}-{{ .Build.Number }}/%s' %(name) - return genericCache(name, action, [path], cache_key) + cache_key = '%s/%s/%s' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}', name) + if action == "rebuild" or action == "restore": + return genericCache(name, action, [path], cache_key) + if action == "purge": + return genericCachePurge(name, cache_key) + return [] + +def restoreBuildArtifactCache(ctx, name, path): + return [genericBuildArtifactCache(ctx, name, 'restore', path)] -def restoreBuildArtifactCache(name, path): - return [genericBuildArtifactCache(name, 'restore', path)] +def rebuildBuildArtifactCache(ctx, name, path): + return [genericBuildArtifactCache(ctx, name, 'rebuild', path)] -def rebuildBuildArtifactCache(name, path): - return [genericBuildArtifactCache(name, 'rebuild', path)] +def purgeBuildArtifactCache(ctx, name): + return genericBuildArtifactCache(ctx, name, 'purge', []) From 0ec05a4f5eb2eb82ea30c1baae5aae73c0709ee3 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Sat, 28 Nov 2020 11:53:27 +0100 Subject: [PATCH 4/8] fix name which led to stuck drone ci --- .drone.star | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.star b/.drone.star index 5de941433ae..791f341ec97 100644 --- a/.drone.star +++ b/.drone.star @@ -145,7 +145,7 @@ def main(ctx): badges(ctx), docs(ctx), updateDeployment(ctx), - #purge, + purge, ] if ctx.build.event == "cron": @@ -1620,7 +1620,7 @@ def genericCachePurge(name, cache_key): return { 'kind': 'pipeline', 'type': 'docker', - 'name': 'purge_%s_build_artifact_cache' %(name), + 'name': 'purge_%s' %(name), 'platform': { 'os': 'linux', 'arch': 'amd64', From cba60d73c8f8781725d5916081548a8a8569ba93 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Sat, 28 Nov 2020 12:47:47 +0100 Subject: [PATCH 5/8] always trigger cache prune, even if previous pipelines failed --- .drone.star | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.star b/.drone.star index 791f341ec97..bcfb212aeb6 100644 --- a/.drone.star +++ b/.drone.star @@ -1645,6 +1645,10 @@ def genericCachePurge(name, cache_key): 'refs/tags/v*', 'refs/pull/**', ], + 'status': [ + 'success', + 'failure', + ] }, } From a3a6b1206e471cca3944ef7e105bff3ceaf08206 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Sat, 28 Nov 2020 13:23:28 +0100 Subject: [PATCH 6/8] fix cache key --- .drone.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index bcfb212aeb6..99abf4426f6 100644 --- a/.drone.star +++ b/.drone.star @@ -1635,7 +1635,7 @@ def genericCachePurge(name, cache_key): } }, 'commands': [ - 'mc rm --force cache/%s' % (cache_key), + 'mc rm --recursive --force %s' % (cache_key), ] }, ], From e10c79c653b49be751ac8f2590c4779c6a82beaa Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Sat, 28 Nov 2020 15:08:20 +0100 Subject: [PATCH 7/8] fix path --- .drone.star | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.drone.star b/.drone.star index 99abf4426f6..2d22abee5f1 100644 --- a/.drone.star +++ b/.drone.star @@ -1616,7 +1616,7 @@ def genericCache(name, action, mounts, cache_key): } return step -def genericCachePurge(name, cache_key): +def genericCachePurge(ctx, name, cache_key): return { 'kind': 'pipeline', 'type': 'docker', @@ -1629,13 +1629,14 @@ def genericCachePurge(name, cache_key): { 'name': 'purge-cache', 'image': 'minio/mc', + 'failure': 'ignore', 'environment': { 'MC_HOST_cache': { 'from_secret': 'cache_s3_connection_url' } }, 'commands': [ - 'mc rm --recursive --force %s' % (cache_key), + 'mc rm --recursive --force %s/%s' % (ctx.repo.name, cache_key), ] }, ], @@ -1658,7 +1659,7 @@ def genericBuildArtifactCache(ctx, name, action, path): if action == "rebuild" or action == "restore": return genericCache(name, action, [path], cache_key) if action == "purge": - return genericCachePurge(name, cache_key) + return genericCachePurge(ctx, name, cache_key) return [] def restoreBuildArtifactCache(ctx, name, path): From 2394de10f79f60ce8c551ffe1399214ea262477e Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 30 Nov 2020 10:39:08 +0100 Subject: [PATCH 8/8] fix cache purge --- .drone.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 2d22abee5f1..7f208890254 100644 --- a/.drone.star +++ b/.drone.star @@ -1636,7 +1636,7 @@ def genericCachePurge(ctx, name, cache_key): } }, 'commands': [ - 'mc rm --recursive --force %s/%s' % (ctx.repo.name, cache_key), + 'mc rm --recursive --force cache/cache/%s/%s' % (ctx.repo.name, cache_key), ] }, ],