diff --git a/.drone.star b/.drone.star index 88be56d670a..72445ec71ff 100644 --- a/.drone.star +++ b/.drone.star @@ -107,7 +107,6 @@ def getPipelineNames(pipelines=[]): names.append(pipeline['name']) return names - def main(ctx): """main is the entrypoint for drone @@ -117,9 +116,11 @@ def main(ctx): Returns: none """ + pipelines = [] before = \ + [ buildOcisBinaryForTesting(ctx) ] + \ testOcisModules(ctx) + \ testPipelines(ctx) @@ -134,6 +135,9 @@ def main(ctx): releaseSubmodule(ctx), ] + purge = purgeBuildArtifactCache(ctx, 'ocis-binary-amd64') + purge['depends_on'] = getPipelineNames(testPipelines(ctx)) + after = [ manifest(ctx), changelog(ctx), @@ -141,6 +145,7 @@ def main(ctx): badges(ctx), docs(ctx), updateDeployment(ctx), + purge, ] if ctx.build.event == "cron": @@ -150,7 +155,12 @@ def main(ctx): pipelines = before + [ notify_pipeline ] - elif '[docs-only]' in (ctx.build.title + ctx.build.message): + elif \ + (ctx.build.event == "pull" and '[docs-only]' in ctx.build.title) \ + or \ + (ctx.build.event != "pull" and '[docs-only]' in (ctx.build.title + ctx.build.message)): + # [docs-only] is not taken from PR messages, but from commit messages + docs_pipeline = docs(ctx) docs_pipeline['depends_on'] = [] docs_pipelines = [ docs_pipeline ] @@ -182,7 +192,6 @@ def testOcisModules(ctx): return pipelines + [coverage_upload] - def testPipelines(ctx): pipelines = [ localApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], 'owncloud', 'apiOcisSpecific'), @@ -200,7 +209,7 @@ def testPipelines(ctx): return pipelines def testOcisModule(ctx, module): - steps = generate(module) + [ + steps = makeGenerate(module) + [ { 'name': 'vet', 'image': 'webhippie/golang:1.14', @@ -307,6 +316,34 @@ def testOcisModule(ctx, module): ], } +def buildOcisBinaryForTesting(ctx): + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'build_ocis_binary_for_testing', + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': + makeGenerate('ocis') + + build() + + rebuildBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis'), + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/v*', + 'refs/pull/**', + ], + }, + 'volumes': [ + { + 'name': 'gopath', + 'temp': {}, + }, + ], + } + def uploadCoverage(ctx): return { 'kind': 'pipeline', @@ -385,8 +422,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo 'arch': 'amd64', }, 'steps': - generate('ocis') + - build() + + restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + cloneCoreRepos(coreBranch, coreCommit) + [ { @@ -421,6 +457,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo 'temp': {}, }, ], + 'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]), 'trigger': { 'ref': [ 'refs/heads/master', @@ -440,8 +477,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n 'arch': 'amd64', }, 'steps': - generate('ocis') + - build() + + restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + cloneCoreRepos(coreBranch, coreCommit) + [ { @@ -477,6 +513,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n 'temp': {}, }, ], + 'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]), 'trigger': { 'ref': [ 'refs/heads/master', @@ -509,8 +546,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '', 'arch': 'amd64', }, 'steps': - generate('ocis') + - build() + + restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + [ { 'name': 'webUITests', @@ -562,6 +598,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '', 'temp': {} } ], + 'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]), 'trigger': { 'ref': [ 'refs/heads/master', @@ -581,8 +618,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc 'arch': 'amd64', }, 'steps': - generate('ocis') + - build() + + restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') + ocisServer(storage, accounts_hash_difficulty) + [ { 'name': 'WebUIAcceptanceTests', @@ -655,6 +691,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc 'temp': {} } ], + 'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]), 'trigger': { 'ref': [ 'refs/heads/master', @@ -674,7 +711,7 @@ def docker(ctx, arch): 'arch': arch, }, 'steps': - generate('ocis') + + makeGenerate('ocis') + build() + [ { 'name': 'dryrun', @@ -747,7 +784,7 @@ def dockerEos(ctx): 'arch': 'amd64', }, 'steps': - generate('ocis') + + makeGenerate('ocis') + build() + [ { 'name': 'dryrun-eos-ocis', @@ -858,7 +895,7 @@ def binary(ctx, name): 'arch': 'amd64', }, 'steps': - generate('ocis') + [ + makeGenerate('ocis') + [ { 'name': 'build', 'image': 'webhippie/golang:1.14', @@ -1312,7 +1349,7 @@ def docs(ctx): }, } -def generate(module): +def makeGenerate(module): return [ { 'name': 'generate', @@ -1541,3 +1578,95 @@ def build(): ], }, ] + +def genericCache(name, action, mounts, cache_key): + rebuild = 'false' + restore = 'false' + if action == 'rebuild': + rebuild = 'true' + action = 'rebuild' + else: + restore = 'true' + action = 'restore' + + step = { + 'name': '%s_%s' %(action, name), + 'image': 'meltwater/drone-cache:v1', + 'pull': 'always', + 'environment': { + 'AWS_ACCESS_KEY_ID': { + 'from_secret': 'cache_s3_access_key', + }, + 'AWS_SECRET_ACCESS_KEY': { + 'from_secret': 'cache_s3_secret_key', + }, + }, + 'settings': { + 'endpoint': { + 'from_secret': 'cache_s3_endpoint' + }, + 'bucket': 'cache', + 'region': 'us-east-1', # not used at all, but failes if not given! + 'path_style': 'true', + 'cache_key': cache_key, + 'rebuild': rebuild, + 'restore': restore, + 'mount': mounts, + }, + } + return step + +def genericCachePurge(ctx, name, cache_key): + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'purge_%s' %(name), + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': [ + { + 'name': 'purge-cache', + 'image': 'minio/mc', + 'failure': 'ignore', + 'environment': { + 'MC_HOST_cache': { + 'from_secret': 'cache_s3_connection_url' + } + }, + 'commands': [ + 'mc rm --recursive --force cache/cache/%s/%s' % (ctx.repo.name, cache_key), + ] + }, + ], + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/v*', + 'refs/pull/**', + ], + 'status': [ + 'success', + 'failure', + ] + }, + } + +def genericBuildArtifactCache(ctx, name, action, path): + name = '%s_build_artifact_cache' %(name) + cache_key = '%s/%s/%s' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}', name) + if action == "rebuild" or action == "restore": + return genericCache(name, action, [path], cache_key) + if action == "purge": + return genericCachePurge(ctx, name, cache_key) + return [] + +def restoreBuildArtifactCache(ctx, name, path): + return [genericBuildArtifactCache(ctx, name, 'restore', path)] + +def rebuildBuildArtifactCache(ctx, name, path): + return [genericBuildArtifactCache(ctx, name, 'rebuild', path)] + +def purgeBuildArtifactCache(ctx, name): + return genericBuildArtifactCache(ctx, name, 'purge', [])