Skip to content

Commit

Permalink
Merge pull request #961 from owncloud/cache_ocis_build_artifact_for_t…
Browse files Browse the repository at this point in the history 
…esting

[tests-only] cache ocis build artifact for testing
  • Loading branch information
@wkloucek
wkloucek authored Nov 30, 2020
2 parents 50382ee + 2394de1 commit d856d54
Showing 1 changed file with 145 additions and 16 deletions.
161 changes: 145 additions & 16 deletions .drone.star
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ def getPipelineNames(pipelines=[]):
names.append(pipeline['name'])
return names


def main(ctx):
"""main is the entrypoint for drone
Expand All @@ -117,9 +116,11 @@ def main(ctx):
Returns:
none
"""

pipelines = []

before = \
[ buildOcisBinaryForTesting(ctx) ] + \
testOcisModules(ctx) + \
testPipelines(ctx)

Expand All @@ -134,13 +135,17 @@ def main(ctx):
releaseSubmodule(ctx),
]

purge = purgeBuildArtifactCache(ctx, 'ocis-binary-amd64')
purge['depends_on'] = getPipelineNames(testPipelines(ctx))

after = [
manifest(ctx),
changelog(ctx),
readme(ctx),
badges(ctx),
docs(ctx),
updateDeployment(ctx),
purge,
]

if ctx.build.event == "cron":
Expand All @@ -150,7 +155,12 @@ def main(ctx):

pipelines = before + [ notify_pipeline ]

elif '[docs-only]' in (ctx.build.title + ctx.build.message):
elif \
(ctx.build.event == "pull" and '[docs-only]' in ctx.build.title) \
or \
(ctx.build.event != "pull" and '[docs-only]' in (ctx.build.title + ctx.build.message)):
# [docs-only] is not taken from PR messages, but from commit messages

docs_pipeline = docs(ctx)
docs_pipeline['depends_on'] = []
docs_pipelines = [ docs_pipeline ]
Expand Down Expand Up @@ -182,7 +192,6 @@ def testOcisModules(ctx):

return pipelines + [coverage_upload]


def testPipelines(ctx):
pipelines = [
localApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], 'owncloud', 'apiOcisSpecific'),
Expand All @@ -200,7 +209,7 @@ def testPipelines(ctx):
return pipelines

def testOcisModule(ctx, module):
steps = generate(module) + [
steps = makeGenerate(module) + [
{
'name': 'vet',
'image': 'webhippie/golang:1.14',
Expand Down Expand Up @@ -307,6 +316,34 @@ def testOcisModule(ctx, module):
],
}

def buildOcisBinaryForTesting(ctx):
return {
'kind': 'pipeline',
'type': 'docker',
'name': 'build_ocis_binary_for_testing',
'platform': {
'os': 'linux',
'arch': 'amd64',
},
'steps':
makeGenerate('ocis') +
build() +
rebuildBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis'),
'trigger': {
'ref': [
'refs/heads/master',
'refs/tags/v*',
'refs/pull/**',
],
},
'volumes': [
{
'name': 'gopath',
'temp': {},
},
],
}

def uploadCoverage(ctx):
return {
'kind': 'pipeline',
Expand Down Expand Up @@ -385,8 +422,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo
'arch': 'amd64',
},
'steps':
generate('ocis') +
build() +
restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') +
ocisServer(storage, accounts_hash_difficulty) +
cloneCoreRepos(coreBranch, coreCommit) + [
{
Expand Down Expand Up @@ -421,6 +457,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo
'temp': {},
},
],
'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]),
'trigger': {
'ref': [
'refs/heads/master',
Expand All @@ -440,8 +477,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n
'arch': 'amd64',
},
'steps':
generate('ocis') +
build() +
restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') +
ocisServer(storage, accounts_hash_difficulty) +
cloneCoreRepos(coreBranch, coreCommit) + [
{
Expand Down Expand Up @@ -477,6 +513,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n
'temp': {},
},
],
'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]),
'trigger': {
'ref': [
'refs/heads/master',
Expand Down Expand Up @@ -509,8 +546,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '',
'arch': 'amd64',
},
'steps':
generate('ocis') +
build() +
restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') +
ocisServer(storage, accounts_hash_difficulty) + [
{
'name': 'webUITests',
Expand Down Expand Up @@ -562,6 +598,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '',
'temp': {}
}
],
'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]),
'trigger': {
'ref': [
'refs/heads/master',
Expand All @@ -581,8 +618,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc
'arch': 'amd64',
},
'steps':
generate('ocis') +
build() +
restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') +
ocisServer(storage, accounts_hash_difficulty) + [
{
'name': 'WebUIAcceptanceTests',
Expand Down Expand Up @@ -655,6 +691,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc
'temp': {}
}
],
'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]),
'trigger': {
'ref': [
'refs/heads/master',
Expand All @@ -674,7 +711,7 @@ def docker(ctx, arch):
'arch': arch,
},
'steps':
generate('ocis') +
makeGenerate('ocis') +
build() + [
{
'name': 'dryrun',
Expand Down Expand Up @@ -747,7 +784,7 @@ def dockerEos(ctx):
'arch': 'amd64',
},
'steps':
generate('ocis') +
makeGenerate('ocis') +
build() + [
{
'name': 'dryrun-eos-ocis',
Expand Down Expand Up @@ -858,7 +895,7 @@ def binary(ctx, name):
'arch': 'amd64',
},
'steps':
generate('ocis') + [
makeGenerate('ocis') + [
{
'name': 'build',
'image': 'webhippie/golang:1.14',
Expand Down Expand Up @@ -1312,7 +1349,7 @@ def docs(ctx):
},
}

def generate(module):
def makeGenerate(module):
return [
{
'name': 'generate',
Expand Down Expand Up @@ -1541,3 +1578,95 @@ def build():
],
},
]

def genericCache(name, action, mounts, cache_key):
rebuild = 'false'
restore = 'false'
if action == 'rebuild':
rebuild = 'true'
action = 'rebuild'
else:
restore = 'true'
action = 'restore'

step = {
'name': '%s_%s' %(action, name),
'image': 'meltwater/drone-cache:v1',
'pull': 'always',
'environment': {
'AWS_ACCESS_KEY_ID': {
'from_secret': 'cache_s3_access_key',
},
'AWS_SECRET_ACCESS_KEY': {
'from_secret': 'cache_s3_secret_key',
},
},
'settings': {
'endpoint': {
'from_secret': 'cache_s3_endpoint'
},
'bucket': 'cache',
'region': 'us-east-1', # not used at all, but failes if not given!
'path_style': 'true',
'cache_key': cache_key,
'rebuild': rebuild,
'restore': restore,
'mount': mounts,
},
}
return step

def genericCachePurge(ctx, name, cache_key):
return {
'kind': 'pipeline',
'type': 'docker',
'name': 'purge_%s' %(name),
'platform': {
'os': 'linux',
'arch': 'amd64',
},
'steps': [
{
'name': 'purge-cache',
'image': 'minio/mc',
'failure': 'ignore',
'environment': {
'MC_HOST_cache': {
'from_secret': 'cache_s3_connection_url'
}
},
'commands': [
'mc rm --recursive --force cache/cache/%s/%s' % (ctx.repo.name, cache_key),
]
},
],
'trigger': {
'ref': [
'refs/heads/master',
'refs/tags/v*',
'refs/pull/**',
],
'status': [
'success',
'failure',
]
},
}

def genericBuildArtifactCache(ctx, name, action, path):
name = '%s_build_artifact_cache' %(name)
cache_key = '%s/%s/%s' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}', name)
if action == "rebuild" or action == "restore":
return genericCache(name, action, [path], cache_key)
if action == "purge":
return genericCachePurge(ctx, name, cache_key)
return []

def restoreBuildArtifactCache(ctx, name, path):
return [genericBuildArtifactCache(ctx, name, 'restore', path)]

def rebuildBuildArtifactCache(ctx, name, path):
return [genericBuildArtifactCache(ctx, name, 'rebuild', path)]

def purgeBuildArtifactCache(ctx, name):
return genericBuildArtifactCache(ctx, name, 'purge', [])

0 comments on commit d856d54

Please sign in to comment.