diff --git a/OPERATOR.rst b/OPERATOR.rst index 0b528751a8..c7da79debe 100644 --- a/OPERATOR.rst +++ b/OPERATOR.rst @@ -569,6 +569,40 @@ backport PR first. The new PR will include the changes from the old one. .. _#team-boardwalk Slack channel: https://ucsc-gi.slack.com/archives/C705Y6G9Z + +Deploying the Data Browser +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Data Browser is deployed two steps. The first step is building the +``ucsc/data-browser`` project on GitLab. This is initiated by pushing a branch +whose name matches ``ucsc/*/*`` to one of our GitLab instances. The resulting +pipeline produces a tarball stored in the package registry on that GitLab +instance. The second step is running the ``deploy_browser`` job of the +``ucsc/azul`` project pipeline on that same instance. This job creates or +updates the necessary cloud infrastructure (CloudFront, S3, ACM, Route 53), +downloads the tarball from the package registry and unpacks that tarball to the +S3 bucket backing the Data Browser's CloudFront distribution. + +Typically, CC requests the deployment of a Data Browser instance on Slack, +specifying the commit they wish to be deployed. After the system administrator +approves that request, the operator merges the specified commit into one of the +``ucsc/{atlas}/{deployment}`` branches and then pushes that branch to the +``DataBiosphere/data-browser`` project on GitHub, and the ``ucsc/data-browser`` +project on the GitLab instance for the Azul ``{deployment}`` that backs the Data +Browser instance to be deployed. For the merge commit title, SmartGit's default +can be used, as long as the title reflects the commit (branch, tag, or sha1) +specified by CC. + +The ``{atlas}`` placeholder can be ``hca``, ``anvil`` or ``lungmap``. Not all +combinations of ``{atlas}`` and ``{deployment}`` are valid. Valid combinations +are ``ucsc/anvil/anvildev``, ``ucsc/anvil/anvilprod``, ``ucsc/hca/dev``, +``ucsc/hca/prod``, ``ucsc/lungmap/dev`` or ``ucsc/lungmap/prod``, for example. +The ``ucsc/data-browser`` pipeline on GitLab blindly builds any branch, but +Azul's ``deploy_browser`` job is configured to only use the tarball from exactly +one branch (see ``deployments/*.browser/environment.py``) and it will always use +the tarball from the most recent pipeline on that branch. + + Troubleshooting --------------- diff --git a/deployments/anvildev.browser/environment.py b/deployments/anvildev.browser/environment.py index 33b68d53d8..a09943e23d 100644 --- a/deployments/anvildev.browser/environment.py +++ b/deployments/anvildev.browser/environment.py @@ -28,15 +28,14 @@ def env() -> Mapping[str, Optional[str]]: return { 'azul_terraform_component': 'browser', 'azul_browser_sites': json.dumps({ - 'ucsc/data-browser': { - 'main': { - 'anvil': { - 'domain': '{AZUL_DOMAIN_NAME}', - 'bucket': 'browser', - 'tarball_path': 'out', - 'real_path': '' - } - } + 'browser': { + 'zone': '{AZUL_DOMAIN_NAME}', + 'domain': '{AZUL_DOMAIN_NAME}', + 'project': 'ucsc/data-browser', + 'branch': 'ucsc/anvil/anvildev', + 'tarball_name': 'anvil', + 'tarball_path': 'out', + 'real_path': '' } }) } diff --git a/deployments/anvilprod.browser/environment.py b/deployments/anvilprod.browser/environment.py index 33b68d53d8..39cc1f5732 100644 --- a/deployments/anvilprod.browser/environment.py +++ b/deployments/anvilprod.browser/environment.py @@ -28,15 +28,14 @@ def env() -> Mapping[str, Optional[str]]: return { 'azul_terraform_component': 'browser', 'azul_browser_sites': json.dumps({ - 'ucsc/data-browser': { - 'main': { - 'anvil': { - 'domain': '{AZUL_DOMAIN_NAME}', - 'bucket': 'browser', - 'tarball_path': 'out', - 'real_path': '' - } - } + 'browser': { + 'zone': '{AZUL_DOMAIN_NAME}', + 'domain': '{AZUL_DOMAIN_NAME}', + 'project': 'ucsc/data-browser', + 'branch': 'ucsc/anvil/anvilprod', + 'tarball_name': 'anvil', + 'tarball_path': 'out', + 'real_path': '' } }) } diff --git a/deployments/dev.browser/environment.py b/deployments/dev.browser/environment.py new file mode 100644 index 0000000000..917f348496 --- /dev/null +++ b/deployments/dev.browser/environment.py @@ -0,0 +1,41 @@ +from collections.abc import ( + Mapping, +) +import json +from typing import ( + Optional, +) + + +def env() -> Mapping[str, Optional[str]]: + """ + Returns a dictionary that maps environment variable names to values. The + values are either None or strings. String values can contain references to + other environment variables in the form `{FOO}` where FOO is the name of an + environment variable. See + + https://docs.python.org/3.11/library/string.html#format-string-syntax + + for the concrete syntax. These references will be resolved *after* the + overall environment has been compiled by merging all relevant + `environment.py` and `environment.local.py` files. + + Entries with a `None` value will be excluded from the environment. They + can be used to document a variable without a default value in which case + other, more specific `environment.py` or `environment.local.py` files must + provide the value. + """ + return { + 'azul_terraform_component': 'browser', + 'azul_browser_sites': json.dumps({ + 'browser': { + 'zone': '{AZUL_DOMAIN_NAME}', + 'domain': 'explore.{AZUL_DOMAIN_NAME}', + 'project': 'ucsc/data-browser', + 'branch': 'ucsc/hca/dev', + 'tarball_name': 'hca', + 'tarball_path': 'out', + 'real_path': '' + } + }) + } diff --git a/deployments/tempdev.browser/environment.py b/deployments/tempdev.browser/environment.py index 33b68d53d8..8eacfba33e 100644 --- a/deployments/tempdev.browser/environment.py +++ b/deployments/tempdev.browser/environment.py @@ -28,15 +28,14 @@ def env() -> Mapping[str, Optional[str]]: return { 'azul_terraform_component': 'browser', 'azul_browser_sites': json.dumps({ - 'ucsc/data-browser': { - 'main': { - 'anvil': { - 'domain': '{AZUL_DOMAIN_NAME}', - 'bucket': 'browser', - 'tarball_path': 'out', - 'real_path': '' - } - } + 'browser': { + 'zone': '{AZUL_DOMAIN_NAME}', + 'domain': '{AZUL_DOMAIN_NAME}', + 'project': 'ucsc/data-browser', + 'branch': 'ucsc/anvil/tempdev', + 'tarball_name': 'anvil', + 'tarball_path': 'out', + 'real_path': '' } }) } diff --git a/deployments/tempdev/environment.py b/deployments/tempdev/environment.py index f56cad84f4..3b6f97e5a8 100644 --- a/deployments/tempdev/environment.py +++ b/deployments/tempdev/environment.py @@ -145,7 +145,7 @@ def env() -> Mapping[str, Optional[str]]: 'GOOGLE_PROJECT': 'platform-temp-dev', - 'AZUL_DEPLOYMENT_INCARNATION': '0', + 'AZUL_DEPLOYMENT_INCARNATION': '1', 'AZUL_GOOGLE_OAUTH2_CLIENT_ID': '807674395527-erth0gf1m7qme5pe6bu384vpdfjh06dg.apps.googleusercontent.com', } diff --git a/environment b/environment index 18085f0031..14a0a8b7c9 100644 --- a/environment +++ b/environment @@ -320,6 +320,36 @@ _update_clone() { . } +# Destroy the most expensive resources in a main deployment and its components. +# Compared to complete destruction, hibernation has the advantage of needing +# less time to come back from, and not requiring incrementing the incarnation +# counter or adding service accounts to Terra groups. This function was written +# from memory. The `terraform` commands were tested individually but the +# function as a whole was not. +# +_hibernate() { + # shellcheck disable=SC2154 + if test -z "$azul_terraform_component"; then + make -C lambdas && { + cd terraform && + make validate + terraform destroy -target aws_elasticsearch_domain.index && { + cd gitlab && + _select "$AZUL_DEPLOYMENT_STAGE.gitlab" && + make validate && + terraform destroy \ + -target aws_ec2_client_vpn_endpoint.gitlab \ + -target aws_instance.gitlab \ + -target aws_nat_gateway.gitlab_0 \ + -target aws_nat_gateway.gitlab_1 + } + } + else + echo "Must have main component selected" + return 1 + fi +} + # We disable `envhook.py` to avoid redundancy. The `envhook.py` script imports # `export_environment.py`, too. We could also pass -S to `python3` but that # causes problems on Travis (`importlib.util` failing to import `contextlib`). diff --git a/environment.py b/environment.py index 556396dda8..f7200b2327 100644 --- a/environment.py +++ b/environment.py @@ -806,38 +806,44 @@ def env() -> Mapping[str, Optional[str]]: # managed by the `browser` TF component of the current Azul deployment. # # { - # 'ucsc/data-browser': { // The path of the GitLab project hosting - # // the source code for the site. The - # // project must exist on the GitLab - # // instance managing the current Azul - # // deployment. - # - # 'main': { // The name of the branch (in that project) from - # // which the site's content tarball was built - # - # 'anvil': { // The site name. Typically corresponds to an - # // Azul atlas as defined in the AZUL_CATALOGS - # // and a child directory of - # // .gitlab/sites/$AZUL_DEPLOYMENT_STAGE in the - # // source of the project referenced by the - # // top-level key in this structure. - # - # 'domain': '{AZUL_DOMAIN_NAME}', // The domain name of - # // the site - # - # 'bucket': 'browser', // The TF resource name (in the - # // `browser` component) of the - # // S3 bucket hosting the site - # // ('portal' or 'browser') - # - # 'tarball_path': 'explore', // The path to the site's - # // content in the tarball - # - # 'real_path': 'explore/anvil-cmg' // The path of that - # // same content in - # // the bucket - # } - # } + # 'browser': { // The TF resource name of per-site resources in the + # // `browser` component and unqualified name of the + # // S3 bucket hosting the site + # + # 'domain': '{AZUL_DOMAIN_NAME}', // The domain name of the + # // site + # + # 'zone': '{AZUL_DOMAIN_NAME}', // The name of the Route53 + # // hosted zone containing the + # // A record for the domain name + # // of the site. The zone must + # // already exist before the + # + # 'project': 'ucsc/data-browser', // The path of the GitLab + # // project hosting the source + # // code for the site. The + # // project must exist on the + # // GitLab instance managing + # // the current Azul + # // deployment. + # + # 'branch': 'main', // The name of the branch (in that project) + # // from which the site's content tarball was + # // built + # + # 'tarball_name': 'anvil' // Typically corresponds to an Azul + # // atlas as defined in AZUL_CATALOGS + # // and a child directory of + # // .gitlab/sites/$AZUL_DEPLOYMENT_STAGE + # // in the source of the project + # // referenced by the top-level key in + # // this structure. + # + # 'tarball_path': 'explore', // The path to the site's content + # // in the tarball + # + # 'real_path': 'explore/anvil-cmg' // The path of that same + # // content in the bucket # } # } # diff --git a/scripts/rename_resources.py b/scripts/rename_resources.py index 9bd779bf47..86d7161680 100644 --- a/scripts/rename_resources.py +++ b/scripts/rename_resources.py @@ -1,6 +1,5 @@ import argparse import logging -import subprocess import sys from typing import ( Optional, @@ -25,19 +24,6 @@ } -def terraform_state_list() -> list[str]: - try: - output = terraform.run('state', 'list') - except subprocess.CalledProcessError as e: - if e.returncode == 1 and 'No state file was found' in e.stderr: - log.info('No state file was found, assuming empty list of resources.') - return [] - else: - raise - else: - return output.splitlines() - - def main(argv: list[str]): configure_script_logging(log) parser = argparse.ArgumentParser(description=__doc__, @@ -49,7 +35,7 @@ def main(argv: list[str]): args = parser.parse_args(argv) if renamed: - current_names = terraform_state_list() + current_names = terraform.run_state_list() for current_name in current_names: try: new_name = renamed[current_name] diff --git a/src/azul/__init__.py b/src/azul/__init__.py index 0d75a32128..78de48500b 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -1111,14 +1111,16 @@ def shared_deployments_for_branch(self, return None if branch is None else deployments.get(None) class BrowserSite(TypedDict): + zone: str domain: str - bucket: str + project: str + branch: str + tarball_name: str tarball_path: str real_path: str @property - def browser_sites(self - ) -> Mapping[str, Mapping[str, Mapping[str, BrowserSite]]]: + def browser_sites(self) -> Mapping[str, BrowserSite]: import json return json.loads(self.environ['azul_browser_sites']) diff --git a/src/azul/terraform.py b/src/azul/terraform.py index 4c553eff21..cd9afe287f 100644 --- a/src/azul/terraform.py +++ b/src/azul/terraform.py @@ -96,15 +96,17 @@ def run(self, *args: str, **kwargs) -> str: **kwargs) return cmd.stdout - def run_state_list(self): + def run_state_list(self) -> list[str]: try: stdout = self.run('state', 'list', stderr=subprocess.PIPE) - return stdout.splitlines() except subprocess.CalledProcessError as e: - if 'No state file was found!' in e.stderr: + if e.returncode == 1 and 'No state file was found' in e.stderr: + log.info('No state file was found, assuming empty list of resources.') return [] else: raise + else: + return stdout.splitlines() schema_path = Path(config.project_root) / 'terraform' / '_schema.json.gz' diff --git a/terraform/browser/browser.tf.json.template.py b/terraform/browser/browser.tf.json.template.py index 56f75b6d81..efbd8fda63 100644 --- a/terraform/browser/browser.tf.json.template.py +++ b/terraform/browser/browser.tf.json.template.py @@ -26,6 +26,7 @@ ) from azul.collections import ( adict, + dict_merge, ) from azul.deployment import ( aws, @@ -37,12 +38,7 @@ set_empty_s3_bucket_lifecycle_config, ) -buckets = { - site['bucket']: aws.qualified_bucket_name(site['bucket']) - for project, branches in config.browser_sites.items() - for branch, sites in branches.items() - for site_name, site in sites.items() -} +sites = config.browser_sites #: Whether to emit a Google custom search instance and a CF origin for it provision_custom_search = False @@ -71,36 +67,37 @@ def emit(): } }, 'aws_route53_zone': { - 'browser': { - 'name': config.domain_name + '.', + name: { + 'name': site['zone'] + '.', 'private_zone': False } + for name, site in sites.items() } }, 'resource': { 'aws_s3_bucket': { - bucket: { - 'bucket': name, + name: { + 'bucket': aws.qualified_bucket_name(name), 'force_destroy': True, 'lifecycle': { 'prevent_destroy': False } } - for bucket, name in buckets.items() + for name in sites.keys() }, 'aws_s3_bucket_logging': { - bucket: { - 'bucket': '${aws_s3_bucket.%s.id}' % bucket, + name: { + 'bucket': '${aws_s3_bucket.%s.id}' % name, 'target_bucket': '${data.aws_s3_bucket.logs.id}', # Other S3 log deliveries, like ELB, implicitly put a slash # after the prefix. S3 doesn't, so we add one explicitly. - 'target_prefix': config.s3_access_log_path_prefix(bucket) + '/' + 'target_prefix': config.s3_access_log_path_prefix(name) + '/' } - for bucket in buckets + for name in sites.keys() }, 'aws_s3_bucket_policy': { - bucket: { - 'bucket': '${aws_s3_bucket.%s.id}' % bucket, + name: { + 'bucket': '${aws_s3_bucket.%s.id}' % name, 'policy': json.dumps({ 'Version': '2008-10-17', 'Id': 'PolicyForCloudFrontPrivateContent', @@ -116,22 +113,22 @@ def emit(): 's3:ListBucket' ], 'Resource': [ - '${aws_s3_bucket.%s.arn}' % bucket, - '${aws_s3_bucket.%s.arn}/*' % bucket + '${aws_s3_bucket.%s.arn}' % name, + '${aws_s3_bucket.%s.arn}/*' % name ], 'Condition': { 'StringEquals': { - 'AWS:SourceArn': '${aws_cloudfront_distribution.browser.arn}' + 'AWS:SourceArn': '${aws_cloudfront_distribution.%s.arn}' % name } } } ] }) } - for bucket in buckets + for name in sites.keys() }, 'aws_cloudfront_distribution': { - 'browser': { + name: { 'enabled': True, 'restrictions': { 'geo_restriction': { @@ -140,29 +137,30 @@ def emit(): } }, 'price_class': 'PriceClass_100', - 'aliases': [config.domain_name], + 'aliases': [site['domain']], 'default_root_object': 'index.html', 'is_ipv6_enabled': True, 'ordered_cache_behavior': [ *iif(provision_custom_search, [google_search_behavior()]) ], 'default_cache_behavior': - bucket_behaviour('browser', + bucket_behaviour(name, bucket_path_mapper=True, add_response_headers=False), 'viewer_certificate': { - 'acm_certificate_arn': '${aws_acm_certificate.browser.arn}', + 'acm_certificate_arn': '${aws_acm_certificate.%s.arn}' % name, 'minimum_protocol_version': 'TLSv1.2_2021', 'ssl_support_method': 'sni-only' }, 'origin': [ *( { - 'origin_id': bucket_origin_id(bucket), - 'domain_name': bucket_regional_domain_name(bucket), - 'origin_access_control_id': '${aws_cloudfront_origin_access_control.%s.id}' % bucket + 'origin_id': bucket_origin_id(name), + 'domain_name': bucket_regional_domain_name(name), + 'origin_access_control_id': + '${aws_cloudfront_origin_access_control.%s.id}' % name } - for bucket in buckets + for name in sites.keys() ), *iif(provision_custom_search, [google_search_origin()]) ], @@ -176,24 +174,25 @@ def emit(): for error_code in [403, 404] ] } + for name, site in sites.items() }, 'aws_cloudfront_origin_access_control': { - bucket: { - 'name': bucket_origin_id(bucket), + name: { + 'name': bucket_origin_id(name), 'description': '', # becomes 'Managed by Terraform' if omitted 'origin_access_control_origin_type': 's3', 'signing_behavior': 'always', 'signing_protocol': 'sigv4' } - for bucket in buckets + for name in sites.keys() }, 'aws_cloudfront_function': { script.stem: cloudfront_function(script) for script in Path(__file__).parent.glob('*.js') }, 'aws_cloudfront_response_headers_policy': { - 'browser': { - 'name': 'browser', + name: { + 'name': name, 'security_headers_config': { 'content_security_policy': { 'override': True, @@ -224,47 +223,53 @@ def emit(): } } } + for name in sites.keys() }, 'aws_acm_certificate': { - 'browser': { - 'domain_name': config.domain_name, + name: { + 'domain_name': site['domain'], 'validation_method': 'DNS', 'lifecycle': { 'create_before_destroy': True } } + for name, site in sites.items() }, 'aws_acm_certificate_validation': { - 'browser': { - 'certificate_arn': '${aws_acm_certificate.browser.arn}', - 'validation_record_fqdns': '${[for r in aws_route53_record.browser_validation : r.fqdn]}', + name: { + 'certificate_arn': '${aws_acm_certificate.%s.arn}' % name, + 'validation_record_fqdns': '${[for r in aws_route53_record.%s_validation : r.fqdn]}' % name, } + for name in sites.keys() }, - 'aws_route53_record': { - 'browser': { - 'zone_id': '${data.aws_route53_zone.browser.id}', - 'name': config.domain_name, - 'type': 'A', - 'alias': { - 'name': '${aws_cloudfront_distribution.browser.domain_name}', - 'zone_id': '${aws_cloudfront_distribution.browser.hosted_zone_id}', - 'evaluate_target_health': False + 'aws_route53_record': dict_merge( + { + name: { + 'zone_id': '${data.aws_route53_zone.%s.id}' % name, + 'name': site['domain'], + 'type': 'A', + 'alias': { + 'name': '${aws_cloudfront_distribution.%s.domain_name}' % name, + 'zone_id': '${aws_cloudfront_distribution.%s.hosted_zone_id}' % name, + 'evaluate_target_health': False + } + }, + name + '_validation': { + 'for_each': '${{' + 'for o in aws_acm_certificate.%s.domain_validation_options : ' + 'o.domain_name => o' + '}}' % name, + 'name': '${each.value.resource_record_name}', + 'type': '${each.value.resource_record_type}', + 'zone_id': '${data.aws_route53_zone.%s.id}' % name, + 'records': [ + '${each.value.resource_record_value}', + ], + 'ttl': 60 } - }, - 'browser_validation': { - 'for_each': '${{' - 'for o in aws_acm_certificate.browser.domain_validation_options : ' - 'o.domain_name => o' - '}}', - 'name': '${each.value.resource_record_name}', - 'type': '${each.value.resource_record_type}', - 'zone_id': '${data.aws_route53_zone.browser.id}', - 'records': [ - '${each.value.resource_record_value}', - ], - 'ttl': 60 } - }, + for name, site in sites.items() + ), **iif(provision_custom_search, { 'aws_cloudfront_origin_request_policy': { 'google_search': { @@ -334,22 +339,22 @@ def emit(): # object, or rather its `etag` attribute (some hash of the content) # then serves as a stand-in for a bucket identifier which we can # then use to trigger site deployment and CloudFront invalidation. - bucket + '_bucket_id': { - 'bucket': '${aws_s3_bucket.%s.id}' % bucket, + name + '_bucket_id': { + 'bucket': '${aws_s3_bucket.%s.id}' % name, 'key': bucket_id_key, 'content': str(uuid.uuid4()), 'lifecycle': { 'ignore_changes': ['content'] } } - for bucket in buckets + for name in sites.keys() }, 'null_resource': { **{ - f'deploy_site_{i}': { + f'deploy_site_{name}': { 'triggers': { - 'tarball_hash': gitlab_helper.tarball_hash(project, branch, site_name), - 'bucket_id': '${aws_s3_object.%s_bucket_id.etag}' % site['bucket'], + 'tarball_hash': gitlab_helper.tarball_hash(site), + 'bucket_id': '${aws_s3_object.%s_bucket_id.etag}' % name, 'tarball_path': site['tarball_path'], 'real_path': site['real_path'] }, @@ -360,15 +365,15 @@ def emit(): 'command': ' && '.join([ # TF uses concurrent workers so we need to keep the directories # separate between the null_resource resources. - f'rm -rf out_{i}', - f'mkdir out_{i}', + f'rm -rf out_{name}', + f'mkdir out_{name}', ' | '.join([ ' '.join([ 'curl', '--fail', '--silent', gitlab_helper.curl_auth_flags(), - quote(gitlab_helper.tarball_url(project, branch, site_name)) + quote(gitlab_helper.tarball_url(site)) ]), ' '.join([ # --transform is specific to GNU Tar, which, on macOS must be installed @@ -377,46 +382,45 @@ def emit(): '-xvjf -', f'--transform s#^{site["tarball_path"]}/#{site["real_path"]}/#', '--show-transformed-names', - f'-C out_{i}' + f'-C out_{name}' ]) ]), ' '.join([ 'aws', 's3', 'sync', '--exclude', bucket_id_key, '--delete', - f'out_{i}/', - 's3://${aws_s3_bucket.%s.id}/' % site['bucket'] + f'out_{name}/', + 's3://${aws_s3_bucket.%s.id}/' % name ]), - f'rm -rf out_{i}', + f'rm -rf out_{name}', ]) } } } - for i, (project, branches) in enumerate(config.browser_sites.items()) - for branch, sites in branches.items() - for site_name, site in sites.items() + for name, site in sites.items() }, - 'invalidate_cloudfront': { - 'depends_on': [ - f'null_resource.deploy_site_{i}' - for i, _ in enumerate(config.browser_sites) - ], - 'triggers': { - f'{trigger}_{i}': '${null_resource.deploy_site_%i.triggers.%s}' % (i, trigger) - for i, _ in enumerate(config.browser_sites) - for trigger in ['tarball_hash', 'bucket_id'] - }, - 'provisioner': { - 'local-exec': { - 'when': 'create', - 'command': ' '.join([ - 'aws', - 'cloudfront create-invalidation', - '--distribution-id ${aws_cloudfront_distribution.browser.id}', - '--paths "/*"' - ]) + **{ + 'invalidate_cloudfront_' + name: { + 'depends_on': [ + f'null_resource.deploy_site_{name}' + ], + 'triggers': { + f'{trigger}_{name}': '${null_resource.deploy_site_%s.triggers.%s}' % (name, trigger) + for trigger in ['tarball_hash', 'bucket_id'] + }, + 'provisioner': { + 'local-exec': { + 'when': 'create', + 'command': ' '.join([ + 'aws', + 'cloudfront create-invalidation', + '--distribution-id ${aws_cloudfront_distribution.%s.id}' % name, + '--paths "/*"' + ]) + } } } + for name in sites.keys() } } } @@ -434,7 +438,7 @@ def bucket_behaviour(origin, *, path_pattern: str = None, **functions: bool) -> cached_methods=['GET', 'HEAD'], cache_policy_id='${data.aws_cloudfront_cache_policy.caching_optimized.id}', response_headers_policy_id=( - '${aws_cloudfront_response_headers_policy.browser.id}' + '${aws_cloudfront_response_headers_policy.%s.id}' % origin ), viewer_protocol_policy='redirect-to-https', target_origin_id=bucket_origin_id(origin), @@ -461,7 +465,7 @@ def bucket_regional_domain_name(bucket): # FIXME: Remove workaround for # https://github.com/hashicorp/terraform-provider-aws/issues/15102 # https://github.com/DataBiosphere/azul/issues/5257 - return buckets[bucket] + '.s3.us-east-1.amazonaws.com' + return aws.qualified_bucket_name(bucket) + '.s3.us-east-1.amazonaws.com' def cloudfront_function(script: Path): @@ -615,34 +619,34 @@ def curl_auth_flags(self) -> str: header = quote(f'{token_type}-TOKEN: {token}') return '--header ' + header - def tarball_hash(self, project_path: str, branch: str, site_name: str) -> str: - project = self.client.projects.get(project_path, lazy=True) + def tarball_hash(self, site: config.BrowserSite) -> str: + project = self.client.projects.get(site['project'], lazy=True) packages = project.packages.list(iterator=True, package_type='generic') - version = self.tarball_version(branch) + version = self.tarball_version(site['branch']) package = one(p for p in packages if p.version == version) package_files = ( pf for pf in package.package_files.list(iterator=True) - if pf.file_name == self.tarball_file_name(project_path, site_name) + if pf.file_name == self.tarball_file_name(site) ) package_file = max(package_files, key=attrgetter('created_at')) return package_file.file_sha256 - def tarball_url(self, project_path: str, branch: str, site_name: str) -> str: + def tarball_url(self, site: config.BrowserSite) -> str: # GET /projects/:id/packages/generic/:package_name/:package_version/:file_name return str(furl(self.gitlab_url, path=[ - 'api', 'v4', 'projects', project_path, + 'api', 'v4', 'projects', site['project'], 'packages', 'generic', 'tarball', - self.tarball_version(branch), - self.tarball_file_name(project_path, site_name) + self.tarball_version(site['branch']), + self.tarball_file_name(site) ])) - def tarball_file_name(self, project_path: str, site_name: str) -> str: + def tarball_file_name(self, site: config.BrowserSite) -> str: return '_'.join([ - project_path.split('/')[-1], # just the project name + site['project'].split('/')[-1], # just the project name config.deployment_stage, - site_name, + site['tarball_name'], 'distribution', ]) + '.tar.bz2'