diff --git a/hail/python/test/hail/fs/test_worker_driver_fs.py b/hail/python/test/hail/fs/test_worker_driver_fs.py index 344c670a7eb..d8100a6b3db 100644 --- a/hail/python/test/hail/fs/test_worker_driver_fs.py +++ b/hail/python/test/hail/fs/test_worker_driver_fs.py @@ -8,7 +8,7 @@ @skip_in_azure def test_requester_pays_no_settings(): try: - hl.import_table('gs://hail-services-requester-pays/hello') + hl.import_table('gs://hail-test-requester-pays-fds32/hello') except Exception as exc: assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0] else: @@ -17,7 +17,7 @@ def test_requester_pays_no_settings(): @skip_in_azure def test_requester_pays_write_no_settings(): - random_filename = 'gs://hail-services-requester-pays/test_requester_pays_on_worker_driver_' + secret_alnum_string(10) + random_filename = 'gs://hail-test-requester-pays-fds32/test_requester_pays_on_worker_driver_' + secret_alnum_string(10) try: hl.utils.range_table(4, n_partitions=4).write(random_filename, overwrite=True) except Exception as exc: @@ -32,7 +32,7 @@ def test_requester_pays_write_no_settings(): def test_requester_pays_write_with_project(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - random_filename = 'gs://hail-services-requester-pays/test_requester_pays_on_worker_driver_' + secret_alnum_string(10) + random_filename = 'gs://hail-test-requester-pays-fds32/test_requester_pays_on_worker_driver_' + secret_alnum_string(10) try: hl.utils.range_table(4, n_partitions=4).write(random_filename, overwrite=True) finally: @@ -43,20 +43,20 @@ def test_requester_pays_write_with_project(): def test_requester_pays_with_project(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')] + assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')] hl.stop() - hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays'])) - assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')] + hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32'])) + assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')] hl.stop() - hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays', 'other-bucket'])) - assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')] + hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32', 'other-bucket'])) + assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')] hl.stop() hl.init(gcs_requester_pays_configuration=('hail-vdc', ['other-bucket'])) try: - hl.import_table('gs://hail-services-requester-pays/hello') + hl.import_table('gs://hail-test-requester-pays-fds32/hello') except Exception as exc: assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0] else: @@ -64,7 +64,7 @@ def test_requester_pays_with_project(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')] + assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')] @skip_in_azure @@ -93,20 +93,20 @@ def test_requester_pays_with_project_more_than_one_partition(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents + assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents hl.stop() - hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays'])) - assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents + hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32'])) + assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents hl.stop() - hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays', 'other-bucket'])) - assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents + hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32', 'other-bucket'])) + assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents hl.stop() hl.init(gcs_requester_pays_configuration=('hail-vdc', ['other-bucket'])) try: - hl.import_table('gs://hail-services-requester-pays/zero-to-nine', min_partitions=8) + hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', min_partitions=8) except Exception as exc: assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0] else: @@ -114,7 +114,7 @@ def test_requester_pays_with_project_more_than_one_partition(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents + assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents @run_if_azure diff --git a/hail/python/test/hailtop/batch/test_batch.py b/hail/python/test/hailtop/batch/test_batch.py index d94bc910956..e4ea03eada6 100644 --- a/hail/python/test/hailtop/batch/test_batch.py +++ b/hail/python/test/hailtop/batch/test_batch.py @@ -748,7 +748,7 @@ def test_cloudfuse_empty_string_bucket_fails(self): def test_fuse_requester_pays(self): b = self.batch(requester_pays_project='hail-vdc') j = b.new_job() - j.cloudfuse('hail-services-requester-pays', '/fuse-bucket') + j.cloudfuse('hail-test-requester-pays-fds32', '/fuse-bucket') j.command('cat /fuse-bucket/hello') res = b.run() res_status = res.status() @@ -776,7 +776,7 @@ def test_fuse_non_requester_pays_bucket_when_requester_pays_project_specified(se @skip_in_azure def test_requester_pays(self): b = self.batch(requester_pays_project='hail-vdc') - input = b.read_input('gs://hail-services-requester-pays/hello') + input = b.read_input('gs://hail-test-requester-pays-fds32/hello') j = b.new_job() j.command(f'cat {input}') res = b.run() diff --git a/infra/gcp-broad/main.tf b/infra/gcp-broad/main.tf index b208cfe5896..d198e8d149f 100644 --- a/infra/gcp-broad/main.tf +++ b/infra/gcp-broad/main.tf @@ -628,6 +628,21 @@ resource "google_storage_bucket" "hail_test_bucket" { timeouts {} } +resource "random_string" "hail_test_requester_pays_bucket_suffix" { + length = 5 +} + +resource "google_storage_bucket" "hail_test_requester_pays_bucket" { + name = "hail-test-requester-pays-${random_string.hail_test_requester_pays_bucket_suffix.result}" + location = var.hail_test_gcs_bucket_location + force_destroy = false + storage_class = var.hail_test_gcs_bucket_storage_class + uniform_bucket_level_access = true + requester_pays = true + + timeouts {} +} + resource "google_dns_managed_zone" "dns_zone" { description = "" name = "hail"