Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[batch] use regional bucket for requester pays tests #12964

Merged
merged 2 commits into from
May 3, 2023
Merged

[batch] use regional bucket for requester pays tests #12964

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a8cc991
[batch] use regional bucket for requester pays tests
May 2, 2023
9fd409a
forgot to set requester pays
May 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions hail/python/test/hail/fs/test_worker_driver_fs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
@skip_in_azure
def test_requester_pays_no_settings():
try:
hl.import_table('gs://hail-services-requester-pays/hello')
hl.import_table('gs://hail-test-requester-pays-fds32/hello')
except Exception as exc:
assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0]
else:
Expand All @@ -17,7 +17,7 @@ def test_requester_pays_no_settings():

@skip_in_azure
def test_requester_pays_write_no_settings():
random_filename = 'gs://hail-services-requester-pays/test_requester_pays_on_worker_driver_' + secret_alnum_string(10)
random_filename = 'gs://hail-test-requester-pays-fds32/test_requester_pays_on_worker_driver_' + secret_alnum_string(10)
try:
hl.utils.range_table(4, n_partitions=4).write(random_filename, overwrite=True)
except Exception as exc:
Expand All @@ -32,7 +32,7 @@ def test_requester_pays_write_no_settings():
def test_requester_pays_write_with_project():
hl.stop()
hl.init(gcs_requester_pays_configuration='hail-vdc')
random_filename = 'gs://hail-services-requester-pays/test_requester_pays_on_worker_driver_' + secret_alnum_string(10)
random_filename = 'gs://hail-test-requester-pays-fds32/test_requester_pays_on_worker_driver_' + secret_alnum_string(10)
try:
hl.utils.range_table(4, n_partitions=4).write(random_filename, overwrite=True)
finally:
Expand All @@ -43,28 +43,28 @@ def test_requester_pays_write_with_project():
def test_requester_pays_with_project():
hl.stop()
hl.init(gcs_requester_pays_configuration='hail-vdc')
assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')]

hl.stop()
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays']))
assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32']))
assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')]

hl.stop()
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays', 'other-bucket']))
assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32', 'other-bucket']))
assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')]

hl.stop()
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['other-bucket']))
try:
hl.import_table('gs://hail-services-requester-pays/hello')
hl.import_table('gs://hail-test-requester-pays-fds32/hello')
except Exception as exc:
assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0]
else:
assert False

hl.stop()
hl.init(gcs_requester_pays_configuration='hail-vdc')
assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')]


@skip_in_azure
Expand Down Expand Up @@ -93,28 +93,28 @@ def test_requester_pays_with_project_more_than_one_partition():

hl.stop()
hl.init(gcs_requester_pays_configuration='hail-vdc')
assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents

hl.stop()
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays']))
assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32']))
assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents

hl.stop()
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays', 'other-bucket']))
assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32', 'other-bucket']))
assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents

hl.stop()
hl.init(gcs_requester_pays_configuration=('hail-vdc', ['other-bucket']))
try:
hl.import_table('gs://hail-services-requester-pays/zero-to-nine', min_partitions=8)
hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', min_partitions=8)
except Exception as exc:
assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0]
else:
assert False

hl.stop()
hl.init(gcs_requester_pays_configuration='hail-vdc')
assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents


@run_if_azure
Expand Down
4 changes: 2 additions & 2 deletions hail/python/test/hailtop/batch/test_batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,7 +748,7 @@ def test_cloudfuse_empty_string_bucket_fails(self):
def test_fuse_requester_pays(self):
b = self.batch(requester_pays_project='hail-vdc')
j = b.new_job()
j.cloudfuse('hail-services-requester-pays', '/fuse-bucket')
j.cloudfuse('hail-test-requester-pays-fds32', '/fuse-bucket')
j.command('cat /fuse-bucket/hello')
res = b.run()
res_status = res.status()
Expand Down Expand Up @@ -776,7 +776,7 @@ def test_fuse_non_requester_pays_bucket_when_requester_pays_project_specified(se
@skip_in_azure
def test_requester_pays(self):
b = self.batch(requester_pays_project='hail-vdc')
input = b.read_input('gs://hail-services-requester-pays/hello')
input = b.read_input('gs://hail-test-requester-pays-fds32/hello')
j = b.new_job()
j.command(f'cat {input}')
res = b.run()
Expand Down
15 changes: 15 additions & 0 deletions infra/gcp-broad/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -623,6 +623,21 @@ resource "google_storage_bucket" "hail_test_bucket" {
timeouts {}
}

resource "random_string" "hail_test_requester_pays_bucket_suffix" {
length = 5
}

resource "google_storage_bucket" "hail_test_requester_pays_bucket" {
name = "hail-test-requester-pays-${random_string.hail_test_requester_pays_bucket_suffix.result}"
location = var.hail_test_gcs_bucket_location
force_destroy = false
storage_class = var.hail_test_gcs_bucket_storage_class
uniform_bucket_level_access = true
requester_pays = true

timeouts {}
}

resource "google_dns_managed_zone" "dns_zone" {
description = ""
name = "hail"
Expand Down