[batch] use regional bucket for requester pays tests (#12964)

I updated terraform but

1. GCP Terraform state is still local on my laptop.

2. GCP Terraform appears to not configure global-config. As such, I
cannot thread the name of the bucket through to the tests the way we do
with TEST_STORAGE_URI. For now, I've hardcoded the name (which is what
we were doing previously). When we eventually get to testing recreation
of GCP in a new project we'll have to address the global config then.

hail/python/test/hail/fs/test_worker_driver_fs.py

@@ -8,7 +8,7 @@
 @skip_in_azure
 def test_requester_pays_no_settings():
     try:
-        hl.import_table('gs://hail-services-requester-pays/hello')
+        hl.import_table('gs://hail-test-requester-pays-fds32/hello')
     except Exception as exc:
         assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0]
     else:
@@ -17,7 +17,7 @@ def test_requester_pays_no_settings():
 
 @skip_in_azure
 def test_requester_pays_write_no_settings():
-    random_filename = 'gs://hail-services-requester-pays/test_requester_pays_on_worker_driver_' + secret_alnum_string(10)
+    random_filename = 'gs://hail-test-requester-pays-fds32/test_requester_pays_on_worker_driver_' + secret_alnum_string(10)
     try:
         hl.utils.range_table(4, n_partitions=4).write(random_filename, overwrite=True)
     except Exception as exc:
@@ -32,7 +32,7 @@ def test_requester_pays_write_no_settings():
 def test_requester_pays_write_with_project():
     hl.stop()
     hl.init(gcs_requester_pays_configuration='hail-vdc')
-    random_filename = 'gs://hail-services-requester-pays/test_requester_pays_on_worker_driver_' + secret_alnum_string(10)
+    random_filename = 'gs://hail-test-requester-pays-fds32/test_requester_pays_on_worker_driver_' + secret_alnum_string(10)
     try:
         hl.utils.range_table(4, n_partitions=4).write(random_filename, overwrite=True)
     finally:
@@ -43,28 +43,28 @@ def test_requester_pays_write_with_project():
 def test_requester_pays_with_project():
     hl.stop()
     hl.init(gcs_requester_pays_configuration='hail-vdc')
-    assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
+    assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
 
     hl.stop()
-    hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays']))
-    assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
+    hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32']))
+    assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
 
     hl.stop()
-    hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays', 'other-bucket']))
-    assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
+    hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32', 'other-bucket']))
+    assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
 
     hl.stop()
     hl.init(gcs_requester_pays_configuration=('hail-vdc', ['other-bucket']))
     try:
-        hl.import_table('gs://hail-services-requester-pays/hello')
+        hl.import_table('gs://hail-test-requester-pays-fds32/hello')
     except Exception as exc:
         assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0]
     else:
         assert False
 
     hl.stop()
     hl.init(gcs_requester_pays_configuration='hail-vdc')
-    assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
+    assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')]
 
 
 @skip_in_azure
@@ -93,28 +93,28 @@ def test_requester_pays_with_project_more_than_one_partition():
 
     hl.stop()
     hl.init(gcs_requester_pays_configuration='hail-vdc')
-    assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
+    assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
 
     hl.stop()
-    hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays']))
-    assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
+    hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32']))
+    assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
 
     hl.stop()
-    hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays', 'other-bucket']))
-    assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
+    hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32', 'other-bucket']))
+    assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
 
     hl.stop()
     hl.init(gcs_requester_pays_configuration=('hail-vdc', ['other-bucket']))
     try:
-        hl.import_table('gs://hail-services-requester-pays/zero-to-nine', min_partitions=8)
+        hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', min_partitions=8)
     except Exception as exc:
         assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0]
     else:
         assert False
 
     hl.stop()
     hl.init(gcs_requester_pays_configuration='hail-vdc')
-    assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
+    assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents
 
 
 @run_if_azure

hail/python/test/hailtop/batch/test_batch.py

@@ -748,7 +748,7 @@ def test_cloudfuse_empty_string_bucket_fails(self):
     def test_fuse_requester_pays(self):
         b = self.batch(requester_pays_project='hail-vdc')
         j = b.new_job()
-        j.cloudfuse('hail-services-requester-pays', '/fuse-bucket')
+        j.cloudfuse('hail-test-requester-pays-fds32', '/fuse-bucket')
         j.command('cat /fuse-bucket/hello')
         res = b.run()
         res_status = res.status()
@@ -776,7 +776,7 @@ def test_fuse_non_requester_pays_bucket_when_requester_pays_project_specified(se
     @skip_in_azure
     def test_requester_pays(self):
         b = self.batch(requester_pays_project='hail-vdc')
-        input = b.read_input('gs://hail-services-requester-pays/hello')
+        input = b.read_input('gs://hail-test-requester-pays-fds32/hello')
         j = b.new_job()
         j.command(f'cat {input}')
         res = b.run()

infra/gcp-broad/main.tf

@@ -628,6 +628,21 @@ resource "google_storage_bucket" "hail_test_bucket" {
   timeouts {}
 }
 
+resource "random_string" "hail_test_requester_pays_bucket_suffix" {
+  length = 5
+}
+
+resource "google_storage_bucket" "hail_test_requester_pays_bucket" {
+  name = "hail-test-requester-pays-${random_string.hail_test_requester_pays_bucket_suffix.result}"
+  location = var.hail_test_gcs_bucket_location
+  force_destroy = false
+  storage_class = var.hail_test_gcs_bucket_storage_class
+  uniform_bucket_level_access = true
+  requester_pays = true
+
+  timeouts {}
+}
+
 resource "google_dns_managed_zone" "dns_zone" {
   description = ""
   name = "hail"