From 84a06fbe341fbd90afa2d3b325cbe0dc64c55853 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Wed, 28 Sep 2022 11:38:05 -0400 Subject: [PATCH 01/14] [hailctl] Add hailctl batch submit --- hail/python/hailtop/hailctl/batch/cli.py | 12 ++++- hail/python/hailtop/hailctl/batch/submit.py | 58 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 hail/python/hailtop/hailctl/batch/submit.py diff --git a/hail/python/hailtop/hailctl/batch/cli.py b/hail/python/hailtop/hailctl/batch/cli.py index 9082f4f2e73..f42a148fb05 100644 --- a/hail/python/hailtop/hailctl/batch/cli.py +++ b/hail/python/hailtop/hailctl/batch/cli.py @@ -10,6 +10,7 @@ from . import log from . import job from . import billing +from . import submit def parser(): @@ -39,6 +40,11 @@ def parser(): help='Delete a batch', description='Delete a batch' ) + submit_parser = subparsers.add_parser( + 'submit', + help='Submit a batch', + description='Submit a batch', + ) log_parser = subparsers.add_parser( 'log', help='Get log for a job', @@ -69,6 +75,9 @@ def parser(): delete_parser.set_defaults(module='delete') delete.init_parser(delete_parser) + submit_parser.set_defaults(module='submit') + submit.init_parser(submit_parser) + log_parser.set_defaults(module='log') log.init_parser(log_parser) @@ -93,7 +102,8 @@ def main(args): 'cancel': cancel, 'log': log, 'job': job, - 'wait': wait + 'wait': wait, + 'submit': submit, } args, pass_through_args = parser().parse_known_args(args=args) diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py new file mode 100644 index 00000000000..78a9386e8ac --- /dev/null +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -0,0 +1,58 @@ +import asyncio + +import hailtop.batch as hb +import hailtop.batch_client.client as bc +from hailtop import pip_version +from hailtop.aiotools.copy import copy_from_dict +from hailtop.config import get_remote_tmpdir, get_user_config_path, get_deploy_config +from hailtop.utils import unpack_comma_delimited_inputs + +def init_parser(parser): + parser.add_argument('name', type=str, help='Batch name') + parser.add_argument('script', type=str, help='Path to script') + parser.add_argument('--image-name', type=str, required=False, + help='Name for Docker image. Defaults to hailgenetics/hail') + parser.add_argument('--files', nargs='+', action='append', default=[], + help='Comma-separated list of files or directories to add to the working directory of job') + + +async def async_main(args): + script = args.script + files = unpack_comma_delimited_inputs(args.files) + user_config = get_user_config_path() + + remote_tmpdir = get_remote_tmpdir('hailctl batch submit') + def cloud_prefix(path): + return f'{remote_tmpdir}/{path}' + + b = hb.Batch(name=args.name, backend=hb.ServiceBackend()) + j = b.new_bash_job() + j.image(args.image_name or f'hailgenetics/hail:{pip_version()}') + + local_files_to_cloud_files = [{'from': local, 'to': cloud_prefix(local)} for local in files] + await copy_from_dict(files=[ + {'from': script, 'to': cloud_prefix(script)}, + {'from': str(user_config), 'to': cloud_prefix(user_config)}, + *local_files_to_cloud_files, + ]) + for file in local_files_to_cloud_files: + local_file = file['from'] + cloud_file = file['to'] + in_file = b.read_input(cloud_file) + j.command(f'ln -s {in_file} {local_file}') + + script_file = b.read_input(cloud_prefix(script)) + config_file = b.read_input(cloud_prefix(user_config)) + j.command(f'mkdir -p $HOME/.config/hail && ln -s {config_file} $HOME/.config/hail/config.ini') + + command = 'python3' if script.endswith('.py') else 'bash' + j.command(f'{command} {script_file}') + batch_handle: bc.Batch = b.run(wait=False) # type: ignore + + deploy_config = get_deploy_config() + url = deploy_config.external_url('batch', f'/batches/{batch_handle.id}/jobs/1') + print(f'Submitted batch {batch_handle.id}, see {url}') + + +def main(args, pass_through_args, client): # pylint: disable=unused-argument + asyncio.run(async_main(args)) From 42409c7ca3ff091ada1b72d019376b41c1a0a538 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 15 Nov 2022 17:36:50 -0800 Subject: [PATCH 02/14] make name an option not an argument --- hail/python/hailtop/hailctl/batch/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index 78a9386e8ac..c2637b80913 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -8,8 +8,8 @@ from hailtop.utils import unpack_comma_delimited_inputs def init_parser(parser): - parser.add_argument('name', type=str, help='Batch name') parser.add_argument('script', type=str, help='Path to script') + parser.add_argument('--name', type=str, default='', help='Batch name') parser.add_argument('--image-name', type=str, required=False, help='Name for Docker image. Defaults to hailgenetics/hail') parser.add_argument('--files', nargs='+', action='append', default=[], From 4e4bfc06da93b60a2fe5653ac4bfdce52bfcdbd1 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Thu, 17 Nov 2022 11:19:58 -0800 Subject: [PATCH 03/14] lint' --- hail/python/hailtop/hailctl/batch/submit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index c2637b80913..d3b7cd0d6dd 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -7,6 +7,7 @@ from hailtop.config import get_remote_tmpdir, get_user_config_path, get_deploy_config from hailtop.utils import unpack_comma_delimited_inputs + def init_parser(parser): parser.add_argument('script', type=str, help='Path to script') parser.add_argument('--name', type=str, default='', help='Batch name') @@ -22,6 +23,7 @@ async def async_main(args): user_config = get_user_config_path() remote_tmpdir = get_remote_tmpdir('hailctl batch submit') + def cloud_prefix(path): return f'{remote_tmpdir}/{path}' From 1185df367634d4632712b1dc326c09124272a989 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 22 Nov 2022 08:32:36 -0800 Subject: [PATCH 04/14] use relative file paths for files --- hail/python/hailtop/hailctl/batch/submit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index d3b7cd0d6dd..1d14dcb012a 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -1,4 +1,5 @@ import asyncio +import os import hailtop.batch as hb import hailtop.batch_client.client as bc @@ -31,7 +32,8 @@ def cloud_prefix(path): j = b.new_bash_job() j.image(args.image_name or f'hailgenetics/hail:{pip_version()}') - local_files_to_cloud_files = [{'from': local, 'to': cloud_prefix(local)} for local in files] + rel_file_paths = [os.path.relpath(file) for file in files] + local_files_to_cloud_files = [{'from': local, 'to': cloud_prefix(local)} for local in rel_file_paths] await copy_from_dict(files=[ {'from': script, 'to': cloud_prefix(script)}, {'from': str(user_config), 'to': cloud_prefix(user_config)}, From 301b9f20440b40012f0afdc26bb51631344d05c8 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 22 Nov 2022 09:45:47 -0800 Subject: [PATCH 05/14] add basic test --- build.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/build.yaml b/build.yaml index 2fa7078bf43..048869f735d 100644 --- a/build.yaml +++ b/build.yaml @@ -2642,6 +2642,49 @@ steps: - merge_code - service_base_image - deploy_batch + - kind: runImage + name: test_hailctl_batch + numSplits: 5 + image: + valueFrom: hailgenetics_hail_image.image + script: | + mkdir -p foo + echo "bar" > foo/baz.txt + + cat >simple_hail.py < Date: Tue, 22 Nov 2022 11:20:47 -0800 Subject: [PATCH 06/14] use the current hailgenetics/hail image not the dockerhub one --- build.yaml | 2 ++ hail/python/hailtop/hailctl/batch/submit.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/build.yaml b/build.yaml index 048869f735d..990d33ec7bf 100644 --- a/build.yaml +++ b/build.yaml @@ -2648,6 +2648,8 @@ steps: image: valueFrom: hailgenetics_hail_image.image script: | + export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}" + mkdir -p foo echo "bar" > foo/baz.txt diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index 1d14dcb012a..c57bae70ea0 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -8,6 +8,8 @@ from hailtop.config import get_remote_tmpdir, get_user_config_path, get_deploy_config from hailtop.utils import unpack_comma_delimited_inputs +HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') + def init_parser(parser): parser.add_argument('script', type=str, help='Path to script') @@ -30,7 +32,7 @@ def cloud_prefix(path): b = hb.Batch(name=args.name, backend=hb.ServiceBackend()) j = b.new_bash_job() - j.image(args.image_name or f'hailgenetics/hail:{pip_version()}') + j.image(args.image_name or HAIL_GENETICS_HAIL_IMAGE) rel_file_paths = [os.path.relpath(file) for file in files] local_files_to_cloud_files = [{'from': local, 'to': cloud_prefix(local)} for local in rel_file_paths] From e37b6acfda27b28e2960bf78a61568f7ea04369c Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 22 Nov 2022 12:03:28 -0800 Subject: [PATCH 07/14] get quiet mode working to fully test hailctl batch submit --- build.yaml | 10 +++++++++- hail/python/hailtop/aiotools/copy.py | 3 ++- hail/python/hailtop/batch_client/aioclient.py | 2 +- hail/python/hailtop/hailctl/batch/submit.py | 15 +++++++++++---- hail/python/hailtop/hailctl/batch/wait.py | 6 +++++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/build.yaml b/build.yaml index 990d33ec7bf..88f867e35b9 100644 --- a/build.yaml +++ b/build.yaml @@ -2648,6 +2648,8 @@ steps: image: valueFrom: hailgenetics_hail_image.image script: | + set -ex + export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}" mkdir -p foo @@ -2662,7 +2664,13 @@ steps: hl.utils.range_table(10).collect() EOF - hailctl batch submit simple_hail.py --name=test-hailctl-submit --files=foo + BATCH_ID=$(hailctl batch submit simple_hail.py --name=test-hailctl-batch-submit --files=foo -o json | jq '.id') + STATUS=$(hailctl batch wait --quiet $BATCH_ID | jq -jr '.state') + if [ "$STATUS" == "success" ]; then + exit 0; + else + exit 1; + fi secrets: - name: worker-deploy-config namespace: diff --git a/hail/python/hailtop/aiotools/copy.py b/hail/python/hailtop/aiotools/copy.py index 7ccff4060d1..dd8938ec626 100644 --- a/hail/python/hailtop/aiotools/copy.py +++ b/hail/python/hailtop/aiotools/copy.py @@ -62,7 +62,8 @@ async def copy(*, transfers, files_listener=make_listener(progress, file_tid), bytes_listener=make_listener(progress, bytes_tid)) - copy_report.summarize() + if verbose: + copy_report.summarize() def make_transfer(json_object: Dict[str, str]) -> Transfer: diff --git a/hail/python/hailtop/batch_client/aioclient.py b/hail/python/hailtop/batch_client/aioclient.py index 1d66af6bea5..d602a82362e 100644 --- a/hail/python/hailtop/batch_client/aioclient.py +++ b/hail/python/hailtop/batch_client/aioclient.py @@ -443,7 +443,7 @@ async def wait(self, description += ': ' if progress is not None: return await self._wait(description, progress, disable_progress_bar) - with BatchProgressBar() as progress2: + with BatchProgressBar(disable=disable_progress_bar) as progress2: return await self._wait(description, progress2, disable_progress_bar) async def debug_info(self): diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index c57bae70ea0..0211075f9e9 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -1,4 +1,5 @@ import asyncio +import orjson import os import hailtop.batch as hb @@ -18,12 +19,14 @@ def init_parser(parser): help='Name for Docker image. Defaults to hailgenetics/hail') parser.add_argument('--files', nargs='+', action='append', default=[], help='Comma-separated list of files or directories to add to the working directory of job') + parser.add_argument('-o', type=str, default='text', choices=['text', 'json']) async def async_main(args): script = args.script files = unpack_comma_delimited_inputs(args.files) user_config = get_user_config_path() + quiet = args.o != 'text' remote_tmpdir = get_remote_tmpdir('hailctl batch submit') @@ -53,11 +56,15 @@ def cloud_prefix(path): command = 'python3' if script.endswith('.py') else 'bash' j.command(f'{command} {script_file}') - batch_handle: bc.Batch = b.run(wait=False) # type: ignore + batch_handle: bc.Batch = b.run(wait=False, disable_progress_bar=quiet) # type: ignore - deploy_config = get_deploy_config() - url = deploy_config.external_url('batch', f'/batches/{batch_handle.id}/jobs/1') - print(f'Submitted batch {batch_handle.id}, see {url}') + if args.o == 'text': + deploy_config = get_deploy_config() + url = deploy_config.external_url('batch', f'/batches/{batch_handle.id}/jobs/1') + print(f'Submitted batch {batch_handle.id}, see {url}') + else: + assert args.o == 'json' + print(orjson.dumps({'id': batch_handle.id}).decode('utf-8')) def main(args, pass_through_args, client): # pylint: disable=unused-argument diff --git a/hail/python/hailtop/hailctl/batch/wait.py b/hail/python/hailtop/hailctl/batch/wait.py index 1999aa9213c..d3a5ca9a1b2 100644 --- a/hail/python/hailtop/hailctl/batch/wait.py +++ b/hail/python/hailtop/hailctl/batch/wait.py @@ -1,9 +1,13 @@ +import json import sys from .batch_cli_utils import get_batch_if_exists def init_parser(parser): parser.add_argument('batch_id', type=int) + parser.add_argument("--quiet", "-q", + action="store_true", + help="Do not print a progress bar for the batch") def main(args, pass_through_args, client): # pylint: disable=unused-argument @@ -13,4 +17,4 @@ def main(args, pass_through_args, client): # pylint: disable=unused-argument sys.exit(1) batch = maybe_batch - print(batch.wait()) + print(json.dumps(batch.wait(disable_progress_bar=True))) From 402855f74f89e02726f6ea2fc58c9d0a9b78eb47 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 22 Nov 2022 13:25:17 -0800 Subject: [PATCH 08/14] fix --- build.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.yaml b/build.yaml index 88f867e35b9..4843d4c7652 100644 --- a/build.yaml +++ b/build.yaml @@ -2650,8 +2650,14 @@ steps: script: | set -ex + sudo apt-get update && apt-get install jq + export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}" + hailctl config set query/backend batch + hailctl config set batch/billing_project test + hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }} + mkdir -p foo echo "bar" > foo/baz.txt From 7671054635cce8e56aee88925a4083c9fab3a509 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 22 Nov 2022 13:28:07 -0800 Subject: [PATCH 09/14] cleanup --- build.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/build.yaml b/build.yaml index 4843d4c7652..fa23b943a24 100644 --- a/build.yaml +++ b/build.yaml @@ -2644,7 +2644,6 @@ steps: - deploy_batch - kind: runImage name: test_hailctl_batch - numSplits: 5 image: valueFrom: hailgenetics_hail_image.image script: | From ed472581b6261e4809fffff4b83ea05419e104ed Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 22 Nov 2022 13:42:52 -0800 Subject: [PATCH 10/14] submit sets HAIL_QUERY_BACKEND to batch --- hail/python/hailtop/hailctl/batch/submit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index 0211075f9e9..116570d8e8a 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -54,6 +54,8 @@ def cloud_prefix(path): config_file = b.read_input(cloud_prefix(user_config)) j.command(f'mkdir -p $HOME/.config/hail && ln -s {config_file} $HOME/.config/hail/config.ini') + j.env('HAIL_QUERY_BACKEND', 'batch') + command = 'python3' if script.endswith('.py') else 'bash' j.command(f'{command} {script_file}') batch_handle: bc.Batch = b.run(wait=False, disable_progress_bar=quiet) # type: ignore From 70a0b062091abb213bc11f58d2ded66452640db6 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 22 Nov 2022 17:23:21 -0800 Subject: [PATCH 11/14] fix --- build.yaml | 5 ++--- docker/hailgenetics/hail/Dockerfile | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.yaml b/build.yaml index fa23b943a24..11d906f2d8b 100644 --- a/build.yaml +++ b/build.yaml @@ -2649,13 +2649,12 @@ steps: script: | set -ex - sudo apt-get update && apt-get install jq - export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}" + export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json hailctl config set query/backend batch hailctl config set batch/billing_project test - hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }} + hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/{{ token }}/hailctl-test mkdir -p foo echo "bar" > foo/baz.txt diff --git a/docker/hailgenetics/hail/Dockerfile b/docker/hailgenetics/hail/Dockerfile index e703f0b5f84..8d49bea23a0 100644 --- a/docker/hailgenetics/hail/Dockerfile +++ b/docker/hailgenetics/hail/Dockerfile @@ -5,6 +5,7 @@ ENV MAKEFLAGS -j2 RUN hail-apt-get-install \ curl \ git \ + jq \ liblapack3 \ openjdk-8-jre-headless \ rsync \ From f44f7832e630ab2857fde9d68c229f81143edb2b Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Mon, 28 Nov 2022 14:23:40 -0500 Subject: [PATCH 12/14] fixes --- build.yaml | 3 +-- hail/python/hailtop/hailctl/batch/submit.py | 5 +++-- hail/python/hailtop/hailctl/batch/wait.py | 8 +++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/build.yaml b/build.yaml index c50859d0ebc..d16bc86492b 100644 --- a/build.yaml +++ b/build.yaml @@ -2661,7 +2661,6 @@ steps: export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}" export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json - hailctl config set query/backend batch hailctl config set batch/billing_project test hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/{{ token }}/hailctl-test @@ -2678,7 +2677,7 @@ steps: EOF BATCH_ID=$(hailctl batch submit simple_hail.py --name=test-hailctl-batch-submit --files=foo -o json | jq '.id') - STATUS=$(hailctl batch wait --quiet $BATCH_ID | jq -jr '.state') + STATUS=$(hailctl batch wait -o json $BATCH_ID | jq -jr '.state') if [ "$STATUS" == "success" ]; then exit 0; else diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index 116570d8e8a..36b5940c9f5 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -7,7 +7,7 @@ from hailtop import pip_version from hailtop.aiotools.copy import copy_from_dict from hailtop.config import get_remote_tmpdir, get_user_config_path, get_deploy_config -from hailtop.utils import unpack_comma_delimited_inputs +from hailtop.utils import secret_alnum_string, unpack_comma_delimited_inputs HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') @@ -30,8 +30,9 @@ async def async_main(args): remote_tmpdir = get_remote_tmpdir('hailctl batch submit') + tmpdir_path_prefix = secret_alnum_string() def cloud_prefix(path): - return f'{remote_tmpdir}/{path}' + return f'{remote_tmpdir}/{tmpdir_path_prefix}/{path}' b = hb.Batch(name=args.name, backend=hb.ServiceBackend()) j = b.new_bash_job() diff --git a/hail/python/hailtop/hailctl/batch/wait.py b/hail/python/hailtop/hailctl/batch/wait.py index d3a5ca9a1b2..0aad2e3ab6a 100644 --- a/hail/python/hailtop/hailctl/batch/wait.py +++ b/hail/python/hailtop/hailctl/batch/wait.py @@ -8,6 +8,7 @@ def init_parser(parser): parser.add_argument("--quiet", "-q", action="store_true", help="Do not print a progress bar for the batch") + parser.add_argument('-o', type=str, default='text', choices=['text', 'json']) def main(args, pass_through_args, client): # pylint: disable=unused-argument @@ -17,4 +18,9 @@ def main(args, pass_through_args, client): # pylint: disable=unused-argument sys.exit(1) batch = maybe_batch - print(json.dumps(batch.wait(disable_progress_bar=True))) + quiet = args.quiet or args.o != 'text' + out = batch.wait(disable_progress_bar=quiet) + if args.o == 'json': + print(json.dumps(out)) + else: + print(out) From b27465d41b788381c330d36b84678e8078c469c3 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Mon, 28 Nov 2022 14:25:29 -0500 Subject: [PATCH 13/14] lint --- hail/python/hailtop/hailctl/batch/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index 36b5940c9f5..40309d59c40 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -29,8 +29,8 @@ async def async_main(args): quiet = args.o != 'text' remote_tmpdir = get_remote_tmpdir('hailctl batch submit') - tmpdir_path_prefix = secret_alnum_string() + def cloud_prefix(path): return f'{remote_tmpdir}/{tmpdir_path_prefix}/{path}' From db8a50c4cd2525724b280b8056ab539602ed3160 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Mon, 28 Nov 2022 14:28:33 -0500 Subject: [PATCH 14/14] name the batch that the submitted job spawns --- build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build.yaml b/build.yaml index d16bc86492b..ced55efe0c2 100644 --- a/build.yaml +++ b/build.yaml @@ -2673,6 +2673,7 @@ steps: with open('foo/baz.txt') as f: print(f.read()) + hl.init(app_name='test-hailctl-batch-submit-query') hl.utils.range_table(10).collect() EOF