From f72d76426a57e56d3a852aa8697f1d25887603fe Mon Sep 17 00:00:00 2001 From: Dan King Date: Fri, 8 Dec 2023 15:02:45 -0500 Subject: [PATCH 1/3] [hailctl] make hailctl work on windows 1. Non 64-bit Windows uses "Program Files" not "Program Files (x86)" 2. Windows Subsystem for Linux looks like GNU/Linux but will not have chromium on its path. 3. The removed arguments are no longer supported. They produce a warning message in my version of Chrome and appear to not work in the version of Chrome that this user was using. Instead, I bind to 0.0.0.0 and access the Notebook using the machine DNS name. This is how Google recommend accessing the Spark UI anyway. --- .../hailtop/hailctl/dataproc/connect.py | 30 +++++++++++-------- .../dataproc/resources/init_notebook.py | 2 +- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/hail/python/hailtop/hailctl/dataproc/connect.py b/hail/python/hailtop/hailctl/dataproc/connect.py index 4da5bb66e45..3668cd901d1 100755 --- a/hail/python/hailtop/hailctl/dataproc/connect.py +++ b/hail/python/hailtop/hailctl/dataproc/connect.py @@ -36,16 +36,25 @@ def get_chrome_path(): if system == 'Darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' - if system == 'Linux': - for c in ['chromium', 'chromium-browser']: + release = platform.uname().release + is_wsl = 'Microsoft' in release or 'microsoft' in release + + if system == 'Linux' and not is_wsl: + for c in ['chromium', 'chromium-browser', 'chrome.exe']: chrome = shutil.which(c) if chrome: return chrome - raise EnvironmentError("cannot find 'chromium' or 'chromium-browser' on path") + raise EnvironmentError("cannot find 'chromium', 'chromium-browser', or 'chrome.exe' on path") - if system == 'Windows': - return '/mnt/c/Program Files (x86)/Google/Chrome/Application/chrome.exe' + if system == 'Windows' or (system == 'Linux' and is_wsl): + fnames = [ + '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe' + '/mnt/c/Program Files (x86)/Google/Chrome/Application/chrome.exe' + ] + for fname in fnames: + if os.path.exists(fname): + return fname raise ValueError(f"unsupported system: {system}, set environment variable HAILCTL_CHROME to a chrome executable") @@ -109,18 +118,15 @@ def connect( gcloud.run(cmd) chrome = os.environ.get('HAILCTL_CHROME') or get_chrome_path() + data_dir = os.path.join(tempfile.gettempdir(), 'hailctl-dataproc-connect-' + secret_alnum_string(6)) # open Chrome with SOCKS proxy configuration with subprocess.Popen( [ # pylint: disable=consider-using-with chrome, - 'http://localhost:{}'.format(connect_port_and_path), - '--proxy-server=socks5://localhost:{}'.format(port), - '--host-resolver-rules=MAP * 0.0.0.0 , EXCLUDE localhost', - '--proxy-bypass-list=<-loopback>', # https://chromium.googlesource.com/chromium/src/+/da790f920bbc169a6805a4fb83b4c2ab09532d91 - '--user-data-dir={}'.format( - os.path.join(tempfile.gettempdir(), 'hailctl-dataproc-connect-' + secret_alnum_string(6)) - ), + f'http://{name}-m:{connect_port_and_path}', + f'--proxy-server=socks5://localhost:{port}' + f'--user-data-dir={data_dir}', ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, diff --git a/hail/python/hailtop/hailctl/dataproc/resources/init_notebook.py b/hail/python/hailtop/hailctl/dataproc/resources/init_notebook.py index 8b473b9a67d..c0a0c1a31ea 100644 --- a/hail/python/hailtop/hailctl/dataproc/resources/init_notebook.py +++ b/hail/python/hailtop/hailctl/dataproc/resources/init_notebook.py @@ -163,7 +163,7 @@ def mkdir_if_not_exists(path): with open('/opt/conda/default/etc/jupyter/jupyter_notebook_config.py', 'w') as f: opts = [ 'c.Application.log_level = "DEBUG"', - 'c.NotebookApp.ip = "127.0.0.1"', + 'c.NotebookApp.ip = "0.0.0.0"', 'c.NotebookApp.open_browser = False', 'c.NotebookApp.port = 8123', 'c.NotebookApp.token = ""', From a83d72b6369fbb9da06571580af5d0d394ff63cb Mon Sep 17 00:00:00 2001 From: Dan King Date: Fri, 8 Dec 2023 16:41:05 -0500 Subject: [PATCH 2/3] adjust tests for new domain name --- hail/python/test/hailtop/hailctl/dataproc/test_connect.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/hail/python/test/hailtop/hailctl/dataproc/test_connect.py b/hail/python/test/hailtop/hailctl/dataproc/test_connect.py index 7a1dbad7eb4..12536f7d4a8 100644 --- a/hail/python/test/hailtop/hailctl/dataproc/test_connect.py +++ b/hail/python/test/hailtop/hailctl/dataproc/test_connect.py @@ -57,11 +57,9 @@ def test_connect(gcloud_run, subprocess): popen_args = subprocess.Popen.call_args[0][0] assert popen_args[0] == "chromium" - assert popen_args[1].startswith("http://localhost") + assert popen_args[1].startswith("http://test-cluster-m") - assert "--proxy-server=socks5://localhost:10000" in popen_args - assert "--host-resolver-rules=MAP * 0.0.0.0 , EXCLUDE localhost" in popen_args - assert "--proxy-bypass-list=<-loopback>" in popen_args + assert "--proxy-server=socks5://test-cluster-m:10000" in popen_args assert any(arg.startswith("--user-data-dir=") for arg in popen_args) @@ -77,7 +75,7 @@ def test_service_port_and_path(subprocess, service, expected_port_and_path): runner.invoke(cli.app, ['connect', 'test-cluster', service]) popen_args = subprocess.Popen.call_args[0][0] - assert popen_args[1] == f"http://localhost:{expected_port_and_path}" + assert popen_args[1] == f"http://test-cluster-m:{expected_port_and_path}" def test_hailctl_chrome(subprocess, monkeypatch): From 92c81b6f90598a2b2c1e101dd19092ed776463d8 Mon Sep 17 00:00:00 2001 From: Dan King Date: Fri, 8 Dec 2023 17:42:15 -0500 Subject: [PATCH 3/3] fix --- hail/python/hailtop/hailctl/dataproc/connect.py | 2 +- hail/python/test/hailtop/hailctl/dataproc/test_connect.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hail/python/hailtop/hailctl/dataproc/connect.py b/hail/python/hailtop/hailctl/dataproc/connect.py index 3668cd901d1..d3c4c6d4a17 100755 --- a/hail/python/hailtop/hailctl/dataproc/connect.py +++ b/hail/python/hailtop/hailctl/dataproc/connect.py @@ -125,7 +125,7 @@ def connect( [ # pylint: disable=consider-using-with chrome, f'http://{name}-m:{connect_port_and_path}', - f'--proxy-server=socks5://localhost:{port}' + f'--proxy-server=socks5://localhost:{port}', f'--user-data-dir={data_dir}', ], stdout=subprocess.DEVNULL, diff --git a/hail/python/test/hailtop/hailctl/dataproc/test_connect.py b/hail/python/test/hailtop/hailctl/dataproc/test_connect.py index 12536f7d4a8..36212906ee8 100644 --- a/hail/python/test/hailtop/hailctl/dataproc/test_connect.py +++ b/hail/python/test/hailtop/hailctl/dataproc/test_connect.py @@ -59,7 +59,7 @@ def test_connect(gcloud_run, subprocess): assert popen_args[0] == "chromium" assert popen_args[1].startswith("http://test-cluster-m") - assert "--proxy-server=socks5://test-cluster-m:10000" in popen_args + assert "--proxy-server=socks5://localhost:10000" in popen_args assert any(arg.startswith("--user-data-dir=") for arg in popen_args)