From 258b26f6044296f0e5e2362d1e9231564497e1d5 Mon Sep 17 00:00:00 2001
From: Daniel Goldstein <danielgold95@gmail.com>
Date: Mon, 13 Nov 2023 12:06:27 -0700
Subject: [PATCH] [query] Remove unused and redundant requirements (#13988)

`asyncinit` is unused AFAICT and the `frozenlist` requirement is already
inherited from hailtop (though it is not used in `hailtop` only in query
code, so am also happy to move it out of hailtop fully and into query).
---
 Makefile                                    |  2 +-
 batch/pinned-requirements.txt               |  6 +--
 benchmark/python/pinned-requirements.txt    |  8 ++--
 ci/pinned-requirements.txt                  |  2 +-
 gear/pinned-requirements.txt                | 16 +++----
 hail/python/dev/pinned-requirements.txt     | 52 +++++++++------------
 hail/python/hailtop/config/deploy_config.py |  2 +-
 hail/python/hailtop/pinned-requirements.txt | 16 +++----
 hail/python/pinned-requirements.txt         | 27 +++++------
 hail/python/requirements.txt                |  2 -
 web_common/pinned-requirements.txt          |  2 +-
 11 files changed, 62 insertions(+), 73 deletions(-)

diff --git a/Makefile b/Makefile
index d87f9950f78..d810b90aa6f 100644
--- a/Makefile
+++ b/Makefile
@@ -118,7 +118,7 @@ hail/python/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt
 hail/python/dev/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/requirements.txt
 	./generate-linux-pip-lockfile.sh hail/python/dev
 
-benchmark/python/pinned-requirements.txt: benchmark/python/requirements.txt
+benchmark/python/pinned-requirements.txt: benchmark/python/requirements.txt hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt
 	./generate-linux-pip-lockfile.sh benchmark/python
 
 gear/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
diff --git a/batch/pinned-requirements.txt b/batch/pinned-requirements.txt
index cfd624921db..35dadef8203 100644
--- a/batch/pinned-requirements.txt
+++ b/batch/pinned-requirements.txt
@@ -37,7 +37,7 @@ attrs==23.1.0
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   -c hail/batch/../web_common/pinned-requirements.txt
     #   aiohttp
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   -c hail/batch/../gear/pinned-requirements.txt
     #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
@@ -78,11 +78,11 @@ packaging==23.2
     #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   plotly
-pandas==2.1.1
+pandas==2.1.2
     # via
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   -r hail/batch/requirements.txt
-plotly==5.17.0
+plotly==5.18.0
     # via
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   -r hail/batch/requirements.txt
diff --git a/benchmark/python/pinned-requirements.txt b/benchmark/python/pinned-requirements.txt
index 7a15b1905dc..9fc6c11ba50 100644
--- a/benchmark/python/pinned-requirements.txt
+++ b/benchmark/python/pinned-requirements.txt
@@ -4,19 +4,19 @@
 #
 #    pip-compile --output-file=hail/benchmark/python/pinned-requirements.txt hail/benchmark/python/requirements.txt
 #
-contourpy==1.1.1
+contourpy==1.2.0
     # via
     #   -c hail/benchmark/python/../../hail/python/pinned-requirements.txt
     #   matplotlib
 cycler==0.12.1
     # via matplotlib
-fonttools==4.43.1
+fonttools==4.44.0
     # via matplotlib
-importlib-resources==6.1.0
+importlib-resources==6.1.1
     # via matplotlib
 kiwisolver==1.4.5
     # via matplotlib
-matplotlib==3.8.0
+matplotlib==3.8.1
     # via -r hail/benchmark/python/requirements.txt
 numpy==1.26.1
     # via
diff --git a/ci/pinned-requirements.txt b/ci/pinned-requirements.txt
index 0a5dccb0c44..7acf28dbfd2 100644
--- a/ci/pinned-requirements.txt
+++ b/ci/pinned-requirements.txt
@@ -15,7 +15,7 @@ cffi==1.16.0
     #   -c hail/ci/../hail/python/dev/pinned-requirements.txt
     #   -c hail/ci/../hail/python/pinned-requirements.txt
     #   cryptography
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   -c hail/ci/../gear/pinned-requirements.txt
     #   -c hail/ci/../hail/python/dev/pinned-requirements.txt
diff --git a/gear/pinned-requirements.txt b/gear/pinned-requirements.txt
index fcba6b39f8d..11492abab7e 100644
--- a/gear/pinned-requirements.txt
+++ b/gear/pinned-requirements.txt
@@ -41,7 +41,7 @@ attrs==23.1.0
     #   aiomonitor
 backports-strenum==1.2.8
     # via aiomonitor
-cachetools==5.3.1
+cachetools==5.3.2
     # via
     #   -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
     #   -c hail/gear/../hail/python/pinned-requirements.txt
@@ -53,7 +53,7 @@ certifi==2023.7.22
     #   -c hail/gear/../hail/python/pinned-requirements.txt
     #   kubernetes-asyncio
     #   requests
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   -c hail/gear/../hail/python/dev/pinned-requirements.txt
     #   -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
@@ -73,11 +73,11 @@ frozenlist==1.4.0
     #   -c hail/gear/../hail/python/pinned-requirements.txt
     #   aiohttp
     #   aiosignal
-google-api-core==2.12.0
+google-api-core==2.13.0
     # via google-api-python-client
-google-api-python-client==2.105.0
+google-api-python-client==2.107.0
     # via google-cloud-profiler
-google-auth==2.23.3
+google-auth==2.23.4
     # via
     #   -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
     #   -c hail/gear/../hail/python/pinned-requirements.txt
@@ -128,14 +128,14 @@ multidict==6.0.4
     #   -c hail/gear/../hail/python/pinned-requirements.txt
     #   aiohttp
     #   yarl
-orjson==3.9.9
+orjson==3.9.10
     # via
     #   -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
     #   -c hail/gear/../hail/python/pinned-requirements.txt
     #   -r hail/gear/requirements.txt
 prometheus-async==19.2.0
     # via -r hail/gear/requirements.txt
-prometheus-client==0.17.1
+prometheus-client==0.18.0
     # via
     #   -c hail/gear/../hail/python/dev/pinned-requirements.txt
     #   -r hail/gear/requirements.txt
@@ -223,7 +223,7 @@ urllib3==1.26.18
     #   -c hail/gear/../hail/python/pinned-requirements.txt
     #   kubernetes-asyncio
     #   requests
-wcwidth==0.2.8
+wcwidth==0.2.9
     # via
     #   -c hail/gear/../hail/python/dev/pinned-requirements.txt
     #   prompt-toolkit
diff --git a/hail/python/dev/pinned-requirements.txt b/hail/python/dev/pinned-requirements.txt
index 87b171648fd..759e4be5ad9 100644
--- a/hail/python/dev/pinned-requirements.txt
+++ b/hail/python/dev/pinned-requirements.txt
@@ -49,8 +49,6 @@ babel==2.13.1
     # via
     #   jupyterlab-server
     #   sphinx
-backcall==0.2.0
-    # via ipython
 beautifulsoup4==4.12.2
     # via nbconvert
 black==22.12.0
@@ -67,7 +65,7 @@ cffi==1.16.0
     #   argon2-cffi-bindings
 cfgv==3.4.0
     # via pre-commit
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   -c hail/hail/python/dev/../pinned-requirements.txt
     #   aiohttp
@@ -79,7 +77,7 @@ click==8.1.7
     #   aiohttp-devtools
     #   black
     #   curlylint
-comm==0.1.4
+comm==0.2.0
     # via
     #   ipykernel
     #   ipywidgets
@@ -113,13 +111,13 @@ exceptiongroup==1.1.3
     #   pytest
 execnet==2.0.2
     # via pytest-xdist
-executing==2.0.0
+executing==2.0.1
     # via
     #   devtools
     #   stack-data
 fastjsonschema==2.18.1
     # via nbformat
-filelock==3.12.4
+filelock==3.13.1
     # via virtualenv
 fqdn==1.5.1
     # via jsonschema
@@ -130,7 +128,7 @@ frozenlist==1.4.0
     #   aiosignal
 fswatch==0.1.1
     # via -r hail/hail/python/dev/requirements.txt
-identify==2.5.30
+identify==2.5.31
     # via pre-commit
 idna==3.4
     # via
@@ -157,13 +155,11 @@ ipykernel==6.26.0
     #   jupyter-console
     #   jupyterlab
     #   qtconsole
-ipython==8.16.1
+ipython==8.17.2
     # via
     #   ipykernel
     #   ipywidgets
     #   jupyter-console
-ipython-genutils==0.2.0
-    # via qtconsole
 ipywidgets==8.1.1
     # via jupyter
 isoduration==20.11.0
@@ -185,7 +181,7 @@ json5==0.9.14
     # via jupyterlab-server
 jsonpointer==2.4
     # via jsonschema
-jsonschema[format-nongpl]==4.19.1
+jsonschema[format-nongpl]==4.19.2
     # via
     #   jupyter-events
     #   jupyterlab-server
@@ -194,7 +190,7 @@ jsonschema-specifications==2023.7.1
     # via jsonschema
 jupyter==1.0.0
     # via -r hail/hail/python/dev/requirements.txt
-jupyter-client==8.5.0
+jupyter-client==8.6.0
     # via
     #   ipykernel
     #   jupyter-console
@@ -203,7 +199,7 @@ jupyter-client==8.5.0
     #   qtconsole
 jupyter-console==6.6.3
     # via jupyter
-jupyter-core==5.4.0
+jupyter-core==5.5.0
     # via
     #   ipykernel
     #   jupyter-client
@@ -214,11 +210,11 @@ jupyter-core==5.4.0
     #   nbconvert
     #   nbformat
     #   qtconsole
-jupyter-events==0.8.0
+jupyter-events==0.9.0
     # via jupyter-server
 jupyter-lsp==2.2.0
     # via jupyterlab
-jupyter-server==2.9.1
+jupyter-server==2.10.0
     # via
     #   jupyter-lsp
     #   jupyterlab
@@ -227,7 +223,7 @@ jupyter-server==2.9.1
     #   notebook-shim
 jupyter-server-terminals==0.4.4
     # via jupyter-server
-jupyterlab==4.0.7
+jupyterlab==4.0.8
     # via notebook
 jupyterlab-pygments==0.2.2
     # via nbconvert
@@ -259,9 +255,9 @@ multidict==6.0.4
     #   yarl
 mypy-extensions==1.0.0
     # via black
-nbclient==0.8.0
+nbclient==0.9.0
     # via nbconvert
-nbconvert==7.9.2
+nbconvert==7.11.0
     # via
     #   jupyter
     #   jupyter-server
@@ -314,8 +310,6 @@ pathspec==0.11.2
     #   curlylint
 pexpect==4.8.0
     # via ipython
-pickleshare==0.7.5
-    # via ipython
 pillow==10.1.0
     # via
     #   -c hail/hail/python/dev/../pinned-requirements.txt
@@ -330,7 +324,7 @@ pluggy==1.3.0
     # via pytest
 pre-commit==3.5.0
     # via -r hail/hail/python/dev/requirements.txt
-prometheus-client==0.17.1
+prometheus-client==0.18.0
     # via jupyter-server
 prompt-toolkit==3.0.39
     # via
@@ -362,7 +356,7 @@ pygments==2.16.1
     #   sphinx
 pylint==2.17.7
     # via -r hail/hail/python/dev/requirements.txt
-pyright==1.1.333
+pyright==1.1.334
     # via -r hail/hail/python/dev/requirements.txt
 pytest==7.4.3
     # via
@@ -411,7 +405,7 @@ pyzmq==25.1.1
     #   jupyter-console
     #   jupyter-server
     #   qtconsole
-qtconsole==5.4.4
+qtconsole==5.5.0
     # via jupyter
 qtpy==2.4.1
     # via qtconsole
@@ -433,11 +427,11 @@ rfc3986-validator==0.1.1
     # via
     #   jsonschema
     #   jupyter-events
-rpds-py==0.10.6
+rpds-py==0.12.0
     # via
     #   jsonschema
     #   referencing
-ruff==0.1.2
+ruff==0.1.4
     # via -r hail/hail/python/dev/requirements.txt
 send2trash==1.8.2
     # via jupyter-server
@@ -503,7 +497,7 @@ tomli==2.0.1
     #   jupyterlab
     #   pylint
     #   pytest
-tomlkit==0.12.1
+tomlkit==0.12.2
     # via pylint
 tornado==6.3.3
     # via
@@ -514,7 +508,7 @@ tornado==6.3.3
     #   jupyterlab
     #   notebook
     #   terminado
-traitlets==5.12.0
+traitlets==5.13.0
     # via
     #   comm
     #   ipykernel
@@ -576,7 +570,7 @@ virtualenv==20.24.6
     # via pre-commit
 watchfiles==0.21.0
     # via aiohttp-devtools
-wcwidth==0.2.8
+wcwidth==0.2.9
     # via prompt-toolkit
 webcolors==1.13
     # via jsonschema
@@ -586,7 +580,7 @@ webencodings==0.5.1
     #   tinycss2
 websocket-client==1.6.4
     # via jupyter-server
-wheel==0.41.2
+wheel==0.41.3
     # via -r hail/hail/python/dev/requirements.txt
 widgetsnbextension==4.0.9
     # via ipywidgets
diff --git a/hail/python/hailtop/config/deploy_config.py b/hail/python/hailtop/config/deploy_config.py
index f13a00a57ef..d6b66c7adc3 100644
--- a/hail/python/hailtop/config/deploy_config.py
+++ b/hail/python/hailtop/config/deploy_config.py
@@ -62,7 +62,7 @@ def with_default_namespace(self, default_namespace):
     def with_location(self, location):
         return DeployConfig(location, self._default_namespace, self._domain)
 
-    def default_namespace(self):
+    def default_namespace(self) -> str:
         return self._default_namespace
 
     def location(self):
diff --git a/hail/python/hailtop/pinned-requirements.txt b/hail/python/hailtop/pinned-requirements.txt
index fe82b17a3b9..220a3440e19 100644
--- a/hail/python/hailtop/pinned-requirements.txt
+++ b/hail/python/hailtop/pinned-requirements.txt
@@ -22,7 +22,7 @@ azure-core==1.29.5
     #   azure-mgmt-core
     #   azure-storage-blob
     #   msrest
-azure-identity==1.14.1
+azure-identity==1.15.0
     # via -r hail/hail/python/hailtop/requirements.txt
 azure-mgmt-core==1.4.0
     # via azure-mgmt-storage
@@ -30,14 +30,14 @@ azure-mgmt-storage==20.1.0
     # via -r hail/hail/python/hailtop/requirements.txt
 azure-storage-blob==12.18.3
     # via -r hail/hail/python/hailtop/requirements.txt
-boto3==1.28.69
+boto3==1.28.80
     # via -r hail/hail/python/hailtop/requirements.txt
-botocore==1.31.69
+botocore==1.31.80
     # via
     #   -r hail/hail/python/hailtop/requirements.txt
     #   boto3
     #   s3transfer
-cachetools==5.3.1
+cachetools==5.3.2
     # via google-auth
 certifi==2023.7.22
     # via
@@ -47,7 +47,7 @@ cffi==1.16.0
     # via
     #   cryptography
     #   pycares
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   aiohttp
     #   requests
@@ -68,7 +68,7 @@ frozenlist==1.4.0
     #   -r hail/hail/python/hailtop/requirements.txt
     #   aiohttp
     #   aiosignal
-google-auth==2.23.3
+google-auth==2.23.4
     # via
     #   -r hail/hail/python/hailtop/requirements.txt
     #   google-auth-oauthlib
@@ -92,7 +92,7 @@ jmespath==1.0.1
     #   botocore
 jproperties==2.1.1
     # via -r hail/hail/python/hailtop/requirements.txt
-msal==1.24.1
+msal==1.25.0
     # via
     #   azure-identity
     #   msal-extensions
@@ -108,7 +108,7 @@ nest-asyncio==1.5.8
     # via -r hail/hail/python/hailtop/requirements.txt
 oauthlib==3.2.2
     # via requests-oauthlib
-orjson==3.9.9
+orjson==3.9.10
     # via -r hail/hail/python/hailtop/requirements.txt
 portalocker==2.8.2
     # via msal-extensions
diff --git a/hail/python/pinned-requirements.txt b/hail/python/pinned-requirements.txt
index c50706333a6..db965402254 100644
--- a/hail/python/pinned-requirements.txt
+++ b/hail/python/pinned-requirements.txt
@@ -20,8 +20,6 @@ async-timeout==4.0.3
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   aiohttp
-asyncinit==0.2.4
-    # via -r hail/hail/python/requirements.txt
 attrs==23.1.0
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
@@ -39,7 +37,7 @@ azure-core==1.29.5
     #   azure-mgmt-core
     #   azure-storage-blob
     #   msrest
-azure-identity==1.14.1
+azure-identity==1.15.0
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   -r hail/hail/python/hailtop/requirements.txt
@@ -57,17 +55,17 @@ azure-storage-blob==12.18.3
     #   -r hail/hail/python/hailtop/requirements.txt
 bokeh==3.3.0
     # via -r hail/hail/python/requirements.txt
-boto3==1.28.69
+boto3==1.28.80
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   -r hail/hail/python/hailtop/requirements.txt
-botocore==1.31.69
+botocore==1.31.80
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   -r hail/hail/python/hailtop/requirements.txt
     #   boto3
     #   s3transfer
-cachetools==5.3.1
+cachetools==5.3.2
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   google-auth
@@ -81,7 +79,7 @@ cffi==1.16.0
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   cryptography
     #   pycares
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   aiohttp
@@ -94,7 +92,7 @@ commonmark==0.9.1
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   rich
-contourpy==1.1.1
+contourpy==1.2.0
     # via bokeh
 cryptography==41.0.5
     # via
@@ -115,10 +113,9 @@ frozenlist==1.4.0
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   -r hail/hail/python/hailtop/requirements.txt
-    #   -r hail/hail/python/requirements.txt
     #   aiohttp
     #   aiosignal
-google-auth==2.23.3
+google-auth==2.23.4
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   -r hail/hail/python/hailtop/requirements.txt
@@ -158,7 +155,7 @@ jproperties==2.1.1
     #   -r hail/hail/python/hailtop/requirements.txt
 markupsafe==2.1.3
     # via jinja2
-msal==1.24.1
+msal==1.25.0
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   azure-identity
@@ -191,7 +188,7 @@ oauthlib==3.2.2
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   requests-oauthlib
-orjson==3.9.9
+orjson==3.9.10
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   -r hail/hail/python/hailtop/requirements.txt
@@ -199,7 +196,7 @@ packaging==23.2
     # via
     #   bokeh
     #   plotly
-pandas==2.1.1
+pandas==2.1.2
     # via
     #   -r hail/hail/python/requirements.txt
     #   bokeh
@@ -207,7 +204,7 @@ parsimonious==0.10.0
     # via -r hail/hail/python/requirements.txt
 pillow==10.1.0
     # via bokeh
-plotly==5.17.0
+plotly==5.18.0
     # via -r hail/hail/python/requirements.txt
 portalocker==2.8.2
     # via
@@ -335,7 +332,7 @@ uvloop==0.19.0 ; sys_platform != "win32"
     #   -r hail/hail/python/hailtop/requirements.txt
 wrapt==1.15.0
     # via deprecated
-xyzservices==2023.10.0
+xyzservices==2023.10.1
     # via bokeh
 yarl==1.9.2
     # via
diff --git a/hail/python/requirements.txt b/hail/python/requirements.txt
index 35f4a3edfcf..9f8492a2a07 100644
--- a/hail/python/requirements.txt
+++ b/hail/python/requirements.txt
@@ -1,12 +1,10 @@
 -c hailtop/pinned-requirements.txt
 -r hailtop/requirements.txt
 
-asyncinit>=0.2.4,<0.3
 avro>=1.10,<1.12
 bokeh>=3,<4
 decorator<5
 Deprecated>=1.2.10,<1.3
-frozenlist>=1.3.1,<2
 numpy<2
 pandas>=2,<3
 parsimonious<1
diff --git a/web_common/pinned-requirements.txt b/web_common/pinned-requirements.txt
index 217f030afc6..7fa1d0ff664 100644
--- a/web_common/pinned-requirements.txt
+++ b/web_common/pinned-requirements.txt
@@ -30,7 +30,7 @@ attrs==23.1.0
     #   -c hail/web_common/../hail/python/dev/pinned-requirements.txt
     #   -c hail/web_common/../hail/python/pinned-requirements.txt
     #   aiohttp
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via
     #   -c hail/web_common/../gear/pinned-requirements.txt
     #   -c hail/web_common/../hail/python/dev/pinned-requirements.txt