Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[hailtop][batch] update aiohttp to 3.9.X, remove aiomonitor #14040

Merged
merged 4 commits into from
Nov 29, 2023
Merged

[hailtop][batch] update aiohttp to 3.9.X, remove aiomonitor #14040

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d6dd9d3
[hailtop][batch] update aiohttp to 3.9.X, remove aiomonitor
Nov 28, 2023
86144e6
fix typing
Nov 28, 2023
a7ba560
sort the imports
Nov 28, 2023
7ee1bae
fix set is list
Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 18 additions & 20 deletions batch/batch/worker/worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import aiodocker.images
import aiohttp
import aiohttp.client_exceptions
import aiomonitor
import aiorwlock
import async_timeout
import orjson
Expand Down Expand Up @@ -3473,25 +3472,24 @@ async def async_main():
await network_allocator.reserve()

worker = Worker(httpx.client_session())
with aiomonitor.start_monitor(asyncio.get_event_loop(), locals=locals()):
try:
async with AsyncExitStack() as cleanup:
cleanup.push_async_callback(docker.close)
cleanup.push_async_callback(network_allocator_task_manager.shutdown_and_wait)
cleanup.push_async_callback(CLOUD_WORKER_API.close)
cleanup.push_async_callback(worker.shutdown)
await worker.run()
finally:
asyncio.get_event_loop().set_debug(True)
other_tasks = [t for t in asyncio.all_tasks() if t != asyncio.current_task()]
if other_tasks:
log.warning('Tasks immediately after docker close')
dump_all_stacktraces()
_, pending = await asyncio.wait(other_tasks, timeout=10 * 60, return_when=asyncio.ALL_COMPLETED)
for t in pending:
log.warning('Dangling task:')
t.print_stack()
t.cancel()
try:
async with AsyncExitStack() as cleanup:
cleanup.push_async_callback(docker.close)
cleanup.push_async_callback(network_allocator_task_manager.shutdown_and_wait)
cleanup.push_async_callback(CLOUD_WORKER_API.close)
cleanup.push_async_callback(worker.shutdown)
await worker.run()
finally:
asyncio.get_event_loop().set_debug(True)
other_tasks = [t for t in asyncio.all_tasks() if t != asyncio.current_task()]
if other_tasks:
log.warning('Tasks immediately after docker close')
dump_all_stacktraces()
_, pending = await asyncio.wait(other_tasks, timeout=10 * 60, return_when=asyncio.ALL_COMPLETED)
for t in pending:
log.warning('Dangling task:')
t.print_stack()
t.cancel()


loop = asyncio.get_event_loop()
Expand Down
18 changes: 5 additions & 13 deletions batch/pinned-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#
aiodocker==0.21.0
# via -r hail/batch/requirements.txt
aiohttp==3.8.6
aiohttp==3.9.1
# via
# -c hail/batch/../gear/pinned-requirements.txt
# -c hail/batch/../hail/python/dev/pinned-requirements.txt
Expand Down Expand Up @@ -37,13 +37,6 @@ attrs==23.1.0
# -c hail/batch/../hail/python/pinned-requirements.txt
# -c hail/batch/../web_common/pinned-requirements.txt
# aiohttp
charset-normalizer==3.3.2
# via
# -c hail/batch/../gear/pinned-requirements.txt
# -c hail/batch/../hail/python/dev/pinned-requirements.txt
# -c hail/batch/../hail/python/pinned-requirements.txt
# -c hail/batch/../web_common/pinned-requirements.txt
# aiohttp
dictdiffer==0.9.0
# via -r hail/batch/requirements.txt
frozenlist==1.4.0
Expand All @@ -54,7 +47,7 @@ frozenlist==1.4.0
# -c hail/batch/../web_common/pinned-requirements.txt
# aiohttp
# aiosignal
idna==3.4
idna==3.6
# via
# -c hail/batch/../gear/pinned-requirements.txt
# -c hail/batch/../hail/python/dev/pinned-requirements.txt
Expand All @@ -69,7 +62,7 @@ multidict==6.0.4
# -c hail/batch/../web_common/pinned-requirements.txt
# aiohttp
# yarl
numpy==1.26.1
numpy==1.26.2
# via
# -c hail/batch/../hail/python/pinned-requirements.txt
# pandas
Expand All @@ -78,7 +71,7 @@ packaging==23.2
# -c hail/batch/../hail/python/dev/pinned-requirements.txt
# -c hail/batch/../hail/python/pinned-requirements.txt
# plotly
pandas==2.1.2
pandas==2.1.3
# via
# -c hail/batch/../hail/python/pinned-requirements.txt
# -r hail/batch/requirements.txt
Expand Down Expand Up @@ -108,15 +101,14 @@ tenacity==8.2.3
# plotly
typing-extensions==4.8.0
# via
# -c hail/batch/../gear/pinned-requirements.txt
# -c hail/batch/../hail/python/dev/pinned-requirements.txt
# -c hail/batch/../hail/python/pinned-requirements.txt
# aiodocker
tzdata==2023.3
# via
# -c hail/batch/../hail/python/pinned-requirements.txt
# pandas
yarl==1.9.2
yarl==1.9.3
# via
# -c hail/batch/../gear/pinned-requirements.txt
# -c hail/batch/../hail/python/dev/pinned-requirements.txt
Expand Down
6 changes: 3 additions & 3 deletions benchmark/python/pinned-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,15 @@ contourpy==1.2.0
# matplotlib
cycler==0.12.1
# via matplotlib
fonttools==4.44.0
fonttools==4.45.1
# via matplotlib
importlib-resources==6.1.1
# via matplotlib
kiwisolver==1.4.5
# via matplotlib
matplotlib==3.8.1
matplotlib==3.8.2
# via -r hail/benchmark/python/requirements.txt
numpy==1.26.1
numpy==1.26.2
# via
# -c hail/benchmark/python/../../hail/python/pinned-requirements.txt
# contourpy
Expand Down
6 changes: 3 additions & 3 deletions ci/ci/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import logging
from collections import Counter, defaultdict
from shlex import quote as shq
from typing import Dict, List, Optional, Sequence, TypedDict
from typing import Dict, List, Optional, Sequence, Set, TypedDict

import jinja2
import yaml
Expand Down Expand Up @@ -217,8 +217,8 @@ def deps_parents(self):
return flatten([d.wrapped_job() for d in self.deps])

def all_deps(self):
visited = set([self])
frontier = [self]
visited: Set[Step] = set([self])
frontier: List[Step] = [self]

while frontier:
current = frontier.pop()
Expand Down
15 changes: 4 additions & 11 deletions ci/pinned-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#
# pip-compile --output-file=hail/ci/pinned-requirements.txt hail/ci/requirements.txt
#
certifi==2023.7.22
certifi==2023.11.17
# via
# -c hail/ci/../gear/pinned-requirements.txt
# -c hail/ci/../hail/python/dev/pinned-requirements.txt
Expand All @@ -20,31 +20,27 @@ charset-normalizer==3.3.2
# -c hail/ci/../gear/pinned-requirements.txt
# -c hail/ci/../hail/python/dev/pinned-requirements.txt
# -c hail/ci/../hail/python/pinned-requirements.txt
# -c hail/ci/../web_common/pinned-requirements.txt
# requests
click==8.1.7
# via
# -c hail/ci/../gear/pinned-requirements.txt
# -c hail/ci/../hail/python/dev/pinned-requirements.txt
# -c hail/ci/../hail/python/pinned-requirements.txt
# zulip
cryptography==41.0.5
cryptography==41.0.7
# via
# -c hail/ci/../hail/python/pinned-requirements.txt
# pyjwt
distro==1.8.0
# via zulip
gidgethub==5.3.0
# via -r hail/ci/requirements.txt
idna==3.4
idna==3.6
# via
# -c hail/ci/../gear/pinned-requirements.txt
# -c hail/ci/../hail/python/dev/pinned-requirements.txt
# -c hail/ci/../hail/python/pinned-requirements.txt
# -c hail/ci/../web_common/pinned-requirements.txt
# requests
matrix-client==0.4.0
# via zulip
pycparser==2.21
# via
# -c hail/ci/../hail/python/dev/pinned-requirements.txt
Expand All @@ -59,11 +55,9 @@ requests[security]==2.31.0
# -c hail/ci/../gear/pinned-requirements.txt
# -c hail/ci/../hail/python/dev/pinned-requirements.txt
# -c hail/ci/../hail/python/pinned-requirements.txt
# matrix-client
# zulip
typing-extensions==4.8.0
# via
# -c hail/ci/../gear/pinned-requirements.txt
# -c hail/ci/../hail/python/dev/pinned-requirements.txt
# -c hail/ci/../hail/python/pinned-requirements.txt
# zulip
Expand All @@ -76,7 +70,6 @@ urllib3==1.26.18
# -c hail/ci/../gear/pinned-requirements.txt
# -c hail/ci/../hail/python/dev/pinned-requirements.txt
# -c hail/ci/../hail/python/pinned-requirements.txt
# matrix-client
# requests
zulip==0.8.2
zulip==0.9.0
# via -r hail/ci/requirements.txt
67 changes: 9 additions & 58 deletions gear/pinned-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,15 @@
#
# pip-compile --output-file=hail/gear/pinned-requirements.txt hail/gear/requirements.txt
#
aioconsole==0.6.2
# via aiomonitor
aiohttp==3.8.6
aiohttp==3.9.1
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# aiohttp-session
# aiomonitor
# kubernetes-asyncio
aiohttp-session==2.12.0
# via -r hail/gear/requirements.txt
aiomonitor==0.6.0
# via -r hail/gear/requirements.txt
aiomysql==0.2.0
# via -r hail/gear/requirements.txt
aiosignal==1.3.1
Expand All @@ -38,15 +33,12 @@ attrs==23.1.0
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# aiohttp
# aiomonitor
backports-strenum==1.2.8
# via aiomonitor
cachetools==5.3.2
# via
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# google-auth
certifi==2023.7.22
certifi==2023.11.17
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
Expand All @@ -58,24 +50,17 @@ charset-normalizer==3.3.2
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# aiohttp
# requests
click==8.1.7
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# aiomonitor
frozenlist==1.4.0
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# aiohttp
# aiosignal
google-api-core==2.13.0
google-api-core==2.14.0
# via google-api-python-client
google-api-python-client==2.107.0
google-api-python-client==2.108.0
# via google-cloud-profiler
google-auth==2.23.4
# via
Expand All @@ -97,30 +82,15 @@ httplib2==0.22.0
# via
# google-api-python-client
# google-auth-httplib2
idna==3.4
idna==3.6
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# requests
# yarl
janus==1.0.0
# via
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# aiomonitor
jinja2==3.1.2
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# aiomonitor
kubernetes-asyncio==19.15.1
# via -r hail/gear/requirements.txt
markupsafe==2.1.3
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# jinja2
multidict==6.0.4
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
Expand All @@ -135,22 +105,18 @@ orjson==3.9.10
# -r hail/gear/requirements.txt
prometheus-async==19.2.0
# via -r hail/gear/requirements.txt
prometheus-client==0.18.0
prometheus-client==0.19.0
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -r hail/gear/requirements.txt
# prometheus-async
prompt-toolkit==3.0.39
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# aiomonitor
protobuf==3.20.2
# via
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# google-api-core
# google-cloud-profiler
pyasn1==0.5.0
pyasn1==0.5.1
# via
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
Expand Down Expand Up @@ -203,17 +169,6 @@ sortedcontainers==2.4.0
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# -r hail/gear/requirements.txt
terminaltables==3.1.10
# via aiomonitor
trafaret==2.1.1
# via aiomonitor
typing-extensions==4.8.0
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# aiomonitor
# janus
uritemplate==4.1.1
# via google-api-python-client
urllib3==1.26.18
Expand All @@ -223,16 +178,12 @@ urllib3==1.26.18
# -c hail/gear/../hail/python/pinned-requirements.txt
# kubernetes-asyncio
# requests
wcwidth==0.2.9
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# prompt-toolkit
wrapt==1.15.0
wrapt==1.16.0
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/pinned-requirements.txt
# prometheus-async
yarl==1.9.2
yarl==1.9.3
# via
# -c hail/gear/../hail/python/dev/pinned-requirements.txt
# -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
Expand Down
1 change: 0 additions & 1 deletion gear/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
-c ../hail/python/dev/pinned-requirements.txt

aiohttp_session>=2.7,<2.13
aiomonitor>=0.4.5,<1
aiomysql>=0.0.20,<1
google-cloud-profiler<4.0.0
kubernetes-asyncio>=19.15.1,<20
Expand Down
Loading