-
-
Notifications
You must be signed in to change notification settings - Fork 4.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Frontend] Factor out code for running uvicorn (#6828)
- Loading branch information
1 parent
d09b94c
commit 981b0d5
Showing
4 changed files
with
116 additions
and
75 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
from .launch import serve_http | ||
|
||
__all__ = ["serve_http"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
import asyncio | ||
import signal | ||
from typing import Any | ||
|
||
import uvicorn | ||
from fastapi import FastAPI | ||
|
||
from vllm.logger import init_logger | ||
|
||
logger = init_logger(__name__) | ||
|
||
|
||
async def serve_http(app: FastAPI, **uvicorn_kwargs: Any) -> None: | ||
logger.info("Available routes are:") | ||
for route in app.routes: | ||
methods = getattr(route, "methods", None) | ||
path = getattr(route, "path", None) | ||
|
||
if methods is None or path is None: | ||
continue | ||
|
||
logger.info("Route: %s, Methods: %s", path, ', '.join(methods)) | ||
|
||
config = uvicorn.Config(app, **uvicorn_kwargs) | ||
server = uvicorn.Server(config) | ||
|
||
loop = asyncio.get_running_loop() | ||
|
||
server_task = loop.create_task(server.serve()) | ||
|
||
def signal_handler() -> None: | ||
# prevents the uvicorn signal handler to exit early | ||
server_task.cancel() | ||
|
||
loop.add_signal_handler(signal.SIGINT, signal_handler) | ||
loop.add_signal_handler(signal.SIGTERM, signal_handler) | ||
|
||
try: | ||
await server_task | ||
except asyncio.CancelledError: | ||
logger.info("Gracefully stopping http server") | ||
await server.shutdown() |