squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Frontend] Fix tcp port reservation for api server (#10012)

Browse Source 
Signed-off-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
Russell Bryant 2024-11-05 10:50:57 -05:00 committed by GitHub
parent 93dee88f6b
commit 5952d81139
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/entrypoints/openai/api_server.py
View File

@ -569,7 +569,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
# This avoids race conditions with ray. # This avoids race conditions with ray.
# see https://github.com/vllm-project/vllm/issues/8204 # see https://github.com/vllm-project/vllm/issues/8204
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("", args.port)) sock.bind((args.host or "", args.port))
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
def signal_handler(*_) -> None: def signal_handler(*_) -> None:
# Interrupt server on sigterm while initializing # Interrupt server on sigterm while initializing
@ -593,13 +594,14 @@ async def run_server(args, **uvicorn_kwargs) -> None:
ssl_certfile=args.ssl_certfile, ssl_certfile=args.ssl_certfile,
ssl_ca_certs=args.ssl_ca_certs, ssl_ca_certs=args.ssl_ca_certs,
ssl_cert_reqs=args.ssl_cert_reqs, ssl_cert_reqs=args.ssl_cert_reqs,
fd=sock.fileno(),
**uvicorn_kwargs, **uvicorn_kwargs,
) )
# NB: Await server shutdown only after the backend context is exited # NB: Await server shutdown only after the backend context is exited
await shutdown_task await shutdown_task
sock.close()
if __name__ == "__main__": if __name__ == "__main__":
# NOTE(simon): # NOTE(simon):