squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Misc] Fix Benchmark TTFT Calculation for Chat Completions (#3768)

Browse Source
This commit is contained in:
Roger Wang 2024-04-01 15:24:30 -07:00 committed by GitHub
parent 49782fcb76
commit ccb58b23e6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
benchmarks/backend_request_func.py
View File

@ -334,7 +334,8 @@ async def async_request_openai_chat_completions(
timestamp = time.perf_counter() timestamp = time.perf_counter()
data = json.loads(chunk) data = json.loads(chunk)
if "content" in data["choices"][0]["delta"]: delta = data["choices"][0]["delta"]
if delta.get("content", None):
# First token # First token
if ttft == 0: if ttft == 0:
ttft = time.perf_counter() - st ttft = time.perf_counter() - st
@ -345,8 +346,7 @@ async def async_request_openai_chat_completions(
output.itl.append(timestamp - output.itl.append(timestamp -
most_recent_timestamp) most_recent_timestamp)
generated_text += data["choices"][0]["delta"][ generated_text += delta["content"]
"content"]
most_recent_timestamp = timestamp most_recent_timestamp = timestamp