squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix OpenAI server completion_tokens referenced before assignment (#1996)

Browse Source
This commit is contained in:
Jin Shang 2023-12-10 13:01:21 +08:00 committed by GitHub
parent fe470ae5ad
commit 1aa1361510
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
vllm/entrypoints/openai/api_server.py
View File

@ -332,8 +332,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
# Send token-by-token response for each request.n # Send token-by-token response for each request.n
delta_text = output.text[len(previous_texts[i]):] delta_text = output.text[len(previous_texts[i]):]
previous_texts[i] = output.text previous_texts[i] = output.text
completion_tokens = len(output.token_ids) previous_num_tokens[i] = len(output.token_ids)
previous_num_tokens[i] = completion_tokens
choice_data = ChatCompletionResponseStreamChoice( choice_data = ChatCompletionResponseStreamChoice(
index=i, index=i,
delta=DeltaMessage(content=delta_text), delta=DeltaMessage(content=delta_text),
@ -351,8 +350,8 @@ async def create_chat_completion(request: ChatCompletionRequest,
prompt_tokens = len(res.prompt_token_ids) prompt_tokens = len(res.prompt_token_ids)
final_usage = UsageInfo( final_usage = UsageInfo(
prompt_tokens=prompt_tokens, prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens, completion_tokens=previous_num_tokens[i],
total_tokens=prompt_tokens + completion_tokens, total_tokens=prompt_tokens + previous_num_tokens[i],
) )
choice_data = ChatCompletionResponseStreamChoice( choice_data = ChatCompletionResponseStreamChoice(
index=i, delta=[], finish_reason=output.finish_reason) index=i, delta=[], finish_reason=output.finish_reason)