squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix completion API echo and logprob combo (#1992)

Browse Source
This commit is contained in:
Simon Mo 2023-12-10 13:20:30 -08:00 committed by GitHub
parent dacaf5a400
commit 2e8fc0d4c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
vllm/entrypoints/openai/api_server.py
View File

@ -567,16 +567,21 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
i = output.index i = output.index
delta_text = output.text[len(previous_texts[i]):] delta_text = output.text[len(previous_texts[i]):]
token_ids = output.token_ids[previous_num_tokens[i]:] token_ids = output.token_ids[previous_num_tokens[i]:]
if request.logprobs is not None:
top_logprobs = output.logprobs[previous_num_tokens[i]:] top_logprobs = output.logprobs[previous_num_tokens[i]:]
else:
top_logprobs = None
offsets = len(previous_texts[i]) offsets = len(previous_texts[i])
if request.echo and not has_echoed[i]: if request.echo and not has_echoed[i]:
if not echo_without_generation: if not echo_without_generation:
delta_text = res.prompt + delta_text delta_text = res.prompt + delta_text
token_ids = res.prompt_token_ids + token_ids token_ids = res.prompt_token_ids + token_ids
if top_logprobs:
top_logprobs = res.prompt_logprobs + top_logprobs top_logprobs = res.prompt_logprobs + top_logprobs
else: else: # only just return the prompt
delta_text = res.prompt delta_text = res.prompt
token_ids = res.prompt_token_ids token_ids = res.prompt_token_ids
if top_logprobs:
top_logprobs = res.prompt_logprobs top_logprobs = res.prompt_logprobs
has_echoed[i] = True has_echoed[i] = True
if request.logprobs is not None: if request.logprobs is not None: