squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Speculative decoding] [Bugfix] Fix overallocation in ngram + spec logprobs (#4672)

Browse Source
This commit is contained in:
Cade Daniel 2024-05-08 16:24:46 -07:00 committed by GitHub
parent f942efb5a3
commit 8b9241be3a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/spec_decode/ngram_worker.py
View File

@ -138,7 +138,7 @@ class NGramWorker(LoraNotSupportedWorkerBase):
SamplerOutput( SamplerOutput(
outputs=None, outputs=None,
sampled_token_probs=token_probs[i], sampled_token_probs=token_probs[i],
logprobs=token_logprobs, logprobs=token_logprobs[i],
sampled_token_ids=token_ids[i], sampled_token_ids=token_ids[i],
)) ))
return outputs, False return outputs, False