squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Better error message for MLPSpeculator when num_speculative_tokens is set too high (#5894)

Browse Source 
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
This commit is contained in:
Thomas Parnell 2024-06-28 16:42:17 +02:00 committed by GitHub
parent 57f09a419c
commit ec1ad0046c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/config.py
View File

@ -956,9 +956,9 @@ class SpeculativeConfig:
# Verify provided value doesn't exceed the maximum # Verify provided value doesn't exceed the maximum
# supported by the draft model. # supported by the draft model.
raise ValueError( raise ValueError(
"Expected both speculative_model and " "This speculative model supports a maximum of "
"num_speculative_tokens to be provided, but found " f"num_speculative_tokens={n_predict}, but "
f"{speculative_model=} and {num_speculative_tokens=}.") f"{num_speculative_tokens=} was provided.")
draft_model_config.max_model_len = ( draft_model_config.max_model_len = (
SpeculativeConfig._maybe_override_draft_max_model_len( SpeculativeConfig._maybe_override_draft_max_model_len(