squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Frontend] Avoid creating guided decoding LogitsProcessor unnecessarily (#9521)

Browse Source
This commit is contained in:
Nick Hill 2024-10-19 04:21:01 +01:00 committed by GitHub
parent 380e18639f
commit 1325872ec8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
vllm/sampling_params.py
View File

@ -49,14 +49,17 @@ class GuidedDecodingParams:
@staticmethod @staticmethod
def from_optional( def from_optional(
json: Optional[Union[Dict, BaseModel, str]], json: Optional[Union[Dict, BaseModel, str]] = None,
regex: Optional[str] = None, regex: Optional[str] = None,
choice: Optional[List[str]] = None, choice: Optional[List[str]] = None,
grammar: Optional[str] = None, grammar: Optional[str] = None,
json_object: Optional[bool] = None, json_object: Optional[bool] = None,
backend: Optional[str] = None, backend: Optional[str] = None,
whitespace_pattern: Optional[str] = None, whitespace_pattern: Optional[str] = None,
) -> "GuidedDecodingParams": ) -> Optional["GuidedDecodingParams"]:
if all(arg is None
for arg in (json, regex, choice, grammar, json_object)):
return None
# Extract json schemas from pydantic models # Extract json schemas from pydantic models
if isinstance(json, (BaseModel, type(BaseModel))): if isinstance(json, (BaseModel, type(BaseModel))):
json = json.model_json_schema() json = json.model_json_schema()