squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Misc] Temporarily resolve the error of BitAndBytes (#7308)

Browse Source
This commit is contained in:
Jee Jee Li 2024-08-09 04:42:58 +08:00 committed by GitHub
parent 8334c39f37
commit a049b107e2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/config.py
View File

@ -322,8 +322,9 @@ class ModelConfig:
"BitAndBytes quantization with TP or PP is not supported yet.") "BitAndBytes quantization with TP or PP is not supported yet.")
if self.quantization == "bitsandbytes" and self.enforce_eager is False: if self.quantization == "bitsandbytes" and self.enforce_eager is False:
raise ValueError( logger.warning("CUDA graph is not supported on BitAndBytes yet, "
"BitAndBytes with enforce_eager = False is not supported yet.") "fallback to the eager mode.")
self.enforce_eager = True
def get_hf_config_sliding_window(self) -> Optional[int]: def get_hf_config_sliding_window(self) -> Optional[int]:
"""Get the sliding window size, or None if disabled.""" """Get the sliding window size, or None if disabled."""