squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Fix layer skip logic with bitsandbytes (#9887)

Browse Source 
Signed-off-by: mgoin <michael@neuralmagic.com>
This commit is contained in:
Michael Goin 2024-11-01 01:12:44 -04:00 committed by GitHub
parent 96e0c9cbbd
commit 37a4947dcd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
vllm/model_executor/layers/quantization/bitsandbytes.py
View File

@ -119,7 +119,12 @@ class BitsAndBytesConfig(QuantizationConfig):
def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]): def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
return any(module_name in prefix for module_name in llm_int8_skip_modules) # Split the prefix into its dot-separated components
components = prefix.split('.')
# Check if any of the skip modules exactly matches any component
return any(module_name in components
for module_name in llm_int8_skip_modules)
class BitsAndBytesLinearMethod(LinearMethodBase): class BitsAndBytesLinearMethod(LinearMethodBase):