squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix] Fix lora loading for Compressed Tensors in #9120 (#9179)

Browse Source
This commit is contained in:
Ahmad Fahadh Ilyas 2024-10-09 05:10:44 -07:00 committed by GitHub
parent dc4aea677a
commit 21906a6f50
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/lora/layers.py
View File

@ -39,6 +39,9 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
# unquantizedLinear # unquantizedLinear
if hasattr(base_layer, "weight"): if hasattr(base_layer, "weight"):
return base_layer.weight.device return base_layer.weight.device
# Compressed Tensor
elif hasattr(base_layer, "weight_packed"):
return base_layer.weight_packed.device
# GPTQ/AWQ # GPTQ/AWQ
elif hasattr(base_layer, "qweight"): elif hasattr(base_layer, "qweight"):
return base_layer.qweight.device return base_layer.qweight.device