squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Use head_dim in config if exists (#2622)

Browse Source
This commit is contained in:
Xiang Xu 2024-01-27 10:30:49 -08:00 committed by GitHub
parent beb89f68b4
commit 220a47627b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/config.py
View File

@ -213,6 +213,8 @@ class ModelConfig:
return self.hf_config.hidden_size return self.hf_config.hidden_size
def get_head_size(self) -> int: def get_head_size(self) -> int:
if hasattr(self.hf_config, "head_dim"):
return self.hf_config.head_dim
# FIXME(woosuk): This may not be true for all models. # FIXME(woosuk): This may not be true for all models.
return self.hf_config.hidden_size // self.hf_config.num_attention_heads return self.hf_config.hidden_size // self.hf_config.num_attention_heads