squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: incorrect bigcode attention heads num (#676)

Browse Source
This commit is contained in:
Wen Sun 2023-08-05 01:35:22 +08:00 committed by GitHub
parent aa84c92ef6
commit 621980bdc0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
vllm/config.py
View File

@ -98,9 +98,11 @@ class ModelConfig:
# Note: for falcon, when new_decoder_architecture is True, the # Note: for falcon, when new_decoder_architecture is True, the
# multi_query flag is ignored and we use n_head_kv for the number of # multi_query flag is ignored and we use n_head_kv for the number of
# KV heads. # KV heads.
if (getattr(self.hf_config, "multi_query", False) and new_decoder_arch_falcon = (
(self.hf_config.model_type == "falcon" and self.hf_config.model_type == "falcon"
not getattr(self.hf_config, "new_decoder_architecture", False))): and getattr(self.hf_config, "new_decoder_architecture", False))
if not new_decoder_arch_falcon and getattr(self.hf_config,
"multi_query", False):
# Multi-query attention, only one KV head. # Multi-query attention, only one KV head.
return 1 return 1
# For Falcon: # For Falcon: