squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Model] Add starcoder2 awq support (#3569)

Browse Source
This commit is contained in:
少年 2024-03-25 12:07:36 +08:00 committed by GitHub
parent 56a8652f33
commit b0dfa91dd7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/model_executor/models/starcoder2.py
View File

@ -141,8 +141,9 @@ class Starcoder2MLP(nn.Module):
bias=config.use_bias, bias=config.use_bias,
linear_method=linear_method, linear_method=linear_method,
) )
self.act = get_act_fn(config.hidden_act, quant_config = getattr(linear_method, "quant_config", None)
intermediate_size=config.intermediate_size) self.act = get_act_fn(config.hidden_act, quant_config,
config.intermediate_size)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states, _ = self.c_fc(hidden_states) hidden_states, _ = self.c_fc(hidden_states)