From b0dfa91dd77e954be71e1e21ef8a4041a0e98cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=91=E5=B9=B4?= <48116214+shaonianyr@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:07:36 +0800 Subject: [PATCH] [Model] Add starcoder2 awq support (#3569) --- vllm/model_executor/models/starcoder2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/starcoder2.py b/vllm/model_executor/models/starcoder2.py index e5003361..e72c5cf1 100644 --- a/vllm/model_executor/models/starcoder2.py +++ b/vllm/model_executor/models/starcoder2.py @@ -141,8 +141,9 @@ class Starcoder2MLP(nn.Module): bias=config.use_bias, linear_method=linear_method, ) - self.act = get_act_fn(config.hidden_act, - intermediate_size=config.intermediate_size) + quant_config = getattr(linear_method, "quant_config", None) + self.act = get_act_fn(config.hidden_act, quant_config, + config.intermediate_size) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: hidden_states, _ = self.c_fc(hidden_states)