squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Misc] Medusa supports custom bias (#10361)

Browse Source
This commit is contained in:
Sky Lee 2024-11-17 00:33:01 +08:00 committed by GitHub
parent 8b6725b0cf
commit b98d89efd4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
vllm/model_executor/models/medusa.py
View File

@ -14,11 +14,14 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
class ResidualBlock(nn.Module): class ResidualBlock(nn.Module):
def __init__(self, hidden_size: int, num_layers: int) -> None: def __init__(self, config: VllmConfig, hidden_size: int,
num_layers: int) -> None:
super().__init__() super().__init__()
self.layers = nn.ModuleList([ self.layers = nn.ModuleList([
nn.Linear(hidden_size, hidden_size, bias=False) nn.Linear(hidden_size,
hidden_size,
bias=getattr(config, "medusa_fc_bias", False))
for _ in range(num_layers) for _ in range(num_layers)
]) ])
self.act = nn.SiLU() self.act = nn.SiLU()
@ -49,7 +52,8 @@ class Medusa(nn.Module):
super().__init__() super().__init__()
self.config = config self.config = config
self.blocks = nn.ModuleList([ self.blocks = nn.ModuleList([
ResidualBlock(hidden_size=self.config.hidden_size, ResidualBlock(config=config,
hidden_size=self.config.hidden_size,
num_layers=self.config.num_hidden_layers) num_layers=self.config.num_hidden_layers)
for _ in range(self.config.num_heads) for _ in range(self.config.num_heads)
]) ])