squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[ Bugfix ] Fix AutoFP8 fp8 marlin (#6609)

Browse Source
This commit is contained in:
Robert Shaw 2024-07-20 19:25:56 -04:00 committed by GitHub
parent f952bbc8ff
commit 082ecd80d5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
View File

@ -76,7 +76,8 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
# WEIGHT SCALES # WEIGHT SCALES
# Currently Marlin doesn't support per-tensor scales, so we # Currently Marlin doesn't support per-tensor scales, so we
# expand it to channelwise # expand it to channelwise
is_channelwise = layer.weight_scale.shape[0] == part_size_n is_channelwise = (len(layer.weight_scale.shape) > 0
and layer.weight_scale.shape[0] == part_size_n)
if is_channelwise: if is_channelwise:
scales = layer.weight_scale scales = layer.weight_scale
else: else: