squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Suggest using dtype=half when OOM.

Browse Source
This commit is contained in:
Ben 2024-01-13 04:33:29 +08:00 committed by GitHub
parent 7878958c0d
commit cb7a1c1cbf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/worker/worker.py
View File

@ -239,4 +239,6 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
raise ValueError( raise ValueError(
"Bfloat16 is only supported on GPUs with compute capability " "Bfloat16 is only supported on GPUs with compute capability "
f"of at least 8.0. Your {gpu_name} GPU has compute capability " f"of at least 8.0. Your {gpu_name} GPU has compute capability "
f"{compute_capability[0]}.{compute_capability[1]}.") f"{compute_capability[0]}.{compute_capability[1]}."
f" You can explicitly specify the data type by using the --dtype option, for example: --dtype=half."
)