squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[V1] Use custom ops for piecewise CUDA graphs (#10227)

Browse Source 
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon 2024-11-11 11:26:48 -08:00 committed by GitHub
parent 330e82d34a
commit fe15729a2b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 0 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/v1/worker/gpu_model_runner.py
View File

@ -1,4 +1,3 @@
import os
import time import time
from dataclasses import dataclass from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Set from typing import TYPE_CHECKING, Dict, List, Optional, Set
@ -406,7 +405,6 @@ class GPUModelRunner:
if self.use_cuda_graph: if self.use_cuda_graph:
# FIXME(woosuk): Currently, we do not use inductor to reduce the # FIXME(woosuk): Currently, we do not use inductor to reduce the
# compilation time and any potential issues with the inductor. # compilation time and any potential issues with the inductor.
os.environ["VLLM_CUSTOM_OPS"] = "none"
set_compilation_config( set_compilation_config(
CompilationConfig( CompilationConfig(
use_cudagraph=True, use_cudagraph=True,