From 1f55e0571350f3dd2c04638e13e52d8ed557d93e Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Tue, 12 Nov 2024 13:39:56 -0800 Subject: [PATCH] [V1] Enable Inductor when using piecewise CUDA graphs (#10268) Signed-off-by: Woosuk Kwon --- vllm/v1/worker/gpu_model_runner.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 2c408537..db676e28 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -404,14 +404,17 @@ class GPUModelRunner: def load_model(self) -> None: if self.use_cuda_graph: - # FIXME(woosuk): Currently, we do not use inductor to reduce the - # compilation time and any potential issues with the inductor. - os.environ["VLLM_CUSTOM_OPS"] = "all" + # NOTE(woosuk): Currently, we use inductor because the piecewise + # CUDA graphs do not work properly with the custom CUDA kernels. + # FIXME(woosuk): Disable inductor to reduce the compilation time + # and avoid any potential issues with the inductor. + os.environ["VLLM_CUSTOM_OPS"] = "none" set_compilation_config( CompilationConfig( use_cudagraph=True, non_cudagraph_ops=["vllm.unified_v1_flash_attention"], - use_inductor=False, + use_inductor=True, + enable_fusion=False, )) logger.info("Starting to load model %s...", self.model_config.model)