[Bugfix][CPU] fix missing input intermediate_tensors in the cpu_model_runner (#8733)

2024-09-23 21:15:16 +08:00 · 2024-09-23 21:15:16 +08:00 · 3e83c12b5c
commit 3e83c12b5c
parent e551ca1555
1 changed files with 2 additions and 0 deletions
--- a/vllm/worker/cpu_model_runner.py
+++ b/vllm/worker/cpu_model_runner.py
@ -456,6 +456,8 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
            model_input.attn_metadata,
            **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
                                         device=self.device),
+            "intermediate_tensors":
+            intermediate_tensors,
        }

        hidden_states = model_executable(**execute_model_kwargs)