squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix error when tp > 1 (#2644)

Browse Source 
Co-authored-by: zhaoyang-star <zhao.yang16@zte.com.cn>
This commit is contained in:
zhaoyang-star 2024-01-29 14:47:39 +08:00 committed by GitHub
parent 9090bf02e7
commit b72af8f1ed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/engine/llm_engine.py
View File

@ -236,7 +236,6 @@ class LLMEngine:
model_config = copy.deepcopy(self.model_config) model_config = copy.deepcopy(self.model_config)
parallel_config = copy.deepcopy(self.parallel_config) parallel_config = copy.deepcopy(self.parallel_config)
scheduler_config = copy.deepcopy(self.scheduler_config) scheduler_config = copy.deepcopy(self.scheduler_config)
cache_config = copy.deepcopy(self.cache_config)
for rank, (worker, (node_id, for rank, (worker, (node_id,
_)) in enumerate(zip(self.workers, _)) in enumerate(zip(self.workers,
@ -252,7 +251,7 @@ class LLMEngine:
rank, rank,
distributed_init_method, distributed_init_method,
lora_config=self.lora_config, lora_config=self.lora_config,
cache_config=cache_config, kv_cache_dtype=self.cache_config.cache_dtype,
)) ))
driver_rank = 0 driver_rank = 0
@ -265,7 +264,7 @@ class LLMEngine:
driver_rank, driver_rank,
distributed_init_method, distributed_init_method,
lora_config=self.lora_config, lora_config=self.lora_config,
cache_config=cache_config, kv_cache_dtype=self.cache_config.cache_dtype,
is_driver_worker=True, is_driver_worker=True,
) )