From 89b8db6bb2ce2948073c21231f103c76456844da Mon Sep 17 00:00:00 2001 From: Jacob Schein Date: Mon, 5 Aug 2024 16:35:47 -0700 Subject: [PATCH] [Bugfix] Specify device when loading LoRA and embedding tensors (#7129) Co-authored-by: Jacob Schein --- vllm/lora/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/lora/models.py b/vllm/lora/models.py index 017a1002..27947756 100644 --- a/vllm/lora/models.py +++ b/vllm/lora/models.py @@ -248,7 +248,7 @@ class LoRAModel(AdapterModel): f" target modules in {expected_lora_modules}" f" but received {unexpected_modules}." f" Please verify that the loaded LoRA module is correct") - tensors = torch.load(lora_bin_file_path) + tensors = torch.load(lora_bin_file_path, map_location=device) else: raise ValueError(f"{lora_dir} doesn't contain tensors") @@ -257,7 +257,8 @@ class LoRAModel(AdapterModel): embeddings = safetensors.torch.load_file( new_embeddings_tensor_path) elif os.path.isfile(new_embeddings_bin_file_path): - embeddings = torch.load(new_embeddings_bin_file_path) + embeddings = torch.load(new_embeddings_bin_file_path, + map_location=device) rank = config["r"] lora_alpha = config["lora_alpha"]