squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[misc] add start loading models for users information (#6670)

Browse Source
This commit is contained in:
youkaichao 2024-07-22 20:08:02 -07:00 committed by GitHub
parent 729171ae58
commit 7c2749a4fd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
vllm/worker/model_runner.py
View File

@ -677,6 +677,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
int(self.cache_config.cpu_offload_gb * 1024**3)) int(self.cache_config.cpu_offload_gb * 1024**3))
def load_model(self) -> None: def load_model(self) -> None:
logger.info("Starting to load model %s...", self.model_config.model)
with CudaMemoryProfiler() as m: with CudaMemoryProfiler() as m:
self.model = get_model(model_config=self.model_config, self.model = get_model(model_config=self.model_config,
device_config=self.device_config, device_config=self.device_config,