squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix_tokenizer_snapshot_download_bug (#4493)

Browse Source
This commit is contained in:
fuchen.ljl 2024-05-01 07:38:50 +08:00 committed by GitHub
parent dd1a50a8bc
commit 6ad58f42c5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/transformers_utils/tokenizer.py
View File

@ -79,7 +79,7 @@ def get_tokenizer(
revision=revision, revision=revision,
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
# Ignore weights - we only need the tokenizer. # Ignore weights - we only need the tokenizer.
ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"]) ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
tokenizer_name = tokenizer_path tokenizer_name = tokenizer_path
if tokenizer_mode == "slow": if tokenizer_mode == "slow":