squall/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Core] Optimize evictor-v2 performance (#7193)

Browse Source
This commit is contained in:
xiaobochen123 2024-08-07 03:34:25 +08:00 committed by GitHub
parent 8d59dbb000
commit 660470e5a3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/core/evictor_v2.py
View File

@ -91,8 +91,9 @@ class LRUEvictor(Evictor):
# at the start of OrderedDict. Loop through all these blocks to
# find the one with maximum number of hashed tokens.
for _id, block in self.free_table.items():
if evicted_block.last_accessed > block.last_accessed or (
evicted_block.last_accessed == block.last_accessed and
if evicted_block.last_accessed < block.last_accessed:
break
if (evicted_block.last_accessed == block.last_accessed and
evicted_block.num_hashed_tokens < block.num_hashed_tokens):
evicted_block = block
evicted_block_id = _id
@ -109,6 +110,7 @@ class LRUEvictor(Evictor):
def update(self, block_id: int, last_accessed: float):
self.free_table[block_id].last_accessed = last_accessed
self.free_table.move_to_end(block_id)
def remove(self, block_id: int):
if block_id not in self.free_table: