From 660470e5a36b8e52083615ad7c85e9b4fd4c72ce Mon Sep 17 00:00:00 2001 From: xiaobochen123 <35516720+xiaobochen123@users.noreply.github.com> Date: Wed, 7 Aug 2024 03:34:25 +0800 Subject: [PATCH] [Core] Optimize evictor-v2 performance (#7193) --- vllm/core/evictor_v2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vllm/core/evictor_v2.py b/vllm/core/evictor_v2.py index 3dd12e2e..5b1a208b 100644 --- a/vllm/core/evictor_v2.py +++ b/vllm/core/evictor_v2.py @@ -91,8 +91,9 @@ class LRUEvictor(Evictor): # at the start of OrderedDict. Loop through all these blocks to # find the one with maximum number of hashed tokens. for _id, block in self.free_table.items(): - if evicted_block.last_accessed > block.last_accessed or ( - evicted_block.last_accessed == block.last_accessed and + if evicted_block.last_accessed < block.last_accessed: + break + if (evicted_block.last_accessed == block.last_accessed and evicted_block.num_hashed_tokens < block.num_hashed_tokens): evicted_block = block evicted_block_id = _id @@ -109,6 +110,7 @@ class LRUEvictor(Evictor): def update(self, block_id: int, last_accessed: float): self.free_table[block_id].last_accessed = last_accessed + self.free_table.move_to_end(block_id) def remove(self, block_id: int): if block_id not in self.free_table: