From c0935c96d3472fe153ec849311c78fd8728dbb3c Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Thu, 28 Mar 2024 16:26:30 -0700 Subject: [PATCH] [Bugfix] Set enable_prefix_caching=True in prefix caching example (#3703) --- examples/offline_inference_with_prefix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/offline_inference_with_prefix.py b/examples/offline_inference_with_prefix.py index fbfb384f..7ed0563f 100644 --- a/examples/offline_inference_with_prefix.py +++ b/examples/offline_inference_with_prefix.py @@ -22,7 +22,7 @@ prompts = [ sampling_params = SamplingParams(temperature=0.0) # Create an LLM. -llm = LLM(model="facebook/opt-125m") +llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True) generating_prompts = [prefix + prompt for prompt in prompts]