From 24154f8618aa5da695b18cbbd71704f59803ec6b Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Mon, 12 Aug 2024 20:58:34 +0800 Subject: [PATCH] [Frontend] Disallow passing `model` as both argument and option (#7347) --- vllm/engine/arg_utils.py | 4 ++-- vllm/scripts.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 73698511..d4a3fae7 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -32,7 +32,7 @@ def nullable_str(val: str): @dataclass class EngineArgs: """Arguments for vLLM engine.""" - model: str + model: str = 'facebook/opt-125m' served_model_name: Optional[Union[List[str]]] = None tokenizer: Optional[str] = None skip_tokenizer_init: bool = False @@ -133,7 +133,7 @@ class EngineArgs: parser.add_argument( '--model', type=str, - default='facebook/opt-125m', + default=EngineArgs.model, help='Name or path of the huggingface model to use.') parser.add_argument( '--tokenizer', diff --git a/vllm/scripts.py b/vllm/scripts.py index f45bfe06..a9ddfcf8 100644 --- a/vllm/scripts.py +++ b/vllm/scripts.py @@ -9,6 +9,7 @@ from typing import List, Optional from openai import OpenAI from openai.types.chat import ChatCompletionMessageParam +from vllm.engine.arg_utils import EngineArgs from vllm.entrypoints.openai.api_server import run_server from vllm.entrypoints.openai.cli_args import make_arg_parser from vllm.utils import FlexibleArgumentParser @@ -24,6 +25,12 @@ def register_signal_handlers(): def serve(args: argparse.Namespace) -> None: + # The default value of `--model` + if args.model != EngineArgs.model: + raise ValueError( + "With `vllm serve`, you should provide the model as a " + "positional argument instead of via the `--model` option.") + # EngineArgs expects the model name to be passed as --model. args.model = args.model_tag