From 856c990041bf6cf4b2397401d4b18531382ecb50 Mon Sep 17 00:00:00 2001
From: Roger Wang <136131678+ywang96@users.noreply.github.com>
Date: Mon, 10 Jun 2024 09:53:50 -0700
Subject: [PATCH] [Docs] Add Docs on Limitations of VLM Support (#5383)

---
 docs/source/conf.py        | 1 +
 docs/source/models/vlm.rst | 9 ++++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/source/conf.py b/docs/source/conf.py
index f1a7013e..ee0f6c53 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -92,6 +92,7 @@ autodoc_mock_imports = [
     "vllm._C",
     "PIL",
     "numpy",
+    'triton'
     "tqdm",
     "tensorizer",
 ]
diff --git a/docs/source/models/vlm.rst b/docs/source/models/vlm.rst
index b917688a..33aa8246 100644
--- a/docs/source/models/vlm.rst
+++ b/docs/source/models/vlm.rst
@@ -16,6 +16,13 @@ The following :ref:`engine arguments <engine_args>` are specific to VLMs:
     :prog: -m vllm.entrypoints.openai.api_server
     :nodefaultconst:
 
+.. important::
+    Currently, the support for vision language models on vLLM has the following limitations:
+
+    * Only single image input is supported per text prompt.
+    * Dynamic ``image_input_shape`` is not supported: the input image will be resized to the static ``image_input_shape``. This means model output might not exactly match the huggingface implementation.
+    We are continuously improving user & developer experience for VLMs. Please raise an issue on GitHub if you have any feedback or feature requests.
+
 Offline Batched Inference
 -------------------------
 
@@ -31,7 +38,7 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
         image_feature_size=576,
     )
 
-For now, we only support a single image per text prompt. To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
+To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
 
 * ``prompt``: The prompt should have a number of ``<image>`` tokens equal to ``image_feature_size``.
 * ``multi_modal_data``: This should be an instance of :class:`~vllm.multimodal.image.ImagePixelData` or :class:`~vllm.multimodal.image.ImageFeatureData`.