feat: add support for SmolVLM

Author: dhdaines

examples/granite_docling/main.py

@@ -11,6 +11,7 @@
     filename="granite*Q8_0*",
     chat_handler=chat_handler,
     n_ctx=8192,
+    n_gpu_layers=-1,
 )
 response = llama.create_chat_completion(
     messages=[

examples/smolvlm/main.py

@@ -0,0 +1,36 @@
+from llama_cpp import Llama
+from llama_cpp.llama_chat_format import SmolVLMChatHandler
+
+
+chat_handler = SmolVLMChatHandler.from_pretrained(
+    repo_id="ggml-org/SmolVLM-256M-Instruct-GGUF",
+    filename="mmproj*Q8_0*",
+)
+llama = Llama.from_pretrained(
+    repo_id="ggml-org/SmolVLM-256M-Instruct-GGUF",
+    filename="SmolVLM*Q8_0*",
+    chat_handler=chat_handler,
+    n_ctx=8192,
+    n_gpu_layers=-1,
+)
+response = llama.create_chat_completion(
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": "https://huggingface.co/spaces/ibm-granite/granite-docling-258m-demo/resolve/main/data/images/lake-zurich-switzerland-view-nature-landscapes-7bbda4-1024.jpg"},
+                {"type": "text", "text": "Describe this image."},
+            ],
+        }
+    ],
+    stream=True,
+    special=True,
+)
+
+for chunk in response:
+    delta = chunk["choices"][0]["delta"]
+    if "content" not in delta:
+        continue
+    print(delta["content"], end="", flush=True)
+
+print()

llama_cpp/llama_chat_format.py

@@ -3530,6 +3530,33 @@ def __call__(self, **kwargs):
         return super().__call__(**kwargs)
 
 
+class SmolVLMChatHandler(Llava15ChatHandler):
+    DEFAULT_SYSTEM_MESSAGE = None
+    CHAT_FORMAT = (
+        "<|im_start|>"
+        "{% for message in messages %}"
+        "{{message['role'] | capitalize}}"
+        "{% if message['content'][0]['type'] == 'image' %}"
+        "{{':'}}{% else %}{{': '}}{% endif %}"
+        "{% for line in message['content'] %}"
+        "{% if line['type'] == 'text' %}"
+        "{{line['text']}}"
+        "{% elif line['type'] == 'image_url' %}"
+        "{% if line['image_url'] is string %}"
+        "{{ line['image_url'] }}"
+        "{% elif line['image_url'] is mapping %}"
+        "{{ line['image_url']['url'] }}"
+        "{% endif %}"
+        "{% endif %}"
+        "{% endfor %}"
+        "<end_of_utterance>\n"
+        "{% endfor %}"
+        "{% if add_generation_prompt %}"
+        "{{ 'Assistant:' }}"
+        "{% endif %}"
+    )
+
+
 class GraniteDoclingChatHandler(Llava15ChatHandler):
     DEFAULT_SYSTEM_MESSAGE = None