Skip to content

Commit 2bc4bc0

Browse files
JamePengJamePeng
committed
feat(mtmd): expand vision chat template media support
- Recognize Muse Glimmer's <|patch|> marker in the generic MTMD handler. - Update Qwen3VL rendering to inject image and video URLs for MTMD processing. - Enable Qwen3.5 video inputs and handle string or {"url": ...} values. - Add regression tests for the supported media input formats. Signed-off-by: JamePeng <jame_peng@sina.com>
1 parent 690bce5 commit 2bc4bc0

2 files changed

Lines changed: 212 additions & 69 deletions

File tree

‎llama_cpp/llama_multimodal.py‎

Lines changed: 146 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -2079,6 +2079,7 @@ class GenericMTMDChatHandler(MTMDChatHandler):
20792079
"<|image|>",
20802080
"<|audio|>",
20812081
"<|video|>",
2082+
"<|patch|>", # Muse Glimmer used
20822083

20832084
# LLaVA / LFM / Mistral-style placeholders.
20842085
"<image>",
@@ -4031,84 +4032,157 @@ class Qwen3VLChatHandler(MTMDChatHandler):
40314032
QWEN3_VL_EOS_TOKEN = "<|im_end|>"
40324033

40334034
CHAT_FORMAT = (
4034-
"{{- '<|im_start|>system\n' -}}"
4035-
"{%- if messages[0].content is string and messages[0].role == 'system' -%}"
4036-
"{{- messages[0].content -}}"
4037-
"{%- elif messages[0].role == 'system' -%}"
4038-
"{%- if 'text' in messages[0].content -%}"
4039-
"{{- messages[0].content.text -}}"
4040-
"{%- else -%}"
4041-
"{{- 'You are a helpful assistant.' -}}"
4042-
"{%- endif -%}"
4043-
"{%- endif -%}"
4044-
"{%- if tools -%}"
4045-
"{{- '\n\n' -}}"
4046-
"{{- '# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>' -}}"
4047-
"{%- for tool in tools -%}"
4048-
"{{- '\n' -}}"
4049-
"{{- tool | tojson -}}"
4050-
"{%- endfor -%}"
4051-
"{{- '\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <arguments-json-object>}\n</tool_call>\n\nYou can also return a response for the user alongside a function call:\nRESPONSE FOR THE USER HERE\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <arguments-json-object>}\n</tool_call>' -}}"
4052-
"{%- endif -%}"
4053-
"{{- '<|im_end|>\n' -}}"
40544035
"{%- set image_count = namespace(value=0) -%}"
4055-
#"{%- set video_count = namespace(value=0) -%}"
4056-
"{%- for message in messages -%}"
4057-
"{%- if message.role == 'tool' -%}"
4058-
"{{- '<|im_start|>user\n<tool_response>\n' -}}"
4059-
"{%- elif message.role != 'system' -%}"
4060-
"{{- '<|im_start|>' + message.role + '\n' -}}"
4061-
"{%- endif -%}"
4062-
"{%- if message.content is string and message.role != 'system' -%}"
4063-
"{{- message.content -}}"
4064-
"{%- elif message.role != 'system' -%}"
4065-
"{%- for content in message.content -%}"
4066-
"{%- if 'image_url' in content -%}"
4067-
"{%- set image_count.value = image_count.value + 1 -%}"
4036+
"{%- set video_count = namespace(value=0) -%}"
4037+
4038+
"{%- macro render_content(content, do_vision_count) %}"
4039+
"{%- if content is string %}"
4040+
"{{- content }}"
4041+
"{%- else %}"
4042+
"{%- for item in content -%}"
4043+
" {%- if 'image' in item or 'image_url' in item or item.type == 'image' or item.type == 'image_url' %}"
4044+
"{%- if do_vision_count %}"
4045+
"{%- set image_count.value = image_count.value + 1 %}"
4046+
"{%- endif %}"
40684047
"{%- if add_vision_id -%}"
40694048
"{{- 'Picture ' -}}"
40704049
"{{- image_count.value | string -}}"
40714050
"{{- ': ' -}}"
40724051
"{%- endif -%}"
40734052
"{{- '<|vision_start|>' -}}"
4074-
"{%- if content.image_url is string -%}"
4075-
"{{- content.image_url -}}"
4076-
"{%- else -%}"
4077-
"{{- content.image_url.url -}}"
4053+
"{%- if 'image' in item -%}"
4054+
"{%- if item.image is string -%}"
4055+
"{{- item.image -}}"
4056+
"{%- else -%}"
4057+
"{{- item.image.url -}}"
4058+
"{%- endif -%}"
4059+
"{%- elif 'image_url' in item -%}"
4060+
"{%- if item.image_url is string -%}"
4061+
"{{- item.image_url -}}"
4062+
"{%- else -%}"
4063+
"{{- item.image_url.url -}}"
4064+
"{%- endif -%}"
40784065
"{%- endif -%}"
40794066
"{{- '<|vision_end|>' -}}"
40804067
"{%- endif -%}"
4081-
# Video not supported yet
4082-
"{%- if 'text' in content -%}"
4083-
"{{- content.text -}}"
4084-
"{%- endif -%}"
4085-
"{%- endfor -%}"
4086-
"{%- endif -%}"
4087-
"{%- if message.role == 'assistant' -%}"
4088-
"{%- if message.tool_calls -%}"
4089-
"{%- for tool_call in message.tool_calls -%}"
4090-
"{%- if (loop.first and message.content) or (not loop.first) -%}"
4091-
"{{- '\n' -}}"
4068+
"{%- if 'video_url' in item or 'video' in item -%}"
4069+
"{%- if do_vision_count %}"
4070+
"{%- set video_count.value = video_count.value + 1 %}"
4071+
"{%- endif %}"
4072+
"{%- if add_vision_id -%}"
4073+
"{{- 'Video ' -}}"
4074+
"{{- video_count.value | string -}}"
4075+
"{{- ': ' -}}"
40924076
"{%- endif -%}"
4093-
"{%- if tool_call.function -%}"
4094-
"{%- set tool_call = tool_call.function -%}"
4077+
"{{- '<|vision_start|>' -}}"
4078+
"{%- if 'video_url' in item -%}"
4079+
"{%- if item.video_url is string -%}"
4080+
"{{- item.video_url -}}"
4081+
"{%- else -%}"
4082+
"{{- item.video_url.url -}}"
4083+
"{%- endif -%}"
40954084
"{%- endif -%}"
4096-
"{{- '<tool_call>\n{\"name\": \"' + tool_call.name + '\", \"arguments\": ' -}}"
4097-
"{%- if tool_call.arguments is string -%}"
4098-
"{{- tool_call.arguments -}}"
4099-
"{%- else -%}"
4100-
"{{- tool_call.arguments | tojson -}}"
4085+
"{%- if 'video' in item -%}"
4086+
"{%- if item.video is string -%}"
4087+
"{{- item.video -}}"
4088+
"{%- else -%}"
4089+
"{{- item.video.url -}}"
4090+
"{%- endif -%}"
41014091
"{%- endif -%}"
4102-
"{{- '}\n</tool_call>' -}}"
4103-
"{%- endfor -%}"
4104-
"{%- endif -%}"
4105-
"{%- elif message.role == 'tool' -%}"
4106-
"{{- '</tool_response>' -}}"
4107-
"{%- endif -%}"
4108-
"{%- if message.role != 'system' -%}"
4109-
"{{- '<|im_end|>\n' -}}"
4110-
"{%- endif -%}"
4111-
"{%- endfor -%}"
4092+
"{{- '<|vision_end|>' -}}"
4093+
"{%- endif -%}"
4094+
"{%- if 'text' in item -%}"
4095+
"{{- item.text -}}"
4096+
"{%- endif -%}"
4097+
"{%- endfor -%}"
4098+
"{%- endif %}"
4099+
"{%- endmacro %}"
4100+
4101+
"{%- if tools -%}"
4102+
"{{- '<|im_start|>system\n' }}"
4103+
"{%- if messages[0].role == 'system' %}"
4104+
"{{- render_content(messages[0].content, false) + '\n\n' }}"
4105+
"{%- endif %}"
4106+
"{{- '# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>' -}}"
4107+
"{%- for tool in tools -%}"
4108+
"{{- '\n' -}}"
4109+
"{{- tool | tojson -}}"
4110+
"{%- endfor -%}"
4111+
"{{- '\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n' -}}"
4112+
"{%- else %}"
4113+
"{%- if messages[0].role == 'system' %}"
4114+
"{{- '<|im_start|>system\n' + render_content(messages[0].content, false) + '<|im_end|>\n' }}"
4115+
"{%- endif %}"
4116+
"{%- endif %}"
4117+
4118+
"{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}"
4119+
"{%- for message in messages[::-1] %}"
4120+
"{%- set index = (messages|length - 1) - loop.index0 %}"
4121+
"{%- if ns.multi_step_tool and message.role == \"user\" %}"
4122+
"{%- set content = render_content(message.content, false) %}"
4123+
"{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}"
4124+
"{%- set ns.multi_step_tool = false %}"
4125+
"{%- set ns.last_query_index = index %}"
4126+
"{%- endif %}"
4127+
"{%- endif %}"
4128+
"{%- endfor %}"
4129+
4130+
"{%- for message in messages %}"
4131+
"{%- set content = render_content(message.content, True) %}"
4132+
"{%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}"
4133+
"{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}"
4134+
"{%- elif message.role == \"assistant\" %}"
4135+
"{%- set reasoning_content = '' %}"
4136+
"{%- if message.reasoning_content is string %}"
4137+
"{%- set reasoning_content = message.reasoning_content %}"
4138+
"{%- else %}"
4139+
"{%- if '</think>' in content %}"
4140+
"{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}"
4141+
"{%- set content = content.split('</think>')[-1].lstrip('\n') %}"
4142+
"{%- endif %}"
4143+
"{%- endif %}"
4144+
"{%- if loop.index0 > ns.last_query_index %}"
4145+
"{%- if loop.last or (not loop.last and reasoning_content) %}"
4146+
"{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}"
4147+
"{%- else %}"
4148+
"{{- '<|im_start|>' + message.role + '\n' + content }}"
4149+
"{%- endif %}"
4150+
"{%- else %}"
4151+
"{{- '<|im_start|>' + message.role + '\n' + content }}"
4152+
"{%- endif %}"
4153+
"{%- if message.tool_calls %}"
4154+
"{%- for tool_call in message.tool_calls %}"
4155+
"{%- if (loop.first and content) or (not loop.first) %}"
4156+
"{{- '\n' }}"
4157+
"{%- endif %}"
4158+
"{%- if tool_call.function %}"
4159+
"{%- set tool_call = tool_call.function %}"
4160+
"{%- endif %}"
4161+
"{{- '<tool_call>\n{\"name\": \"' }}"
4162+
"{{- tool_call.name }}"
4163+
"{{- '\", \"arguments\": ' }}"
4164+
"{%- if tool_call.arguments is string %}"
4165+
"{{- tool_call.arguments }}"
4166+
"{%- else %}"
4167+
"{{- tool_call.arguments | tojson }}"
4168+
"{%- endif %}"
4169+
"{{- '}\n</tool_call>' }}"
4170+
"{%- endfor %}"
4171+
"{%- endif %}"
4172+
"{{- '<|im_end|>\n' }}"
4173+
"{%- elif message.role == \"tool\" %}"
4174+
"{%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}"
4175+
"{{- '<|im_start|>user' }}"
4176+
"{%- endif %}"
4177+
"{{- '\n<tool_response>\n' }}"
4178+
"{{- content }}"
4179+
"{{- '\n</tool_response>' }}"
4180+
"{%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}"
4181+
"{{- '<|im_end|>\n' }}"
4182+
"{%- endif %}"
4183+
"{%- endif %}"
4184+
"{%- endfor %}"
4185+
41124186
"{%- if add_generation_prompt -%}"
41134187
"{{- '<|im_start|>assistant\n' -}}"
41144188
"{%- if force_reasoning -%}"
@@ -4153,7 +4227,7 @@ def __call__(self, **kwargs):
41534227

41544228
class Qwen35ChatHandler(MTMDChatHandler):
41554229
"""
4156-
Handler for Qwen3.5/Qwen3.6 models.
4230+
Handler for Qwen3.5/Qwen3.6/Qwen3.8 models.
41574231
"""
41584232
CHAT_FORMAT = (
41594233
"{%- set image_count = namespace(value=0) -%}"
@@ -4182,8 +4256,7 @@ class Qwen35ChatHandler(MTMDChatHandler):
41824256
" {{- item.image_url.url -}}"
41834257
" {%- endif -%}"
41844258
" {{- '<|vision_end|>' -}}"
4185-
" {%- elif 'video' in item -%}"
4186-
" {{- raise_exception('llama.cpp does not currently support video.') -}}" # Video not supported, raise exception
4259+
" {%- elif 'video' in item or 'video_url' in item or item.type == 'video' or item.type == 'video_url' -%}"
41874260
" {%- if is_system_content -%}"
41884261
" {{- raise_exception('System message cannot contain videos.') -}}"
41894262
" {%- endif -%}"
@@ -4194,7 +4267,11 @@ class Qwen35ChatHandler(MTMDChatHandler):
41944267
" {{- 'Video ' ~ video_count.value ~ ': ' -}}"
41954268
" {%- endif -%}"
41964269
" {{- '<|vision_start|>' -}}"
4197-
" {{- item.video -}}"
4270+
" {%- if 'video' in item -%}"
4271+
" {{- item.video if item.video is string else item.video.url -}}"
4272+
" {%- elif 'video_url' in item -%}"
4273+
" {{- item.video_url if item.video_url is string else item.video_url.url -}}"
4274+
" {%- endif -%}"
41984275
" {{- '<|vision_end|>' -}}"
41994276
" {%- elif 'text' in item -%}"
42004277
" {{- item.text -}}"

‎tests/test_media.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,72 @@ def test_mtmd_chat_template_strftime_now(tmp_path):
370370
handler.close()
371371

372372

373+
@pytest.mark.parametrize(
374+
"video_item",
375+
[
376+
{"type": "video", "video": "/path/to/clip.mp4"},
377+
{"type": "video", "video": {"url": "/path/to/clip.mp4"}},
378+
{"type": "video_url", "video_url": "/path/to/clip.mp4"},
379+
{"type": "video_url", "video_url": {"url": "/path/to/clip.mp4"}},
380+
],
381+
)
382+
def test_qwen35_video_template_uses_mtmd_marker(tmp_path, video_item):
383+
multimodal = importlib.import_module("llama_cpp.llama_multimodal")
384+
handler = multimodal.Qwen35ChatHandler(mmproj_path=str(tmp_path), verbose=False)
385+
handler.mtmd_bos_token = ""
386+
handler.mtmd_eos_token = ""
387+
handler.media_marker = "<__media__>"
388+
handler.is_support_video = True
389+
messages = [{"role": "user", "content": [video_item]}]
390+
try:
391+
media_items = handler._get_media_items(messages)
392+
prompt = handler._render_and_replace_media(
393+
messages=messages,
394+
media_items=media_items,
395+
add_generation_prompt=False,
396+
)
397+
assert media_items == [{"url": "/path/to/clip.mp4", "type": "video"}]
398+
assert "Video 1: <|vision_start|><__media__><|vision_end|>" in prompt
399+
assert "<|video_pad|>" not in prompt
400+
with pytest.raises(TemplateError, match="System message cannot contain videos"):
401+
handler._render_mtmd_prompt(
402+
messages=[{"role": "system", "content": [video_item]}],
403+
add_generation_prompt=False,
404+
)
405+
finally:
406+
handler.close()
407+
408+
409+
@pytest.mark.parametrize(
410+
"image_item",
411+
[
412+
{"image": "/path/to/image.png"},
413+
{"image_url": {"url": "/path/to/image.png"}},
414+
{"type": "image", "image": {"url": "/path/to/image.png"}},
415+
{"type": "image_url", "image_url": "/path/to/image.png"},
416+
],
417+
)
418+
def test_qwen3vl_image_template_injects_media_without_type(tmp_path, image_item):
419+
multimodal = importlib.import_module("llama_cpp.llama_multimodal")
420+
handler = multimodal.Qwen3VLChatHandler(mmproj_path=str(tmp_path), verbose=False)
421+
handler.mtmd_bos_token = ""
422+
handler.mtmd_eos_token = ""
423+
handler.media_marker = "<__media__>"
424+
handler.is_support_vision = True
425+
messages = [{"role": "user", "content": [image_item]}]
426+
try:
427+
media_items = handler._get_media_items(messages)
428+
prompt = handler._render_and_replace_media(
429+
messages=messages,
430+
media_items=media_items,
431+
add_generation_prompt=False,
432+
)
433+
assert media_items == [{"url": "/path/to/image.png", "type": "image"}]
434+
assert "Picture 1: <|vision_start|><__media__><|vision_end|>" in prompt
435+
finally:
436+
handler.close()
437+
438+
373439
def test_mtmd_base_image_loader_uses_subclass_byte_loader():
374440
import io
375441
from PIL import Image

0 commit comments

Comments
 (0)