From 476576302f23945e8d7b6c3bfe856ba8b04813b5 Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Tue, 28 Jul 2026 06:09:55 +0000 Subject: [PATCH] feat: wire up video input modality for MiniMax providers Mirror the existing image input pipeline so MiniMax providers can transmit the video input modality that MiniMax-M3 supports: - Populate ProviderRequest.video_urls from Video attachments so resolved video paths reach the provider instead of only a textual placeholder. - Resolve video_url context blocks into data URLs in the OpenAI source payload materialization. - Convert video_url blocks into Anthropic video blocks in the Anthropic source _prepare_payload so the MiniMax Token Plan provider transmits video. - Add a VideoURLPart content block type for extra user content parts. - Expose "video" as a model modality option and honor it in context assembly and modality sanitization. - Add tests covering assemble_context, modality sanitization, OpenAI payload materialization, and Anthropic payload conversion for video. --- astrbot/core/agent/message.py | 18 +- .../agent/runners/tool_loop_agent_runner.py | 8 +- astrbot/core/astr_main_agent.py | 10 +- astrbot/core/config/default.py | 4 +- astrbot/core/provider/entities.py | 42 ++++- astrbot/core/provider/modalities.py | 12 +- .../core/provider/sources/anthropic_source.py | 33 ++++ .../core/provider/sources/openai_source.py | 65 +++++++ tests/test_video_input_modality.py | 176 ++++++++++++++++++ 9 files changed, 360 insertions(+), 8 deletions(-) create mode 100644 tests/test_video_input_modality.py diff --git a/astrbot/core/agent/message.py b/astrbot/core/agent/message.py index 4292f4c04e..abaeea6f14 100644 --- a/astrbot/core/agent/message.py +++ b/astrbot/core/agent/message.py @@ -21,7 +21,7 @@ class ContentPart(BaseModel): __content_part_registry: ClassVar[dict[str, type["ContentPart"]]] = {} - type: Literal["text", "think", "image_url", "audio_url"] + type: Literal["text", "think", "image_url", "audio_url", "video_url"] _no_save: bool = PrivateAttr(default=False) def __init_subclass__(cls, **kwargs: Any) -> None: @@ -141,6 +141,22 @@ class AudioURL(BaseModel): audio_url: AudioURL +class VideoURLPart(ContentPart): + """ + >>> VideoURLPart(video_url=VideoURLPart.VideoURL(url="https://example.com/video.mp4")).model_dump() + {'type': 'video_url', 'video_url': {'url': 'https://example.com/video.mp4', 'id': None}} + """ + + class VideoURL(BaseModel): + url: str + """The URL of the video, can be a data URI scheme like `data:video/mp4;base64,...`.""" + id: str | None = None + """The ID of the video, to allow LLMs to distinguish different videos.""" + + type: str = "video_url" + video_url: VideoURL + + class ToolCall(BaseModel): """ A tool call requested by the assistant. diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index 98754f9b6a..4ba5b8f417 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -312,6 +312,7 @@ async def reset( request.prompt is not None or request.image_urls or request.audio_urls + or request.video_urls or request.extra_user_content_parts ): m = await self._assemble_request_context_for_provider(request) @@ -341,13 +342,15 @@ async def _assemble_request_context_for_provider( supports_image = "image" in modalities supports_audio = "audio" in modalities - if supports_image and supports_audio: + supports_video = "video" in modalities + if supports_image and supports_audio and supports_video: return await request.assemble_context() adjusted_request = replace( request, image_urls=request.image_urls if supports_image else [], audio_urls=request.audio_urls if supports_audio else [], + video_urls=request.video_urls if supports_video else [], ) context = await adjusted_request.assemble_context() content = context.get("content") @@ -364,6 +367,9 @@ async def _assemble_request_context_for_provider( if not supports_audio: for _ in request.audio_urls: content_blocks.append({"type": "text", "text": "[Audio]"}) + if not supports_video: + for _ in request.video_urls: + content_blocks.append({"type": "text", "text": "[Video]"}) return {"role": "user", "content": content_blocks} diff --git a/astrbot/core/astr_main_agent.py b/astrbot/core/astr_main_agent.py index 9312b8c3af..039e163c02 100644 --- a/astrbot/core/astr_main_agent.py +++ b/astrbot/core/astr_main_agent.py @@ -765,6 +765,7 @@ async def _append_video_attachment( else: text = f"[Video Attachment: name {video_name}, path {video_path}]" + req.video_urls.append(video_path) req.extra_user_content_parts.append(TextPart(text=text)) @@ -1394,6 +1395,7 @@ async def build_main_agent( req.prompt = "" req.image_urls = [] req.audio_urls = [] + req.video_urls = [] if sel_model := event.get_extra("selected_model"): req.model = sel_model if config.provider_wake_prefix and not event.message_str.startswith( @@ -1539,6 +1541,7 @@ async def build_main_agent( ) req.image_urls = normalize_and_dedupe_strings(req.image_urls) req.audio_urls = normalize_and_dedupe_strings(req.audio_urls) + req.video_urls = normalize_and_dedupe_strings(req.video_urls) if config.file_extract_enabled: try: @@ -1548,7 +1551,12 @@ async def build_main_agent( has_reply = any(isinstance(comp, Reply) for comp in event.message_obj.message) - if not req.prompt and not req.image_urls and not req.audio_urls: + if ( + not req.prompt + and not req.image_urls + and not req.audio_urls + and not req.video_urls + ): if has_reply or req.extra_user_content_parts: req.prompt = "" else: diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index ff3ccd4f96..0f913c4d50 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2071,8 +2071,8 @@ "description": "模型能力", "type": "list", "items": {"type": "string"}, - "options": ["text", "image", "audio", "tool_use"], - "labels": ["文本", "图像", "音频", "工具使用"], + "options": ["text", "image", "audio", "video", "tool_use"], + "labels": ["文本", "图像", "音频", "视频", "工具使用"], "render_type": "checkbox", "hint": "模型支持的模态及能力。", }, diff --git a/astrbot/core/provider/entities.py b/astrbot/core/provider/entities.py index 5d20ad65c9..e3e72f5680 100644 --- a/astrbot/core/provider/entities.py +++ b/astrbot/core/provider/entities.py @@ -95,6 +95,8 @@ class ProviderRequest: """图片 URL 列表""" audio_urls: list[str] = field(default_factory=list) """音频 URL 列表，也支持本地路径""" + video_urls: list[str] = field(default_factory=list) + """视频 URL 列表，也支持本地路径""" extra_user_content_parts: list[ContentPart] = field(default_factory=list) """额外的用户消息内容部分列表，用于在用户消息后添加额外的内容块（如系统提醒、指令等）。支持 dict 或 ContentPart 对象""" func_tool: ToolSet | None = None @@ -118,6 +120,7 @@ def __repr__(self) -> str: f"ProviderRequest(prompt={self.prompt}, session_id={self.session_id}, " f"image_count={len(self.image_urls or [])}, " f"audio_count={len(self.audio_urls or [])}, " + f"video_count={len(self.video_urls or [])}, " f"func_tool={self.func_tool}, " f"contexts={self._print_friendly_context()}, " f"system_prompt={self.system_prompt}, " @@ -140,7 +143,8 @@ def _print_friendly_context(self): if not self.contexts: return ( f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}, " - f"audio_count: {len(self.audio_urls or [])}" + f"audio_count: {len(self.audio_urls or [])}, " + f"video_count: {len(self.video_urls or [])}" ) result_parts = [] @@ -157,6 +161,7 @@ def _print_friendly_context(self): msg_parts = [] image_count = 0 audio_count = 0 + video_count = 0 for item in content: item_type = item.get("type", "") @@ -167,6 +172,8 @@ def _print_friendly_context(self): image_count += 1 elif item_type == "audio_url": audio_count += 1 + elif item_type == "video_url": + video_count += 1 if image_count > 0: if msg_parts: @@ -178,6 +185,11 @@ def _print_friendly_context(self): msg_parts.append(f"[+{audio_count} audios]") else: msg_parts.append(f"[{audio_count} audios]") + if video_count > 0: + if msg_parts: + msg_parts.append(f"[+{video_count} videos]") + else: + msg_parts.append(f"[{video_count} videos]") result_parts.append(f"{role}: {''.join(msg_parts)}") @@ -197,6 +209,9 @@ async def assemble_context(self) -> dict: elif self.audio_urls: # 如果没有文本但有音频，添加占位文本 content_blocks.append({"type": "text", "text": "[音频]"}) + elif self.video_urls: + # 如果没有文本但有视频，添加占位文本 + content_blocks.append({"type": "text", "text": "[Video]"}) # 2. 额外的内容块（系统提醒、指令等） if self.extra_user_content_parts: @@ -245,6 +260,30 @@ async def assemble_context(self) -> dict: }, ) + # 5. 视频内容 + if self.video_urls: + for video_url in self.video_urls: + try: + video_data = await MediaResolver( + video_url, + media_type="video", + default_suffix=".mp4", + ).to_base64_data(strict=True) + except Exception as exc: + logger.warning( + "Video preprocessing failed, will skip. Error: %s", exc + ) + continue + if not video_data: + logger.warning("Video preprocessing result is empty, will skip.") + continue + content_blocks.append( + { + "type": "video_url", + "video_url": {"url": video_data.to_data_url()}, + }, + ) + # 只有当只有一个来自 prompt 的文本块且没有额外内容块时，才降级为简单格式以保持向后兼容 if ( len(content_blocks) == 1 @@ -252,6 +291,7 @@ async def assemble_context(self) -> dict: and not self.extra_user_content_parts and not self.image_urls and not self.audio_urls + and not self.video_urls ): return {"role": "user", "content": content_blocks[0]["text"]} diff --git a/astrbot/core/provider/modalities.py b/astrbot/core/provider/modalities.py index 66ac74e9b7..3f89032f76 100644 --- a/astrbot/core/provider/modalities.py +++ b/astrbot/core/provider/modalities.py @@ -13,6 +13,7 @@ class ContextSanitizeStats: fixed_image_blocks: int = 0 fixed_audio_blocks: int = 0 + fixed_video_blocks: int = 0 fixed_tool_messages: int = 0 removed_tool_calls: int = 0 @@ -21,6 +22,7 @@ def changed(self) -> bool: return bool( self.fixed_image_blocks or self.fixed_audio_blocks + or self.fixed_video_blocks or self.fixed_tool_messages or self.removed_tool_calls ) @@ -50,8 +52,9 @@ def sanitize_contexts_by_modalities( supports_image = "image" in modalities supports_audio = "audio" in modalities + supports_video = "video" in modalities supports_tool_use = "tool_use" in modalities - if supports_image and supports_audio and supports_tool_use: + if supports_image and supports_audio and supports_video and supports_tool_use: copied_contexts = [] for msg in contexts: copied_msg = _message_to_dict(msg) @@ -83,7 +86,7 @@ def sanitize_contexts_by_modalities( msg.pop("tool_calls", None) msg.pop("tool_call_id", None) - if not supports_image or not supports_audio: + if not supports_image or not supports_audio or not supports_video: content = msg.get("content") if isinstance(content, list): filtered_parts: list[Any] = [] @@ -104,6 +107,11 @@ def sanitize_contexts_by_modalities( stats.fixed_audio_blocks += 1 filtered_parts.append({"type": "text", "text": "[Audio]"}) continue + if not supports_video and part_type in {"video_url", "video"}: + removed_any_multimodal = True + stats.fixed_video_blocks += 1 + filtered_parts.append({"type": "text", "text": "[Video]"}) + continue filtered_parts.append(part) if removed_any_multimodal: msg["content"] = filtered_parts diff --git a/astrbot/core/provider/sources/anthropic_source.py b/astrbot/core/provider/sources/anthropic_source.py index 27cc459622..696948bdf6 100644 --- a/astrbot/core/provider/sources/anthropic_source.py +++ b/astrbot/core/provider/sources/anthropic_source.py @@ -292,6 +292,39 @@ def _prepare_payload(self, messages: list[dict]): "text": "[Audio Attachment]", } ) + elif part.get("type") == "video_url": + # Convert OpenAI-style video_url format to Anthropic + # video format so models that accept video input + # actually receive the modality. + video_url_data = part.get("video_url", {}) + url = video_url_data.get("url", "") + if url.startswith("data:"): + try: + header, base64_data = url.split(",", 1) + media_type = ( + header.split(":", 1)[1].split(";", 1)[0].strip() + or "video/mp4" + ) + converted_content.append( + { + "type": "video", + "source": { + "type": "base64", + "media_type": media_type, + "data": base64_data, + }, + } + ) + except (ValueError, IndexError): + logger.warning( + "Failed to parse video data URI, skipping: %s", + url[:50], + ) + else: + logger.warning( + "Unsupported video URL format for Anthropic, skipping: %s", + url[:50], + ) else: converted_content.append(part) new_messages.append( diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index f7870b7137..ced657d060 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -24,6 +24,7 @@ ImageURLPart, Message, TextPart, + VideoURLPart, ) from astrbot.core.agent.tool import ToolSet from astrbot.core.exceptions import EmptyModelOutputError @@ -143,6 +144,7 @@ def _context_contains_image(contexts: list[dict]) -> bool: if isinstance(item, dict) and item.get("type") in { "image_url", "audio_url", + "video_url", }: return True return False @@ -266,6 +268,34 @@ async def _resolve_audio_part(self, audio_ref: str) -> dict | None: }, } + def _extract_video_part_info(self, part: dict) -> str | None: + if not isinstance(part, dict) or part.get("type") != "video_url": + return None + + video_url_data = part.get("video_url") + if not isinstance(video_url_data, dict): + logger.warning("Video content block has invalid format, keeping original.") + return None + + url = video_url_data.get("url") + if not isinstance(url, str) or not url: + logger.warning("Video content block has no valid URL, keeping original.") + return None + return url + + async def _resolve_video_part(self, video_ref: str) -> dict | None: + video_data = await resolve_media_ref_to_base64_data( + video_ref, + media_type="video", + ) + if not video_data: + logger.warning("Video preprocessing result is empty, will ignore.") + return None + return { + "type": "video_url", + "video_url": {"url": video_data.to_data_url()}, + } + async def _transform_content_part(self, part: dict) -> dict: if not isinstance(part, dict): return part @@ -296,6 +326,21 @@ async def _transform_content_part(self, part: dict) -> dict: resolved_part = await self._resolve_audio_part(audio_ref) return resolved_part or part + if part.get("type") == "video_url": + video_ref = self._extract_video_part_info(part) + if not video_ref: + return part + try: + resolved_part = await self._resolve_video_part(video_ref) + except Exception as exc: + logger.warning( + "Video %s preprocessing failed, keeping original. Error: %s", + video_ref, + exc, + ) + return part + return resolved_part or part + return part async def _materialize_message_image_parts(self, message: dict) -> dict: @@ -956,6 +1001,7 @@ async def _prepare_chat_payload( tool_calls_result: ToolCallsResult | list[ToolCallsResult] | None = None, model: str | None = None, extra_user_content_parts: list[ContentPart] | None = None, + video_urls: list[str] | None = None, **kwargs, ) -> tuple: """准备聊天所需的有效载荷和上下文""" @@ -968,6 +1014,7 @@ async def _prepare_chat_payload( image_urls, audio_urls, extra_user_content_parts, + video_urls, ) context_query = copy.deepcopy(self._ensure_message_to_dicts(contexts)) if new_record: @@ -1194,6 +1241,7 @@ async def text_chat( tool_calls_result=None, model=None, extra_user_content_parts=None, + video_urls=None, tool_choice: Literal["auto", "required"] = "auto", request_max_retries: int | None = None, **kwargs, @@ -1207,6 +1255,7 @@ async def text_chat( tool_calls_result, model=model, extra_user_content_parts=extra_user_content_parts, + video_urls=video_urls, **kwargs, ) if func_tool and not func_tool.empty(): @@ -1271,6 +1320,7 @@ async def text_chat_stream( system_prompt=None, tool_calls_result=None, model=None, + video_urls=None, tool_choice: Literal["auto", "required"] = "auto", request_max_retries: int | None = None, **kwargs, @@ -1284,6 +1334,7 @@ async def text_chat_stream( system_prompt, tool_calls_result, model=model, + video_urls=video_urls, **kwargs, ) if func_tool and not func_tool.empty(): @@ -1370,6 +1421,7 @@ async def assemble_context( image_urls: list[str] | None = None, audio_urls: list[str] | None = None, extra_user_content_parts: list[ContentPart] | None = None, + video_urls: list[str] | None = None, ) -> dict: """组装成符合 OpenAI 格式的 role 为 user 的消息段""" @@ -1384,6 +1436,8 @@ async def assemble_context( content_blocks.append({"type": "text", "text": "[Image]"}) elif audio_urls: content_blocks.append({"type": "text", "text": "[Audio]"}) + elif video_urls: + content_blocks.append({"type": "text", "text": "[Video]"}) elif extra_user_content_parts: # 如果只有额外内容块，也需要添加占位文本 content_blocks.append({"type": "text", "text": " "}) @@ -1403,6 +1457,10 @@ async def assemble_context( audio_part = await self._resolve_audio_part(part.audio_url.url) if audio_part: content_blocks.append(audio_part) + elif isinstance(part, VideoURLPart): + video_part = await self._resolve_video_part(part.video_url.url) + if video_part: + content_blocks.append(video_part) else: raise ValueError(f"不支持的额外内容块类型: {type(part)}") @@ -1419,12 +1477,19 @@ async def assemble_context( if audio_part: content_blocks.append(audio_part) + if video_urls: + for video_ref in video_urls: + video_part = await self._resolve_video_part(video_ref) + if video_part: + content_blocks.append(video_part) + # 如果只有主文本且没有额外内容块和图片，返回简单格式以保持向后兼容 if ( text and not extra_user_content_parts and not image_urls and not audio_urls + and not video_urls and len(content_blocks) == 1 and content_blocks[0]["type"] == "text" ): diff --git a/tests/test_video_input_modality.py b/tests/test_video_input_modality.py new file mode 100644 index 0000000000..2500092785 --- /dev/null +++ b/tests/test_video_input_modality.py @@ -0,0 +1,176 @@ +import pytest + +from astrbot.core.provider.entities import ProviderRequest +from astrbot.core.provider.modalities import ( + sanitize_contexts_by_modalities, +) +from astrbot.core.provider.sources.openai_source import ProviderOpenAIOfficial +from astrbot.core.utils.media_utils import ResolvedMediaData + + +@pytest.mark.asyncio +async def test_assemble_context_includes_video_url_block(monkeypatch): + """ProviderRequest.assemble_context must emit a video_url content block.""" + captured = {} + + class _FakeVideoData: + def __init__(self, data, mime_type): + self.base64_data = data + self.mime_type = mime_type + self.format = "mp4" + + def to_data_url(self): + return f"data:{self.mime_type};base64,{self.base64_data}" + + async def fake_to_base64_data( + self, + *, + strict=False, + target_format=None, + preserve_mp3=False, + default_mime_type=None, + ): + captured["media_type"] = self.media_type + return _FakeVideoData("abcd", "video/mp4") + + monkeypatch.setattr( + "astrbot.core.provider.entities.MediaResolver.to_base64_data", + fake_to_base64_data, + ) + + req = ProviderRequest(prompt="look", video_urls=["https://example.com/clip.mp4"]) + msg = await req.assemble_context() + + assert msg["role"] == "user" + blocks = msg["content"] + assert blocks[0] == {"type": "text", "text": "look"} + assert blocks[1] == { + "type": "video_url", + "video_url": {"url": "data:video/mp4;base64,abcd"}, + } + assert captured["media_type"] == "video" + + +def test_sanitize_contexts_strips_video_when_unsupported(): + contexts = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + { + "type": "video_url", + "video_url": {"url": "data:video/mp4;base64,abcd"}, + }, + ], + } + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, ["text", "image", "audio", "tool_use"] + ) + assert sanitized[0]["content"] == [ + {"type": "text", "text": "hi"}, + {"type": "text", "text": "[Video]"}, + ] + assert stats.fixed_video_blocks == 1 + + +def test_sanitize_contexts_preserves_video_when_supported(): + block = {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,abcd"}} + contexts = [{"role": "user", "content": [{"type": "text", "text": "hi"}, block]}] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, ["text", "image", "audio", "video", "tool_use"] + ) + assert sanitized[0]["content"][1] == block + assert stats.fixed_video_blocks == 0 + + +@pytest.mark.asyncio +async def test_prepare_chat_payload_materializes_context_video_urls(monkeypatch): + """The OpenAI source resolves video_url context blocks into data URLs.""" + import astrbot.core.provider.sources.openai_source as openai_source_module + + async def fake_resolve_media_ref_to_base64_data( + media_ref, *, media_type, strict=False + ): + assert media_type == "video" + return ResolvedMediaData(base64_data="abcd", mime_type="video/mp4") + + monkeypatch.setattr( + openai_source_module, + "resolve_media_ref_to_base64_data", + fake_resolve_media_ref_to_base64_data, + ) + + provider = ProviderOpenAIOfficial( + provider_config={ + "id": "test-openai", + "type": "openai_chat_completion", + "model": "gpt-4o-mini", + "key": ["test-key"], + }, + provider_settings={}, + ) + try: + contexts = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + { + "type": "video_url", + "video_url": {"url": "https://example.com/clip.mp4"}, + }, + ], + } + ] + payloads, _ = await provider._prepare_chat_payload( + prompt=None, contexts=contexts + ) + assert payloads["messages"][0]["content"] == [ + {"type": "text", "text": "look"}, + { + "type": "video_url", + "video_url": {"url": "data:video/mp4;base64,abcd"}, + }, + ] + finally: + await provider.terminate() + + +def test_anthropic_prepare_payload_converts_video_url_to_anthropic_video_block(): + """The Anthropic source must convert video_url blocks into Anthropic video blocks.""" + from astrbot.core.provider.sources.anthropic_source import ProviderAnthropic + + provider = ProviderAnthropic( + provider_config={ + "id": "test-anthropic", + "type": "anthropic_chat_completion", + "model": "MiniMax-M3", + "key": ["test-key"], + "api_base": "https://api.minimaxi.com/anthropic", + }, + provider_settings={}, + ) + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this clip"}, + { + "type": "video_url", + "video_url": {"url": "data:video/mp4;base64,abcd"}, + }, + ], + } + ] + _, new_messages = provider._prepare_payload(messages) + user_blocks = new_messages[0]["content"] + assert user_blocks[0] == {"type": "text", "text": "describe this clip"} + assert user_blocks[1] == { + "type": "video", + "source": { + "type": "base64", + "media_type": "video/mp4", + "data": "abcd", + }, + }