Skip to content

feat: wire up video input modality for MiniMax providers #9424

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
octo-patch wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat: wire up video input modality for MiniMax providers #9424

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion astrbot/core/agent/message.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class ContentPart(BaseModel):

__content_part_registry: ClassVar[dict[str, type["ContentPart"]]] = {}

type: Literal["text", "think", "image_url", "audio_url"]
type: Literal["text", "think", "image_url", "audio_url", "video_url"]
_no_save: bool = PrivateAttr(default=False)

def __init_subclass__(cls, **kwargs: Any) -> None:
Expand Down Expand Up @@ -141,6 +141,22 @@ class AudioURL(BaseModel):
audio_url: AudioURL


class VideoURLPart(ContentPart):
"""
>>> VideoURLPart(video_url=VideoURLPart.VideoURL(url="https://example.com/video.mp4")).model_dump()
{'type': 'video_url', 'video_url': {'url': 'https://example.com/video.mp4', 'id': None}}
"""

class VideoURL(BaseModel):
url: str
"""The URL of the video, can be a data URI scheme like `data:video/mp4;base64,...`."""
id: str | None = None
"""The ID of the video, to allow LLMs to distinguish different videos."""

type: str = "video_url"
video_url: VideoURL


class ToolCall(BaseModel):
"""
A tool call requested by the assistant.
Expand Down
8 changes: 7 additions & 1 deletion astrbot/core/agent/runners/tool_loop_agent_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,7 @@ async def reset(
request.prompt is not None
or request.image_urls
or request.audio_urls
or request.video_urls
or request.extra_user_content_parts
):
m = await self._assemble_request_context_for_provider(request)
Expand Down Expand Up @@ -341,13 +342,15 @@ async def _assemble_request_context_for_provider(

supports_image = "image" in modalities
supports_audio = "audio" in modalities
if supports_image and supports_audio:
supports_video = "video" in modalities
if supports_image and supports_audio and supports_video:
return await request.assemble_context()

adjusted_request = replace(
request,
image_urls=request.image_urls if supports_image else [],
audio_urls=request.audio_urls if supports_audio else [],
video_urls=request.video_urls if supports_video else [],
)
context = await adjusted_request.assemble_context()
content = context.get("content")
Expand All @@ -364,6 +367,9 @@ async def _assemble_request_context_for_provider(
if not supports_audio:
for _ in request.audio_urls:
content_blocks.append({"type": "text", "text": "[Audio]"})
if not supports_video:
for _ in request.video_urls:
content_blocks.append({"type": "text", "text": "[Video]"})

return {"role": "user", "content": content_blocks}

Expand Down
10 changes: 9 additions & 1 deletion astrbot/core/astr_main_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -765,6 +765,7 @@ async def _append_video_attachment(
else:
text = f"[Video Attachment: name {video_name}, path {video_path}]"

req.video_urls.append(video_path)
req.extra_user_content_parts.append(TextPart(text=text))


Expand Down Expand Up @@ -1394,6 +1395,7 @@ async def build_main_agent(
req.prompt = ""
req.image_urls = []
req.audio_urls = []
req.video_urls = []
if sel_model := event.get_extra("selected_model"):
req.model = sel_model
if config.provider_wake_prefix and not event.message_str.startswith(
Expand Down Expand Up @@ -1539,6 +1541,7 @@ async def build_main_agent(
)
req.image_urls = normalize_and_dedupe_strings(req.image_urls)
req.audio_urls = normalize_and_dedupe_strings(req.audio_urls)
req.video_urls = normalize_and_dedupe_strings(req.video_urls)

if config.file_extract_enabled:
try:
Expand All @@ -1548,7 +1551,12 @@ async def build_main_agent(

has_reply = any(isinstance(comp, Reply) for comp in event.message_obj.message)

if not req.prompt and not req.image_urls and not req.audio_urls:
if (
not req.prompt
and not req.image_urls
and not req.audio_urls
and not req.video_urls
):
if has_reply or req.extra_user_content_parts:
req.prompt = "<attachment>"
else:
Expand Down
4 changes: 2 additions & 2 deletions astrbot/core/config/default.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2071,8 +2071,8 @@
"description": "模型能力",
"type": "list",
"items": {"type": "string"},
"options": ["text", "image", "audio", "tool_use"],
"labels": ["文本", "图像", "音频", "工具使用"],
"options": ["text", "image", "audio", "video", "tool_use"],
"labels": ["文本", "图像", "音频", "视频", "工具使用"],
"render_type": "checkbox",
"hint": "模型支持的模态及能力。",
},
Expand Down
42 changes: 41 additions & 1 deletion astrbot/core/provider/entities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ class ProviderRequest:
"""图片 URL 列表"""
audio_urls: list[str] = field(default_factory=list)
"""音频 URL 列表，也支持本地路径"""
video_urls: list[str] = field(default_factory=list)
"""视频 URL 列表，也支持本地路径"""
extra_user_content_parts: list[ContentPart] = field(default_factory=list)
"""额外的用户消息内容部分列表，用于在用户消息后添加额外的内容块（如系统提醒、指令等）。支持 dict 或 ContentPart 对象"""
func_tool: ToolSet | None = None
Expand All @@ -118,6 +120,7 @@ def __repr__(self) -> str:
f"ProviderRequest(prompt={self.prompt}, session_id={self.session_id}, "
f"image_count={len(self.image_urls or [])}, "
f"audio_count={len(self.audio_urls or [])}, "
f"video_count={len(self.video_urls or [])}, "
f"func_tool={self.func_tool}, "
f"contexts={self._print_friendly_context()}, "
f"system_prompt={self.system_prompt}, "
Expand All @@ -140,7 +143,8 @@ def _print_friendly_context(self):
if not self.contexts:
return (
f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}, "
f"audio_count: {len(self.audio_urls or [])}"
f"audio_count: {len(self.audio_urls or [])}, "
f"video_count: {len(self.video_urls or [])}"
)

result_parts = []
Expand All @@ -157,6 +161,7 @@ def _print_friendly_context(self):
msg_parts = []
image_count = 0
audio_count = 0
video_count = 0

for item in content:
item_type = item.get("type", "")
Expand All @@ -167,6 +172,8 @@ def _print_friendly_context(self):
image_count += 1
elif item_type == "audio_url":
audio_count += 1
elif item_type == "video_url":
video_count += 1

if image_count > 0:
if msg_parts:
Expand All @@ -178,6 +185,11 @@ def _print_friendly_context(self):
msg_parts.append(f"[+{audio_count} audios]")
else:
msg_parts.append(f"[{audio_count} audios]")
if video_count > 0:
if msg_parts:
msg_parts.append(f"[+{video_count} videos]")
else:
msg_parts.append(f"[{video_count} videos]")

result_parts.append(f"{role}: {''.join(msg_parts)}")

Expand All @@ -197,6 +209,9 @@ async def assemble_context(self) -> dict:
elif self.audio_urls:
# 如果没有文本但有音频，添加占位文本
content_blocks.append({"type": "text", "text": "[音频]"})
elif self.video_urls:
# 如果没有文本但有视频，添加占位文本
content_blocks.append({"type": "text", "text": "[Video]"})

# 2. 额外的内容块（系统提醒、指令等）
if self.extra_user_content_parts:
Expand Down Expand Up @@ -245,13 +260,38 @@ async def assemble_context(self) -> dict:
},
)

# 5. 视频内容
if self.video_urls:
for video_url in self.video_urls:
try:
video_data = await MediaResolver(
video_url,
media_type="video",
default_suffix=".mp4",
).to_base64_data(strict=True)
except Exception as exc:
logger.warning(
"Video preprocessing failed, will skip. Error: %s", exc
)
continue
if not video_data:
logger.warning("Video preprocessing result is empty, will skip.")
continue
content_blocks.append(
{
"type": "video_url",
"video_url": {"url": video_data.to_data_url()},
},
)

# 只有当只有一个来自 prompt 的文本块且没有额外内容块时，才降级为简单格式以保持向后兼容
if (
len(content_blocks) == 1
and content_blocks[0]["type"] == "text"
and not self.extra_user_content_parts
and not self.image_urls
and not self.audio_urls
and not self.video_urls
):
return {"role": "user", "content": content_blocks[0]["text"]}

Expand Down
12 changes: 10 additions & 2 deletions astrbot/core/provider/modalities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
class ContextSanitizeStats:
fixed_image_blocks: int = 0
fixed_audio_blocks: int = 0
fixed_video_blocks: int = 0
fixed_tool_messages: int = 0
removed_tool_calls: int = 0

Expand All @@ -21,6 +22,7 @@ def changed(self) -> bool:
return bool(
self.fixed_image_blocks
or self.fixed_audio_blocks
or self.fixed_video_blocks
or self.fixed_tool_messages
or self.removed_tool_calls
)
Expand Down Expand Up @@ -50,8 +52,9 @@ def sanitize_contexts_by_modalities(

supports_image = "image" in modalities
supports_audio = "audio" in modalities
supports_video = "video" in modalities
supports_tool_use = "tool_use" in modalities
if supports_image and supports_audio and supports_tool_use:
if supports_image and supports_audio and supports_video and supports_tool_use:
copied_contexts = []
for msg in contexts:
copied_msg = _message_to_dict(msg)
Expand Down Expand Up @@ -83,7 +86,7 @@ def sanitize_contexts_by_modalities(
msg.pop("tool_calls", None)
msg.pop("tool_call_id", None)

if not supports_image or not supports_audio:
if not supports_image or not supports_audio or not supports_video:
content = msg.get("content")
if isinstance(content, list):
filtered_parts: list[Any] = []
Expand All @@ -104,6 +107,11 @@ def sanitize_contexts_by_modalities(
stats.fixed_audio_blocks += 1
filtered_parts.append({"type": "text", "text": "[Audio]"})
continue
if not supports_video and part_type in {"video_url", "video"}:
removed_any_multimodal = True
stats.fixed_video_blocks += 1
filtered_parts.append({"type": "text", "text": "[Video]"})
continue
filtered_parts.append(part)
if removed_any_multimodal:
msg["content"] = filtered_parts
Expand Down
33 changes: 33 additions & 0 deletions astrbot/core/provider/sources/anthropic_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,39 @@ def _prepare_payload(self, messages: list[dict]):
"text": "[Audio Attachment]",
}
)
elif part.get("type") == "video_url":
# Convert OpenAI-style video_url format to Anthropic
# video format so models that accept video input
# actually receive the modality.
video_url_data = part.get("video_url", {})
url = video_url_data.get("url", "")
if url.startswith("data:"):
try:
header, base64_data = url.split(",", 1)
media_type = (
Comment on lines +295 to +304

@sourcery-ai sourcery-ai Bot Jul 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion: Dropping non-data video URLs entirely may be a surprising degradation for users.

For non-data: video_url parts, the code only logs a warning and skips the part, so the video attachment effectively disappears from the Anthropic request. This silent loss of user content can be surprising when callers send normal HTTP URLs. Please consider a more graceful degradation, e.g., inserting a [Video Attachment] placeholder (as with audio) or preserving the original video_url block so callers can see the unconverted modality is still present.

Suggested implementation:

                            if url.startswith("data:"):
                                try:
                                    header, base64_data = url.split(",", 1)
                                    media_type = (
                                        header.split(":", 1)[1]
                                )
                                # existing logic for decoding the data: URL and
                                # appending Anthropic-formatted video input continues here
                            else:
                                # For non-data URLs, gracefully degrade by inserting
                                # a placeholder text part instead of silently
                                # dropping the attachment.
                                # This mirrors the audio placeholder behavior.
                                logger.warning(
                                    "Non-data video_url encountered; sending '[Video Attachment]' "
                                    "placeholder text instead of video content."
                                )
                                anthropic_message["content"].append(
                                    {
                                        "type": "text",
                                        "text": "[Video Attachment]",
                                    }
                                )

I assumed:

  1. logger is the same logger used elsewhere in this file; if it has a different name (e.g., log, LOGGER), update the logger.warning(...) call accordingly.
  2. anthropic_message["content"] is the list being populated with parts in this loop; if you are collecting parts in a different list (e.g., anthropic_parts or content_parts), append the placeholder to that list instead.
  3. The existing logic below media_type = (...) remains intact and continues to handle the data: URL case; ensure you keep all existing video decoding and Anthropic video-block creation code after that line.

header.split(":", 1)[1].split(";", 1)[0].strip()
or "video/mp4"
)
converted_content.append(
{
"type": "video",
"source": {
"type": "base64",
"media_type": media_type,
"data": base64_data,
},
}
)
except (ValueError, IndexError):
logger.warning(
"Failed to parse video data URI, skipping: %s",
url[:50],
)
else:
logger.warning(
"Unsupported video URL format for Anthropic, skipping: %s",
url[:50],
)
else:
converted_content.append(part)
new_messages.append(
Expand Down
Loading
Loading