From 617b7469a39001f711ef5a847cc9d5ea71d2a17a Mon Sep 17 00:00:00 2001 From: xiaoyuyu6420 <93528429+xiaoyuyu6420@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:45:19 +0800 Subject: [PATCH 1/3] fix: sanitize unsupported image MIME types (e.g. GIF) in context cleaner Fixes #9295. When a provider declares image modality support, the context sanitizer previously kept every image block as-is. Animated GIFs can then be persisted into session history and later cause continuous failures on OpenAI-compatible Gemini gateways that reject image/gif. - Drop image/gif data URLs and .gif http(s) URLs to a [Image] placeholder even when the provider supports image modality. - Keep the existing full image-strip path when image modality is absent. - Preserve JPEG/PNG/WebP and unknown-MIME images when image is supported. - Add unit coverage for the reporter scenario and related edge cases. --- astrbot/core/provider/modalities.py | 87 ++++++++++- tests/unit/test_modalities_sanitize.py | 205 +++++++++++++++++++++++++ 2 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_modalities_sanitize.py diff --git a/astrbot/core/provider/modalities.py b/astrbot/core/provider/modalities.py index 66ac74e9b7..2f0e940703 100644 --- a/astrbot/core/provider/modalities.py +++ b/astrbot/core/provider/modalities.py @@ -34,6 +34,72 @@ def _message_to_dict(message: dict[str, Any] | Message) -> dict[str, Any] | None return None +# Image MIME types that some OpenAI-compatible gateways reject even when the +# model claims image support. Animated GIFs in particular are accepted by raw +# Gemini (which wants image/heif/video/mp4 for animations) but rejected by +# several Gemini-flavored OpenAI proxies with "mime type is not supported". +# Keeping this list narrow avoids over-stripping providers that do accept GIF +# (e.g. Anthropic Claude). See issue #9295. +_UNSUPPORTED_IMAGE_MIMES = frozenset({"image/gif"}) + +# Lazily-built extension → MIME mapping used only for http(s) URL fallback. +_IMAGE_EXT_TO_MIME: dict[str, str] = { + ".gif": "image/gif", +} + + +def _extract_image_mime(part: dict[str, Any]) -> str | None: + """Best-effort extraction of an image MIME type from a multimodal part. + + Handles the OpenAI-style ``{"image_url": {"url": ...}}`` and the Anthropic / + Gemini-style ``{"source": {"media_type": ...}}`` / ``{"mimeType": ...}`` + layouts, as well as a bare ``{"url": ...}`` or ``{"image_url": ""}``. + + Returns: + The lowercased MIME type (e.g. ``image/gif``) if it can be determined, + otherwise ``None``. + """ + image_url = part.get("image_url") + if isinstance(image_url, dict): + url = image_url.get("url") + else: + url = image_url + if not isinstance(url, str): + url = part.get("url") if isinstance(part.get("url"), str) else None + + if isinstance(url, str): + # data URLs look like "data:image/gif;base64,...." + if url.lower().startswith("data:"): + head = url[5:].split(",", 1)[0] + # head is e.g. "image/gif;base64" + mime = head.split(";", 1)[0].strip().lower() + if mime: + return mime + else: + # Fall back to the URL path extension for http(s) URLs. + path = url.split("?", 1)[0].split("#", 1)[0].lower() + for ext, mime in _IMAGE_EXT_TO_MIME.items(): + if path.endswith(ext): + return mime + + source = part.get("source") + if isinstance(source, dict): + media_type = source.get("media_type") + if isinstance(media_type, str): + return media_type.lower() + + mime_type = part.get("mimeType") or part.get("mime_type") + if isinstance(mime_type, str): + return mime_type.lower() + + return None + + +def _is_unsupported_image_mime(mime: str | None) -> bool: + """Return True when the MIME is known to be rejected by some providers.""" + return bool(mime) and mime in _UNSUPPORTED_IMAGE_MIMES + + def sanitize_contexts_by_modalities( contexts: Sequence[dict[str, Any] | Message], modalities: list[str] | None, @@ -51,7 +117,11 @@ def sanitize_contexts_by_modalities( supports_image = "image" in modalities supports_audio = "audio" in modalities supports_tool_use = "tool_use" in modalities - if supports_image and supports_audio and supports_tool_use: + # Even when the provider declares all modalities, we may still need to walk + # the contexts to drop specific image MIME types (e.g. image/gif) that some + # OpenAI-compatible gateways reject. See issue #9295. + needs_mime_pass = supports_image and bool(_UNSUPPORTED_IMAGE_MIMES) + if supports_image and supports_audio and supports_tool_use and not needs_mime_pass: copied_contexts = [] for msg in contexts: copied_msg = _message_to_dict(msg) @@ -83,7 +153,7 @@ def sanitize_contexts_by_modalities( msg.pop("tool_calls", None) msg.pop("tool_call_id", None) - if not supports_image or not supports_audio: + if not supports_image or not supports_audio or needs_mime_pass: content = msg.get("content") if isinstance(content, list): filtered_parts: list[Any] = [] @@ -91,7 +161,18 @@ def sanitize_contexts_by_modalities( for part in content: if isinstance(part, dict): part_type = str(part.get("type", "")).lower() - if not supports_image and part_type in {"image_url", "image"}: + if part_type in {"image_url", "image"} and ( + not supports_image + or _is_unsupported_image_mime(_extract_image_mime(part)) + ): + # Either the model has no image modality at all, or it + # declares image support but the specific MIME (e.g. + # image/gif) is rejected by some OpenAI-compatible + # gateways (notably certain Gemini endpoints that only + # accept JPEG/PNG/WebP). Replacing the block with a + # placeholder prevents the unsupported bytes from being + # persisted into the session history and poisoning all + # subsequent requests. See issue #9295. removed_any_multimodal = True stats.fixed_image_blocks += 1 filtered_parts.append({"type": "text", "text": "[Image]"}) diff --git a/tests/unit/test_modalities_sanitize.py b/tests/unit/test_modalities_sanitize.py new file mode 100644 index 0000000000..a338d8783f --- /dev/null +++ b/tests/unit/test_modalities_sanitize.py @@ -0,0 +1,205 @@ +"""Tests for ``astrbot.core.provider.modalities.sanitize_contexts_by_modalities``. + +These tests focus on the image MIME handling added for issue #9295, where an +animated GIF referenced via a quote could poison the session history and make +subsequent requests to GIF-rejecting Gemini-compatible gateways fail forever. +""" + +from __future__ import annotations + +from astrbot.core.provider.modalities import ( + ContextSanitizeStats, + sanitize_contexts_by_modalities, +) + + +def _image_url_part(url: str) -> dict: + return {"type": "image_url", "image_url": {"url": url}} + + +def _user(*parts: dict) -> dict: + return {"role": "user", "content": list(parts)} + + +GIF_DATA_URL = "data:image/gif;base64,R0lGODlh8ADwAPcAAPxzxg==" +PNG_DATA_URL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB" +JPEG_DATA_URL = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD" +WEBP_DATA_URL = "data:image/webp;base64,UklGRkBAAABXRUJQ" + + +# --------------------------------------------------------------------------- +# Issue #9295: GIF must be stripped even when the model claims image support +# --------------------------------------------------------------------------- + + +def test_gif_data_url_replaced_when_image_supported_but_gif_unsupported() -> None: + """Reproduces the exact reporter scenario. + + Provider declares ``[text, image, audio, tool_use]`` (which used to hit the + fast-path and skip sanitizing), and the context carries a ``data:image/gif`` + block. The GIF must be replaced with ``[Image]``. + """ + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "audio", "tool_use"], + ) + + content = sanitized[0]["content"] + assert content == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + assert stats.changed + + +def test_gif_data_url_replaced_when_only_text_and_image_supported() -> None: + """Stripping also applies to the regular image-supported path.""" + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_gif_http_url_replaced_by_extension_fallback() -> None: + """http(s) URLs ending in ``.gif`` are also detected via extension.""" + contexts = [_user(_image_url_part("https://example.com/animation.gif"))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "tool_use"], + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_gif_http_url_with_query_string_still_detected() -> None: + """Query/fragment suffixes on the URL must not defeat detection.""" + contexts = [ + _user(_image_url_part("https://cdn.example.com/a.GIF?token=abc#frag")), + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +# --------------------------------------------------------------------------- +# Non-GIF images must be preserved when image is supported +# --------------------------------------------------------------------------- + + +def test_png_preserved_when_image_supported() -> None: + contexts = [_user(_image_url_part(PNG_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "audio", "tool_use"], + ) + + assert sanitized[0]["content"] == [_image_url_part(PNG_DATA_URL)] + assert stats.fixed_image_blocks == 0 + assert not stats.changed + + +def test_jpeg_and_webp_preserved_when_image_supported() -> None: + contexts = [ + _user(_image_url_part(JPEG_DATA_URL), _image_url_part(WEBP_DATA_URL)), + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + ) + + assert sanitized[0]["content"] == [ + _image_url_part(JPEG_DATA_URL), + _image_url_part(WEBP_DATA_URL), + ] + assert not stats.changed + + +def test_unknown_mime_image_preserved_when_image_supported() -> None: + """When no MIME can be determined we must not strip the image. + + Stripping on "unknown" would over-aggressively drop legitimate images and + break providers that accept arbitrary image formats. + """ + contexts = [_user({"type": "image_url", "image_url": {"url": "abc-not-a-url"}})] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == contexts[0]["content"] + assert not stats.changed + + +# --------------------------------------------------------------------------- +# Existing behavior: full removal when image modality is not declared +# --------------------------------------------------------------------------- + + +def test_all_images_removed_when_image_not_supported() -> None: + """Pre-existing behavior must remain: no image modality → strip everything.""" + contexts = [ + _user( + _image_url_part(GIF_DATA_URL), + _image_url_part(PNG_DATA_URL), + ) + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text"]) + + assert sanitized[0]["content"] == [ + {"type": "text", "text": "[Image]"}, + {"type": "text", "text": "[Image]"}, + ] + assert stats.fixed_image_blocks == 2 + + +def test_audio_branch_unchanged_by_gif_handling() -> None: + """The audio-stripping branch must keep working independently.""" + contexts = [ + _user( + {"type": "input_audio", "input_audio": {"data": "abc", "format": "mp3"}}, + ) + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Audio]"}] + assert stats.fixed_audio_blocks == 1 + assert stats.fixed_image_blocks == 0 + + +def test_mixed_text_and_gif_only_gif_is_replaced() -> None: + """A GIF between text parts must only drop the GIF, preserving the text.""" + contexts = [ + _user( + {"type": "text", "text": "look at this"}, + _image_url_part(GIF_DATA_URL), + {"type": "text", "text": "isn't it cool"}, + ) + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "audio", "tool_use"], + ) + + assert sanitized[0]["content"] == [ + {"type": "text", "text": "look at this"}, + {"type": "text", "text": "[Image]"}, + {"type": "text", "text": "isn't it cool"}, + ] + assert stats.fixed_image_blocks == 1 + + +def test_empty_modalities_returns_contexts_unchanged() -> None: + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities(contexts, None) + + assert sanitized[0]["content"] == contexts[0]["content"] + assert not stats.changed + + +def test_empty_contexts_returns_empty() -> None: + sanitized, stats = sanitize_contexts_by_modalities([], ["text", "image"]) + assert sanitized == [] + assert isinstance(stats, ContextSanitizeStats) + assert not stats.changed From fee8be24f9c3d109b79235fbe8dd9e774fc7d223 Mon Sep 17 00:00:00 2001 From: xiaoyuyu6420 <93528429+xiaoyuyu6420@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:32:27 +0800 Subject: [PATCH 2/3] fix: normalize MIME params and harden URL parsing in context sanitizer Address Sourcery/Gemini review feedback on #9335 without adding new abstractions. - Strip parameters (e.g. "image/gif; charset=binary") and whitespace from source.media_type and mimeType/mime_type before comparing against _UNSUPPORTED_IMAGE_MIMES, so parameterized GIF values are still detected. - Strip surrounding whitespace from image URLs defensively. - Switch the http(s) URL path extraction from a manual split('?', '#') to urllib.parse.urlsplit, which keeps percent-encoded delimiters inside the path and is more robust for edge cases. - Inline the normalization at both call sites instead of introducing a helper, keeping it consistent with the existing data-URL branch and honoring the repo's No-Unnecessary-Helpers rule. Adds 5 unit tests covering parameterized media_type/mimeType, the encoded path case, and a non-GIF-with-params regression guard. --- astrbot/core/provider/modalities.py | 18 ++++-- tests/unit/test_modalities_sanitize.py | 78 ++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/astrbot/core/provider/modalities.py b/astrbot/core/provider/modalities.py index 2f0e940703..bae20ac31d 100644 --- a/astrbot/core/provider/modalities.py +++ b/astrbot/core/provider/modalities.py @@ -4,6 +4,7 @@ from collections.abc import Sequence from dataclasses import dataclass from typing import Any +from urllib.parse import urlsplit from astrbot import logger from astrbot.core.agent.message import Message @@ -56,7 +57,7 @@ def _extract_image_mime(part: dict[str, Any]) -> str | None: layouts, as well as a bare ``{"url": ...}`` or ``{"image_url": ""}``. Returns: - The lowercased MIME type (e.g. ``image/gif``) if it can be determined, + The normalized MIME type (e.g. ``image/gif``) if it can be determined, otherwise ``None``. """ image_url = part.get("image_url") @@ -68,6 +69,7 @@ def _extract_image_mime(part: dict[str, Any]) -> str | None: url = part.get("url") if isinstance(part.get("url"), str) else None if isinstance(url, str): + url = url.strip() # data URLs look like "data:image/gif;base64,...." if url.lower().startswith("data:"): head = url[5:].split(",", 1)[0] @@ -76,8 +78,10 @@ def _extract_image_mime(part: dict[str, Any]) -> str | None: if mime: return mime else: - # Fall back to the URL path extension for http(s) URLs. - path = url.split("?", 1)[0].split("#", 1)[0].lower() + # Fall back to the URL path extension for http(s) URLs. urlsplit + # robustly separates the path from query/fragment even when those + # delimiters appear percent-encoded inside the path itself. + path = urlsplit(url).path.lower() for ext, mime in _IMAGE_EXT_TO_MIME.items(): if path.endswith(ext): return mime @@ -86,11 +90,15 @@ def _extract_image_mime(part: dict[str, Any]) -> str | None: if isinstance(source, dict): media_type = source.get("media_type") if isinstance(media_type, str): - return media_type.lower() + # Normalize by stripping parameters (e.g. "image/gif; charset=binary" + # -> "image/gif") so it matches _UNSUPPORTED_IMAGE_MIMES. + return media_type.split(";", 1)[0].strip().lower() mime_type = part.get("mimeType") or part.get("mime_type") if isinstance(mime_type, str): - return mime_type.lower() + # Strip parameters (e.g. "image/gif;codec=xyz" -> "image/gif") to align + # with the data-URL and source.media_type handling above. + return mime_type.split(";", 1)[0].strip().lower() return None diff --git a/tests/unit/test_modalities_sanitize.py b/tests/unit/test_modalities_sanitize.py index a338d8783f..966101708a 100644 --- a/tests/unit/test_modalities_sanitize.py +++ b/tests/unit/test_modalities_sanitize.py @@ -86,6 +86,84 @@ def test_gif_http_url_with_query_string_still_detected() -> None: assert stats.fixed_image_blocks == 1 +def test_gif_http_url_path_strips_query_via_urlsplit() -> None: + """A real ``?`` inside the path (percent-encoded) must not be mis-split. + + ``urlsplit`` keeps the percent-encoded ``%3F`` inside the path, so a path + ending in ``.gif`` followed by an encoded delimiter is still detected. + """ + contexts = [ + _user( + _image_url_part( + "https://cdn.example.com/path%3Fextra/anim.gif?sig=1#x", + ), + ), + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_anthropic_media_type_with_params_still_detected() -> None: + """``media_type`` carrying parameters must normalize down to ``image/gif``.""" + contexts = [ + _user( + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/gif; charset=binary", + "data": "R0lGODlh8ADwAPcAAPxzxg==", + }, + }, + ), + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "audio", "tool_use"], + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_gemini_mimetype_field_with_params_still_detected() -> None: + """A bare ``mimeType``/``mime_type`` with params must also be normalized.""" + contexts = [ + _user( + { + "type": "image", + "mimeType": " image/gif;codec=xyz ", + }, + ), + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_non_gif_media_type_with_params_preserved() -> None: + """Non-unsupported MIME with parameters must be preserved, not stripped.""" + contexts = [ + _user( + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png; charset=utf-8", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB", + }, + }, + ), + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == contexts[0]["content"] + assert not stats.changed + + # --------------------------------------------------------------------------- # Non-GIF images must be preserved when image is supported # --------------------------------------------------------------------------- From b0bf6f73bc40226c2c78d0e9a575c4aac1d3a34d Mon Sep 17 00:00:00 2001 From: xiaoyuyu6420 <93528429+xiaoyuyu6420@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:31:14 +0800 Subject: [PATCH 3/3] feat(provider): add configurable image MIME whitelist (supported_image_mimes) Replaces the hardcoded GIF blocklist with a per-provider whitelist, addressing feedback that the global blocklist was too heavy-handed. Changes: - Add 'supported_image_mimes' config field (checkbox: JPEG/PNG/WebP/GIF) in provider settings. Unchecked = fallback blocklist (GIF filtered on known- problematic gateways). Checked = strict whitelist mode. - sanitize_contexts_by_modalities() now accepts supported_image_mimes param. - When whitelist is provided, only images with MIME types in the list are kept. - When whitelist is None/empty, existing GIF blocklist is used as fallback (backward compatible, preserves #9295 fix). - Schema-driven UI: WebUI auto-generates the checkbox group from config schema. - Update i18n (en-US, zh-CN) for the new field. - Add 4 tests: whitelist filtering, whitelist includes GIF, None fallback, empty-list fallback. Fixes suggestion from @sujoshua and Sourcery review on #9335. Tests: 20 passed (16 existing + 4 new whitelist tests). --- astrbot/core/agent/context/compressor.py | 1 + .../agent/runners/tool_loop_agent_runner.py | 1 + astrbot/core/config/default.py | 17 +++++ astrbot/core/provider/modalities.py | 59 ++++++++++----- .../en-US/features/config-metadata.json | 10 +++ .../zh-CN/features/config-metadata.json | 10 +++ tests/unit/test_modalities_sanitize.py | 71 +++++++++++++++++++ 7 files changed, 153 insertions(+), 16 deletions(-) diff --git a/astrbot/core/agent/context/compressor.py b/astrbot/core/agent/context/compressor.py index 759604dd93..685d417426 100644 --- a/astrbot/core/agent/context/compressor.py +++ b/astrbot/core/agent/context/compressor.py @@ -268,6 +268,7 @@ async def __call__(self, messages: list[Message]) -> list[Message]: sanitized_summary_contexts, sanitize_stats = sanitize_contexts_by_modalities( summary_contexts, self.provider.provider_config.get("modalities", None), + self.provider.provider_config.get("supported_image_mimes", None), ) log_context_sanitize_stats(sanitize_stats) diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index 98754f9b6a..44d31484f0 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -595,6 +595,7 @@ def _sanitize_contexts_for_provider( sanitized_contexts, stats = sanitize_contexts_by_modalities( contexts, self.provider.provider_config.get("modalities", None), + self.provider.provider_config.get("supported_image_mimes", None), ) log_context_sanitize_stats(stats) return sanitized_contexts diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index ae6a3e7883..9f43291390 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2090,6 +2090,23 @@ "render_type": "checkbox", "hint": "模型支持的模态及能力。", }, + "supported_image_mimes": { + "description": "支持的图片 MIME 类型", + "type": "list", + "items": {"type": "string"}, + "options": [ + "image/jpeg", + "image/png", + "image/webp", + "image/gif", + ], + "labels": ["JPEG", "PNG", "WebP", "GIF"], + "render_type": "checkbox", + "hint": ( + "模型支持的图片格式。不勾选时保留全部（仅过滤已知不安全格式如 GIF）。" + "勾选后仅保留勾选的格式。" + ), + }, "custom_headers": { "description": "自定义请求头", "type": "dict", diff --git a/astrbot/core/provider/modalities.py b/astrbot/core/provider/modalities.py index bae20ac31d..b5739bed01 100644 --- a/astrbot/core/provider/modalities.py +++ b/astrbot/core/provider/modalities.py @@ -111,7 +111,22 @@ def _is_unsupported_image_mime(mime: str | None) -> bool: def sanitize_contexts_by_modalities( contexts: Sequence[dict[str, Any] | Message], modalities: list[str] | None, + supported_image_mimes: list[str] | None = None, ) -> tuple[list[dict[str, Any]], ContextSanitizeStats]: + """Sanitize message contexts based on provider capabilities. + + Args: + contexts: The message contexts to sanitize. + modalities: List of modalities the provider supports (e.g. ["text", "image"]). + supported_image_mimes: Whitelist of image MIME types the provider accepts. + When None or empty, a fallback blocklist (_UNSUPPORTED_IMAGE_MIMES) is used + to filter known-problematic formats (e.g. image/gif for some Gemini gateways). + When provided, images with MIME types not in this list are replaced with + "[Image]" placeholders. + + Returns: + Tuple of (sanitized contexts, statistics). + """ if not contexts: return [], ContextSanitizeStats() if not modalities or not isinstance(modalities, list): @@ -125,10 +140,12 @@ def sanitize_contexts_by_modalities( supports_image = "image" in modalities supports_audio = "audio" in modalities supports_tool_use = "tool_use" in modalities - # Even when the provider declares all modalities, we may still need to walk - # the contexts to drop specific image MIME types (e.g. image/gif) that some - # OpenAI-compatible gateways reject. See issue #9295. - needs_mime_pass = supports_image and bool(_UNSUPPORTED_IMAGE_MIMES) + # Determine whether we need a MIME-filtering pass. When a whitelist is + # provided, we always filter; otherwise we fall back to the hardcoded + # blocklist for known-problematic MIME types (e.g. image/gif). + needs_mime_pass = supports_image and ( + bool(supported_image_mimes) or bool(_UNSUPPORTED_IMAGE_MIMES) + ) if supports_image and supports_audio and supports_tool_use and not needs_mime_pass: copied_contexts = [] for msg in contexts: @@ -169,18 +186,28 @@ def sanitize_contexts_by_modalities( for part in content: if isinstance(part, dict): part_type = str(part.get("type", "")).lower() - if part_type in {"image_url", "image"} and ( - not supports_image - or _is_unsupported_image_mime(_extract_image_mime(part)) - ): - # Either the model has no image modality at all, or it - # declares image support but the specific MIME (e.g. - # image/gif) is rejected by some OpenAI-compatible - # gateways (notably certain Gemini endpoints that only - # accept JPEG/PNG/WebP). Replacing the block with a - # placeholder prevents the unsupported bytes from being - # persisted into the session history and poisoning all - # subsequent requests. See issue #9295. + # Determine whether this image part should be dropped. + should_drop_image = False + if part_type in {"image_url", "image"}: + if not supports_image: + # Provider declares no image support at all. + should_drop_image = True + else: + # Provider supports image, but may have MIME restrictions. + mime = _extract_image_mime(part) + if supported_image_mimes: + # Whitelist mode: drop if MIME not in the list. + if mime not in supported_image_mimes: + should_drop_image = True + else: + # Fallback blocklist mode: drop if MIME is known + # to be rejected by some gateways (e.g. image/gif). + if _is_unsupported_image_mime(mime): + should_drop_image = True + if should_drop_image: + # Replacing the block with a placeholder prevents unsupported + # bytes from being persisted into the session history and + # poisoning all subsequent requests. See issue #9295. removed_any_multimodal = True stats.fixed_image_blocks += 1 filtered_parts.append({"type": "text", "text": "[Image]"}) diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index 59f2c96944..25e76ba17f 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -1272,6 +1272,16 @@ "Tool use" ] }, + "supported_image_mimes": { + "description": "Supported image MIME types", + "hint": "Image formats the model accepts. When unchecked, all formats are kept (only known-unsafe formats like GIF are filtered on some gateways). When checked, only selected formats are preserved.", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF" + ] + }, "custom_headers": { "description": "Custom request headers", "hint": "Key/value pairs added here are merged into the OpenAI SDK default_headers for custom HTTP headers. Values must be strings." diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index 455587f308..d73feac355 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -1274,6 +1274,16 @@ "工具使用" ] }, + "supported_image_mimes": { + "description": "支持的图片 MIME 类型", + "hint": "模型支持的图片格式。不勾选时保留全部（仅过滤已知不安全格式如 GIF）。勾选后仅保留勾选的格式。", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF" + ] + }, "custom_headers": { "description": "自定义请求头", "hint": "此处添加的键值对将被合并到 OpenAI SDK 的 default_headers 中，用于自定义 HTTP 请求头。值必须为字符串。" diff --git a/tests/unit/test_modalities_sanitize.py b/tests/unit/test_modalities_sanitize.py index 966101708a..0f1c1cba60 100644 --- a/tests/unit/test_modalities_sanitize.py +++ b/tests/unit/test_modalities_sanitize.py @@ -281,3 +281,74 @@ def test_empty_contexts_returns_empty() -> None: assert sanitized == [] assert isinstance(stats, ContextSanitizeStats) assert not stats.changed + + +# --------------------------------------------------------------------------- +# Whitelist mode: supported_image_mimes parameter +# --------------------------------------------------------------------------- + + +def test_whitelist_filters_gif_but_preserves_jpeg_and_png() -> None: + """When whitelist excludes GIF, GIF is replaced but JPEG/PNG are kept.""" + contexts = [ + _user( + _image_url_part(GIF_DATA_URL), + _image_url_part(JPEG_DATA_URL), + _image_url_part(PNG_DATA_URL), + ), + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + supported_image_mimes=["image/jpeg", "image/png"], + ) + + assert sanitized[0]["content"] == [ + {"type": "text", "text": "[Image]"}, # GIF replaced + _image_url_part(JPEG_DATA_URL), # JPEG preserved + _image_url_part(PNG_DATA_URL), # PNG preserved + ] + assert stats.fixed_image_blocks == 1 + + +def test_whitelist_includes_gif_preserves_gif() -> None: + """When whitelist includes GIF, GIF is preserved.""" + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + supported_image_mimes=["image/jpeg", "image/png", "image/gif"], + ) + + assert sanitized[0]["content"] == [_image_url_part(GIF_DATA_URL)] + assert not stats.changed + + +def test_whitelist_none_falls_back_to_blocklist_gif_replaced() -> None: + """When whitelist is None, fallback blocklist applies (GIF replaced). + + This ensures backward compatibility: users who don't configure + supported_image_mimes still get the #9295 GIF fix. + """ + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + supported_image_mimes=None, + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_whitelist_empty_list_same_as_none() -> None: + """Empty list for whitelist behaves the same as None (blocklist fallback).""" + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + supported_image_mimes=[], + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1