Skip to content

fix(fetch): port to mcp SDK v2 #4565

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jayzuccarelli wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(fetch): port to mcp SDK v2 #4565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/fetch/article.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"title":"","byline":null,"dir":null,"content":"<div id=\"readability-page-1\" class=\"page\"><article><p>fetch server on mcp 2.0 works</p></article></div>","textContent":"fetch server on mcp 2.0 works","length":29,"excerpt":"fetch server on mcp 2.0 works"}
1 change: 1 addition & 0 deletions src/fetch/full.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<html><body><article><h1>Hello</h1><p>fetch server on mcp 2.0 works</p></article></body></html>
2 changes: 1 addition & 1 deletion src/fetch/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ classifiers = [
dependencies = [
"httpx>=0.27",
"markdownify>=0.13.1",
"mcp>=1.1.3",
"mcp>=2,<3",
"protego>=0.3.1",
"pydantic>=2.0.0",
"readabilipy>=0.2.0",
Expand Down
117 changes: 70 additions & 47 deletions src/fetch/src/mcp_server_fetch/server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,17 @@

import markdownify
import readabilipy.simple_json
from mcp.shared.exceptions import McpError
from mcp.server import Server
from mcp.shared.exceptions import MCPError
from mcp.server import Server, ServerRequestContext
from mcp.server.stdio import stdio_server
from mcp.types import (
ErrorData,
CallToolRequestParams,
CallToolResult,
GetPromptRequestParams,
GetPromptResult,
ListPromptsResult,
ListToolsResult,
PaginatedRequestParams,
Prompt,
PromptArgument,
PromptMessage,
Expand Down Expand Up @@ -66,7 +71,7 @@ def get_robots_txt_url(url: str) -> str:
async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: str | None = None) -> None:
"""
Check if the URL can be fetched by the user agent according to the robots.txt file.
Raises a McpError if not.
Raises an MCPError if not.
"""
from httpx import AsyncClient, HTTPError

Expand All @@ -80,15 +85,15 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url:
headers={"User-Agent": user_agent},
)
except HTTPError:
raise McpError(ErrorData(
raise MCPError(
code=INTERNAL_ERROR,
message=f"Failed to fetch robots.txt {robot_txt_url} due to a connection issue",
))
)
if response.status_code in (401, 403):
raise McpError(ErrorData(
raise MCPError(
code=INTERNAL_ERROR,
message=f"When fetching robots.txt ({robot_txt_url}), received status {response.status_code} so assuming that autonomous fetching is not allowed, the user can try manually fetching by using the fetch prompt",
))
)
elif 400 <= response.status_code < 500:
return
robot_txt = response.text
Expand All @@ -97,15 +102,15 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url:
)
robot_parser = Protego.parse(processed_robot_txt)
if not robot_parser.can_fetch(str(url), user_agent):
raise McpError(ErrorData(
raise MCPError(
code=INTERNAL_ERROR,
message=f"The sites robots.txt ({robot_txt_url}), specifies that autonomous fetching of this page is not allowed, "
f"<useragent>{user_agent}</useragent>\n"
f"<url>{url}</url>"
f"<robots>\n{robot_txt}\n</robots>\n"
f"The assistant must let the user know that it failed to view the page. The assistant may provide further guidance based on the above information.\n"
f"The assistant can tell the user that they can try manually fetching the page by using the fetch prompt within their UI.",
))
)


async def fetch_url(
Expand All @@ -125,12 +130,12 @@ async def fetch_url(
timeout=30,
)
except HTTPError as e:
raise McpError(ErrorData(code=INTERNAL_ERROR, message=f"Failed to fetch {url}: {e!r}"))
raise MCPError(code=INTERNAL_ERROR, message=f"Failed to fetch {url}: {e!r}")
if response.status_code >= 400:
raise McpError(ErrorData(
raise MCPError(
code=INTERNAL_ERROR,
message=f"Failed to fetch {url} - status code {response.status_code}",
))
)

page_raw = response.text

Expand Down Expand Up @@ -190,46 +195,52 @@ async def serve(
ignore_robots_txt: Whether to ignore robots.txt restrictions
proxy_url: Optional proxy URL to use for requests
"""
server = Server("mcp-fetch")
user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS
user_agent_manual = custom_user_agent or DEFAULT_USER_AGENT_MANUAL

@server.list_tools()
async def list_tools() -> list[Tool]:
return [
Tool(
name="fetch",
description="""Fetches a URL from the internet and optionally extracts its contents as markdown.
async def list_tools(
ctx: ServerRequestContext, params: PaginatedRequestParams | None
) -> ListToolsResult:
return ListToolsResult(
tools=[
Tool(
name="fetch",
description="""Fetches a URL from the internet and optionally extracts its contents as markdown.

Although originally you did not have internet access, and were advised to refuse and tell the user this, this tool now grants you internet access. Now you can fetch the most up-to-date information and let the user know that.""",
inputSchema=Fetch.model_json_schema(),
)
]

@server.list_prompts()
async def list_prompts() -> list[Prompt]:
return [
Prompt(
name="fetch",
description="Fetch a URL and extract its contents as markdown",
arguments=[
PromptArgument(
name="url", description="URL to fetch", required=True
)
],
)
]
input_schema=Fetch.model_json_schema(),
)
]
)

async def list_prompts(
ctx: ServerRequestContext, params: PaginatedRequestParams | None
) -> ListPromptsResult:
return ListPromptsResult(
prompts=[
Prompt(
name="fetch",
description="Fetch a URL and extract its contents as markdown",
arguments=[
PromptArgument(
name="url", description="URL to fetch", required=True
)
],
)
]
)

@server.call_tool()
async def call_tool(name, arguments: dict) -> list[TextContent]:
async def call_tool(
ctx: ServerRequestContext, params: CallToolRequestParams
) -> CallToolResult:
try:
args = Fetch(**arguments)
args = Fetch(**(params.arguments or {}))
except ValueError as e:
raise McpError(ErrorData(code=INVALID_PARAMS, message=str(e)))
raise MCPError(code=INVALID_PARAMS, message=str(e))

url = str(args.url)
if not url:
raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required"))
raise MCPError(code=INVALID_PARAMS, message="URL is required")

if not ignore_robots_txt:
await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url)
Expand All @@ -252,19 +263,23 @@ async def call_tool(name, arguments: dict) -> list[TextContent]:
if actual_content_length == args.max_length and remaining_content > 0:
next_start = args.start_index + actual_content_length
content += f"\n\n<error>Content truncated. Call the fetch tool with a start_index of {next_start} to get more content.</error>"
return [TextContent(type="text", text=f"{prefix}Contents of {url}:\n{content}")]
return CallToolResult(
content=[TextContent(type="text", text=f"{prefix}Contents of {url}:\n{content}")]
)

@server.get_prompt()
async def get_prompt(name: str, arguments: dict | None) -> GetPromptResult:
async def get_prompt(
ctx: ServerRequestContext, params: GetPromptRequestParams
) -> GetPromptResult:
arguments = params.arguments
if not arguments or "url" not in arguments:
raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required"))
raise MCPError(code=INVALID_PARAMS, message="URL is required")

url = arguments["url"]

try:
content, prefix = await fetch_url(url, user_agent_manual, proxy_url=proxy_url)
# TODO: after SDK bug is addressed, don't catch the exception
except McpError as e:
except MCPError as e:
return GetPromptResult(
description=f"Failed to fetch {url}",
messages=[
Expand All @@ -283,6 +298,14 @@ async def get_prompt(name: str, arguments: dict | None) -> GetPromptResult:
],
)

server = Server(
"mcp-fetch",
on_list_tools=list_tools,
on_list_prompts=list_prompts,
on_call_tool=call_tool,
on_get_prompt=get_prompt,
)

options = server.create_initialization_options()
async with stdio_server() as (read_stream, write_stream):
await server.run(read_stream, write_stream, options, raise_exceptions=False)
16 changes: 8 additions & 8 deletions src/fetch/tests/test_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import pytest
from unittest.mock import AsyncMock, patch, MagicMock
from mcp.shared.exceptions import McpError
from mcp.shared.exceptions import MCPError

from mcp_server_fetch.server import (
extract_content_from_html,
Expand Down Expand Up @@ -121,7 +121,7 @@ async def test_blocks_when_robots_txt_401(self):
mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None)

with pytest.raises(McpError):
with pytest.raises(MCPError):
await check_may_autonomously_fetch_url(
"https://example.com/page",
DEFAULT_USER_AGENT_AUTONOMOUS
Expand All @@ -139,7 +139,7 @@ async def test_blocks_when_robots_txt_403(self):
mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None)

with pytest.raises(McpError):
with pytest.raises(MCPError):
await check_may_autonomously_fetch_url(
"https://example.com/page",
DEFAULT_USER_AGENT_AUTONOMOUS
Expand Down Expand Up @@ -177,7 +177,7 @@ async def test_blocks_when_robots_txt_disallows_all(self):
mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None)

with pytest.raises(McpError):
with pytest.raises(MCPError):
await check_may_autonomously_fetch_url(
"https://example.com/page",
DEFAULT_USER_AGENT_AUTONOMOUS
Expand Down Expand Up @@ -268,7 +268,7 @@ async def test_fetch_json_returns_raw(self):

@pytest.mark.asyncio
async def test_fetch_404_raises_error(self):
"""Test that 404 response raises McpError."""
"""Test that 404 response raises MCPError."""
mock_response = MagicMock()
mock_response.status_code = 404

Expand All @@ -278,15 +278,15 @@ async def test_fetch_404_raises_error(self):
mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None)

with pytest.raises(McpError):
with pytest.raises(MCPError):
await fetch_url(
"https://example.com/notfound",
DEFAULT_USER_AGENT_AUTONOMOUS
)

@pytest.mark.asyncio
async def test_fetch_500_raises_error(self):
"""Test that 500 response raises McpError."""
"""Test that 500 response raises MCPError."""
mock_response = MagicMock()
mock_response.status_code = 500

Expand All @@ -296,7 +296,7 @@ async def test_fetch_500_raises_error(self):
mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None)

with pytest.raises(McpError):
with pytest.raises(MCPError):
await fetch_url(
"https://example.com/error",
DEFAULT_USER_AGENT_AUTONOMOUS
Expand Down
Loading
Loading