starred/servers-modelcontextprotocol
1
0
Fork 0
mirror of https://github.com/modelcontextprotocol/servers.git synced 2026-04-20 21:05:17 +02:00
Code Issues Packages Projects Releases Wiki Activity

follows redirects on checking robots.txt

Browse Source
This commit is contained in:
YeongJun
2024-11-30 23:31:30 +09:00
committed by GitHub
parent 168522f34d
commit d5bae8759f
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/fetch/src/mcp_server_fetch/server.py
View File

@@ -74,7 +74,9 @@ async def check_may_autonomously_fetch_url(url: AnyUrl | str, user_agent: str) -
async with AsyncClient() as client: async with AsyncClient() as client:
try: try:
response = await client.get( response = await client.get(
robot_txt_url, headers={"User-Agent": user_agent} robot_txt_url,
follow_redirects=True,
headers={"User-Agent": user_agent},
) )
except HTTPError: except HTTPError:
raise McpError( raise McpError(