starred/servers-modelcontextprotocol
1
0
Fork 0
mirror of https://github.com/modelcontextprotocol/servers.git synced 2026-04-18 16:13:22 +02:00
Code Issues Packages Projects Releases Wiki Activity

make the fetch mcp-server obey robots.txt

Browse Source
This commit is contained in:
Jack Adamson
2024-11-22 18:51:39 +00:00
parent c08d2a3cf6
commit 84321c89cf
3 changed files with 85 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/fetch/pyproject.toml
View File

@@ -1,6 +1,6 @@
[project]
name = "mcp-server-fetch"
version = "0.1.1"
version = "0.1.2"
description = "A Model Context Protocol server providing tools to fetch and convert web content for usage by LLMs"
readme = "README.md"
requires-python = ">=3.10"
@@ -18,6 +18,7 @@ classifiers = [
dependencies = [
"markdownify>=0.13.1",
"mcp>=0.6.0",
"protego>=0.3.1",
"pydantic>=2.0.0",
"readabilipy>=0.2.0",
"requests>=2.32.3",