{
  "id": "web.crawl",
  "object": "tool",
  "name": "Crawl a site",
  "summary": "A whole website to clean Markdown.",
  "description": "Crawls a site breadth-first from a start URL, staying on the same origin, respecting robots.txt, with include and exclude patterns, a page limit (up to 200) and a depth limit. Returns one Markdown file per page and an index.json with each page's URL, title, depth, status and link count. Uses fast plain fetching by default, or a real browser for JavaScript sites.",
  "category": "web",
  "hosted_by": "Opentools",
  "runtime": "browser",
  "side_effects": false,
  "status": "live",
  "credits": 4,
  "price": "$0.0040",
  "providers": [
    {
      "id": "opentools",
      "name": "Opentools",
      "price": "$0.0040"
    }
  ],
  "mcp_name": "web_crawl",
  "input_schema": {
    "type": "object",
    "properties": {
      "url": {
        "type": "string",
        "format": "uri",
        "description": "Where to start"
      },
      "max_pages": {
        "default": 50,
        "description": "Stop after this many pages",
        "type": "integer",
        "minimum": 1,
        "maximum": 200
      },
      "max_depth": {
        "default": 3,
        "description": "How many links away from the start page to go",
        "type": "integer",
        "minimum": 0,
        "maximum": 10
      },
      "scope": {
        "default": "origin",
        "description": "origin: same scheme and host only. subdomains: also follow links to subdomains.",
        "type": "string",
        "enum": [
          "origin",
          "subdomains"
        ]
      },
      "include": {
        "description": "Only crawl paths matching one of these globs, e.g. [\"/docs/**\"]. * matches within a segment, ** across.",
        "maxItems": 50,
        "type": "array",
        "items": {
          "type": "string",
          "maxLength": 300
        }
      },
      "exclude": {
        "description": "Skip paths matching these globs, e.g. [\"/blog/tag/**\", \"**/*.pdf\"]",
        "maxItems": 50,
        "type": "array",
        "items": {
          "type": "string",
          "maxLength": 300
        }
      },
      "respect_robots": {
        "default": true,
        "description": "Obey robots.txt",
        "type": "boolean"
      },
      "render": {
        "default": false,
        "description": "Render each page in a real browser (slower, for JavaScript sites)",
        "type": "boolean"
      },
      "include_links": {
        "default": true,
        "description": "Keep hyperlinks in the Markdown",
        "type": "boolean"
      }
    },
    "required": [
      "url"
    ]
  },
  "kind": "tool"
}