{
  "id": "web.robots",
  "object": "tool",
  "name": "Check robots.txt",
  "summary": "Can a crawler fetch these paths?",
  "description": "Fetches and parses a site's robots.txt (or text you give) and answers whether each path is allowed for a user agent, using RFC 9309 rules: the most specific user-agent group, longest matching rule, * wildcards and $ anchors. Also lists sitemaps and the crawl delay.",
  "category": "web",
  "hosted_by": "Opentools",
  "runtime": "worker",
  "side_effects": false,
  "status": "live",
  "credits": 1,
  "price": "$0.0010",
  "providers": [
    {
      "id": "opentools",
      "name": "Opentools",
      "price": "$0.0010"
    }
  ],
  "mcp_name": "web_robots",
  "input_schema": {
    "type": "object",
    "properties": {
      "url": {
        "description": "Site or robots.txt URL",
        "type": "string",
        "minLength": 1,
        "maxLength": 2000
      },
      "text": {
        "description": "robots.txt content, instead of a URL",
        "type": "string",
        "maxLength": 1000000
      },
      "user_agent": {
        "default": "*",
        "description": "Crawler name, like \"Googlebot\" or \"GPTBot\"",
        "type": "string",
        "minLength": 1,
        "maxLength": 200
      },
      "paths": {
        "default": [],
        "description": "Paths or URLs to check",
        "maxItems": 200,
        "type": "array",
        "items": {
          "type": "string",
          "maxLength": 2000
        }
      }
    }
  },
  "kind": "tool"
}