{ "id": "contextdev.web.crawl", "object": "tool", "name": "Crawl Website", "summary": "Crawl a website and return clean Markdown for every page crawled.", "description": "Crawl a site from a starting URL and get one clean, LLM-ready Markdown document per page — the ingestion primitive for RAG pipelines, search indexes, and knowledge bases. Controls cover the page cap (maxPages, hard cap 500), link depth, a URL regex filter, subdomain following, per-page Markdown options (links, images, main-content-only, iframes, CSS include/exclude selectors), PDF parsing and OCR, cache reuse, and a soft crawl time budget (stopAfterMs, max 110s). The response reports per-page Markdown and metadata plus a crawl summary (numUrls, numSucceeded, numFailed, numSkipped).\n\nRaw contextdev#web/crawl endpoint: input is { body, queryParams, pathParams } as the provider documents them (https://docs.context.dev/api-reference/web-scraping/crawl).", "category": "lookup", "hosted_by": "Context.dev", "runtime": "partner", "side_effects": false, "status": "live", "credits": 0, "price": null, "providers": [ { "id": "contextdev", "name": "Context.dev", "price": null, "own_key_only": true } ], "mcp_name": "contextdev_web_crawl", "input_schema": { "type": "object", "properties": { "body": { "type": "object", "properties": { "url": { "type": "string", "pattern": "^https?:\\/\\/\\S+$", "description": "Starting URL for the crawl, including the http:// or https:// scheme." }, "maxPages": { "type": "integer", "minimum": 1, "maximum": 500, "description": "Maximum number of pages to crawl (1-500). Default 100. Sizes the hold: one credit per page." }, "maxDepth": { "type": "integer", "minimum": 0, "maximum": 9007199254740991, "description": "Maximum link depth from the starting URL (0 = only the starting page). Unlimited when omitted." }, "urlRegex": { "type": "string", "description": "Only URLs matching this pattern are followed and scraped." }, "includeLinks": { "type": "boolean", "description": "Preserve hyperlinks in the Markdown output. Default true." }, "includeImages": { "type": "boolean", "description": "Include image references in the Markdown output. Default false." }, "shortenBase64Images": { "type": "boolean", "description": "Truncate inline base64 image payloads to keep the output small. Default true." }, "useMainContentOnly": { "type": "boolean", "description": "Keep only the page's main content, dropping headers, footers, sidebars, and navigation when detectable. Default false." }, "followSubdomains": { "type": "boolean", "description": "Follow links on subdomains of the starting URL's domain. www and apex are always equivalent. Default false." }, "pdf": { "type": "object", "properties": { "shouldParse": { "type": "boolean", "description": "Fetch and parse PDF pages. When false, PDFs are skipped entirely. Default true." }, "ocr": { "type": "boolean", "description": "OCR the selected PDF pages that have no usable text layer (scans). Default false. An OCR'd page costs an extra credit upstream." }, "start": { "type": "integer", "minimum": 1, "maximum": 9007199254740991, "description": "First 1-based PDF page to parse." }, "end": { "type": "integer", "minimum": 1, "maximum": 9007199254740991, "description": "Last 1-based PDF page to parse. Must be >= start." } }, "additionalProperties": false, "description": "PDF parsing controls." }, "includeFrames": { "type": "boolean", "description": "Render iframe contents into the output. Default false." }, "includeSelectors": { "maxItems": 50, "type": "array", "items": { "type": "string", "maxLength": 2048 }, "description": "CSS selectors. When provided, only matching subtrees are kept before each page is converted to Markdown." }, "excludeSelectors": { "maxItems": 50, "type": "array", "items": { "type": "string", "maxLength": 2048 }, "description": "CSS selectors to remove before conversion. Applied after includeSelectors; exclusion wins on a double match." }, "maxAgeMs": { "type": "integer", "minimum": 0, "maximum": 2592000000, "description": "Reuse a cached result younger than this many milliseconds. Default 86400000 (1 day), max 2592000000 (30 days). Set 0 to always fetch fresh." }, "waitForMs": { "type": "integer", "minimum": 0, "maximum": 30000, "description": "Extra browser wait in milliseconds after page load before the content is captured (0-30000). Useful for JavaScript-heavy pages." }, "settleAnimations": { "type": "boolean", "description": "Wait briefly for animations to settle before capturing each page. Default false." }, "stopAfterMs": { "type": "integer", "minimum": 10000, "maximum": 110000, "description": "Soft time budget for the whole crawl in milliseconds (10000-110000, default 80000). When exceeded, the pages collected so far are returned." }, "country": { "type": "string", "enum": [ "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw" ], "description": "Two-letter ISO 3166-1 alpha-2 country code, lowercase (e.g. 'us', 'gb', 'de'). Must be one of Context.dev's supported countries." }, "timeoutOpts": { "type": "object", "properties": { "milliseconds": { "type": "integer", "minimum": 1000, "maximum": 300000, "description": "Request deadline in milliseconds (1000-300000)." }, "behavior": { "type": "string", "enum": [ "fail", "return-partial" ], "description": "What to do at the deadline: 'fail' returns 408 at no charge (default); 'return-partial' returns the usable results collected so far and bills them." } }, "required": [ "milliseconds" ], "additionalProperties": false, "description": "Request deadline and behavior on timeout." }, "zdr": { "type": "string", "enum": [ "enabled", "disabled" ], "description": "Set 'enabled' to bypass Context.dev's shared caches and omit request and response content from its retained usage logs. Requires zero data retention on the Context.dev organization, otherwise the call fails with ZDR_NOT_ENABLED." } }, "required": [ "url", "maxPages" ], "additionalProperties": false } }, "required": [ "body" ], "additionalProperties": false }, "kind": "endpoint" }