{"uid":"cap_A3cH8fC4jjM_JplVmXdAq","slug":"x402-orth-sh-web-crawler-93de2788","name":"x402.orth.sh Web Crawler","description":"Performs a crawl starting from a given URL, extracts page content as Markdown, and returns results for all crawled pages. Only follows links within the same domain as the starting URL. Costs 1 credit per successful page crawled.","url":"https://x402.orth.sh/context-dev/web/crawl","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","description":"The starting URL for the crawl (must include http:// or https:// protocol)"},"maxDepth":{"type":"integer","description":"Maximum link depth from the starting URL (0 = only the starting page)"},"maxPages":{"type":"integer","description":"Maximum number of pages to crawl. Hard cap: 500."},"urlRegex":{"type":"string","description":"Regex pattern. Only URLs matching this pattern will be followed and scraped."},"includeLinks":{"type":"boolean","description":"Preserve hyperlinks in the Markdown output"},"includeImages":{"type":"boolean","description":"Include image references in the Markdown output"},"followSubdomains":{"type":"boolean","description":"When true, follow links on subdomains of the starting URL's domain (e.g. docs.example.com when starting from example.com). www and apex are always treated as equivalent."},"useMainContentOnly":{"type":"boolean","description":"Extract only the main content, stripping headers, footers, sidebars, and navigation"},"shortenBase64Images":{"type":"boolean","description":"Truncate base64-encoded image data in the Markdown output"}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.03","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.03/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.03","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.03","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_n3uP3a2USldXWmO-DMPJK","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.03","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a website starting from a given URL, follows internal links up to a configurable depth, and returns all crawled pages as Markdown content.","exampleAgentPrompt":"Crawl https://docs.example.com up to 3 levels deep, limit to 50 pages, and return the content of each page as Markdown — only extract the main content and skip headers and footers.","exampleUseCases":null,"resultDescription":"An array of crawled page results, each containing the page URL and its full content converted to Markdown format. Links and images can optionally be preserved in the output. Up to 500 pages may be returned per request.","failureModes":["Invalid or unreachable starting URL returns an error","maxPages cap of 500 enforced — requests exceeding this are capped silently","urlRegex syntax errors may reject or ignore the filter","Crawl may return fewer pages than requested if the domain has fewer internal links","Pages behind authentication or JavaScript-heavy SPAs may return empty or partial content","Rate limiting or bot-blocking by the target site may yield incomplete results","Insufficient credits causes the request to fail mid-crawl"],"whenToPreferThis":"Use this endpoint when you need to extract content from multiple pages of a single website in one call, rather than scraping individual pages one at a time. It is ideal for indexing documentation sites, blogs, or product catalogs where following internal links systematically is required. Prefer this over single-page scrapers when depth and breadth of crawl coverage matter. Choose it over LLM-driven extraction when you want raw Markdown output across many pages without a specific extraction prompt.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T00:34:55.443Z","isFirstParty":false}