{"uid":"cap_sU7RMBFvjuV3Ddz4bvLWj","slug":"x402-orthogonal-com-web-crawler-59106162","name":"x402.orthogonal.com Web Crawler","description":"Performs a crawl starting from a given URL, extracts page content as Markdown, and returns results for all crawled pages. Only follows links within the same domain as the starting URL. Costs 1 credit per successful page crawled.","url":"https://x402.orthogonal.com/context-dev/web/crawl?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","description":"The starting URL for the crawl (must include http:// or https:// protocol)"},"maxDepth":{"type":"integer","description":"Maximum link depth from the starting URL (0 = only the starting page)"},"maxPages":{"type":"integer","description":"Maximum number of pages to crawl. Hard cap: 500."},"urlRegex":{"type":"string","description":"Regex pattern. Only URLs matching this pattern will be followed and scraped."},"includeLinks":{"type":"boolean","description":"Preserve hyperlinks in the Markdown output"},"includeImages":{"type":"boolean","description":"Include image references in the Markdown output"},"followSubdomains":{"type":"boolean","description":"When true, follow links on subdomains of the starting URL's domain (e.g. docs.example.com when starting from example.com). www and apex are always treated as equivalent."},"useMainContentOnly":{"type":"boolean","description":"Extract only the main content, stripping headers, footers, sidebars, and navigation"},"shortenBase64Images":{"type":"boolean","description":"Truncate base64-encoded image data in the Markdown output"}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.03","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.03/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.03","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.03","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_k3R44Wk0mjUncqu9EoaVV","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.03","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a website starting from a given URL, follows internal links up to a configurable depth, and returns all page content as Markdown","exampleAgentPrompt":"Crawl https://docs.example.com up to 2 levels deep, limit it to 50 pages, only follow URLs matching /docs/, and give me all the content as clean Markdown with links included but without headers, footers, or navigation.","exampleUseCases":[{"title":"Documentation site indexing","prompt":"Crawl the entire docs site at https://docs.myproduct.io — go up to 3 levels deep, cap it at 100 pages, strip out navigation and sidebars so I only get the main content, and return everything as Markdown."},{"title":"Competitor content audit","prompt":"Crawl https://www.competitor.com starting from the homepage, follow links up to 2 levels deep with a max of 30 pages, and return all the page text as Markdown — I want to review what they're publishing."},{"title":"Blog archive extraction","prompt":"Pull all the content from https://blog.example.com — follow links up to depth 2, limit to 80 pages, only follow URLs that match /blog/, and include links in the Markdown output so I can see how posts reference each other."}],"resultDescription":"Returns a list of crawled pages, each with its URL and extracted content rendered as Markdown. Optionally includes hyperlinks and image references within the Markdown. Only pages within the same domain (and optionally subdomains) as the starting URL are included. Costs 1 credit per successfully crawled page.","failureModes":["Invalid or unreachable starting URL returns an error","maxPages cap of 500 hard-limits results regardless of site size","URL regex pattern errors may silently exclude pages","Sites blocking crawlers (robots.txt or rate-limiting) may yield partial results","Deep or large sites may time out before reaching maxPages","base64 images may inflate response size if shortenBase64Images is false"],"whenToPreferThis":"Use this endpoint when you need to extract content from multiple pages of a website automatically, following internal links rather than scraping a single page. It is ideal for documentation indexing, competitive research, content audits, and knowledge base construction where you need breadth across an entire domain. Prefer it over a single-page scraper when the target content spans many URLs. Avoid it for real-time lookups where latency matters, since multi-page crawls are inherently batch operations.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-19T00:55:53.588Z","isFirstParty":false,"canonicalSlug":"x402-orthogonal-com-web-crawler-59106162"}