{"uid":"cap_sZ1s2QJt4_G6E00alYJqk","slug":"site-crawler-b685651f","name":"Site Crawler","description":"Crawl a website (sitemap first, then same-site links), respecting robots.txt, and return up to 20 pages as clean Markdown. Use path_prefix to stay inside a docs section.","url":"https://164.160.90.241.sslip.io/api/web-crawl-site?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Start URL"},"max_pages":{"type":"integer","description":"Pages to return, 1-20 (default 10)"},"path_prefix":{"type":"string","description":"Only crawl URLs whose path starts with this"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.05","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.05/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.05","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.05","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_nwBPZqNcCgBrbr8qFuia3","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.05","costPer":"request","priority":0,"asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a website (sitemap first, then same-site links) respecting robots.txt and returns up to 20 pages as clean Markdown","exampleAgentPrompt":"Can you crawl https://docs.example.com and return up to 15 pages of content as Markdown, staying only within the /guides path?","exampleUseCases":[{"title":"Documentation site ingestion","prompt":"Crawl https://docs.stripe.com and pull back up to 20 pages of content as Markdown, but only from the /payments path prefix so I just get the payments docs."},{"title":"Competitor website content audit","prompt":"Crawl https://www.acmecorp.com and return up to 10 pages as clean Markdown so I can review what they say about their product features."},{"title":"Internal wiki content extraction","prompt":"Grab all the pages you can from https://wiki.mycompany.io/engineering — up to 20 pages, staying within the /engineering path — and give me the text as Markdown."}],"resultDescription":"Returns up to 20 pages from the target website as clean Markdown text. The crawler starts from the sitemap if available, then follows same-site links, and respects robots.txt. Each page's content is converted to readable Markdown with titles and URLs preserved.","failureModes":["robots.txt blocks crawling of the target domain — returns restricted or empty results","Start URL is unreachable or returns non-200 status — crawl fails with an error","Site has no sitemap and no crawlable links — returns only the start page or nothing","path_prefix filters out all discovered URLs — returns zero pages","Site requires JavaScript rendering to expose links — crawler may miss dynamically generated pages","max_pages set to 0 or above 20 — validation error"],"whenToPreferThis":"Choose this endpoint when you need to crawl multiple pages from a site automatically rather than fetching a single URL. It is ideal for ingesting documentation sites, gathering competitive intelligence across a whole domain, or building a Markdown corpus from a structured website. Prefer it over a single-page reader when you don't know the exact URLs in advance and need the crawler to discover them via sitemap or link traversal. Use path_prefix to scope the crawl to a specific section like /docs or /blog.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-12T00:41:31.752Z","isFirstParty":false,"canonicalSlug":"site-crawler-b685651f"}