{"uid":"cap_ouOxjn1042RtCnNILya0D","slug":"readfetch-site-mapper-34501f40","name":"ReadFetch Site Mapper","description":"Discover a site's crawlable surface without crawling it: parse robots.txt allow/disallow rules, collect URLs from sitemap.xml and sitemap indexes, find RSS, Atom and JSON feeds, and count internal and external links on the homepage. Use the returned URLs to plan readfetch_scrape, readfetch_batch, or readfetch_crawl — not readfetch_map again on each URL. Prefer readfetch_probe on one URL before paying for bodies.","url":"https://readfetch.com/v1/map?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","title":"Url","format":"uri","maxLength":2083,"minLength":1,"description":"Public http(s) URL of the site to map."},"limit":{"type":"integer","title":"Limit","default":200,"maximum":1000,"minimum":1,"description":"Maximum number of URLs to return (1-1000)."},"max_bytes":{"type":"integer","title":"Max Bytes","default":5000000,"maximum":20000000,"minimum":1024,"description":"Abort if any single document exceeds this many bytes."},"timeout_s":{"type":"number","title":"Timeout S","default":60,"maximum":95,"minimum":1,"description":"Total budget for the whole call in seconds; every sitemap fetch and the browser fallback share it. 95 s is the edge ceiling (Cloudflare closes at 100 s)."}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.005","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.005/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_fdFJzMc94KzsKcRzxTXLI","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.005","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Maps a website's crawlable surface by parsing robots.txt, collecting URLs from sitemaps, finding RSS/Atom/JSON feeds, and counting homepage links — without full crawling.","exampleAgentPrompt":"Map the crawlable surface of https://techcrunch.com — give me up to 500 URLs from its sitemaps, any RSS or Atom feeds you find, and note what robots.txt allows or disallows, using direct egress.","exampleUseCases":[{"title":"Pre-crawl URL discovery for scraper","prompt":"Before I start scraping https://arstechnica.com, map it for me — pull up to 300 URLs from its sitemaps and list any RSS feeds so I know what to target."},{"title":"Feed aggregator onboarding","prompt":"I want to add The Verge to my feed reader — can you check https://theverge.com for any RSS, Atom, or JSON feeds and return all the feed URLs you find? Limit to 100 URLs total."},{"title":"SEO audit site structure check","prompt":"I need to understand the site structure of https://shopify.com for an SEO audit — map its robots.txt rules, collect up to 1000 sitemap URLs, and tell me how many internal and external links are on the homepage."}],"resultDescription":"Returns a structured payload including: URLs discovered from sitemap.xml and sitemap indexes (up to the requested limit), robots.txt allow/disallow rules, discovered RSS/Atom/JSON feed URLs, and counts of internal and external links found on the homepage.","failureModes":["Site has no sitemap.xml or robots.txt — returns empty URL lists","Target blocks datacenter IPs — retry with residential egress","Timeout exceeded before all sitemaps fetched — partial results returned","Sitemap or robots.txt document exceeds max_bytes — fetch aborted","Invalid or unreachable URL — HTTP error response","Sitemap index references too many child sitemaps to fetch within timeout"],"whenToPreferThis":"Use this endpoint when you need to understand a site's crawlable structure before committing to a full scrape or crawl. It is ideal for planning /v1/scrape or /v1/crawl calls, discovering feeds for aggregation, or auditing robots.txt compliance — much cheaper and faster than crawling all pages to find URLs organically.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-31T18:39:53.922Z","isFirstParty":false,"canonicalSlug":"readfetch-site-mapper-34501f40"}