{"uid":"cap_A56ENRdeABM98aR1iXGo5","slug":"agent-read-proxy-sitemap-url-extractor-c711e89e","name":"Agent Read Proxy - Sitemap URL Extractor","description":"Web scraping API for AI agents: turn any public web page into clean text or Markdown with title, author, date and language metadata. $0.002 per page, paid in USDC on Base via x402. No API key or signup.","url":"https://agent-read-proxy.agent-read-proxy.workers.dev/v1/sitemap?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri","description":"Any page on the site, or the sitemap URL itself"},"limit":{"type":"integer","default":1000,"maximum":5000,"minimum":1,"description":"Most URLs to return"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"site":"https://www.cloudflare.com","urls":[{"url":"https://www.cloudflare.com/learning/","lastmod":"2026-10-01"}],"count":1,"sitemaps_read":["https://www.cloudflare.com/sitemap.xml"]}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Y1CaurzyviDgQ22JQm7xd","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's sitemap to return a structured list of all discoverable page URLs with optional last-modified dates.","exampleAgentPrompt":"Can you fetch the sitemap for cloudflare.com and give me a list of all their pages — up to 500 URLs?","exampleUseCases":[{"title":"Audit all pages on a competitor site","prompt":"Pull the full sitemap for hubspot.com and list every URL you find, up to 2000 pages — I want to see what content they've published."},{"title":"Find all blog posts for content scraping","prompt":"Get me the sitemap for nytimes.com and extract all article URLs from it so I can read through their recent stories."},{"title":"Index a documentation site for search","prompt":"Fetch the sitemap from docs.stripe.com and return all the URLs — I want to build a searchable index of their documentation pages."}],"resultDescription":"Returns a JSON object containing the site's root URL, an array of discovered page URLs (each with optional lastmod date), the total count of URLs found, and a list of sitemap files that were read to produce the result.","failureModes":["Site has no sitemap.xml — returns empty or error response","Sitemap is blocked by robots.txt or requires authentication","Malformed or non-standard sitemap format causes parsing failure","limit parameter exceeds maximum of 5000 — validation error","URL parameter missing or invalid URI format — schema validation error","Sitemap is very large and processing times out on Cloudflare Workers"],"whenToPreferThis":"Use this endpoint when you need to discover all pages on a website systematically without crawling link-by-link. It is ideal when you want a quick, structured index of a site's content, especially for content audits, SEO analysis, documentation indexing, or preparing a list of pages to subsequently scrape. It is preferred over manual crawling when the site publishes a standard sitemap, and over general web search when you need comprehensive URL enumeration rather than search-relevance ranking.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-11-16T18:33:34.313Z","isFirstParty":false,"canonicalSlug":"agent-read-proxy-sitemap-url-extractor-c711e89e"}