{"uid":"cap_6znoBefTJTdLWYQKF7tCg","slug":"scrapfly-sitemap-crawler-af92cd59","name":"ScrapFly Sitemap Crawler","description":"69 endpoints for data, content generation, and web scraping. Pay per request via x402 on Base chain.","url":"https://scrapfly.vercel.app/api/crawl/sitemap?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"url"}}},"responseSchema":{"type":"object","required":["success"],"properties":{"data":{"type":"object"},"success":{"type":"boolean"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.02","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.02/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_jyoYXpaOubI_qEd2ieiTT","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.02","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses the XML sitemap of a given website URL, returning all discovered page URLs and metadata","exampleAgentPrompt":"Can you fetch the sitemap for https://example.com and give me a list of all the pages they have indexed?","exampleUseCases":[{"title":"SEO audit page discovery","prompt":"I need to audit the SEO of techcrunch.com — can you crawl their sitemap and pull every URL they have listed so I can see what pages are indexed?"},{"title":"Competitor content inventory","prompt":"Pull the sitemap from shopify.com so I can see what blog posts and product pages they're publishing — I want a full list of their indexed URLs."},{"title":"Broken link detection prep","prompt":"Fetch the sitemap for my website at https://mybrand.io so I can run a broken link check across every page we have listed."}],"resultDescription":"Returns a success boolean and a data object containing the parsed sitemap entries, including all discovered page URLs and any associated metadata such as last-modified dates, change frequencies, or priority values present in the sitemap XML.","failureModes":["URL not provided or malformed — returns success: false","Target site has no sitemap.xml or robots.txt-referenced sitemap — returns empty or error","Sitemap is behind authentication or bot protection — may return blocked or empty response","Network timeout reaching the target URL — returns success: false","Sitemap is extremely large (millions of URLs) — may truncate or time out"],"whenToPreferThis":"Use this endpoint when you need to discover all pages on a website programmatically without manually browsing — ideal for SEO audits, content inventories, competitor analysis, or setting up a crawl pipeline. Prefer this over general web scraping endpoints when the goal is to enumerate a site's full URL structure rather than extract content from individual pages.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-24T12:30:08.200Z","isFirstParty":false,"canonicalSlug":"scrapfly-sitemap-crawler-af92cd59"}