{"uid":"cap_pKJvuMzVAn4zF4JKs2iw5","slug":"scrapfly-sitemap-crawler-6f78a014","name":"ScrapFly Sitemap Crawler","description":"69 endpoints for data, content generation, and web scraping. Pay per request via x402 on Base chain.","url":"https://scravfly.vercel.app/api/crawl/sitemap","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"url"}}},"responseSchema":{"type":"object","required":["success"],"properties":{"data":{"type":"object"},"success":{"type":"boolean"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.06","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.06/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.06","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.06","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_UeOveN8ZLHpuiSRwehgw_","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.06","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses the XML sitemap of a given URL, returning all discovered page URLs and metadata.","exampleAgentPrompt":"Can you crawl the sitemap for https://example.com and give me a list of all the pages it indexes?","exampleUseCases":[{"title":"Competitor website page discovery","prompt":"Pull the sitemap from https://competitor.com and list all the URLs they have indexed so I can see what content they're publishing."},{"title":"SEO audit page inventory","prompt":"Fetch the sitemap for https://mybusiness.com so I can get a full list of every page on the site for an SEO audit."},{"title":"Content pipeline seeding","prompt":"Grab the sitemap from https://techcrunch.com and extract all the article URLs so I can queue them up for summarization."}],"resultDescription":"Returns a JSON object with a success boolean and a data object containing the parsed sitemap entries, including all discovered page URLs and any associated metadata such as last-modified dates, change frequencies, or priorities found in the sitemap XML.","failureModes":["URL does not have a sitemap (404 on sitemap.xml) — success may be false or data empty","Invalid or malformed URL input — likely returns success: false","Website blocks crawlers via robots.txt or rate limiting — may return empty or error response","Sitemap is dynamically generated and not accessible — data may be incomplete","Network timeout on large or slow-loading sitemaps"],"whenToPreferThis":"Use this endpoint when you need to discover all pages on a website quickly without crawling individual pages. Ideal for SEO audits, content inventories, competitor research, or seeding a content pipeline. Prefer this over full-site crawling when you only need the URL list rather than page content, as it is faster and cheaper.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T00:40:19.454Z","isFirstParty":false}