{"uid":"cap_4Cc9Xs5sf9U3NAYQ_OfqG","slug":"sitemap-crawler-x402-orthogonal-com-5777305e","name":"Sitemap Crawler - x402.orthogonal.com","description":"Crawls the sitemap of the given domain and returns all discovered page URLs. Supports sitemap index files (recursive), parallel fetching with concurrency control, deduplication, and filters out non-page resources (images, PDFs, etc.).","url":"https://x402.orthogonal.com/context-dev/web/scrape/sitemap","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","properties":{"domain":{"type":"string","description":"Domain name to crawl sitemaps for (e.g., 'example.com'). The domain will be automatically normalized and validated."},"maxLinks":{"type":"integer","description":"Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000."}}}},"additionalProperties":false}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.03","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.03/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.03","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.03","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_4kQwDdm40yRvtZU1uKIC9","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.03","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a domain's sitemap (including recursive sitemap index files) and returns all discovered page URLs, deduplicated and filtered to pages only.","exampleAgentPrompt":"Can you crawl the sitemap for stripe.com and give me all the page URLs you discover, up to 5,000 links?","exampleUseCases":[{"title":"Full site audit preparation","prompt":"Pull all the page URLs from the sitemap of shopify.com — I want to see the full list of indexed pages, up to 50,000 links."},{"title":"Competitor content mapping","prompt":"Crawl the sitemap for hubspot.com and return every page URL you find so I can analyze their content structure."},{"title":"Broken link detection setup","prompt":"Get me all the URLs from the sitemap of myblog.com, maximum 10,000 links, so I can check which ones return 404 errors."}],"resultDescription":"A list of all discovered page URLs found across the domain's sitemap and any nested sitemap index files. Results are deduplicated and filtered to exclude non-page resources such as images, PDFs, and other media files. The response respects the maxLinks cap and is fetched with parallel concurrency control.","failureModes":["Domain has no sitemap.xml — returns empty or error","Invalid or malformed domain name — validation error","Domain is unreachable or returns non-200 — fetch failure","Sitemap is password-protected or blocked by robots.txt","maxLinks exceeds 100,000 or is less than 1 — validation error","Very large sitemap index may hit timeout or concurrency limits"],"whenToPreferThis":"Use this endpoint when you need to enumerate all pages of a website via its sitemap, especially for SEO audits, content mapping, link checking, or building a crawl queue. It is preferable over generic URL scrapers when you want structured, deduplicated page discovery at scale with recursive sitemap index support. Best for domains that maintain a sitemap.xml.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-16T12:58:37.156Z","isFirstParty":false}