{"uid":"cap_ykq7lUvghmGoNfNfRAW0k","slug":"x402-orth-sh-sitemap-crawler-4236637a","name":"x402.orth.sh Sitemap Crawler","description":"Crawls the sitemap of the given domain and returns all discovered page URLs. Supports sitemap index files (recursive), parallel fetching with concurrency control, deduplication, and filters out non-page resources (images, PDFs, etc.).","url":"https://x402.orth.sh/context-dev/web/scrape/sitemap","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","properties":{"domain":{"type":"string","description":"Domain name to crawl sitemaps for (e.g., 'example.com'). The domain will be automatically normalized and validated."},"maxLinks":{"type":"integer","description":"Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000."}}}},"additionalProperties":false}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.03","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.03/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.03","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.03","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Q89BAxzsiGTqm9bHrYgEe","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.03","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls the sitemap of a given domain and returns all discovered page URLs, supporting recursive sitemap index files, parallel fetching, deduplication, and non-page resource filtering.","exampleAgentPrompt":"Can you crawl the sitemap for nytimes.com and give me all the page URLs you find — up to 5,000 links?","exampleUseCases":null,"resultDescription":"A deduplicated list of page URLs discovered by crawling the domain's sitemap (and any linked sitemap index files), filtered to exclude non-page resources like images and PDFs. Returns up to the specified maxLinks count (default 10,000).","failureModes":["Domain has no sitemap.xml or robots.txt pointing to a sitemap — returns empty result or error","Invalid or unresolvable domain name — validation error returned","maxLinks out of range (below 1 or above 100,000) — schema validation failure","Sitemap server is unreachable or returns non-200 status — fetch error","Rate limiting or timeout on large sitemap index files — partial results or timeout error"],"whenToPreferThis":"Use this endpoint when you need a comprehensive list of all publicly indexed pages on a domain, especially when the site has a structured sitemap. Prefer this over general web scraping when you want structured URL discovery rather than content extraction. Ideal for SEO audits, content indexing, site monitoring, or feeding URLs into downstream scraping workflows.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T12:44:27.779Z","isFirstParty":false}