{"uid":"cap_2KEtDdizl8-kyj030DL-z","slug":"fastx402-robots-txt-sitemap-checker-52df0c4e","name":"FastX402 Robots.txt & Sitemap Checker","description":"200 pay-per-call tools for AI agents, from $0.001 per request in USDC on Base or Solana with x402. No account, no API key.","url":"https://fastx402.tools/robots-sitemap?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Site URL or bare domain"},"path":{"type":"string","description":"Path to check, starting with /, default /"},"agent":{"type":"string","description":"User-agent name to check, default *"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","properties":{"query":{"type":"object","properties":{"path":{"type":"string"},"agent":{"type":"string"}}},"origin":{"type":"string"},"robots":{"type":"object","properties":{"found":{"type":"boolean"},"groups":{"type":"array","items":{"type":"object","properties":{"rules":{"type":"integer"},"agents":{"type":"array","items":{"type":"string"}},"crawlDelay":{"type":"null"}}}},"status":{"type":"integer"}}},"allowed":{"type":"boolean"},"sitemap":{"type":"object","properties":{"url":{"type":"string"},"error":{"type":"string"}}},"sitemaps":{"type":"array","items":{"type":"string"}},"robotsUrl":{"type":"string"},"crawlDelay":{"type":"null"},"matchedRule":{"type":"string"},"matchedGroup":{"type":"array","items":{"type":"string"}}}}}}}},"responseSchema":{"type":"json","example":{"query":{"path":"/wiki/Special:Search","agent":"MyBot"},"origin":"https://wikipedia.org","robots":{"found":true,"groups":[{"rules":1,"agents":["mj12bot"],"crawlDelay":null},{"rules":0,"agents":["semrushbot"],"crawlDelay":5},{"rules":432,"agents":["*"],"crawlDelay":null}],"status":200},"allowed":false,"sitemap":{"url":"https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0","error":"no sitemap XML (HTTP 403)"},"sitemaps":["https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0"],"robotsUrl":"https://wikipedia.org/robots.txt","crawlDelay":null,"matchedRule":"Disallow: /wiki/Special:","matchedGroup":["*"]}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_dspCSHXQpvIf90F0pYuDF","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt to determine if a given path is allowed for a given user-agent, and retrieves its sitemap URLs.","exampleAgentPrompt":"Can you check whether a bot called 'MyBot' is allowed to crawl the path /wiki/Special:Search on wikipedia.org, and also grab any sitemap URLs listed there?","exampleUseCases":[{"title":"Pre-crawl compliance check for a scraper","prompt":"Before my scraper starts, check if a bot named 'DataBot' is allowed to access /products on shopify.com — I need to know the matched rule and whether there's a crawl delay."},{"title":"Sitemap discovery for SEO audit","prompt":"Find all the sitemap URLs listed in the robots.txt for nytimes.com so I can start indexing their content structure."},{"title":"Verify Googlebot access to a blog path","prompt":"Check if Googlebot is allowed to crawl /blog/private-posts on example.com and tell me which robots.txt rule applies."}],"resultDescription":"Returns a JSON object with: whether the path is allowed for the given user-agent, the matched rule and group, the full robots.txt parse result (groups, rules count, crawl delays), all discovered sitemap URLs with their status, the robots.txt URL fetched, and the origin domain.","failureModes":["robots.txt not found (HTTP 404) — found:false returned","sitemap URL returns non-XML or HTTP error — error field populated in sitemap object","domain unreachable or invalid URL — error response","rate limiting or timeout on target domain fetch","sitemap HTTP 403 — noted in sitemap error field"],"whenToPreferThis":"Choose this endpoint when you need to programmatically check crawl permissions before scraping a site, discover sitemap URLs without manual inspection, or audit robots.txt compliance for a specific bot and path — all without needing an API key, using micro-payments via x402 on Base or Solana.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-27T19:22:54.056Z","isFirstParty":false,"canonicalSlug":"fastx402-robots-txt-sitemap-checker-52df0c4e"}