{"uid":"cap_rQf2nmWfsx9L736kPm77o","slug":"robots-txt-sitemap-checker-a7f01d34","name":"Robots.txt & Sitemap Checker","description":"A site's robots.txt rules per user agent, whether AI crawlers (GPTBot, ClaudeBot, Google-Extended, CCBot, PerplexityBot) are blocked, and the URLs listed in its sitemap. Use before crawling a site or for SEO audits.","url":"https://bess-x402.onrender.com/v1/web/robots?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET","HEAD","DELETE"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"limit":{"type":"integer","description":"Sitemap URLs to return (1-1000)"},"domain":{"type":"string","description":"Domain name"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.003","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_lkOJgDmpVJFTwuujr8V-J","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches a domain's robots.txt crawl rules (including AI crawler blocking status) and sitemap URLs in a single call.","exampleAgentPrompt":"Before I crawl techcrunch.com, check its robots.txt to see if GPTBot and ClaudeBot are blocked, and also pull up to 50 URLs from its sitemap.","exampleUseCases":[{"title":"Pre-crawl AI bot compliance check","prompt":"I'm about to scrape venturebeat.com with my AI agent — can you check if GPTBot or PerplexityBot are blocked in its robots.txt first?"},{"title":"SEO sitemap discovery audit","prompt":"Pull the sitemap for shopify.com and give me up to 100 URLs so I can audit what pages they're indexing."},{"title":"Competitor crawl policy research","prompt":"Check the robots.txt for openai.com and tell me which user agents are disallowed and whether Google-Extended or CCBot are blocked."}],"resultDescription":"Returns the parsed robots.txt rules per user agent, a clear flag for whether each major AI crawler (GPTBot, ClaudeBot, Google-Extended, CCBot, PerplexityBot) is blocked or allowed, and a list of URLs from the domain's sitemap (up to the requested limit).","failureModes":["Domain not found or unreachable — returns error indicating the host could not be resolved","No robots.txt present — may return empty rules or a not-found indicator","Sitemap not declared in robots.txt — sitemap URLs list may be empty","Invalid domain format — returns validation error","Limit parameter out of range (not 1-1000) — returns validation error"],"whenToPreferThis":"Use this endpoint when you need to check crawl permissions before scraping a site, perform an SEO audit of robots.txt directives, or discover sitemap URLs without manually fetching and parsing these files yourself. It is especially valuable when you need quick answers about whether specific AI crawlers (GPTBot, ClaudeBot, PerplexityBot, CCBot, Google-Extended) are blocked on a target domain.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-30T07:11:49.106Z","isFirstParty":false,"canonicalSlug":"robots-txt-sitemap-checker-a7f01d34"}