{"uid":"cap_siI4Z2dot7a1HEiyqfwH6","slug":"foundry-robots-txt-ai-crawler-checker-0ed004eb","name":"Foundry robots.txt AI Crawler Checker","description":"robots.txt checker: whether GPTBot, ClaudeBot, CCBot and other AI crawlers may crawl a site, plus sitemaps. Input: a domain or URL.","url":"https://minty-sprout.onrender.com/v1/robots-txt?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"input":{"type":"string","maxLength":8000,"description":"A domain or URL"}}},"responseSchema":{"type":"json","example":{"host":"nytimes.com","found":true,"lines":364,"status":200,"sitemaps":["https://www.nytimes.com/sitemaps/new/news.xml.gz","https://www.nytimes.com/sitemaps/new/sitemap.xml.gz","https://www.nytimes.com/sitemaps/new/collections.xml.gz"],"ai_crawlers":{"CCBot":false,"GPTBot":false,"ClaudeBot":false,"Bytespider":false,"Claude-User":false,"ChatGPT-User":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"meta-externalagent":false},"crawl_delay":null,"path_checked":"/","googlebot_allowed":true}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_BV4dLkuTg-RrrSlFz63eq","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether GPTBot, ClaudeBot, CCBot and other AI crawlers are allowed to crawl a site, plus extracts sitemaps, crawl delay, and robots.txt details.","exampleAgentPrompt":"Can you check whether GPTBot, ClaudeBot, and CCBot are allowed to crawl nytimes.com, and also grab any sitemaps listed in its robots.txt?","exampleUseCases":[{"title":"AI crawler compliance audit","prompt":"Check whether GPTBot, ClaudeBot, CCBot, and PerplexityBot are allowed to crawl openai.com — I need to know if they block any AI training crawlers."},{"title":"Sitemap discovery for a domain","prompt":"Pull the sitemaps listed in the robots.txt for techcrunch.com so I can feed them into my content indexer."},{"title":"Competitive AI access research","prompt":"I'm researching which news sites block AI crawlers — can you check robots.txt for bbc.com and tell me which bots like GPTBot, ClaudeBot, and Google-Extended are allowed or blocked?"}],"resultDescription":"Returns a JSON object with: the resolved host, whether a robots.txt was found, HTTP status, total line count, a list of sitemap URLs, per-bot allow/deny flags for major AI crawlers (GPTBot, ClaudeBot, CCBot, Bytespider, Claude-User, ChatGPT-User, PerplexityBot, Google-Extended, Applebot-Extended, meta-externalagent), any crawl delay, the path checked, and whether Googlebot is allowed.","failureModes":["Domain does not exist or is unreachable — returns found:false or HTTP error status","robots.txt not present — returns found:false with status 404","Malformed URL input — may return validation error","Timeout on slow or unresponsive servers","Non-standard robots.txt format may cause incomplete parsing"],"whenToPreferThis":"Use this endpoint when you need to quickly check AI crawler permissions for any domain or URL without fetching and parsing the robots.txt yourself. It is the right choice when you want structured, per-bot allow/deny flags for all major LLM crawlers (GPTBot, ClaudeBot, CCBot, etc.) plus sitemaps in a single call. Prefer it over generic HTTP fetch tools when AI crawler compliance, content licensing research, or sitemap discovery is the goal.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-11-01T06:57:41.817Z","isFirstParty":false,"canonicalSlug":"foundry-robots-txt-ai-crawler-checker-0ed004eb"}