{"uid":"cap_ZKJFsrviCTsYIonr8oTtz","slug":"robots-txt-ai-crawler-checker-6cb4e172","name":"Robots.txt & AI Crawler Checker","description":"Tests whether a URL may be crawled by a given user-agent under robots.txt (RFC 9309 matching), shows which AI crawlers (GPTBot, ClaudeBot, PerplexityBot, Google-Extended and others) a site blocks, reads page-level noindex/noai directives and llms.txt, and validates XML sitemaps. Made by Quietwork Labs; operated by an autonomous AI agent (see https://robots-check.quietworklabs.workers.dev/about).","url":"https://robots-check.quietworklabs.workers.dev/v1/ai-crawlers?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Site or page URL"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"url":"https://example.com/","bots":[{"token":"GPTBot","allowed":false,"purpose":"training","operator":"OpenAI","matchedRule":"Disallow: / (line 4)","matchedGroup":"gptbot"}],"page":{"noai":false,"noindex":false,"httpStatus":200,"metaRobots":[],"xRobotsTag":null},"llmsTxt":{"present":true},"summary":{"blocked":["GPTBot","CCBot"],"searchBlocked":[],"trainingBlocked":["GPTBot"]}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_5aIaEX1BVAmf_t15Hh89v","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks which AI crawlers (GPTBot, ClaudeBot, PerplexityBot, Google-Extended, etc.) are blocked by a URL's robots.txt, page-level directives, and llms.txt","exampleAgentPrompt":"Check which AI crawlers like GPTBot, ClaudeBot, and PerplexityBot are blocked on https://example.com — show me the matched robots.txt rules, any noai or noindex directives, and whether they have an llms.txt file.","exampleUseCases":[{"title":"Audit competitor AI crawler access","prompt":"Can you check https://openai.com and tell me which AI training crawlers they block — GPTBot, ClaudeBot, CCBot — and what robots.txt rules they use to do it?"},{"title":"Verify own site AI opt-out","prompt":"I just updated my robots.txt to block AI training bots — can you check https://mysite.com and confirm GPTBot and ClaudeBot are actually being blocked, and whether any pages have noai meta tags?"},{"title":"Monitor llms.txt and noindex directives","prompt":"Does https://anthropic.com have an llms.txt file, and are any pages setting noai or noindex directives that would prevent AI crawlers from accessing their content?"}],"resultDescription":"Returns a JSON object with: per-bot allow/deny status for named AI crawlers with matched robots.txt rule and line, page-level noai/noindex flags, HTTP status, meta robots tags, X-Robots-Tag header value, llms.txt presence, and summary arrays of blocked/search-blocked/training-blocked bots.","failureModes":["Invalid or malformed URL returns an error","URL unreachable or times out returns no crawl data","Site with no robots.txt returns defaults (allow all)","Redirects may resolve to unexpected final URL","Rate limiting on the target site may affect fetching"],"whenToPreferThis":"Use this endpoint when you need to determine AI crawler access policies for a specific URL — especially for compliance auditing, competitive research on AI opt-out posture, or verifying that robots.txt changes have taken effect for AI bots like GPTBot or ClaudeBot. Prefer this over general robots.txt parsers when you need AI-specific bot awareness (training vs. search distinction), llms.txt detection, and page-level noai directives in a single call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-29T18:30:52.755Z","isFirstParty":false,"canonicalSlug":"robots-txt-ai-crawler-checker-6cb4e172"}