{"uid":"cap_fQBcP6Xgb9mSIa9S6Lq_u","slug":"ai-crawler-access-checker-2c5b150c","name":"AI Crawler Access Checker","description":"Pay-per-call data & tool APIs","url":"https://api.timzinin.com/api/ai-crawler-access-checker","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"websites":{"type":"array","maxItems":100},"maxConcurrency":{"type":"integer","maximum":50,"minimum":1}}},"responseSchema":{"type":"json","example":{"meta":{"run_ms":1200,"settled":true,"item_count":1},"results":[{"bots":[{"bot":"GPTBot","vendor":"OpenAI","purpose":"training","verdict":"partial","matchedRule":"wildcard","restrictedPaths":["/microsoft-for-startups/"]},{"bot":"OAI-SearchBot","vendor":"OpenAI","purpose":"search","verdict":"partial","matchedRule":"wildcard","restrictedPaths":["/microsoft-for-startups/"]},{"bot":"ClaudeBot","vendor":"Anthropic","purpose":"training","verdict":"partial","matchedRule":"wildcard","restrictedPaths":["/microsoft-for-startups/"]},{"bot":"PerplexityBot","vendor":"Perplexity","purpose":"search","verdict":"partial","matchedRule":"wildcard","restrictedPaths":["/microsoft-for-startups/"]}],"error":null,"found":true,"llmsTxt":{"exists":false,"status":403},"summary":"All audited AI crawlers are allowed.","website":"https://openai.com","sitemaps":["https://openai.com/sitemap.xml"],"checkedAt":"2026-07-26T13:55:53.146Z","robotsTxt":{"error":null,"exists":true,"status":200},"blockedBots":[],"llmsFullTxt":{"exists":false,"status":403},"aiAccessScore":100,"aiSearchScore":100}]}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_eewdgdoiLQNwzvbbkMdq1","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether AI bots and crawlers (GPTBot, ClaudeBot, etc.) are allowed or blocked by a website's robots.txt and llms.txt, returning per-bot verdicts and AI access scores","exampleAgentPrompt":"Can you check whether AI crawlers like GPTBot, ClaudeBot, and PerplexityBot are allowed to access https://openai.com, and give me their verdicts plus the overall AI access score?","exampleUseCases":[{"title":"Audit competitor site for AI crawlability","prompt":"Check if AI bots like GPTBot and ClaudeBot are blocked on https://anthropic.com — I want to know which ones are allowed, which paths are restricted, and what the AI access score is."},{"title":"Verify own site before LLM indexing","prompt":"Can you audit https://myblog.com to see if it allows AI training and search crawlers? Tell me if robots.txt exists, whether llms.txt is set up, and which bots have full vs partial access."},{"title":"Research AI content policy of a news site","prompt":"I want to know how https://nytimes.com handles AI crawler access — which bots are allowed or blocked, what paths they're restricted from, and whether there's an llms.txt file."}],"resultDescription":"A JSON object containing per-bot verdicts (allowed/partial/blocked) for major AI crawlers (GPTBot, ClaudeBot, OAI-SearchBot, PerplexityBot, etc.), matched robots.txt rules, restricted paths, robots.txt and llms.txt existence and HTTP status, a list of fully blocked bots, sitemaps found, an aiAccessScore (0–100), an aiSearchScore (0–100), a human-readable summary, and run metadata including elapsed time and item count.","failureModes":["Website is unreachable or returns a non-200 status for robots.txt — error field will be populated","Domain does not exist — results in a network error or null response","llms.txt returns 403 or 404 — reflected in llmsFullTxt.status and llmsTxt.status fields","Malformed URL input — may cause a 400-level error or empty results","Rate limiting by the target website during robots.txt fetch — may cause incomplete results"],"whenToPreferThis":"Use this endpoint when you need to determine whether AI crawlers and LLM training bots can access a specific website, including per-bot granularity, path-level restrictions, and presence of llms.txt. Prefer this over manually parsing robots.txt when you need structured, scored output with vendor attribution (OpenAI, Anthropic, Perplexity) and a normalized AI access score rather than raw text parsing.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T00:53:21.943Z","isFirstParty":false}