{"uid":"cap_hIOtzbn0nl0MGmJcLuq99","slug":"ai-crawler-access-test-92645808","name":"AI Crawler Access Test","description":"Pay-per-call data and tools for AI agents, $0.002-0.01 per call in USDC via x402 (Base, Polygon or Solana), no account or API key; MAX tiers ($0.05-0.25) run deeper jobs. Agent toolkit: web search, search + read top pages, any page as clean markdown with a prompt-injection check, which-x402-API-to-call recommender, seller check. Data: Google Maps places with contacts, YouTube video data and transcripts, job listings (Indeed, LinkedIn, 13 boards), Google Ads Transparency, TikTok comments, Pinterest, Airbnb, Redfin, AliExpress, app stores, live sports, website and SEO checks, Shopify store tech stacks, and onchain token / address risk checks.","url":"https://agent-data.centorprat.workers.dev/tools/ai-crawler-access-test?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"urls":{"type":"array","items":{"type":"string"},"maxItems":3,"description":"e.g. \"example.com\" or \"https://example.com/pricing\". One report per URL (the path is checked against robots.txt rules)."},"maxSites":{"type":"integer","description":"Cap how many URLs to test this run."}}},"responseSchema":{"type":"json","example":{"items":[{"url":"https://react.dev","bots":[{"bot":"OAI-SearchBot","major":true,"owner":"OpenAI","purpose":"search","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"ChatGPT-User","major":true,"owner":"OpenAI","purpose":"user-fetch","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"GPTBot","major":false,"owner":"OpenAI","purpose":"training","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"Claude-SearchBot","major":true,"owner":"Anthropic","purpose":"search","robotsRule":"no rule matches","robotsAllowed":true},{"bot":"Claude-User","major":true,"owner":"Anthropic","purpose":"user-fetch","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"ClaudeBot","major":false,"owner":"Anthropic","purpose":"training","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"PerplexityBot","major":true,"owner":"Perplexity","purpose":"search","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"Perplexity-User","major":true,"owner":"Perplexity","purpose":"user-fetch","robotsRule":"no rule matches","robotsAllowed":true},{"bot":"Googlebot","major":true,"owner":"Google","purpose":"search","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"Google-Extended","major":false,"owner":"Google","purpose":"training","robotsRule":"no rule matches","robotsAllowed":true},{"bot":"Bingbot","major":true,"owner":"Microsoft","purpose":"search","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"Applebot","major":false,"owner":"Apple","purpose":"search","robotsRule":"no rule matches","robotsAllowed":true},{"bot":"Applebot-Extended","major":false,"owner":"Apple","purpose":"training","robotsRule":"no rule matches","robotsAllowed":true},{"bot":"Meta-ExternalAgent","major":false,"owner":"Meta","purpose":"training","robotsRule":"no rule matches","robotsAllowed":true},{"bot":"CCBot","major":false,"owner":"Common Crawl","purpose":"training","liveAccess":"ok","liveStatus":200,"robotsRule":"no rule matches","blockReason":null,"liveTextChars":8342,"robotsAllowed":true},{"bot":"Bytespider","major":false,"owner":"ByteDance","purpose":"training","robotsRule":"no rule matches","robotsAllowed":true},{"bot":"Amazonbot","major":false,"owner":"Amazon","purpose":"search","robotsRule":"no rule matches","robotsAllowed":true},{"bot":"DuckAssistBot","major":false,"owner":"DuckDuckGo","purpose":"user-fetch","robotsRule":"no rule matches","robotsAllowed":true}],"input":"react.dev","scores":{"llmsTxt":100,"botAccess":100,"indexHygiene":75,"contentInHtml":100,"entitySignals":20},"summary":"react.dev -> ai-ready (88/100): major AI search/answer bots 8/8 not blocked; content content in html; llms.txt valid","verdict":"ai-ready","finalUrl":"https://react.dev","liveTest":{"baselineOk":true,"inconclusive":[],"controlBlocked":false},"rendering":{"ssrData":true,"verdict":"content-in-html","spaShell":false,"scriptTags":15,"htmlTextChars":8342},"httpStatus":200,"aiReadinessScore":88,"minorBotsBlocked":[],"trainingBotsBlocked":[],"majorSearchBotsBlocked":[]}]}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.02","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.02/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_XzhGfCdZxkc2uIIDKrT_r","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.02","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Tests whether AI bots (OpenAI, Anthropic, Perplexity, Google, Bing) can actually crawl a website by checking robots.txt rules, live fetch simulation per bot, CDN/WAF blocking, JS-only content, llms.txt, and entity schema.","exampleAgentPrompt":"Check whether AI bots like GPTBot, ClaudeBot, and PerplexityBot can actually access https://example.com and https://example.com/pricing — tell me if robots.txt is blocking any of them, if the CDN is rejecting bot fetches, and whether llms.txt is present.","exampleUseCases":[{"title":"Diagnosing AI crawler blocks on a SaaS site","prompt":"Can you test whether AI search bots like GPTBot and ClaudeBot can actually reach my site at saasproduct.com? I want to know if robots.txt is blocking them, if the WAF is rejecting their fetches, and whether my llms.txt is set up correctly."},{"title":"Pre-launch AI visibility audit","prompt":"Before I launch my new site at mybrand.io, check if AI crawlers from OpenAI, Anthropic, and Perplexity will be able to read it — flag any robots.txt issues, CDN blocks, or missing entity schema that might stop them."},{"title":"Debugging why a specific page isn't in AI answers","prompt":"My pricing page at mycompany.com/pricing never shows up in Perplexity or ChatGPT answers. Can you test whether AI bots can actually fetch that URL, check the robots.txt rules for it, and tell me if it's JS-only content that they can't read?"}],"resultDescription":"Per-URL report covering: which AI bots are allowed or blocked by robots.txt (separately for search vs training), live fetch result as each bot vs a real browser (to catch CDN/WAF blocks), detection of JS-only content that bots can't render, presence and validity of llms.txt and entity schema, plus actionable fix recommendations. Failed lookups are not billed.","failureModes":["URL is unreachable or returns non-200 — endpoint reports the fetch failure without charging","robots.txt is missing or malformed — reported as absent/unparseable","More than 3 URLs submitted — rejected due to maxItems limit","Malformed URL format — may cause lookup failure","CDN timeout during live fetch — reported as a failed fetch result"],"whenToPreferThis":"Choose this endpoint when you need to diagnose why AI crawlers cannot read a specific website or page, rather than general SEO audits. It uniquely simulates live fetches as individual AI bots (GPTBot, ClaudeBot, PerplexityBot, Bingbot, Googlebot-AI) and compares them to browser fetches, making it the right tool for pinpointing CDN/WAF blocks, robots.txt misconfigurations, JS-only rendering issues, and missing llms.txt — all in a single call per URL.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-28T06:32:46.586Z","isFirstParty":false,"canonicalSlug":"ai-crawler-access-test-92645808"}