{"uid":"cap_K0WI4am0THJn7N3PrAnGs","slug":"setu-anantasutra-ai-crawler-robots-txt-audit-6830914b","name":"Setu Anantasutra: AI Crawler robots.txt Audit","description":"Pay-per-call JSON tools for AI agents: Indian identifier validation (GSTIN, PAN, TAN, CIN, LLPIN, IFSC, PIN code, UPI ID, vehicle registration), AI-visibility (GEO) audits, AI crawler robots.txt matrices and llms.txt checks. x402 (USDC on Base) or RapidAPI.","url":"https://setu.xyz23110928.workers.dev/v1/geo/ai-crawlers?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"paths":{"type":"string","maxLength":600,"description":"Optional comma-separated extra paths to test, e.g. /blog/,/docs/"},"domain":{"type":"string","maxLength":253,"minLength":3,"description":"Domain or URL, e.g. example.com"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","required":["origin","summary","crawlers"],"properties":{"origin":{"type":"string"},"summary":{"type":"object"},"crawlers":{"type":"array","items":{"type":"object","properties":{"bot":{"type":"string"},"category":{"enum":["training","search","user","other"],"type":"string"},"operator":{"type":"string"},"allowedRoot":{"type":"boolean"},"decidingRule":{"type":["string","null"]}}}}}}}}}},"responseSchema":{"type":"json","example":{"origin":"https://example.com","summary":{"blocked":["GPTBot","CCBot"],"searchCrawlersAllowed":{"total":9,"allowed":9},"trainingCrawlersAllowed":{"total":12,"allowed":0}},"crawlers":[{"bot":"GPTBot","category":"training","operator":"OpenAI","allowedRoot":false,"decidingRule":"Disallow: /"}],"robotsTxt":{"state":"ok","httpStatus":200},"pathsChecked":["/","/blog/"]}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_bdG8mGIU5CBru0ZybY2Nx","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a domain's robots.txt to produce a structured matrix showing which AI training and search crawlers are allowed or blocked, with per-bot decisions and summary counts.","exampleAgentPrompt":"Check which AI crawlers and training bots are allowed or blocked on openai.com — I want to see the full breakdown by bot name, operator, and whether they can access the root path.","exampleUseCases":[{"title":"AI training crawler access audit","prompt":"Can you check robots.txt on nytimes.com and tell me which AI training crawlers like GPTBot and CCBot are allowed or blocked, with the deciding rule for each?"},{"title":"Competitor AI-visibility benchmark","prompt":"Audit hubspot.com for AI crawler access — I want to know how many search bots versus training bots are allowed, and a per-bot breakdown including the operator name."},{"title":"Own site GEO compliance check","prompt":"Check my domain mybrand.io and also the /blog/ and /docs/ paths — show me which AI crawlers can reach each path and which are blocked so I can fix our robots.txt."}],"resultDescription":"A JSON object with: the canonical origin URL, a summary object with counts of allowed vs blocked training and search crawlers, an array of per-bot objects (bot name, category, operator, allowedRoot boolean, decidingRule), robots.txt HTTP status and state, and a list of paths checked.","failureModes":["Domain not found or DNS failure — robots.txt state will reflect fetch error","robots.txt returns non-200 status — reported in robotsTxt.httpStatus field","Domain missing robots.txt — state may be 'missing' with all crawlers treated as allowed","Malformed domain input — validation error returned","Rate limit or payment failure via x402 — 402 Payment Required response"],"whenToPreferThis":"Use this endpoint when you need a structured, machine-readable audit of which AI and search crawlers a domain's robots.txt permits or denies, broken down by bot and operator — especially useful for GEO (generative engine optimization) workflows, content licensing research, or compliance checks. Prefer it over manual robots.txt parsing when you need per-bot categorization (training vs search vs user) and summary statistics in a single call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-23T12:32:51.383Z","isFirstParty":false,"canonicalSlug":"setu-anantasutra-ai-crawler-robots-txt-audit-6830914b"}