{"uid":"cap_oS5Y9Kx6yiZcQudV2pkJ2","slug":"vextorium-robots-txt-analyzer-5a537ae5","name":"Vextorium Robots.txt Analyzer","description":"Parse a site's robots.txt: whether a user-agent may crawl a path, which AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot...) are blocked, and declared sitemaps. Check before your agent scrapes.","url":"https://api.vextorium.com/v1/robots-txt?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"required":["url"],"properties":{"url":{"type":"string"},"agent":{"type":"string","description":"User-agent to test (optional)"},"route":{"type":"string","description":"Path to test (default /)"}}},"type":{"type":"string","const":"http"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json","form-data","text"],"type":"string"}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":["object","null"],"properties":{"site":{"type":["string","null"]},"query":{"type":["object","null"],"properties":{"agent":{"type":["string","null"]},"route":{"type":["string","null"]},"allowed":{"type":["boolean","null"]},"applied_group":{"type":["array","null"],"items":{"type":["null","string"]}}}},"groups":{"type":["number","null"]},"ai_bots":{"type":["object","null"],"properties":{"CCBot":{"type":["string","null"]},"GPTBot":{"type":["string","null"]},"YouBot":{"type":["string","null"]},"Diffbot":{"type":["string","null"]},"Amazonbot":{"type":["string","null"]},"ClaudeBot":{"type":["string","null"]},"cohere-ai":{"type":["string","null"]},"Bytespider":{"type":["string","null"]},"Claude-User":{"type":["string","null"]},"ChatGPT-User":{"type":["string","null"]},"anthropic-ai":{"type":["string","null"]},"DuckAssistBot":{"type":["string","null"]},"OAI-SearchBot":{"type":["string","null"]},"PerplexityBot":{"type":["string","null"]},"MistralAI-User":{"type":["string","null"]},"Google-Extended":{"type":["string","null"]},"Perplexity-User":{"type":["string","null"]},"Claude-SearchBot":{"type":["string","null"]},"Applebot-Extended":{"type":["string","null"]},"meta-externalagent":{"type":["string","null"]}}},"sitemaps":{"type":["array","null"],"items":{"type":["null","string"]}},"robots_txt":{"type":["boolean","null"]},"blocked_ai_bots":{"type":["number","null"]}}}}}}},"responseSchema":{"type":"json","example":{"site":"https://www.nytimes.com","query":{"agent":"GPTBot","route":"/section/world","allowed":false,"applied_group":["gptbot"]},"groups":63,"ai_bots":{"CCBot":"bloqueado","GPTBot":"bloqueado","YouBot":"bloqueado","Diffbot":"bloqueado","Amazonbot":"permitido","ClaudeBot":"bloqueado","cohere-ai":"bloqueado","Bytespider":"bloqueado","Claude-User":"bloqueado","ChatGPT-User":"bloqueado","anthropic-ai":"bloqueado","DuckAssistBot":"bloqueado","OAI-SearchBot":"bloqueado","PerplexityBot":"bloqueado","MistralAI-User":"permitido","Google-Extended":"bloqueado","Perplexity-User":"bloqueado","Claude-SearchBot":"bloqueado","Applebot-Extended":"bloqueado","meta-externalagent":"bloqueado"},"sitemaps":["https://www.nytimes.com/sitemaps/new/news.xml.gz","https://www.nytimes.com/sitemaps/new/sitemap.xml.gz"],"robots_txt":true,"blocked_ai_bots":18}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.003","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.003/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_6IpGsr-HogkbuJ9Vvvvg2","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.003","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt to determine AI bot permissions, allowed/blocked routes, and crawl rules for specific user-agents","exampleAgentPrompt":"Check if GPTBot is allowed to crawl https://www.nytimes.com/section/world — I want to know the full robots.txt AI bot permissions including which bots are blocked or allowed.","exampleUseCases":[{"title":"AI bot crawl compliance check","prompt":"Can you check if ClaudeBot and GPTBot are allowed to crawl https://www.wired.com? I want to see all the AI bots listed in their robots.txt and whether each one is blocked or permitted."},{"title":"Route-specific access validation","prompt":"Check the robots.txt for https://www.reddit.com and tell me if PerplexityBot is allowed to access the /r/technology/ path specifically."},{"title":"Sitemap discovery via robots.txt","prompt":"Fetch the robots.txt for https://www.bbc.com and pull out any sitemaps listed there, plus the full list of AI bots and their access status."}],"resultDescription":"Returns a structured JSON with: the parsed robots.txt existence flag, total number of rule groups, a map of well-known AI bots (GPTBot, ClaudeBot, CCBot, PerplexityBot, etc.) with their allowed/blocked status, a list of sitemaps, and — if a specific agent and route are queried — whether that agent is allowed on that path and which rule group applies.","failureModes":["Invalid or unreachable URL returns an error or null robots.txt fields","Site has no robots.txt — robots_txt field returns false with no rules","User-agent not found in robots.txt — allowed status may default to true (permissive)","Malformed robots.txt on target site may cause partial parse results","Network timeout fetching remote robots.txt"],"whenToPreferThis":"Use this endpoint when you need to programmatically check whether an AI crawler or agent is permitted to access a specific website or route before scraping, indexing, or crawling. It is especially useful for compliance checks before web data collection, building ethical scraping pipelines, or auditing a site's AI bot policy across multiple known AI user-agents simultaneously. Prefer this over raw HTTP fetches of robots.txt when you need structured, parsed output with per-bot status rather than raw text.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-11T18:52:00.073Z","isFirstParty":false,"canonicalSlug":"vextorium-robots-txt-analyzer-5a537ae5"}