{"uid":"cap_7F0nPb2bvnQ3dSpant-_d","slug":"robots-allowed-for-bot-93ad2824","name":"robots-allowed-for-bot","description":"Is a specific bot allowed to fetch a URL? Fetches the site's robots.txt and evaluates the path for the named user-agent (GPTBot, ClaudeBot, Googlebot, Bingbot, CCBot, PerplexityBot or any token), returning allowed/disallowed, the matching rule and group, crawl-delay, and whether the bot has a dedicated group or falls back to *. Pre-flight for compliant scraping agents. $0.01 per check.","url":"https://intel.rallylive.ca/dev/robots-for-bot","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","properties":{}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_CC2Brl2YYte6ee62SsYb0","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a named bot (e.g. GPTBot, ClaudeBot, Googlebot) is allowed to fetch a specific URL by evaluating the site's robots.txt rules, returning allow/disallow status, matching rule, crawl-delay, and group membership.","exampleAgentPrompt":"Before I scrape https://example.com/blog/article-123, check if GPTBot is allowed to fetch that URL by looking up their robots.txt.","exampleUseCases":[{"title":"Pre-flight check before AI scraping","prompt":"I'm about to crawl https://techcrunch.com/ai/ — can you check if ClaudeBot is allowed to access that URL according to their robots.txt before I proceed?"},{"title":"Audit site's crawler access policy","prompt":"Check whether Googlebot is allowed to fetch https://shop.example.com/products and tell me the matching rule, any crawl-delay, and whether Googlebot has its own dedicated robots.txt group."},{"title":"Verify AI bot compliance for content aggregator","prompt":"Is PerplexityBot allowed to access https://nytimes.com/2024/science/article.html? I need to know the exact matching rule and whether it falls back to the wildcard group or has a dedicated entry."}],"resultDescription":"Returns an allowed/disallowed boolean, the exact matching rule from robots.txt, the group (user-agent block) that matched, the crawl-delay if specified, and a flag indicating whether the bot has a dedicated group or falls back to the wildcard (*) group.","failureModes":["robots.txt not found or unreachable (site returns 4xx/5xx on /robots.txt)","URL is malformed or unreachable","bot name not recognized as a known token (may fall back to wildcard)","robots.txt is too large or malformed to parse","network timeout fetching the remote robots.txt"],"whenToPreferThis":"Use this endpoint when an AI agent or scraping workflow needs to perform compliant, polite crawling and must verify robots.txt permissions before fetching any page. Prefer this over manual parsing when you need a structured response that explicitly names the matching rule, group, and crawl-delay for a specific user-agent token.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T13:07:50.778Z","isFirstParty":false}