{"uid":"cap_PzrdZt06oKGQImX-Y2DpO","slug":"robots-txt-fetcher-with-ai-crawler-detection-ab3f3da8","name":"Robots.txt Fetcher with AI Crawler Detection","description":"Fetch robots.txt for a public site and report common AI crawler mentions plus the robots content.","url":"https://remarkable-creation-production-8797.up.railway.app/v1/robots","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET","HEAD","DELETE"],"type":"string"},"queryParams":{"type":"object","required":["target"],"properties":{"target":{"type":"string","description":"Public HTTP(S) URL or hostname, e.g. example.com"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_7NrB_xhrh-Yllvh9mNgTi","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and returns the robots.txt file for a given public website, and reports which common AI crawlers (e.g. GPTBot, ClaudeBot) are mentioned in it.","exampleAgentPrompt":"Can you fetch the robots.txt for openai.com and tell me which AI crawlers are mentioned and whether any of them are blocked?","exampleUseCases":[{"title":"Check if site blocks AI crawlers","prompt":"Pull up the robots.txt for nytimes.com and tell me if GPTBot or ClaudeBot are blocked from crawling it."},{"title":"Pre-scrape crawl policy check","prompt":"Before I scrape data from techcrunch.com, can you grab their robots.txt and tell me what paths are disallowed and if any AI bots are specifically called out?"},{"title":"Audit own site's AI crawler policy","prompt":"Fetch the robots.txt for my site at myblog.com and report exactly what it says about AI crawlers — I want to know if I accidentally blocked any of them."}],"resultDescription":"Returns the raw robots.txt content for the given domain, along with a structured report of which common AI crawlers (such as GPTBot, ClaudeBot, PerplexityBot, etc.) are explicitly mentioned, whether they are allowed or disallowed, and any relevant path-level directives.","failureModes":["Target URL is not publicly accessible or returns a non-200 status — robots.txt may not exist","Malformed or missing 'target' query parameter returns a validation error","Site returns a robots.txt that is very large or non-standard, which may produce incomplete parsing","Private or localhost URLs may be rejected or fail to resolve","Some sites serve robots.txt with incorrect content types, which may affect parsing accuracy"],"whenToPreferThis":"Use this endpoint when you need to programmatically check a site's robots.txt content and specifically want to see how AI crawlers are treated — it saves you from manually fetching and parsing robots.txt yourself. Prefer it over a generic HTTP fetch when you want structured AI-crawler mention extraction alongside the raw content. It pairs well with the sibling URL inspection and header-checking endpoints for a complete site audit.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T18:30:26.444Z","isFirstParty":false}