{"uid":"cap_LVJ6ZbMaRShuwFlfK_Dik","slug":"api-trustsource-cc-5efec1b6","name":"TrustSource Robots.txt Intelligence API","description":"robots.txt and AI crawler policy check. Tells an agent whether a website permits crawling and whether it blocks AI bots such as GPTBot, ClaudeBot, Google-Extended, PerplexityBot and CCBot, before scraping, RAG ingestion, training data collection or archiving. Parses robots.txt and returns a crawl-policy tier with per-bot allow and disallow rules and sitemap URLs.","url":"https://api.trustsource.cc/robots","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"domain":{"type":"string","description":"Domain whose robots.txt to fetch and parse (e.g. example.com); https is tried first, then http."}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":{"request":{"input":{"type":"http","method":"GET","queryParams":{"domain":"example.com"}}},"response":{"ai":null,"meta":{"cached":false,"paidWith":"x402/USDC","checkedAt":"2026-05-29T04:40:36.981Z","apiVersion":"1.0"},"tier":"NO_ROBOTS_TXT","domain":"example.com","exists":false,"summary":null,"response":{"status":404},"sitemaps":[],"aiFriendly":true,"userAgents":[]}},"exampleRequest":{"domain":"example.com"},"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"settled","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_vlOlLDhw_v532Y9XRV2pQ","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Parses a domain's robots.txt file and returns AI bot crawl policies, access tiers, per-bot allow/disallow rules, and sitemap URLs","exampleAgentPrompt":"Can you check whether openai.com allows GPTBot and ClaudeBot to crawl it — I want to know the full robots.txt tier and any per-bot allow or disallow rules before I start crawling.","exampleUseCases":[{"title":"Verify crawler permissions before RAG ingestion","prompt":"Before we ingest this website into our knowledge base, check if they block AI crawlers like GPTBot and ClaudeBot. What's their robots.txt tier and do they have any specific rules for training data collection?"},{"title":"Audit AI bot access across competitors","prompt":"Pull the robots.txt policies for our top five competitors and tell me which ones allow PerplexityBot and Google-Extended. I need to know if they're open to AI crawling or if they're blocking it."},{"title":"Compliance check for automated archiving","prompt":"We're archiving news articles from this domain for our research tool. Check the robots.txt to see if they block all bots or just AI crawlers, and grab any sitemaps so we know what pages they're exposing."}],"resultDescription":"Returns a tier label (OPEN, SELECTIVE, BLOCKED_AI, BLOCKED_ALL, or NO_ROBOTS_TXT), per-bot allow/disallow analysis for known AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, etc.), and a list of sitemap URLs found in robots.txt","failureModes":["Domain does not exist or is unreachable — may return NO_ROBOTS_TXT tier or error","robots.txt file is malformed or non-standard — partial parsing results","Domain missing 'domain' query parameter — 400 bad request","Upstream fetch timeout if target server is slow","Payment not attached — 402 Payment Required response"],"whenToPreferThis":"Use this endpoint when an AI agent or crawler needs to programmatically determine whether it is permitted to crawl a given domain, especially for checking AI-specific bot policies (GPTBot, ClaudeBot, PerplexityBot, Google-Extended). Prefer over manual robots.txt parsing when you need structured tier classification and per-bot analysis in a single call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T12:37:07.957Z","isFirstParty":false}