{"uid":"cap_0p51F3VH-Fjg3cQvBBkYb","slug":"robots-txt-llms-txt-checker-9206682b","name":"Robots.txt & LLMs.txt Checker","description":"Guides and free-to-run Apify actors for bulk tech stack lookup, sitemap URL export to CSV, TED and UK Find a Tender search, and RSS-to-JSON feed monitoring.","url":"https://tools.adamhaleworks.workers.dev/x402/v1/robots-check?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["urls"],"properties":{"urls":{"type":"string","description":"Up to 5 URLs, comma-separated. A bare domain means its https:// homepage. Public hosts only (no IPs, no internal names), ports 80/443."},"agents":{"type":"string","description":"User-agent product tokens to check, comma-separated, up to 15 (default: GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-User, PerplexityBot, Google-Extended, Googlebot, Bingbot, CCBot, Applebot-Extended, Bytespider, *)."}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"count":1,"failed":0,"results":[{"ok":true,"input":"https://github.com/about","result":{"url":"https://github.com/about","agents":{"*":{"allowed":true,"crawlDelay":null},"GPTBot":{"allowed":true,"crawlDelay":1},"ClaudeBot":{"allowed":true,"crawlDelay":1}},"llmsTxt":{"url":"https://github.com/llms.txt","status":200,"present":true,"firstLine":"# GitHub"},"robotsTxt":{"url":"https://github.com/robots.txt","state":"rules","groups":8,"status":200,"sitemaps":[]}}}],"succeeded":1}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_I7aF-NDpVH53r6K5KSLNf","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks robots.txt crawl permissions and llms.txt presence for up to 5 URLs across configurable AI and web crawler user-agents.","exampleAgentPrompt":"Can you check whether github.com and openai.com allow GPTBot, ClaudeBot, and PerplexityBot to crawl them, and also tell me if either site has an llms.txt file?","exampleUseCases":[{"title":"AI crawler access audit for competitor sites","prompt":"Check if stripe.com, shopify.com, and squarespace.com block any AI crawlers like GPTBot or ClaudeBot in their robots.txt, and let me know if any of them have an llms.txt file."},{"title":"Pre-launch robots.txt verification for own site","prompt":"Before I launch my new site at mycompany.com, can you verify that GPTBot, Bingbot, and Googlebot are all allowed to crawl it according to robots.txt, and check if llms.txt is present?"},{"title":"LLM discoverability check for content strategy","prompt":"I'm building an LLM-friendly site — can you check whether anthropic.com and mistral.ai have llms.txt files and what crawl permissions they grant to ClaudeBot and Google-Extended?"}],"resultDescription":"Returns a JSON object with a count of results, success/failure counts, and per-URL details including: crawl permission (allowed/denied) and crawl delay per user-agent, robots.txt fetch status and group count, any sitemaps listed in robots.txt, and llms.txt presence with its URL, HTTP status, and first line of content.","failureModes":["Private/internal hostnames or IP addresses are rejected","More than 5 URLs in a single request causes an error","Invalid URL format returns a failed result for that entry","Site unreachable or DNS failure marks the URL as failed","Ports other than 80 or 443 are rejected"],"whenToPreferThis":"Choose this endpoint when you need to programmatically verify how a website's robots.txt treats AI or web crawlers, especially when checking for llms.txt presence simultaneously. It supports batch checking of up to 5 URLs and a configurable set of up to 15 user-agent tokens, making it ideal for auditing multiple domains at once. Prefer it over manual robots.txt parsing or generic HTTP fetchers when you need structured, per-agent crawl permission data in a single call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-13T12:30:02.310Z","isFirstParty":false,"canonicalSlug":"robots-txt-llms-txt-checker-9206682b"}