{"uid":"cap_5A4IawEKlRuSYIXPG7TYB","slug":"robots-txt-llms-txt-checker-19b1c8ab","name":"robots.txt + llms.txt Checker","description":"WebberSites – Branding, Web Design, Photography, Marketing Strategy - Youngstown Ohio","url":"https://api.webbersites.com/api/seo/robots-check?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Domain or any page on it"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","properties":{"origin":{"type":"string"},"robots":{"type":"object"},"llms_txt":{"type":"object"}}}}}}},"responseSchema":{"type":"json","example":{"origin":"https://example.com","robots":{"bots":{"GPTBot":{"kind":"ai","root_blocked":true},"Googlebot":{"kind":"search","root_blocked":false}},"found":true,"sitemaps_declared":["https://example.com/sitemap.xml"]},"llms_txt":{"found":false}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":{"p10Cents":"0.1000","medianCents":"0.1000","p90Cents":"0.1000","minCents":"0.1000","maxCents":"0.1000","p95Cents":"0.1000","sampleCount":1,"varies":false,"failureChargeRate":1},"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_DN3EFFwORNZy71hmX-B9h","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks a website's robots.txt and llms.txt files, returning crawler access verdicts for major search and AI bots, declared sitemaps, syntax warnings, and llms.txt structure summary.","exampleAgentPrompt":"Check the robots.txt and llms.txt files for openai.com — I want to know if GPTBot, ClaudeBot, Googlebot, and PerplexityBot are allowed or blocked, any declared sitemaps, syntax warnings, and whether they have an llms.txt.","exampleUseCases":[{"title":"Audit competitor AI bot policies","prompt":"Pull up the robots.txt and llms.txt for anthropic.com — I want to see whether GPTBot, ClaudeBot, and PerplexityBot are allowed or blocked, what sitemaps they've declared, and if they have an llms.txt file set up."},{"title":"Check site before AI training crawl","prompt":"Before we start crawling nytimes.com for training data, check their robots.txt — I need to know if CCBot, GPTBot, and Google-Extended are allowed, any syntax warnings we should be aware of, and whether they have an llms.txt."},{"title":"Diagnose missing search engine traffic","prompt":"My site mybusiness.com has dropped out of Google and Bing search results — can you check the robots.txt to see if Googlebot or Bingbot are being blocked, flag any syntax errors, and tell me what sitemaps are declared?"}],"resultDescription":"Returns crawler access verdicts (allow/disallow) for major search bots (Googlebot, Bingbot) and AI bots (GPTBot, ClaudeBot, PerplexityBot, CCBot, Google-Extended, and more), a list of sitemaps declared in robots.txt, any syntax warnings found, and whether llms.txt and llms-full.txt are present along with a structural summary of their contents.","failureModes":["URL parameter missing — returns 400 error","Unreachable domain or invalid URL — returns error or empty robots result","robots.txt returns non-200 status — noted in response","llms.txt absent — field returns null or absent indicator","Malformed robots.txt — syntax warnings returned but parsing continues"],"whenToPreferThis":"Use this endpoint when you need to audit a website's crawler access policies in one call — specifically when you need verdicts for both traditional search bots AND AI training/inference bots simultaneously, and also want to check llms.txt presence. Prefer this over a generic HTTP fetch when you need structured, interpreted bot-by-bot verdicts rather than raw file content.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-10T18:40:26.348Z","isFirstParty":false,"canonicalSlug":"robots-txt-llms-txt-checker-19b1c8ab"}