{"uid":"cap_v-uJ-OIfRKWi_90GFDuEX","slug":"fiatdock-robots-txt-llms-txt-ai-crawler-check-f3e08455","name":"FiatDock Robots.txt & llms.txt AI Crawler Check","description":"Pay-per-call: true402 Robots Check — robots.txt & llms.txt: which AI crawlers are allowed","url":"https://fiatdock.com/s/svc_5c3f119b-73f0-4ef0-b456-ddd188c2e701?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET","HEAD","DELETE"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.003","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.003/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_kOPPO85olef5gEdU9X8Bw","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.003","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks a given URL's robots.txt and llms.txt files to determine which AI crawlers are permitted or blocked","exampleAgentPrompt":"Can you check whether https://example.com allows AI crawlers like GPTBot — look at both its robots.txt and llms.txt files and tell me what's permitted?","exampleUseCases":[{"title":"Pre-crawl AI bot permission check","prompt":"Before my AI agent scrapes openai.com, can you check its robots.txt and llms.txt to see if AI crawlers are actually allowed there?"},{"title":"Competitor site crawler policy audit","prompt":"I want to know if our competitor at techcrunch.com has blocked AI crawlers — check their robots.txt and llms.txt and tell me which bots are permitted or restricted."},{"title":"LLM training data sourcing compliance","prompt":"We're building a dataset from web sources — can you verify whether wikipedia.org's robots.txt and llms.txt permit AI crawlers to access their content?"}],"resultDescription":"Returns parsed information from the target URL's robots.txt and llms.txt files, indicating which AI crawlers (e.g. GPTBot, CCBot, ClaudeBot) are allowed or disallowed, along with any relevant directives or llms.txt declarations governing AI access.","failureModes":["URL not reachable or returns non-200 status — robots.txt or llms.txt may be absent or inaccessible","Domain does not have a robots.txt or llms.txt — result will reflect absence of rules","Invalid URL format provided — request will fail input validation","Timeout fetching remote file — transient network error","Site returns unexpected content type for robots.txt path"],"whenToPreferThis":"Use this endpoint when you need to determine AI crawler permissions for a specific URL before scraping, indexing, or training on content. Prefer this over manually parsing robots.txt when you need quick, structured, pay-per-call access without managing HTTP logic yourself. Especially valuable for compliance checks, dataset curation pipelines, or agentic crawlers that must respect llms.txt conventions.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-27T06:38:21.439Z","isFirstParty":false,"canonicalSlug":"fiatdock-robots-txt-llms-txt-ai-crawler-check-f3e08455"}