{"uid":"cap_bNE3upcMLcRMVnabVuTLD","slug":"robots-txt-llms-txt-fetcher-and-parser-4672e643","name":"robots.txt / llms.txt Fetcher and Parser","description":"robots.txt / llms.txt — Fetch and parse a site's robots.txt and llms.txt.","url":"https://mcpfax-utility.bowling-anthony.workers.dev/v1/robots","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object"}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.005","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.005/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_pF5XehVWM9jM-eNo0QkM9","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.005","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt and llms.txt files, returning structured crawl rules and LLM access directives.","exampleAgentPrompt":"Can you fetch and parse the robots.txt and llms.txt files for openai.com and tell me which paths are disallowed and whether they have any LLM-specific access rules?","exampleUseCases":[{"title":"Check if site blocks AI crawlers","prompt":"Look up the robots.txt and llms.txt for anthropic.com and tell me whether they allow or block AI model training crawlers."},{"title":"Audit crawl permissions before scraping","prompt":"Before I scrape data from nytimes.com, can you fetch their robots.txt and tell me which paths are off-limits and what user-agents they restrict?"},{"title":"Verify LLM policy for a content site","prompt":"Does medium.com have an llms.txt file, and if so, what does it say about how AI agents are allowed to use their content?"}],"resultDescription":"Returns the raw and parsed contents of the site's robots.txt and llms.txt files, including structured crawl directives (allow/disallow rules, user-agent groups, crawl-delay) and any LLM-specific access policies defined in llms.txt.","failureModes":["Site does not have a robots.txt (returns 404 or empty)","Site does not have an llms.txt file","Domain is unreachable or times out","Malformed robots.txt that cannot be parsed","Invalid or non-existent domain provided"],"whenToPreferThis":"Use this endpoint when you need to programmatically check a site's crawl permissions or LLM access policy before scraping, indexing, or training on its content. Prefer this over manual fetching when you need structured, parsed output from both robots.txt and llms.txt in a single call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T06:52:37.788Z","isFirstParty":false}