{"uid":"cap_lSLlDm-1ibXkFdO6u2v4H","slug":"api-x402node-dev-394ad214","name":"Robots.txt Fetcher and Parser","description":"Fetch and parse robots.txt for a domain. Returns parsed allow/disallow rules per user-agent, sitemap URLs, crawl-delay directives, and raw content. Built for AI agents planning ethical web crawling and SEO audit. Use ?domain=example.com. Accepts payment on Base or Solana — either network works.","url":"https://api.x402node.dev/web/robots","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","properties":{"domain":{"type":"string","description":"Domain name (optional)"}}}},"additionalProperties":false}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.005","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.005/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_nuS6VD4_u9vL44xcIOgo5","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.005","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses robots.txt for a given domain, returning allow/disallow rules per user-agent, sitemap URLs, crawl-delay directives, and raw content.","exampleAgentPrompt":"Can you fetch and parse the robots.txt for example.com and tell me which paths are disallowed for all crawlers, what the crawl delay is, and list any sitemap URLs?","exampleUseCases":null,"resultDescription":"Returns a structured breakdown of the robots.txt file including: allow and disallow rules grouped by user-agent, sitemap URLs, crawl-delay values per agent, and the raw robots.txt text content.","failureModes":["Domain not found or unreachable — network or DNS error returned","No robots.txt file exists at the domain — empty or 404 response","Malformed robots.txt that cannot be parsed — partial or empty result","Invalid domain parameter format — validation error returned","Timeout fetching the remote file — timeout error returned"],"whenToPreferThis":"Use this endpoint when an AI agent needs to plan ethical web crawling by checking a site's crawl permissions before scraping, or when performing an SEO audit that requires inspecting robots.txt directives. Prefer this over manual fetching when structured parsing of allow/disallow rules, sitemap discovery, and crawl-delay extraction is needed in a single call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T00:57:16.131Z","isFirstParty":false}