{"uid":"cap_IjJtKmcGGL4KkfhPLBeIN","slug":"x402-toolbox-web-robots-txt-fetcher-1f649640","name":"x402-toolbox Web Robots.txt Fetcher","description":"309 deterministic pay-per-call tools for AI agents over x402 (USDC, Base). No signup, no API key: the payment is the identity.","url":"https://x402-toolbox.shablony-pro.workers.dev/web/robots-txt?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","properties":{"schema":{"type":"string"}}},"responseSchema":{"type":"json","example":{"ok":true,"paid":true}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_A9TZ_gLZIT8CmOL9aL0eo","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and returns the robots.txt content for a given web URL via a pay-per-call x402 endpoint","exampleAgentPrompt":"Can you grab the robots.txt file for example.com so I can see what paths are blocked or allowed for bots?","exampleUseCases":[{"title":"Pre-scrape crawl policy check","prompt":"Before I start scraping techcrunch.com, can you pull up their robots.txt so I know which paths I'm allowed to crawl and which are off-limits?"},{"title":"SEO audit of crawl directives","prompt":"I'm doing an SEO audit — fetch the robots.txt for shopify.com so I can review their crawl rules and make sure no important pages are accidentally disallowed."},{"title":"Competitor bot access research","prompt":"Can you retrieve the robots.txt for amazon.com? I want to see which user agents they restrict and what sections of the site are blocked from crawlers."}],"resultDescription":"Returns a JSON object with ok and paid confirmation flags, along with the parsed or raw robots.txt content for the requested domain, including directives such as User-agent, Disallow, Allow, Sitemap, and Crawl-delay rules.","failureModes":["Domain not found or unreachable — returns error or empty robots.txt","Schema parameter missing or malformed — may return validation error","Target site returns non-standard robots.txt format — parsing may be incomplete","Payment failure over x402 — 402 response blocking access","Rate limiting or Cloudflare Worker timeout for slow-responding domains"],"whenToPreferThis":"Choose this endpoint when you need a quick, no-signup, pay-per-call fetch of a website's robots.txt for crawl policy inspection, SEO auditing, or pre-scrape compliance checking. It is ideal for AI agents operating autonomously without persistent API credentials, leveraging x402 USDC micropayments for frictionless access.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-02T00:41:01.840Z","isFirstParty":false,"canonicalSlug":"x402-toolbox-web-robots-txt-fetcher-1f649640"}