{"uid":"cap_5PqqwQN6QtsJBXhjrUEqE","slug":"conway-replicatio-robots-txt-fetcher-c3435110","name":"Conway Replicatio Robots.txt Fetcher","description":"Autonomous pay-per-call machine utilities for AI agents, automation and developers. Payments settle in USDC on Base through x402.","url":"https://conway-replicatio-cloudflare.nikitamakiel1.workers.dev/api/x402/robots?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri"}},"additionalProperties":false},"type":{"type":"string","const":"http"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json","form-data","text"],"type":"string"}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"ok":true,"data":{},"service":"robots"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_QqJv9ysyUOMfBD2MBGSIM","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and returns the robots.txt content from a given URL via a pay-per-call HTTP utility service","exampleAgentPrompt":"Can you fetch the robots.txt file for https://example.com and tell me which paths are disallowed for bots?","exampleUseCases":[{"title":"Pre-scrape crawl permission check","prompt":"Before I start scraping, fetch the robots.txt from https://news.ycombinator.com and tell me which paths bots are not allowed to access."},{"title":"SEO audit robots.txt review","prompt":"Pull the robots.txt from https://mystore.com so I can audit whether we're accidentally blocking Googlebot from important pages."},{"title":"Sitemap discovery from robots.txt","prompt":"Grab the robots.txt file from https://techcrunch.com and extract any Sitemap URLs listed in it so I can start indexing their content."}],"resultDescription":"Returns a JSON object with an 'ok' boolean indicating success, a 'data' object containing the robots.txt content or parsed response from the target URL, and a 'service' field confirming the robots service was used.","failureModes":["Invalid or unreachable URL returns an error response with ok: false","Malformed URI in the body causes schema validation failure","Target site blocks the fetch request (403/404) resulting in an empty or error data payload","Network timeout if the target server is slow to respond","Missing required 'url' field in the body causes a 400-level error"],"whenToPreferThis":"Use this endpoint when you need a quick, pay-per-call fetch of a website's robots.txt without managing your own HTTP infrastructure. It's ideal for AI agents that need to check crawl permissions before web scraping, or for automated pipelines that audit SEO crawl rules on demand. It settles payments in USDC on Base via x402, making it suitable for autonomous agent workflows that operate without manual billing setup.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-11T06:46:02.375Z","isFirstParty":false,"canonicalSlug":"conway-replicatio-robots-txt-fetcher-c3435110"}