{"uid":"cap_3zEW61j9yJX-vCzerEnj-","slug":"agent-services-x402-web-tools-robots-txt-sitemap-lookup-6a2583ef","name":"agent-services x402 web tools – robots.txt / sitemap lookup","description":"Paid per-call web tools for AI agents: DNS, TLS, robots.txt, hashing, JavaScript rendering and screenshots. Payment over x402 (USDC on Base).","url":"https://49112-hjhjtioxvirqvcvo.splox.app/sitemap?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"site URL, e.g. https://example.com"}}},"responseSchema":{"type":"object","required":["robots_url"],"properties":{"note":{"type":"string"},"error":{"type":"string"},"rules":{"type":"array","items":{"type":"object","additionalProperties":true}},"exists":{"type":"boolean"},"status":{"type":"integer"},"sitemaps":{"type":"array","items":{"type":"string"}},"robots_url":{"type":"string","format":"uri"},"crawl_delay":{"type":"number"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.0005","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.0005/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.0005","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.0005","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_5eB58nUtgf5OhdJP8vzGq","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.0005","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt file, returning crawl rules, allowed/disallowed paths, sitemap URLs, crawl delay, and existence status for a given site URL.","exampleAgentPrompt":"Can you check the robots.txt for https://example.com and tell me whether crawling is allowed, what the crawl delay is, and list any sitemap URLs it references?","exampleUseCases":[{"title":"Pre-scrape crawl permission check","prompt":"Before I scrape https://news.ycombinator.com, can you pull its robots.txt and tell me which paths are disallowed and whether there's a crawl delay I need to respect?"},{"title":"Sitemap discovery for SEO audit","prompt":"I'm doing an SEO audit of https://www.shopify.com — can you fetch their robots.txt and give me all the sitemap URLs listed there?"},{"title":"Bot policy verification for AI agent","prompt":"I'm building a crawler and need to know if https://reddit.com blocks AI user agents in its robots.txt — can you retrieve it and show me the relevant rules?"}],"resultDescription":"Returns a JSON object containing the canonical robots.txt URL, whether the file exists, its HTTP status code, an array of parsed crawl rules (with user-agent, allow/disallow paths), discovered sitemap URLs, crawl delay (if specified), and any error or note messages.","failureModes":["Site does not have a robots.txt (exists=false, empty rules)","Target site is unreachable or times out (error field populated)","Site returns non-200 status for robots.txt (status field reflects actual HTTP code)","Malformed robots.txt that cannot be parsed (rules array may be empty or partial)","Payment failure over x402 protocol preventing the call from being processed"],"whenToPreferThis":"Choose this endpoint when an AI agent needs to programmatically check a website's crawl permissions, discover sitemap URLs, or verify bot policies before crawling — especially when you want a structured, parsed result rather than raw text. It is paid per-call via x402/USDC on Base, making it suitable for lightweight, on-demand checks without a subscription.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-02T12:39:17.476Z","isFirstParty":false,"canonicalSlug":"agent-services-x402-web-tools-robots-txt-sitemap-lookup-6a2583ef"}