{"uid":"cap_bWAn-09AC5wolvC768JlY","slug":"public-robots-txt-crawler-policy-lookup-17eaaf53","name":"Public Robots.txt Crawler Policy Lookup","description":"Purchase: Public Robots.txt Crawler Policy — Rules, Delay & Sitemaps","url":"https://api.the402.ai/v1/services/svc_049286bebcf3424b/purchase","method":"POST","headers":{},"bodySchema":null,"responseSchema":{"type":"object","properties":{"job_id":{"type":"string"},"status":{"type":"string"},"thread_id":{"type":"string"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.053","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.053/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.053","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.053","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_FAh7qNIxO2nBDH7oJTIPq","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.053","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt file, returning crawler rules, crawl delay settings, and sitemap URLs","exampleAgentPrompt":"Can you pull the robots.txt crawler policy for example.com and tell me which paths are blocked, what the crawl delay is, and where the sitemaps are?","exampleUseCases":null,"resultDescription":"Returns a job_id, status, and thread_id representing an asynchronous job that, when complete, will contain the parsed robots.txt data including allow/disallow rules per user-agent, crawl delay values, and sitemap URLs discovered from the target domain's robots.txt file.","failureModes":["Domain has no robots.txt file — returns empty or 404-equivalent result","Domain is unreachable or times out — job fails with error status","Invalid or malformed URL input — request rejected","Payment of 0.053 USDC not fulfilled — 402 Payment Required response","Rate limiting by target server prevents robots.txt retrieval"],"whenToPreferThis":"Use this endpoint when an AI agent needs to programmatically check a website's crawler policy before scraping, indexing, or accessing site content — especially when building respectful web crawlers, SEO tools, or compliance-checking workflows. Prefer this over manual fetch when you need structured parsing of robots.txt rules including per-user-agent directives, delay values, and sitemap discovery in a single paid call via x402 micropayment.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-13T18:39:42.965Z","isFirstParty":false}