{"uid":"cap_gydEllKQyJueifG7Q8bO-","slug":"search-readiness-lab-crawler-policy-audit-c88f0c9d","name":"Search Readiness Lab – Crawler Policy Audit","description":"Paid deterministic audits for Google, conventional search, and AI-agent crawler discoverability.","url":"https://search-readiness.inboxtzdjqv.workers.dev/v1/crawler-policy?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","format":"uri"},"userAgents":{"type":"array","items":{"type":"string"},"maxItems":20}}},"responseSchema":{"type":"json","example":{"agents":[],"origin":"https://example.com","sitemaps":[],"robotsStatus":200}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_MY62jY2xNpILiZUl4F1kE","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Audits a URL's robots.txt, sitemaps, and crawler access policies for Google, traditional search engines, and AI-agent crawlers.","exampleAgentPrompt":"Can you audit https://www.myshop.com for crawler access? I want to know whether Googlebot, GPTBot, and ClaudeBot are allowed in, and whether there are any sitemaps discoverable.","exampleUseCases":[{"title":"AI crawler readiness check","prompt":"Check whether https://www.techstartup.io allows AI crawlers like GPTBot and PerplexityBot — I want to know if they're being blocked or allowed through robots.txt and whether there's a sitemap listed."},{"title":"SEO pre-launch site audit","prompt":"Before we go live, can you audit https://staging.myblog.com for Googlebot and Bingbot access? I need to know the robots.txt status, any sitemaps, and whether those crawlers are being let through."},{"title":"Competitor crawler policy analysis","prompt":"Can you pull the crawler policy for https://www.competitorsite.com? I want to see what user agents they allow, whether they have a sitemap, and how their robots.txt is configured."}],"resultDescription":"Returns a JSON object with the HTTP status of the robots.txt file, a list of discovered sitemaps, the origin URL, and per-agent access rules indicating which of the requested user agents are allowed or disallowed from crawling the site.","failureModes":["Invalid or unreachable URL returns an error or empty agent array","robots.txt not found (non-200 status) may result in no policy data","More than 20 user agents specified will be rejected due to maxItems constraint","Network timeouts fetching the target URL's robots.txt","Malformed URL format fails schema validation"],"whenToPreferThis":"Choose this endpoint when you need a deterministic, structured audit of a specific URL's crawler access policies — especially when you need to verify AI-agent crawler discoverability alongside traditional search engine access. It is ideal for pre-launch SEO checks, competitive analysis, or diagnosing why a site might not be indexed by AI search platforms.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-10T12:31:02.779Z","isFirstParty":false,"canonicalSlug":"search-readiness-lab-crawler-policy-audit-c88f0c9d"}