{"uid":"cap_BKJr7M8As0vKDMynIzRRJ","slug":"sitesignal-robots-policy-snapshot-904888f3","name":"SiteSignal Robots Policy Snapshot","description":"Parse a public robots.txt into crawler groups, effective allow/disallow rules, crawl delay, sitemap references, and a response hash.","url":"https://trinity-throw-thursday-gravity.trycloudflare.com/x402/robots-policy","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri"},"userAgent":{"type":"string","default":"*","maxLength":100}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.015","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.015/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.015","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.015","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_coCN_PlZqx9oDkJLgtF-j","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.015","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a public robots.txt file into structured crawler groups, allow/disallow rules, crawl delays, sitemap references, and a response hash.","exampleAgentPrompt":"Can you fetch and parse the robots.txt for https://example.com and tell me which crawlers are blocked, what paths are disallowed, any crawl delays set, and what sitemaps are listed?","exampleUseCases":[{"title":"SEO audit of crawler permissions","prompt":"Pull the robots.txt from https://shopify.com and give me a breakdown of all the crawler groups, which paths are disallowed, and any sitemaps it references — I want to understand their crawl policy."},{"title":"Checking if Googlebot is blocked","prompt":"Can you parse the robots.txt at https://mysite.io and tell me specifically what rules apply to Googlebot — is it allowed everywhere, or are certain paths disallowed?"},{"title":"Monitoring robots.txt for changes","prompt":"Fetch the robots.txt for https://competitor.com right now and give me the response hash — I want to compare it against what I saw last week to see if anything changed."}],"resultDescription":"Returns a structured breakdown of the robots.txt including: all crawler/user-agent groups with their effective allow and disallow rules, any crawl delay directives, a list of sitemap URLs referenced, and a hash of the raw response for change detection purposes.","failureModes":["URL not provided or malformed — returns validation error","Target site returns non-200 status for robots.txt (e.g. 404, 403) — returns error indicating file unavailable","robots.txt exists but is empty — returns empty groups and rules","Target site is unreachable or times out — returns network/timeout error","Malformed robots.txt that cannot be parsed — returns partial or error response"],"whenToPreferThis":"Use this endpoint when you need structured, machine-readable extraction of a public robots.txt file rather than raw text. It is ideal for SEO auditing, crawler compliance checks, competitive research on site access policies, and change detection via response hashing. Prefer this over manual fetching when you need normalized rule sets per user-agent group and sitemap enumeration without writing your own parser.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-16T12:33:04.109Z","isFirstParty":false}