{"uid":"cap_udZXUGE4-TENctFash1Yg","slug":"sitesignal-robots-policy-snapshot-f5ff93bd","name":"SiteSignal Robots Policy Snapshot","description":"Parse a public robots.txt into crawler groups, effective allow/disallow rules, crawl delay, sitemap references, and a response hash.","url":"https://phases-prot-shine-royal.trycloudflare.com/x402/robots-policy","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri"},"userAgent":{"type":"string","default":"*","maxLength":100}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.015","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"down","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.015/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.015","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.015","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_OBz8UlLeHutjKXfA3ABqG","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.015","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a public robots.txt file into structured crawler groups, allow/disallow rules, crawl delay, sitemap references, and a response hash.","exampleAgentPrompt":"Can you fetch and parse the robots.txt for https://example.com and show me the crawler groups, disallowed paths, crawl delay, and any sitemap references listed?","exampleUseCases":[{"title":"SEO audit of crawler access rules","prompt":"Pull the robots.txt for https://shopify.com and break down which crawlers are restricted, what paths are blocked, and what crawl delay is set — I want to understand their crawler policy before auditing their SEO setup."},{"title":"Monitoring competitor robots policy changes","prompt":"Grab the robots.txt snapshot for https://competitor.com including the response hash — I want to track whether their crawler rules have changed since last week."},{"title":"Verifying sitemap discoverability","prompt":"Check the robots.txt at https://myblog.com and tell me all the sitemap URLs referenced in it — I need to confirm my sitemaps are properly declared for search engines."}],"resultDescription":"A structured breakdown of the parsed robots.txt including named crawler groups with their effective allow and disallow path rules, crawl delay values, sitemap URLs referenced in the file, and a hash of the raw response for change detection.","failureModes":["URL is not publicly accessible or returns a non-200 HTTP status","robots.txt does not exist at the given domain (404 response)","Malformed robots.txt that cannot be parsed into structured groups","Network timeout or DNS resolution failure for the provided URL","URL parameter missing or not a valid URI format"],"whenToPreferThis":"Use this endpoint when you need structured, machine-readable extraction of robots.txt crawler policies rather than just fetching the raw file. It is ideal for SEO audits, crawler policy monitoring, sitemap discovery, and change detection via response hashing. Prefer it over general HTTP fetchers when you specifically need per-agent allow/disallow rule parsing and crawl delay extraction.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T10:26:33.082Z","isFirstParty":false}