{"uid":"cap_mQOlHtYfM9L3Wp_QGpw9-","slug":"sitesignal-robots-policy-snapshot-9acae6c1","name":"SiteSignal Robots Policy Snapshot","description":"Paid SiteSignal robots policy resource with bounded public-data inputs and direct Base USDC settlement.","url":"https://specialty-chemical-refer-comparisons.trycloudflare.com/x402/robots-policy?url=https%3A%2F%2Fexample.com&userAgent=*","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri"},"userAgent":{"type":"string","default":"*","maxLength":100}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.015","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.015/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.015","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.015","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_LyWoU6Uh4kqQg5Kl6mYnb","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.015","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and returns the robots.txt crawl policy for a given website URL, indicating what crawlers and user-agents are allowed or disallowed.","exampleAgentPrompt":"Can you check the robots.txt policy for https://example.com and tell me whether the wildcard user-agent (*) is allowed to crawl it?","exampleUseCases":[{"title":"Pre-scrape compliance check","prompt":"Before I start scraping nytimes.com, can you pull up their robots.txt policy and tell me which sections are blocked for all user agents?"},{"title":"SEO audit robots policy review","prompt":"I'm doing an SEO audit — can you fetch the robots policy for shopify.com and summarize what's disallowed for Googlebot?"},{"title":"Bot access verification for automation","prompt":"I'm building a price-monitoring bot — check if my custom user agent is allowed to crawl amazon.com by looking up their robots.txt rules."}],"resultDescription":"Returns a structured snapshot of the target site's robots.txt policy, including allow/disallow rules, crawl-delay directives, and applicable user-agent scopes, reflecting what crawlers are permitted or blocked from accessing.","failureModes":["Target URL is unreachable or returns a non-200 status — policy snapshot unavailable","Website has no robots.txt file — empty or default permissive policy returned","Malformed URL input — validation error returned","Rate limiting or access restrictions on the target domain — partial or no data returned","Very large robots.txt files may be truncated"],"whenToPreferThis":"Use this endpoint when you need a quick, paid-per-call lookup of a website's robots crawl policy without building your own fetch-and-parse pipeline. Ideal for compliance checks before scraping, SEO tooling, or automated agent workflows that need to verify crawl permissions on demand. Prefer this over building custom robots.txt parsers when low-latency, on-demand lookups with structured output are needed.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T18:30:27.808Z","isFirstParty":false}