{"uid":"cap_yh8R6nKcXiB8ag-kmFx07","slug":"relay402-robots-txt-check-5e8b4751","name":"relay402 Robots.txt Check","description":"Checks whether a given user agent may fetch a given path on a host, per the site's robots.txt (specific-agent groups, wildcard rules, Allow/Disallow longest-match). Also returns crawl-delay, sitemap URLs and whether the site publishes llms.txt or ai.txt. For scraping and research agents that want to crawl politely and prove compliance.","url":"https://relay402.georgespring.workers.dev/api/robots-check","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["host"],"properties":{"host":{"type":"string","pattern":"^[a-z0-9]([a-z0-9.-]{2,251})[a-z0-9]$"},"path":{"type":"string","maxLength":500},"user_agent":{"type":"string","maxLength":100}},"additionalProperties":false}},"additionalProperties":false}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_dxILyXjM2dt0cLxW4KyQQ","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt to determine crawl permissions for a given host, path, and user agent before acting on that site.","exampleAgentPrompt":"Before scraping docs.example.com, check whether the path /pricing is allowed for a generic crawler bot by looking up their robots.txt.","exampleUseCases":[{"title":"Pre-scrape permission check","prompt":"Before I start pulling content from techcrunch.com/articles, can you check if their robots.txt allows crawling that path for a generic user agent?"},{"title":"Verify Googlebot access to a page","prompt":"Check whether robots.txt on shopify.com permits Googlebot to access the path /products — I need to know if that page is indexable."},{"title":"Agent self-check before web access","prompt":"My agent is about to fetch data from api-docs.stripe.com/reference — first verify that the site's robots.txt allows automated access to that path."}],"resultDescription":"Returns parsed robots.txt rules for the specified host, indicating whether the given path is allowed or disallowed for the specified user agent, along with the relevant directives from the robots.txt file.","failureModes":["Host does not have a robots.txt file — returns a permissive default or 404 signal","Invalid hostname format rejected by schema pattern validation","Host is unreachable or times out — network error response","Path or user_agent exceeds maximum length constraints"],"whenToPreferThis":"Use this endpoint when an AI agent or automated crawler needs to verify crawl permissions before accessing a website, especially in agentic workflows that must respect site policies. Prefer this over manual robots.txt parsing when you need a pay-per-call, keyless solution that works in autonomous pipelines without API key management.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T00:34:18.175Z","isFirstParty":false}