{"uid":"cap_Qt6qwrEc1PzduTCywez_1","slug":"cited-brief-robots-txt-check-a3ef7b97","name":"Cited Brief Robots.txt Check","description":"Pay-per-call web research tools for AI agents, paid in USDC over x402.","url":"https://cited-brief.cited-brief.workers.dev/robots-check?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","description":"Public http(s) URL you want to fetch"},"user_agent":{"type":"string","description":"Your crawler's name, for example MyBot (default: any crawler)"}}},"responseSchema":{"type":"json","example":{"url":"https://example.com/private/report.html","allowed":false,"sitemaps":["https://example.com/sitemap.xml"],"robots_url":"https://example.com/robots.txt","user_agent":"mybot","crawl_delay":null,"matched_rule":"Disallow: /private/","matched_group":"*","robots_status":200}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm__xiGBWFz3k1XH-6pLuzlW","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL is allowed or disallowed by the target site's robots.txt for a specified crawler user agent","exampleAgentPrompt":"Before scraping https://example.com/private/report.html with my bot called ResearchBot, check whether robots.txt actually allows me to fetch that page.","exampleUseCases":[{"title":"Pre-crawl compliance check for agent","prompt":"I'm about to scrape https://news-site.com/articles/2024-elections to gather research — can you check if robots.txt allows my crawler called CitedBot to access that URL?"},{"title":"Sitemap discovery during web research","prompt":"Before my agent starts crawling https://shop.example.com/products, verify the robots.txt permissions for my bot called ShopScraper and also tell me what sitemaps are listed there."},{"title":"Debugging a blocked crawler","prompt":"My crawler MyBot is getting blocked somewhere on https://techblog.io/internal/drafts — can you check the robots.txt to see which rule is actually disallowing it?"}],"resultDescription":"Returns a JSON object indicating whether the URL is allowed for the given user agent, the matched robots.txt rule and group, any crawl delay, the robots.txt URL, a list of sitemaps found, and the HTTP status code of the robots.txt response.","failureModes":["URL is not reachable or returns a non-200 status — robots.txt fetch may fail","robots.txt does not exist on the target domain — allowed defaults may apply","malformed URL input causes a validation error","target site blocks the fetcher's IP, preventing robots.txt retrieval"],"whenToPreferThis":"Use this endpoint when an AI agent needs to verify crawl permissions before fetching a URL, especially in automated research or scraping workflows where robots.txt compliance is required. Prefer this over parsing robots.txt manually since it handles rule matching, user-agent grouping, and crawl-delay extraction automatically.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-18T06:39:28.610Z","isFirstParty":false,"canonicalSlug":"cited-brief-robots-txt-check-a3ef7b97"}