{"uid":"cap_VFg8Uf8NPdyJm402Lf3Az","slug":"robots-txt-inspector-b134a106","name":"Robots.txt Inspector","description":"Quality-scored paid utility APIs for AI agents over x402 on Base USDC.","url":"https://agent-utility-network.frosty-sound-4560.workers.dev/v1/paid/robots-inspect","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","maxLength":8192,"minLength":8}}},"responseSchema":{"type":"json","example":{"result":{"groups":[],"status":200,"sitemaps":[],"robotsUrl":"https://example.com/robots.txt"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.003","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.003/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_D62t7LfF4wD_s2dNUKOhD","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.003","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt file, returning crawl rules, allowed/disallowed groups, sitemaps, and HTTP status.","exampleAgentPrompt":"Can you check the robots.txt for https://example.com and tell me what paths are blocked for bots, what crawl groups are defined, and whether there are any sitemaps listed?","exampleUseCases":[{"title":"Pre-scrape crawl permission check","prompt":"Before I start scraping https://techcrunch.com, can you pull its robots.txt and tell me which paths are off-limits for automated bots and if there are any sitemaps I should use?"},{"title":"SEO sitemap discovery","prompt":"I'm doing an SEO audit for https://shopify.com — can you inspect its robots.txt and pull out all the sitemap URLs listed in there?"},{"title":"Competitor crawl policy research","prompt":"Can you check what crawl rules https://openai.com has set in its robots.txt? I want to know which user agents are restricted and what directories they can't access."}],"resultDescription":"Returns a JSON object with the resolved robots.txt URL, the HTTP status of the fetch, a list of crawl rule groups (each with user-agent and allow/disallow directives), and an array of sitemap URLs found in the file.","failureModes":["URL is unreachable or returns a non-200 HTTP status — result will reflect the actual status code","robots.txt does not exist at the domain (404) — groups and sitemaps will be empty","Malformed URL input causes a validation error","The target server blocks the inspector's requests (e.g. firewall or rate limiting)","Very large robots.txt files may be truncated or time out"],"whenToPreferThis":"Use this endpoint when an agent needs to programmatically check a site's robots.txt before crawling, scraping, or indexing — especially when you need parsed, structured output (groups and sitemaps) rather than raw text. Preferred over manual fetch+parse when you want instant structured data without building your own parser.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T18:44:52.493Z","isFirstParty":false}