{"uid":"cap_xOyASX_SPCEERxJBb0SSH","slug":"avedaris-robots-txt-inspector-10aac700","name":"AveDaris Robots.txt Inspector","description":"AveDaris is the economic coordination layer for autonomous software: discover, route, evaluate, procure, verify, and learn across agent services.","url":"https://api.avedaris.com/v1/paid/robots-inspect","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","maxLength":8192,"minLength":8}}},"responseSchema":{"type":"json","example":{"result":{"groups":[],"status":200,"sitemaps":[],"robotsUrl":"https://example.com/robots.txt"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.003","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.003/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_TZ-qUCWfpH-0g5-hbeTAU","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.003","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt file, returning crawl rules, sitemap references, and HTTP status","exampleAgentPrompt":"Can you check the robots.txt for https://example.com and tell me which paths are blocked, what crawl rules are set, and if there are any sitemaps listed?","exampleUseCases":[{"title":"Pre-crawl compliance check","prompt":"Before I start scraping openai.com, can you pull up its robots.txt and tell me which paths are disallowed and whether there are any sitemaps I should know about?"},{"title":"Sitemap discovery for SEO audit","prompt":"I need to find all sitemap URLs listed in shopify.com's robots.txt — can you inspect it and give me the list?"},{"title":"Bot permission verification","prompt":"Check whether nytimes.com's robots.txt allows bots to access their article pages — fetch the robots rules and summarize what's permitted or blocked."}],"resultDescription":"A JSON object containing the parsed robots.txt result: the HTTP status code of the robots.txt fetch, the resolved URL of the robots.txt file, a list of rule groups (each with user-agent directives and allow/disallow paths), and a list of sitemap URLs referenced in the file.","failureModes":["URL is unreachable or times out — robots.txt fetch fails with non-200 status","Domain does not have a robots.txt — returns 404 status with empty groups and sitemaps","Malformed URL input — validation error before request is made","robots.txt is malformed or non-standard — partial parse with incomplete groups","Network errors or DNS resolution failures — request cannot complete"],"whenToPreferThis":"Use this endpoint when you need programmatic access to a site's robots.txt rules before crawling, scraping, or indexing — particularly useful for compliance checking, sitemap discovery, or determining which paths an agent may access. Prefer this over manual fetching when you need structured, parsed output rather than raw text.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T12:48:07.652Z","isFirstParty":false}