{"uid":"cap_Bh1vWu9AEzLDq6oPEmE3V","slug":"delx-commerce-robots-txt-inspector-ddc57728","name":"Delx Commerce — robots.txt Inspector","description":"Pay-per-result APIs for agents. No signup. Exact price. Verifiable delivery. USDC on Base + Solana via x402.","url":"https://commerce.delx.ai/api/v1/x402/robots-inspect?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string"},"timeout":{"type":"integer","maximum":15,"minimum":1}}},"responseSchema":{"type":"json","example":{"result":{"allow":[],"disallow":[],"sitemaps":[],"robots_url":"https://example.com/robots.txt"},"tool_name":"util_robots_inspect"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Dh0SBueCmh5HPXFMSkBFm","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt file, returning allowed paths, disallowed paths, and sitemap URLs","exampleAgentPrompt":"Can you inspect the robots.txt for https://example.com and tell me which paths are disallowed and what sitemaps are listed? Use a timeout of 10 seconds.","exampleUseCases":[{"title":"Pre-scrape crawl permission check","prompt":"Before I start scraping https://shop.example.com, can you check its robots.txt and tell me which paths I'm not allowed to crawl?"},{"title":"Sitemap discovery for SEO audit","prompt":"I need to find all the sitemaps listed in the robots.txt for https://mybusiness.com — can you fetch and parse that for me?"},{"title":"Bot policy review for a competitor site","prompt":"Can you pull the robots.txt from https://competitor.io and show me all the disallowed paths and any sitemaps it references? Give it up to 10 seconds to respond."}],"resultDescription":"A JSON object containing arrays of allowed paths, disallowed paths, sitemap URLs, and the resolved robots.txt URL for the given domain","failureModes":["Timeout if the site does not respond within the specified timeout window (max 15 seconds)","robots.txt not found (404) resulting in empty or null result","Network errors or DNS resolution failures for unreachable domains","Malformed robots.txt that cannot be parsed cleanly","Invalid URL input causing request rejection"],"whenToPreferThis":"Use this endpoint when you need a fast, pay-per-call, no-signup way to fetch and parse a robots.txt file before crawling or scraping a site. Prefer it over writing custom HTTP fetch logic when you need structured output (allowed, disallowed, sitemaps) rather than raw text, and when operating in an agent context where micropayment-based access via x402/USDC is acceptable.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-01T00:53:45.384Z","isFirstParty":false,"canonicalSlug":"delx-commerce-robots-txt-inspector-ddc57728"}