{"uid":"cap_zWIewSjKPH-U4b_ua58hk","slug":"api-delx-ai-4a3ab788","name":"Robots.txt & Sitemap Inspector","description":"Read robots.txt rules and sitemap declarations before crawling or indexing a domain.","url":"https://api.delx.ai/api/v1/x402/robots-inspect","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string"},"timeout":{"type":"integer","maximum":15,"minimum":1}}},"responseSchema":{"type":"json","example":{"result":{"allow":[],"disallow":[],"sitemaps":[],"robots_url":"https://example.com/robots.txt"},"tool_name":"util_robots_inspect"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_MT1dJaxB1UPDmlw7AjKcD","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a domain's robots.txt rules and sitemap declarations to reveal crawlability, disallow paths, and indexing directives before scraping or indexing.","exampleAgentPrompt":"Before I start scraping example.com, check its robots.txt to see which paths are disallowed and list any sitemap URLs it declares.","exampleUseCases":null,"resultDescription":"Returns the parsed contents of the domain's robots.txt including disallow and allow rules per user-agent, crawl-delay directives, and a list of sitemap URLs declared within the file — giving the agent a clear picture of what is and isn't permitted to crawl.","failureModes":["Domain has no robots.txt — returns empty or 404 signal","Domain is unreachable or times out — network error response","Malformed robots.txt — partial parse with best-effort output","Invalid URL input — validation error returned","Rate limit exceeded — HTTP 429 response"],"whenToPreferThis":"Use this endpoint when you need to respect or audit crawl policies before scraping, indexing, or building a sitemap graph for a domain. It is the right choice when an agent must check legal crawl boundaries, discover sitemap entry points, or verify whether a specific path is disallowed — especially as a pre-flight step before running a broader crawl.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T00:57:33.509Z","isFirstParty":false}