{"uid":"cap_c5ttHHa_Mj0o_6_L1QY6M","slug":"delx-commerce-robots-txt-path-budget-parser-2b8c3712","name":"Delx Commerce — robots.txt Path Budget Parser","description":"Pay-per-result APIs for agents. No signup. Exact price. Verifiable delivery. USDC on Base + Solana via x402.","url":"https://commerce.delx.ai/api/v1/x402/robots-path-budget?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"robots":{"type":"string","description":"robots.txt body text"}}},"responseSchema":{"type":"json","example":{"schema":"delx/util-robots-path-budget/v1","allow_n":1,"sitemaps":["https://ex.com/sitemap.xml"],"disallow_n":1}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_sslHteQqO7UjKlwQtaWJx","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Parses a robots.txt body and returns a structured summary of allowed/disallowed path counts and sitemap URLs","exampleAgentPrompt":"Parse this robots.txt content and tell me how many paths are allowed vs disallowed, and list any sitemaps it references: 'User-agent: * Disallow: /private/ Allow: / Sitemap: https://example.com/sitemap.xml'","exampleUseCases":[{"title":"SEO audit of robots.txt rules","prompt":"I just fetched the robots.txt from our site — can you parse it and tell me how many paths we're blocking vs allowing and what sitemaps are listed? Here's the content: 'User-agent: * Disallow: /admin/ Disallow: /checkout/ Allow: / Sitemap: https://mysite.com/sitemap.xml'"},{"title":"Crawler budget check before scraping","prompt":"Before I start scraping this site, parse this robots.txt and give me a count of allowed and disallowed paths so I know how restricted it is: 'User-agent: * Disallow: /search Disallow: /login Allow: /blog Sitemap: https://example.com/sitemap_index.xml'"},{"title":"Automated site discovery pipeline","prompt":"I'm building a web index — take this robots.txt body and extract all the sitemap URLs and the total number of disallow rules so I can decide whether to proceed: 'User-agent: Googlebot Disallow: /private/ Allow: / Sitemap: https://news.example.com/news-sitemap.xml Sitemap: https://news.example.com/sitemap.xml'"}],"resultDescription":"Returns a JSON object with a schema identifier ('delx/util-robots-path-budget/v1'), the count of allowed paths (allow_n), the count of disallowed paths (disallow_n), and an array of sitemap URLs found in the robots.txt body.","failureModes":["Empty or missing 'robots' field returns an error or zero counts","Malformed robots.txt with non-standard directives may produce inaccurate counts","Very large robots.txt bodies may cause parsing timeouts","Non-UTF-8 encoded input may cause parsing failures","Payment failure via x402 protocol prevents the call from being processed"],"whenToPreferThis":"Choose this endpoint when you need a fast, structured parse of a robots.txt body into counts and sitemaps without building your own parser. Ideal for automated web crawling pipelines, SEO audits, or any agent that needs to programmatically understand a site's crawl budget. At $0.001 USDC per call with no signup, it's suited for lightweight, pay-as-you-go usage inside agentic workflows.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-02T02:51:14.950Z","isFirstParty":false,"canonicalSlug":"delx-commerce-robots-txt-path-budget-parser-2b8c3712"}