{"uid":"cap_n_g4yHo4q_96ZH4dtAoIQ","slug":"robots-directive-parse-6b1ab932","name":"Robots Directive Parse","description":"Machine-callable x402-paid utilities for autonomous agents.","url":"https://creator-desk-x402-isolated.onrender.com/v1/utility/robots-directive-parse?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"value":{},"options":{"type":"object"}}},"responseSchema":{"type":"json","example":{"result":"example","operation":"robots_directive_parse"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_12RXYVqjbv6I8TrPW1dsF","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Parses and interprets robots.txt directives for a given URL or robots.txt content, returning structured crawl permission data.","exampleAgentPrompt":"Can you parse this robots.txt content for me and tell me which paths are disallowed and what the crawl-delay is set to?","exampleUseCases":[{"title":"Pre-crawl permission check","prompt":"Before my scraper visits this site, parse its robots.txt and tell me which paths are off-limits for the Googlebot user-agent."},{"title":"SEO audit of crawl directives","prompt":"I need to audit our website's robots.txt — can you parse it and give me a structured breakdown of all the allow and disallow rules, plus any sitemap references?"},{"title":"Autonomous agent compliance check","prompt":"My agent is about to index pages from a third-party domain — parse their robots.txt directive and confirm whether the /api/ and /private/ paths are permitted for a generic bot."}],"resultDescription":"A JSON object with the parsed robots.txt directive broken into structured components, including user-agent rules, allowed and disallowed path patterns, crawl-delay values, and sitemap URLs where present.","failureModes":["Malformed or empty robots.txt content returns an error or empty result","Unrecognized directive syntax may be skipped silently","Network-fetched robots.txt URLs that return non-200 responses may cause fetch failures","Overly large robots.txt files may time out or be truncated"],"whenToPreferThis":"Use this endpoint when an autonomous agent or crawler needs to programmatically check robots.txt compliance before accessing web resources, or when building SEO tooling that must interpret crawl directives. Prefer this over manual string parsing when you need structured, machine-readable output from raw robots.txt content.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-13T00:41:16.370Z","isFirstParty":false,"canonicalSlug":"robots-directive-parse-6b1ab932"}