{"uid":"cap_mzZgwMK3Oa98Rq0y3va8-","slug":"api-strale-io-315c3817","name":"robots.txt Parser","description":"Fetch and parse a website's robots.txt. Returns rules per user-agent, sitemaps, crawl delays, and whether all crawlers are blocked.","url":"https://api.strale.io/x402/robots-txt-parse?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string"}}},"responseSchema":{"rules":{"type":"array"},"exists":{"type":"boolean"},"sitemaps":{"type":"array"},"user_agents":{"type":"array"},"blocks_all_crawlers":{"type":"boolean"}},"example":{"request":{"url":"https://example.com"},"response":{"url":"https://example.com/robots.txt","_meta":{"payment":{"method":"x402","price_usd":0.0216,"settlement_id":"0xf9930276be754b28f2c8e0cad34001707107dff587d3fcf431470d04b429fcaf"},"capability":"robots-txt-parse","latency_ms":15,"provenance":{"source":"http-fetch","fetched_at":"2026-06-13T04:23:02.354Z"}},"exists":false,"message":"No robots.txt found."}},"exampleRequest":{"url":"https://example.com"},"tags":["x402"],"displayCostAmount":"0.0216","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"settled","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.0216/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.0216","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.0216","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_3-NqtKPz-XY2kZ-MkY-Mx","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.0216","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt file, returning structured rules per user-agent, sitemaps, crawl delays, and global block status","exampleAgentPrompt":"Can you fetch and parse the robots.txt for https://example.com and tell me which paths are blocked, what crawl delays are set, and whether all bots are disallowed?","exampleUseCases":[{"title":"Validate competitor website crawling permissions","prompt":"I need to check if we're allowed to crawl our competitor's website. Can you pull their robots.txt and tell me what paths they're blocking and what their crawl delay requirements are?"},{"title":"Audit third-party vendor compliance before integration","prompt":"Before we integrate with this new data vendor, can you check their robots.txt file and let me know if they block all crawlers or if there are specific rules we need to follow to respect their site?"},{"title":"Discover sitemaps for SEO monitoring","prompt":"I'm setting up SEO monitoring for a client. Can you parse their robots.txt, pull out all the sitemaps they've listed, and tell me if Googlebot is allowed to crawl their site?"}],"resultDescription":"Returns a structured breakdown of the robots.txt file including per-user-agent allow/disallow rules, crawl delays, sitemap URLs, and a boolean flag indicating whether all crawlers are blocked.","failureModes":["robots.txt not found at the given domain (404 response)","Invalid or unreachable URL returns an error","Domain has no robots.txt file","Malformed robots.txt that cannot be parsed","Network timeout reaching the target domain"],"whenToPreferThis":"Use this endpoint when you need to programmatically check whether a website permits crawling, understand crawl rules for specific bots, discover sitemaps, or validate that your crawler respects the site's robots.txt directives before scraping.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-10T18:45:12.798Z","isFirstParty":false,"canonicalSlug":"api-strale-io-315c3817"}