{"uid":"cap_7gHS9jGOT4VKSy5_cIK9W","slug":"netintel-robots-txt-analyzer-e236b856","name":"NetIntel Robots.txt Analyzer","description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","url":"https://netintel.dev/robots-txt/analyze?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"path":{"type":"string","description":"URL path to check permission for (e.g. /api/data)"},"domain":{"type":"string","description":"Domain name to fetch robots.txt from (e.g. example.com)"},"user_agent":{"type":"string","description":"Bot name to check rules for (default: *)"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","properties":{"found":{"type":"boolean"},"grade":{"type":"string"},"rules":{"type":"array","items":{"type":"object"}},"score":{"type":"number"},"domain":{"type":"string"},"findings":{"type":"array","items":{"type":"object"}},"sitemaps":{"type":"array","items":{"type":"string"}},"path_check":{"type":"object"},"robots_url":{"type":"string"},"status_code":{"type":"number"},"raw_content_preview":{"type":"string"}}}}}}},"responseSchema":{"type":"json","example":{"found":true,"grade":"B","rules":[{"allow":["/public/"],"disallow":["/private/","/admin/"],"user_agent":"*","crawl_delay":null}],"score":85,"domain":"example.com","findings":[],"sitemaps":["https://example.com/sitemap.xml"],"path_check":{"path":"/private/page","allowed":false,"rule_type":"disallow","user_agent":"*","matched_rule":"Disallow: /private/"},"robots_url":"https://example.com/robots.txt","status_code":200,"raw_content_preview":"User-agent: *\nDisallow: /private/\n..."}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_kuCiSpP6j9MTZMu0gWmwd","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and analyzes a domain's robots.txt file, returning parsed rules, crawl permissions, sitemaps, a compliance score, and an optional path-permission check for a specific bot and URL path.","exampleAgentPrompt":"Can you fetch and analyze the robots.txt for example.com and tell me whether a bot called 'MySpider' is allowed to access /private/page — include the grade, rules, and any sitemaps listed?","exampleUseCases":null,"resultDescription":"A JSON object containing: whether robots.txt was found, the domain, the full parsed rules array (per user-agent: allow/disallow/crawl-delay), a numeric compliance score (0-100), a letter grade (A-F), a list of sitemap URLs, a path_check object indicating if the requested path is allowed or blocked and which rule matched, the robots.txt URL, HTTP status code, and a raw content preview.","failureModes":["Domain has no robots.txt (found: false, status_code may be 404)","Domain is unreachable or DNS resolution fails (network error)","Malformed robots.txt that cannot be parsed (partial rules returned)","Invalid domain format in query parameter (400 bad request)","Payment failure via x402 micropayment protocol (402 payment required)"],"whenToPreferThis":"Use this endpoint when you need to programmatically check crawl permissions before scraping a site, audit a domain's robots.txt for SEO compliance, verify whether a specific bot or user-agent is blocked from a given path, or extract sitemap locations. Prefer this over manual fetching when you also need a structured grade/score and per-path permission checks in one call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-12T18:40:06.564Z","isFirstParty":false,"canonicalSlug":"netintel-robots-txt-analyzer-e236b856"}