{"uid":"cap_R0X0KmLDOjSTdU9Zb0eZw","slug":"netintel-production-440c-up-railway-app-e1cf02df","name":"Robots.txt Analyzer","description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","url":"https://netintel-production-440c.up.railway.app/robots-txt/analyze","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"path":{"type":"string","description":"URL path to check permission for (e.g. /api/data)"},"domain":{"type":"string","description":"Domain name to fetch robots.txt from (e.g. example.com)"},"user_agent":{"type":"string","description":"Bot name to check rules for (default: *)"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","properties":{"found":{"type":"boolean"},"grade":{"type":"string"},"rules":{"type":"array","items":{"type":"object"}},"score":{"type":"number"},"domain":{"type":"string"},"findings":{"type":"array","items":{"type":"object"}},"sitemaps":{"type":"array","items":{"type":"string"}},"path_check":{"type":"object"},"robots_url":{"type":"string"},"status_code":{"type":"number"},"raw_content_preview":{"type":"string"}}}}}}},"responseSchema":{"type":"json","example":{"found":true,"grade":"B","rules":[{"allow":["/public/"],"disallow":["/private/","/admin/"],"user_agent":"*","crawl_delay":null}],"score":85,"domain":"example.com","findings":[],"sitemaps":["https://example.com/sitemap.xml"],"path_check":{"path":"/private/page","allowed":false,"rule_type":"disallow","user_agent":"*","matched_rule":"Disallow: /private/"},"robots_url":"https://example.com/robots.txt","status_code":200,"raw_content_preview":"User-agent: *\nDisallow: /private/\n..."}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"settled","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_bnVo9IcHQeiA3rmz-VTQy","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a domain's robots.txt file, returning crawl rules by user-agent, sitemap URLs, crawl delay settings, and path permission checks.","exampleAgentPrompt":"Before I start scraping nytimes.com, can you check their robots.txt file and tell me whether the path /articles is allowed for a generic bot, and list any sitemaps they have?","exampleUseCases":null,"resultDescription":"Returns whether robots.txt was found, parsed crawl rules grouped by user-agent, a list of sitemap URLs, crawl delay settings, a path permission check result for the specified path and user-agent, the raw content preview, the robots.txt URL, HTTP status code, a compliance grade, and a numeric score.","failureModes":["Domain has no robots.txt file — found=false returned with 404 status","Domain is unreachable or DNS resolution fails — connection error","Invalid domain format provided — validation error on input","Path check returns ambiguous result when no matching user-agent rule exists — defaults to allowed","Robots.txt file is malformed or non-standard — partial parse with best-effort results"],"whenToPreferThis":"Use this endpoint when an agent needs to respect crawl policies before scraping a website, locate sitemap URLs for a domain, check whether a specific path is accessible to a particular bot, or audit a domain's robots.txt configuration for compliance grading. Prefer this over manual HTTP fetching when structured parsing, multi-field extraction, and path permission evaluation are needed in a single call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-13T12:59:29.343Z","isFirstParty":false}