{"uid":"cap_a725_Az3cGcZDxmfWFbug","slug":"agent402-tools-robots-txt-crawl-permission-checker-6cd23395","name":"agent402.tools Robots.txt Crawl Permission Checker","description":"Fetch a site's robots.txt and answer: may this user-agent crawl this path? Returns the matched rule and all declared sitemaps.","url":"https://agent402.tools/api/robots-check","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","description":"URL whose path to check"},"userAgent":{"type":"string","description":"User-agent token (default *)"}}},"responseSchema":{"type":"json","example":{"allowed":true,"sitemaps":["https://example.com/sitemap.xml"],"matchedRule":null}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_XIPAkmhTiICG-N7Wxcsft","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches a site's robots.txt and determines whether a given user-agent is permitted to crawl a specific URL path, returning the matched rule and declared sitemaps.","exampleAgentPrompt":"Check whether the user-agent 'MyBot' is allowed to crawl the path '/products/deals' on example.com — fetch the robots.txt and tell me the matched rule and any sitemaps declared.","exampleUseCases":null,"resultDescription":"Returns a crawl permission decision (allowed/disallowed), the specific matched rule from robots.txt (e.g. Disallow: /products/), and a list of all sitemap URLs declared in the robots.txt file.","failureModes":["robots.txt not found (404) — site may have no robots.txt, permission assumed allowed by convention","network timeout fetching robots.txt — transient failure, retry recommended","malformed URL input — returns validation error","robots.txt is inaccessible due to server error (5xx) — unclear permission state","user-agent token not found in robots.txt — falls back to wildcard (*) rules"],"whenToPreferThis":"Use this endpoint when an agent needs to programmatically verify crawl compliance before scraping or indexing a URL, especially when operating multiple bots with distinct user-agent identifiers. Ideal for respecting crawl ethics at scale without manually parsing robots.txt files. Prefer this over generic HTTP fetch when you also need sitemap discovery in the same call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T07:06:52.133Z","isFirstParty":false}