{"uid":"cap_veXCyK-EoSW7sCtrl568T","slug":"swift-bot-labs-toolbox-web-robots-txt-checker-b36652ae","name":"Swift Bot Labs Toolbox – Web Robots.txt Checker","description":"62 small, deterministic utility APIs (DNS and e-mail checks, web extraction, EVM decoding, x402 checks, IP math, JSON/CSV, hashing, conversions). No account or API key: pay per call in USDC with x402.","url":"https://x402-toolbox.swiftbotlab.workers.dev/v1/web/robots?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET","HEAD"],"type":"string"},"headers":{"type":"object","additionalProperties":{"type":"string"}},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Page URL (its origin's robots.txt is read)"},"userAgent":{"type":"string"}},"additionalProperties":false},"discoverable":{"type":"boolean"}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"ok":true,"tool":"web-robots","result":{"path":"/search?q=x","allowed":false,"sitemaps":["https://www.google.com/sitemap.xml"],"robotsUrl":"https://www.google.com/robots.txt","userAgent":"GPTBot","matchedRule":"Disallow: /search"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.005","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.005/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_etN1Q2iQRW-cU5SX75XJ3","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.005","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches a site's robots.txt and checks whether a given URL path is allowed or disallowed for a specified user agent","exampleAgentPrompt":"Can you check whether GPTBot is allowed to crawl https://www.example.com/search?q=test — look up the site's robots.txt and tell me if that path is permitted or blocked, and what rule matched.","exampleUseCases":[{"title":"Pre-crawl robots compliance check","prompt":"Before I scrape https://news.ycombinator.com/newest, can you check if robots.txt allows my bot — call it 'MyScraperBot' — to access that path?"},{"title":"SEO audit for blocked Googlebot paths","prompt":"Check whether Googlebot is allowed to access https://shop.example.com/checkout according to that site's robots.txt, and show me the matching rule and any sitemaps listed."},{"title":"AI crawler access verification","prompt":"Is GPTBot allowed to fetch https://www.openai.com/blog? Pull the robots.txt and tell me the rule that applies."}],"resultDescription":"A JSON object with a boolean 'allowed' field indicating whether the path is permitted for the given user agent, the matched robots.txt rule string, the URL of the robots.txt file that was read, the user agent string that was matched, and an array of sitemap URLs declared in the file.","failureModes":["URL unreachable or domain does not exist — ok:false with error message","robots.txt returns non-200 HTTP status — treated as allow-all or error","malformed URL input — validation error","No robots.txt present — typically treated as allow-all","Rate limiting or timeout on the target server — connection error returned"],"whenToPreferThis":"Use this endpoint when you need a fast, deterministic, pay-per-call check of robots.txt permissions for a specific URL and user agent, with no API key or account required. Prefer it over rolling your own robots.txt parser when you need a verified, structured result including matched rule, sitemaps, and allow/disallow decision in a single call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-31T18:40:45.363Z","isFirstParty":false,"canonicalSlug":"swift-bot-labs-toolbox-web-robots-txt-checker-b36652ae"}