{"uid":"cap_nInhxvlIZ7DecUTN1uUo7","slug":"toll402-robots-txt-check-c87dfd0a","name":"Toll402 Robots.txt Check","description":"The OpenRouter for AI-agent tools: one gateway, one wallet, every capability. Built-in tools, forged tools, a global verified business directory and 2,000+ aggregated x402 services, all paid per call in USDC via x402 (HTTP 402). No signup, no API key.","url":"https://toll402.dev/v1/t/robots_txt_check?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Site URL or domain, e.g. https://example.com"},"path":{"type":"string","default":"/","description":"URL path to test, e.g. /some/page"},"userAgent":{"type":"string","default":"*","description":"User-agent token to check rules for"}}},"type":{"type":"string","const":"http"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json","form-data","text"],"type":"string"}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"ok":true}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_c8_M6rIx33CL_ZNCWIZlf","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a site's robots.txt to determine whether a given URL path is allowed or disallowed for a specified user-agent","exampleAgentPrompt":"Check whether my crawler with user-agent 'MyBot' is allowed to access the path /blog/private on https://example.com according to their robots.txt.","exampleUseCases":[{"title":"Pre-crawl permission check","prompt":"Before I start scraping https://techcrunch.com, check if the path /search is allowed for a user-agent called 'ResearchBot' according to their robots.txt."},{"title":"SEO audit robots blocking","prompt":"I'm doing an SEO audit — can you check if https://mystore.com has any robots.txt rule that blocks Googlebot from accessing /products?"},{"title":"Automated scraper compliance","prompt":"I need to make sure my bot isn't violating robots.txt rules — check if user-agent '*' is allowed to crawl the path /api/data on https://opendata.gov."}],"resultDescription":"Returns a JSON object indicating whether the request is permitted (ok: true/false) based on the parsed robots.txt rules for the given site, path, and user-agent combination.","failureModes":["Site has no robots.txt (may default to allow-all or return an error)","Unreachable domain causes timeout or connection error","Invalid URL format returns a validation error","Robots.txt has syntax issues that prevent reliable parsing","Rate limiting or blocking by the target server"],"whenToPreferThis":"Use this endpoint when an AI agent or automation workflow needs to programmatically check crawl permissions before accessing a URL, ensuring compliance with a site's robots.txt without implementing a parser locally. Ideal for pre-scrape validation, SEO tooling, and ethical bot workflows. No API key required — pay per call via x402/USDC.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-02T06:47:23.751Z","isFirstParty":false,"canonicalSlug":"toll402-robots-txt-check-c87dfd0a"}