{"uid":"cap_QEsKv3x3ZAViZ559kFDtZ","slug":"robots-txt-path-permission-checker-7dff0c7e","name":"Robots.txt Path Permission Checker","description":"Call when an agent needs a path checked against a site's robots.txt (SSRF-safe). Exact $0.003 USDC. Prefer unpaid POST /v1/sandbox/robots-allowed first.","url":"https://agentshelf.syntexa.ch/v1/robots-allowed?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","examples":["https://example.com/docs"],"maxLength":2048,"minLength":8,"description":"Public page or origin. robots.txt is fetched from this origin."},"path":{"type":"string","maxLength":2048,"minLength":1,"description":"Path to test, such as /docs. Omit it to use the path from url."},"user_agent":{"type":"string","examples":["*"],"maxLength":200,"minLength":1,"description":"User-agent token to match in robots.txt. Default *."}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.003","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.003/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_o5dKBUQSpy7ug0v1qJ7Sd","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.003","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL path is allowed for a specified user agent according to the site's robots.txt rules, with SSRF protection.","exampleAgentPrompt":"Before scraping the product pages, check if robots.txt on https://example.com allows my bot (user agent 'MyBot/1.0') to access the path /products/.","exampleUseCases":[{"title":"Pre-crawl compliance check for scraper","prompt":"I'm about to scrape https://shop.example.com/catalog — can you check if robots.txt allows access to /catalog for the user agent 'MyScraperBot/2.0' before I start?"},{"title":"SEO audit: Googlebot access check","prompt":"Check whether Googlebot is allowed to access the path /admin on https://mysite.com according to their robots.txt rules."},{"title":"Verify bot policy before indexing","prompt":"I need to know if the robots.txt on https://news.example.com permits my crawler with user agent 'NewsAggregator/1.0' to fetch the path /archive/2023/."}],"resultDescription":"Returns whether the specified path is allowed or disallowed for the given user agent based on the site's robots.txt, effectively indicating crawl permission status.","failureModes":["robots.txt file not found or unreachable — may return an error or default allow","Invalid or malformed URL input returns validation error","SSRF-blocked internal/private IP targets return an error","Path or user_agent exceeding max length limits rejected","Network timeout reaching the target site's robots.txt"],"whenToPreferThis":"Use this endpoint when an agent needs to check robots.txt compliance before crawling or scraping a URL, especially when SSRF safety is required. Prefer the free sandbox POST /v1/sandbox/robots-allowed for testing; use this paid endpoint for production compliance checks at $0.003 USDC per call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-01T00:50:27.058Z","isFirstParty":false,"canonicalSlug":"robots-txt-path-permission-checker-7dff0c7e"}