{"uid":"cap__gTCiC4LTT9jlv8_42FBG","slug":"forest-factory-api-robots-context-cdfbf798","name":"Forest Factory API – robots.context","description":"Machine-readable contract for active Forest paid capabilities. Unsigned paid calls return HTTP 402 with a PAYMENT-REQUIRED x402 challenge.","url":"https://http--forest-gas-station-mainnet--lcjl27p8lmjs.code.run/robots/context?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","pattern":"^https?://","maxLength":2048,"minLength":1,"description":"Public HTTP(S) URL to inspect, resolve, or compare."},"user_agent":{"type":"string","maxLength":128,"minLength":1,"description":"Public user agent value used to scope this lookup."}}},"responseSchema":{"type":"json","example":{"ok":true,"data":{"rules":[],"status":1,"decision":"allowed","sitemaps":[],"truncated":false,"robots_url":"example","target_url":"example","user_agent":"example","crawl_delay":0,"matched_rule":{"pattern":"example","directive":"allow","specificity":0},"robots_state":"available","matching_agents":[],"rendered_javascript":false},"meta":{"cache":"miss","provider":"safe-http","checked_at":"2026-01-01T00:00:00.000Z","duration_ms":0},"version":"1","capability":"robots.context"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_klMbMfHVP_e9n_agjPl8u","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and interprets the robots.txt rules for a given URL and user agent, returning crawl permission decisions, matched rules, sitemaps, and crawl delay.","exampleAgentPrompt":"Check whether 'https://example.com/blog/post-1' allows crawling by the user agent 'MyBot/1.0' — I need to know the crawl decision, any matched rule, and the crawl delay before my scraper touches it.","exampleUseCases":[{"title":"Pre-crawl permission check for scraper","prompt":"Before my scraper fetches 'https://shop.example.com/products', check whether the user agent 'ScraperBot/2.0' is allowed by that site's robots.txt and tell me the crawl delay I should respect."},{"title":"SEO audit of bot access rules","prompt":"I'm auditing indexability — can you check if 'Googlebot' is allowed to crawl 'https://mysite.com/pricing' according to its robots.txt rules?"},{"title":"Sitemap discovery via robots.txt","prompt":"Fetch the robots.txt context for 'https://news.example.com/' using user agent 'NewsIndexer/1.0' and list any sitemaps declared in the robots file along with the crawl decision."}],"resultDescription":"Returns a JSON object with: crawl decision (allowed/disallowed), matched robots.txt rule with pattern/directive/specificity, full list of rules, sitemaps declared in robots.txt, crawl delay, robots state (available/unavailable), the resolved robots.txt URL, and metadata including cache status, provider, check timestamp, and duration in milliseconds.","failureModes":["URL is unreachable or returns a non-200 response — robots state may be 'unavailable'","robots.txt does not exist on the target domain — rules list will be empty and state may be 'unavailable'","Malformed URL input returns validation error","Payment not signed — HTTP 402 with x402 PAYMENT-REQUIRED challenge","Rate limits or upstream timeouts may increase duration_ms or return errors"],"whenToPreferThis":"Choose this endpoint when you need a structured, machine-readable interpretation of robots.txt access rules for a specific URL and user agent combination — especially when you need the matched rule, crawl delay, and sitemaps in a single call. Prefer this over manually fetching and parsing robots.txt when you need reliable caching, scoped user-agent matching, and consistent decision logic without writing your own parser.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-20T12:33:42.111Z","isFirstParty":false,"canonicalSlug":"forest-factory-api-robots-context-cdfbf798"}