{"uid":"cap_psDfvUYO3E3ZhC2YrtqkR","slug":"minifetch-url-robots-txt-preflight-check-d9be526e","name":"Minifetch URL Robots.txt Preflight Check","description":"Cheap robots.txt preflight before you pay for other Minifetch endpoints: returns whether the Minifetch user agent may fetch a target URL (allow/deny) plus its crawl-delay. Blocked or setting a crawl delay? https://minifetch.com/tutorials/unblock-minifetch . Checks the URL as given — does not follow redirects. Supports GET & POST","url":"https://minifetch.com/api/v1/x402/preflight/url-check","method":"GET","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"The URL to check against robots.txt"}}},"responseSchema":{"type":"object","properties":{"url":{"type":"string"},"reason":{"type":"string"},"allowed":{"type":"boolean"},"message":{"type":"string"},"success":{"type":"boolean"},"crawlDelay":{"type":"number","description":"How often the URL can be fetched, in seconds."}}},"example":{"request":{"url":"https://example.com/blog/article"},"response":{"results":[{"data":{"url":"https://example.com/blog/article","allowed":true,"message":"robots.txt not found, defaulting to allowed","crawlDelay":1}}],"success":true}},"exampleRequest":{"url":"https://example.com/blog/article"},"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"settled","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_OMOw9mUQHmigIG3kLNaUz","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL is allowed to be fetched according to the target website's robots.txt file, before making a more expensive metadata extraction call.","exampleAgentPrompt":"Before extracting metadata from https://example.com/blog/my-post, quickly check if that URL is allowed to be fetched according to the site's robots.txt so we don't waste money on a blocked page.","exampleUseCases":null,"resultDescription":"Returns a JSON object with: `allowed` (boolean indicating if the URL can be fetched), `reason` (explanation of why it is or isn't allowed), `message` (human-readable summary), `success` (whether the check itself succeeded), and `crawlDelay` (seconds to wait between fetches as specified by robots.txt).","failureModes":["robots.txt file is unreachable or malformed — success may be false with an explanatory message","Invalid or malformed URL input — likely returns an error with success: false","robots.txt does not exist for the domain — typically defaults to allowed: true","Network timeout reaching the target domain's robots.txt"],"whenToPreferThis":"Use this endpoint as a cheap ($0.001) preflight gate before calling any URL metadata extraction endpoint ($0.01+). Ideal when processing large batches of URLs where some may be blocked by robots.txt, to avoid paying for extraction attempts that will fail or return no data. Prefer this over directly attempting extraction whenever respecting crawl rules or cost efficiency matters.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T12:33:30.145Z","isFirstParty":false}