{"uid":"cap_KwCPsvdTVk35XzurPq943","slug":"lordevo-robots-txt-can-i-fetch-33f1b081","name":"Lordevo robots.txt Can-I-Fetch","description":"SEO audits, website security checks, SEO content plans, articles and site fix plans as API endpoints for AI agents. Pay per request in USDC on Base with…","url":"https://shop.lordevo.com/api/quick/can-fetch?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Public http(s) URL or bare domain, e.g. https://example.com"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","required":["tool","url","verdict","allowed","agents","robots_txt"],"properties":{"url":{"type":"string"},"noai":{"type":"boolean"},"page":{"type":["object","null"]},"path":{"type":"string"},"tool":{"type":"string"},"agents":{"type":"object","additionalProperties":{"type":"boolean"}},"origin":{"type":"string"},"allowed":{"type":"boolean"},"summary":{"type":"string"},"verdict":{"enum":["allowed","disallowed","allowed_but_no_ai_use","allowed_with_crawl_delay"],"type":"string"},"sitemaps":{"type":"array","items":{"type":"string"}},"robots_txt":{"enum":["present","missing","unreachable"],"type":"string"},"content_signal":{"type":["object","null"]},"robots_http_status":{"type":["integer","null"]},"crawl_delay_seconds":{"type":["number","null"]}}}}}}},"responseSchema":{"type":"json","example":{"url":"https://example.com/blog/post","noai":true,"page":{"final_url":"https://example.com/blog/post","http_status":200,"content_type":"text/html; charset=utf-8","x_robots_tag":[]},"path":"/blog/post","tool":"can-fetch","agents":{"*":true,"CCBot":false,"GPTBot":false,"ClaudeBot":true,"Claude-User":true,"ChatGPT-User":true,"PerplexityBot":true},"origin":"https://example.com","allowed":true,"summary":"Allowed: example.com/blog/post for generic crawlers/agents. Blocked for: GPTBot, CCBot. The site signals no AI use (noai / Content-Signal).","verdict":"allowed_but_no_ai_use","sitemaps":["https://example.com/sitemap.xml"],"robots_txt":"present","robots_url":"https://example.com/robots.txt","content_signal":{"search":true,"ai-input":false,"ai-train":false},"robots_http_status":200,"crawl_delay_seconds":null}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_II5IjNIEaHyt-2npzyh1x","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL is allowed to be fetched by generic web agents and 16 named AI crawlers according to its robots.txt rules.","exampleAgentPrompt":"Before you scrape https://techcrunch.com/2024/01/ai-trends, check if that URL is allowed for AI crawlers — specifically whether Claude-User and GPTBot are permitted by the site's robots.txt.","exampleUseCases":[{"title":"Pre-scrape robots.txt gate for pipelines","prompt":"Before my scraper fetches https://www.nytimes.com/section/technology, check the robots.txt to see if generic crawlers and GPTBot are allowed to access that URL."},{"title":"AI training data compliance check","prompt":"I want to use content from https://huggingface.co/blog/llm-course for AI training — can you check if the site's robots.txt permits that and whether there's a noai signal blocking it?"},{"title":"Competitor monitoring agent safeguard","prompt":"My agent is about to pull data from https://competitor.com/pricing — first check whether crawling that URL is allowed by their robots.txt, and tell me if there's a crawl delay I should respect."}],"resultDescription":"Returns a structured JSON object with: a verdict enum (allowed, disallowed, allowed_but_no_ai_use, allowed_with_crawl_delay), a boolean allowed flag, per-named-agent boolean permissions (for GPTBot, Claude-User, ChatGPT-User, PerplexityBot, CCBot, ClaudeBot, and others), the robots_txt status (present/missing/unreachable), the HTTP status of the robots.txt response, crawl delay in seconds if specified, a noai boolean, a content_signal object (search/ai-input/ai-train), discovered sitemap URLs, and a human-readable summary string.","failureModes":["URL is unreachable or returns 5xx — treated as disallowed per RFC 9309","Missing or malformed URL query parameter returns validation error","robots.txt itself returns 5xx — interpreted as disallow for all agents","Private or non-HTTP URLs rejected","Rate limiting or timeout on the target domain may delay response"],"whenToPreferThis":"Choose this endpoint when your agent or scraper needs to check robots.txt compliance before fetching a URL, especially when you need per-named-AI-crawler verdicts (ChatGPT-User, Claude-User, GPTBot, PerplexityBot, etc.) rather than just a generic allow/disallow. It is the right choice for pre-fetch gating in agentic pipelines, AI training data sourcing workflows, or any scenario requiring RFC 9309-compliant robots.txt interpretation plus noai/content-signal awareness. Prefer it over raw robots.txt parsing when you need structured, normalized output with named AI agent breakdowns in a single API call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-31T18:40:13.237Z","isFirstParty":false,"canonicalSlug":"lordevo-robots-txt-can-i-fetch-33f1b081"}