{"uid":"cap_ePC934V4V5WP7alrEYNJh","slug":"ai-crawl-permission-checker-2bb35750","name":"AI Crawl Permission Checker","description":"Checks whether a domain's published policy permits AI systems to read, train on, or index its content, parsing the relevant robots.txt directives into structured JSON with a human-readable interpretation. Built for scraping agents, dataset builders, and RAG pipelines that need to verify permission before ingesting a site's pages.","url":"https://x402.forgemesh.io/ai-crawl-permission-check","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"domain":{"type":"string","description":"Domain to check, e.g. example.com"}}},"responseSchema":{"type":"json","example":{"domain":"theverge.com","signals":{"search":"yes","ai-input":"no","ai-train":"no"},"declared":true,"interpretation":"site declares: search=yes, ai-input=no, ai-train=no"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.005","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.005/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_49PkEXnWhY7tQKiMpuge0","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.005","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a domain's robots.txt policy permits AI systems to read, train on, or index its content, returning structured JSON with a human-readable interpretation.","exampleAgentPrompt":"Before we scrape openai.com for our training dataset, check whether its robots.txt policy actually permits AI crawlers to access and train on its content.","exampleUseCases":null,"resultDescription":"A structured JSON object containing the parsed robots.txt AI-related directives for the queried domain, a permission status indicating whether AI systems are allowed to read, train on, or index the site, and a human-readable summary of the policy interpretation.","failureModes":["Domain not found or unreachable — robots.txt cannot be fetched","Domain provided without valid format — returns validation error","robots.txt absent — may return a default permissive interpretation or indicate no policy found","Ambiguous directives — some robots.txt rules may not clearly apply to AI agents, leading to uncertain interpretation","Network timeout — request to fetch robots.txt times out"],"whenToPreferThis":"Use this endpoint when building scraping agents, dataset pipelines, or RAG systems that need to programmatically verify whether a target domain's robots.txt policy permits AI access before ingesting content. Prefer this over manual inspection when processing multiple domains or automating compliance checks for AI training data collection.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-13T18:57:02.055Z","isFirstParty":false}