{"uid":"cap_FewkK-zYLqZIPAWK4EEbv","slug":"citegate-robots-policy-ebfee7ff","name":"CiteGate Robots Policy","description":"CiteGate robots policy — training-vs-answer AI crawler split table for any domain plus a ready-to-paste robots.txt snippet, $0.01 per accepted call","url":"https://mcp-x402.onrender.com/gate/squeezeos/citegate/robots-policy?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input","output"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"type":"string"},"queryParams":{"type":"object","properties":{"domain":{"type":"string"}}}}},"output":{"type":"object","required":["type","example"],"properties":{"type":{"type":"string","const":"json"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_mG6DiSqMQpQuCpGJk2xhP","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Returns a training-vs-answer AI crawler split policy table and a ready-to-paste robots.txt snippet for any given domain","exampleAgentPrompt":"Can you pull up the CiteGate robots policy for openai.com and give me the ready-to-paste robots.txt snippet that separates AI training crawlers from AI answer crawlers?","exampleUseCases":[{"title":"Block AI training crawlers only","prompt":"I want to allow AI assistants like ChatGPT to answer questions using my site, but block bots that scrape for training data — can you get the CiteGate robots policy for myblog.com and show me exactly what to paste into my robots.txt?"},{"title":"Audit competitor AI crawler policy","prompt":"Pull the CiteGate robots policy for nytimes.com so I can see how they split AI training versus AI answer crawlers in their robots.txt."},{"title":"Set up AI bot rules for a new site","prompt":"I just launched techstartup.io and need a robots.txt snippet that properly handles all the major AI crawlers — can you get the CiteGate policy for techstartup.io so I know what to put in there?"}],"resultDescription":"A JSON object containing a policy table that categorizes known AI crawlers into training-bots versus answer-bots for the queried domain, along with a pre-formatted robots.txt text snippet ready to copy-paste into a website's robots.txt file.","failureModes":["Missing or invalid domain query parameter — returns error or empty policy","Unknown or unrecognized domain may return a generic default policy","Payment not accepted or x402 flow fails — call is rejected before processing","Render cold-start latency may cause timeout on first call after idle period"],"whenToPreferThis":"Use this endpoint when you need to quickly generate or audit robots.txt rules specifically targeting the distinction between AI training crawlers and AI answer/retrieval crawlers for a given domain. It is uniquely suited for webmasters, SEO tools, and content policy agents that need to differentiate between bots like GPTBot (training) and ChatGPT-User (answering). Prefer it over generic robots.txt generators when the use case is specifically about AI crawler governance.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-25T12:31:30.194Z","isFirstParty":false,"canonicalSlug":"citegate-robots-policy-ebfee7ff"}