{"uid":"cap__z4l1V8J-2cQQzJf7zwBB","slug":"northline-webkit-robots-txt-check-3b1e1de9","name":"Northline WebKit – robots.txt Check","description":"Pay-per-call web and domain utilities for AI agents: URL to Markdown, link preview metadata, links, uptime/redirect check, DNS, email-domain check, WHOIS/RDAP, robots.txt and a one-call domain report. x402 (USDC on Base), no API key, no signup.","url":"https://northline-webkit.northline-c86.workers.dev/v1/robots/check?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":3,"description":"Public http(s) URL (scheme optional)"},"userAgent":{"type":"string","default":"*"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"allowed":false,"sitemaps":["https://www.nytimes.com/sitemaps/new/news.xml.gz"],"aiCrawlers":{"GPTBot":false,"ClaudeBot":false},"matchedRule":"Disallow: /"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_TXBEIS5f225LgcPpdih2f","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a URL's robots.txt to determine crawl permissions for any user-agent, including known AI crawlers like GPTBot and ClaudeBot.","exampleAgentPrompt":"Check if https://www.nytimes.com is allowed to be crawled — specifically whether GPTBot is permitted according to the site's robots.txt.","exampleUseCases":[{"title":"AI crawler compliance check","prompt":"Before I scrape content from https://techcrunch.com, can you check if ClaudeBot is allowed to crawl it according to their robots.txt?"},{"title":"Sitemap discovery for a domain","prompt":"Pull up the robots.txt for https://www.bbc.com and tell me what sitemaps they expose and whether crawling is allowed at all."},{"title":"Content pipeline pre-flight check","prompt":"I'm building a news aggregator — can you verify whether the default wildcard user-agent is allowed to crawl https://reuters.com before I add it to my pipeline?"}],"resultDescription":"Returns a JSON object indicating whether the given URL path is allowed for the specified user-agent, a list of sitemap URLs declared in robots.txt, per-crawler allow/deny flags for known AI crawlers (GPTBot, ClaudeBot, etc.), and the specific matched rule from the robots.txt file.","failureModes":["URL is unreachable or returns non-2xx — may return error or assume allow-all","Domain has no robots.txt — typically treated as fully allowed","Malformed robots.txt — partial parsing may occur","Invalid or missing URL parameter — returns 400-level error","Private/internal URLs not publicly accessible — connection timeout or error"],"whenToPreferThis":"Use this endpoint when an agent needs to programmatically verify crawl permissions before fetching or indexing web content, especially when AI-specific crawler compliance (GPTBot, ClaudeBot) matters. Ideal for building ethical web scrapers, content pipelines, or SEO tools that must respect robots.txt. Prefer over manual HTTP fetching because it handles parsing, rule matching, and AI-crawler-specific logic in a single call with no signup required.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-31T18:35:55.211Z","isFirstParty":false,"canonicalSlug":"northline-webkit-robots-txt-check-3b1e1de9"}