{"uid":"cap_LLy-9lgGwZrqx---JrJUp","slug":"llms-txt-ai-crawler-audit-api-523d8214","name":"llms.txt & AI-Crawler Audit API","description":"URL-to-clean-markdown extraction API, on-demand broken-link scan API, pre-deploy link audit API, DNS health audit API, llms.txt / AI-crawler audit API, spec-linted llms.txt audit API, email-deliverability (SPF/DKIM/DMARC) audit API, PDF-to-markdown conversion API, technology-stack fingerprinting API, security-headers audit API, and PDF metadata-intel API for coding agents. Paid per call in USDC via x402 on Base.","url":"https://x402-extract-service.onrender.com/llmsaudit","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"domain":{"type":"string","description":"Bare domain to audit (no URL), e.g. \"example.com\"."}}},"responseSchema":{"type":"json","example":{"ok":true,"valid":false,"domain":"example.com","llmsTxt":{"url":"https://example.com/llms.txt","bytes":48210,"present":true},"summary":"FAIL: llms.txt present (48210 bytes), 120 link(s) checked, 1 dead; 1 AI crawler(s) blocked in robots.txt (ClaudeBot).","crawlers":[{"bot":"GPTBot","access":"allowed"},{"bot":"ClaudeBot","access":"blocked"}],"checkedAt":"2026-09-11T00:00:00.000Z","deadLinks":[{"url":"https://example.com/old-page","error":"HTTP 404","status":404}],"durationMs":8120,"linksChecked":120,"pricePaidAtomic":"20000"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.02","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.02/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_j3A9eJ0YHItXuP5t7oSK-","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.02","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Audits a domain's llms.txt file for presence, validity, dead links, and AI crawler access rules in robots.txt","exampleAgentPrompt":"Can you audit example.com to check whether it has a valid llms.txt file, which AI crawlers like GPTBot or ClaudeBot are allowed or blocked, and whether any links in the file are broken?","exampleUseCases":[{"title":"Pre-launch AI accessibility check","prompt":"Before we go live, can you audit acmecorp.com to make sure our llms.txt is present, properly formatted, and that GPTBot and ClaudeBot aren't accidentally blocked in robots.txt?"},{"title":"Competitor AI crawler analysis","prompt":"I want to know if our competitor openai-rival.com blocks any major AI crawlers and whether their llms.txt has dead links — can you run a full audit?"},{"title":"Routine site health monitoring","prompt":"Run an llms.txt and AI crawler audit on myblog.io and tell me if there are any broken links in the file or if any AI bots are being blocked that shouldn't be."}],"resultDescription":"Returns a JSON object with: whether llms.txt is present and its byte size, a list of AI crawlers (e.g. GPTBot, ClaudeBot) with allowed/blocked status, any dead links found in llms.txt with HTTP status codes, a human-readable summary string, total links checked, audit duration in milliseconds, and whether the file passed validation overall.","failureModes":["Domain does not have a llms.txt file — returns present:false with a FAIL summary","Dead links found in llms.txt — listed in deadLinks array with HTTP status codes","AI crawler blocked — reflected in crawlers array with access:blocked","Domain unreachable or DNS failure — likely returns an error or timeout","Invalid domain format provided — may return a validation error","Payment failure via x402 — call does not proceed"],"whenToPreferThis":"Choose this endpoint when you specifically need to audit a domain's llms.txt file for AI crawler compatibility, link health, and robots.txt AI-bot permissions. It is purpose-built for the llms.txt standard and AI crawler access compliance, unlike generic broken-link checkers or robots.txt parsers that don't understand the llms.txt format or AI-specific crawler rules.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-19T06:32:55.139Z","isFirstParty":false}