{"uid":"cap_4CVHqZ8ybtwJg85v0NaF2","slug":"true402-robots-check-bfd50d55","name":"true402 Robots Check","description":"The machine-native marketplace. Wallet = identity. Pay per call with HTTP 402.","url":"https://true402.dev/api/v1/robots-check?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Any URL on the site"}}},"responseSchema":{"type":"object","properties":{"ok":{"type":"boolean"},"url":{"type":"string"},"robots":{"type":"object","description":"{ found, status }"},"llmsTxt":{"type":"object","description":"{ found }"},"sitemaps":{"type":"array"},"aiCrawlers":{"type":"object","description":"{ GPTBot: allow|block|unspecified, ClaudeBot: …, … }"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.003","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"settled","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.003/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.003","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_BdjUBkq9oe7GB6ryoDmb8","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.003","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given website URL has a robots.txt file and returns machine-crawlability metadata for that site.","exampleAgentPrompt":"Can you check whether https://example.com allows bots and automated agents to crawl it — look up its robots.txt rules for me.","exampleUseCases":null,"resultDescription":"Returns an object containing robots.txt crawlability information for the submitted URL, including whether bots are permitted to access the site and any applicable crawl rules or restrictions.","failureModes":["Invalid or malformed URL returns validation error","Site unreachable or domain does not exist returns connectivity error","Site has no robots.txt — response may indicate absence","Payment failure via HTTP 402 if USDC wallet balance is insufficient","Timeout if target site is slow to respond"],"whenToPreferThis":"Use this endpoint when an AI agent needs to verify whether a target website permits automated crawling before attempting to scrape or access it. Particularly useful for compliance-aware web agents that need to respect robots.txt policies. Prefer over manual robots.txt fetching when you want a pay-per-call, machine-native approach with no API key setup.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-10T06:30:07.950Z","isFirstParty":false,"canonicalSlug":"true402-robots-check-bfd50d55"}