{"uid":"cap_II_sPmWCpG1wSWAqf8n5L","slug":"uos-genesis-machine-robots-check-e0780d49","name":"UOS Genesis Machine – robots_check","description":"Autonomous paid machine utilities using x402 and Base USDC.","url":"https://genesis.palsus2023.workers.dev/v1/robots_check?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["input"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri","description":"Public website URL whose robots.txt crawler directives should be fetched and inspected."}}},"type":{"type":"string","const":"http"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json"],"type":"string"}},"additionalProperties":false},"output":{"type":"object"}},"additionalProperties":true},"responseSchema":{"type":"json","example":{"url":"https://example.com/robots.txt","robots":"User-agent: *","status":200}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_d9PeHRveuEUPcIYSZu-HO","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and returns the robots.txt file and crawler directives for any given public website URL.","exampleAgentPrompt":"Can you grab the robots.txt file for https://example.com and tell me what crawler rules it sets?","exampleUseCases":[{"title":"Pre-crawl permission check","prompt":"Before I start scraping https://shopify.com, can you check their robots.txt and tell me which paths are disallowed and whether they block any specific user-agents?"},{"title":"SEO audit of competitor site","prompt":"I'm doing an SEO analysis — can you pull the robots.txt from https://competitor.com so I can see if they're blocking any important pages from indexing?"},{"title":"Automated crawler policy enforcement","prompt":"My crawler is about to index https://techcrunch.com — first check their robots.txt and tell me whether general bots are allowed and which directories are off-limits."}],"resultDescription":"Returns a JSON object containing the full robots.txt content as a string, the URL of the robots.txt file that was fetched, and the HTTP status code of the response (e.g. 200 if found, 404 if missing).","failureModes":["404 status if the website has no robots.txt file","Network timeout if the target domain is unreachable or slow","Non-200 status if the domain blocks the fetching agent","Malformed or empty robots.txt returned for unusual server configurations","Invalid URL input causes schema validation error"],"whenToPreferThis":"Use this endpoint when an agent needs to programmatically verify crawl permissions before scraping or indexing a website, especially in automated pipeline contexts where robots.txt compliance must be checked at scale. Prefer this over manual inspection or full-page scraping tools when you only need crawler directives rather than page content.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-02T18:48:25.604Z","isFirstParty":false,"canonicalSlug":"uos-genesis-machine-robots-check-e0780d49"}