{"uid":"cap_yvq4K2fAV6wfk1GQQSRP6","slug":"robots-txt-check-for-url-1adc88eb","name":"Robots.txt Check for URL","description":"Machine-payable x402 utilities for hashing, Base64, UUID generation, unit and timezone conversion, realtime crypto market data, webpage/document extraction, URL-to-Markdown, metadata, RAG ingestion and deterministic website change detection.","url":"https://x402-gateway-production-1f21.up.railway.app/web/robots-check?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri"},"path":{"type":"string"},"user_agent":{"type":"string"}},"additionalProperties":false},"type":{"type":"string","const":"http"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json","form-data","text"],"type":"string"}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"status":"ready"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_lprNeE7bFB_uhhHI_ljpi","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL is permitted or blocked by the target website's robots.txt rules before crawling or scraping","exampleAgentPrompt":"Before I scrape https://example.com/blog/article-1, can you check if that URL is allowed by their robots.txt for my crawler?","exampleUseCases":[{"title":"Pre-scrape compliance check","prompt":"I want to scrape product prices from https://shop.example.com/products — can you first check if robots.txt allows my bot to access that path?"},{"title":"RAG pipeline ingestion gate","prompt":"Before ingesting https://docs.acme.com/api/reference into our RAG system, verify that the robots.txt on that site permits crawling that URL."},{"title":"Multi-site crawler permission audit","prompt":"Check if https://news.site.com/articles is accessible to a generic crawler according to their robots.txt so I know whether to include it in my data pipeline."}],"resultDescription":"Returns a JSON object with a status field (e.g. 'ready') indicating whether the specified URL and path are permitted by the site's robots.txt rules, enabling agents to make go/no-go crawl decisions before fetching content.","failureModes":["URL is malformed or not a valid URI — input validation error","Target site is unreachable or robots.txt cannot be fetched — network or timeout error","Site has no robots.txt, which may be interpreted as open access","Path not specified and defaults may not match intended crawl target","User agent not specified, defaulting to generic crawler rules that may differ from intended bot identity"],"whenToPreferThis":"Use this endpoint before any web scraping, crawling, or RAG ingestion workflow to confirm you have robots.txt permission for a target URL. Prefer this over manual robots.txt parsing when you need a machine-payable, programmatic compliance check integrated into an automated agent pipeline. Especially valuable when building responsible scraping agents that must respect site policies before extracting content.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-12T12:37:57.930Z","isFirstParty":false,"canonicalSlug":"robots-txt-check-for-url-1adc88eb"}