{"uid":"cap_IKmixdYCaRrPk_GjPjESn","slug":"oromi-robots-txt-agent-access-checker-b82d8f2f","name":"Oromi robots.txt Agent-Access Checker","description":"Machine-payable APIs for AI agents, paid per call in USDC via the x402 protocol: UK business data (Companies House), UK property market data (HM Land Registry), website agent-readiness audits, and crypto market context.","url":"https://agents.oromi.co.uk/api/web/robots-check","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Full URL to check"},"agent":{"type":"string","description":"Crawler user-agent token (default *)"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"allowed":true,"matched_rule":null}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_kW7hdx39NN8uUfj5e6D8u","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL is accessible to a specified web crawler agent by parsing and evaluating the site's robots.txt rules","exampleAgentPrompt":"Before I scrape https://www.bbc.co.uk/news, can you check whether the default wildcard crawler agent is actually allowed to access that URL according to the site's robots.txt?","exampleUseCases":[{"title":"Pre-crawl permission check for news site","prompt":"I'm about to scrape articles from https://www.theguardian.com/technology — can you verify whether a general web crawler is allowed to access that URL before I proceed?"},{"title":"AI agent readiness audit for competitor site","prompt":"Check if Googlebot is allowed to access https://www.competitor.co.uk/products according to their robots.txt rules — I want to know if their content is indexable."},{"title":"Automated workflow gating before data collection","prompt":"Before my pipeline fetches data from https://www.rightmove.co.uk/property-for-sale/, verify that the wildcard crawler agent isn't blocked by robots.txt so I don't get flagged."}],"resultDescription":"Returns a JSON object with an 'allowed' boolean indicating whether the specified crawler agent is permitted to access the given URL under the site's robots.txt rules, and a 'matched_rule' field showing which specific robots.txt directive triggered the decision (null if no specific rule matched).","failureModes":["URL is unreachable or returns a non-200 status — may result in error or default-allow behavior","robots.txt file is missing — typically treated as allow-all","Malformed URL input causes validation error","User-agent token not recognized — falls back to wildcard rules","Network timeout fetching the target site's robots.txt"],"whenToPreferThis":"Use this endpoint when an AI agent or automation pipeline needs to verify robots.txt compliance before crawling or scraping a URL, especially in agentic workflows where legal and ethical crawling behavior must be enforced programmatically. Prefer this over manual robots.txt parsing when you need a reliable, machine-payable micro-service that handles edge cases like missing robots.txt files and wildcard rules. Particularly useful before data collection tasks targeting UK websites alongside Oromi's other UK-focused data services.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T18:34:48.904Z","isFirstParty":false}