{"uid":"cap_Cf1r6qGF64N6rj0qGplYV","slug":"signalharness-robots-policy-parse-adf843ab","name":"SignalHarness Robots Policy Parse","description":"Explore 330 pay-per-call x402 API services and 27 agent-native digital products, with Base USDC pricing, secure Polar checkout, and free discovery.","url":"https://signalharness.ai/api/agent/services/robots_policy_parse/invoke","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","pattern":"^https://","maxLength":2048,"minLength":9},"text":{"type":"string","maxLength":65536,"minLength":0},"userAgent":{"type":"string","maxLength":256,"minLength":1},"targetPath":{"type":"string","maxLength":2048,"minLength":1}}},"responseSchema":{"type":"json","example":{"replay":false,"result":{"groups":[{"rules":[{"path":"example","directive":"allow"}],"crawlDelay":0,"userAgents":["example"]}],"sitemaps":["example"],"warnings":["Verify the caller-supplied data before relying on this result."],"crawlDelay":0,"matchedRule":{},"crawlAllowed":false,"matchedGroup":[null]},"status":"succeeded","receipt":{"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","usage":[],"status":"succeeded","network":"eip155:8453","artifacts":[],"endedAtMs":0,"latencyMs":0,"paymentId":"example-payment","receiptId":"example-receipt","requestId":"example-request","serviceId":"robots_policy_parse","executionId":"example-execution","startedAtMs":0,"amountAtomic":"5000","resultSha256":"9f187257ea9c44cac9c39fc35e42f5352601cd2a27c6b8ee4122b8e15d4a2306","serviceVersion":"1.0.0","settlementReference":"0x0000000000000000000000000000000000000000000000000000000000000000"},"artifacts":[],"requestId":"example-request","executionId":"example-execution"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.005","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.005/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_2a6mVPz2nJxsgqAIrK5Ec","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.005","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt file to determine whether a given user agent is allowed to crawl a specific URL path","exampleAgentPrompt":"Before I scrape https://example.com/blog/article-123, check the robots.txt to see if a user agent called 'MyBot/1.0' is actually allowed to access that path.","exampleUseCases":[{"title":"Pre-scrape compliance check for web agent","prompt":"My crawler 'ResearchBot/2.0' wants to visit https://news.ycombinator.com/item?id=12345 — can you check the robots.txt to see if that path is allowed for my user agent?"},{"title":"SEO audit of competitor crawl rules","prompt":"Pull up the robots.txt for https://www.shopify.com and tell me whether Googlebot is allowed to crawl the /admin/ path, and if there's any crawl delay set."},{"title":"Autonomous agent self-permission check","prompt":"Before my agent fetches content from https://techcrunch.com/2024/01/ai-news, verify against their robots.txt whether a generic 'AI-Agent/1.0' user agent is permitted to access that URL."}],"resultDescription":"Returns a JSON object indicating whether crawling is allowed for the specified user agent and path, the matched robots.txt rule and group, any applicable crawl delay, a list of sitemaps declared, any warnings encountered during parsing, and full execution receipt metadata including payment confirmation and result hash.","failureModes":["URL is unreachable or returns a non-200 status — crawl permission may default to allowed or error","robots.txt is malformed or empty — warnings array will be populated, results may be incomplete","Invalid URL format not matching ^https:// pattern — request rejected with validation error","Target site blocks the fetch request — result may reflect inability to retrieve policy","User agent string too long (>256 chars) or path too long (>2048 chars) — request rejected"],"whenToPreferThis":"Use this endpoint when an AI agent or web crawler needs to programmatically verify robots.txt compliance before accessing any web resource, especially in automated pipelines where legal and ethical scraping compliance must be enforced. Prefer this over manual robots.txt parsing when you need structured, machine-readable output including matched rules, crawl delay, and sitemaps in a single call. Particularly useful for agents that need to self-certify access permissions before taking action on web content.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-13T13:07:01.560Z","isFirstParty":false}