{"uid":"cap_etLUni9UgaMIlWH2yoqWI","slug":"scrape-permit-url-check-dbb5df36","name":"Scrape-Permit URL Check","description":"Scrape-permit API: check whether a website allows automated scraping before you crawl it. Returns robots.txt rules, AI-crawler directives, rate limits, licensing/ToS flags, and a clear 'scrape allowed / restricted' verdict. x402 pay-per-call on Base USDC. Keeps AI agents and data pipelines compliant and avoids bans and legal risk.","url":"https://apiwitchcraft.duckdns.org/scrape-permit/check","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_NuVqLHpLMg4IszZwyhnLz","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL is permitted to be scraped (i.e., allows web scraping based on its robots.txt or access policy).","exampleAgentPrompt":"Before I scrape it, can you check whether https://example.com/products is permitted for scraping?","exampleUseCases":null,"resultDescription":"Returns a permission status indicating whether the specified URL allows scraping via GET requests, based on its access policy or robots.txt rules.","failureModes":["Missing or invalid 'url' query parameter returns a validation error","URL is unreachable or times out, resulting in an error or indeterminate response","Malformed URL format causes a parsing or bad-request error","Payment not provided or insufficient USDC results in a 402 response blocking access"],"whenToPreferThis":"Use this endpoint before initiating any web scraping workflow to programmatically verify that a target URL permits automated GET access, avoiding potential legal or policy violations. Prefer this over manual robots.txt parsing when you need a fast, API-integrated permission check as part of an agent pipeline.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T18:33:36.423Z","isFirstParty":false}