{"uid":"cap_Sr4GJn4_lbOfT7ex55r10","slug":"weblens-crawl-api-e5c8cde4","name":"WebLens Crawl API","description":"Crawl a site and get clean markdown for every page, in one call. Same-host BFS with depth and page-budget limits, robots.txt honoured by default. $0.0015 per requested page (1-25).","url":"https://api.weblens.dev/crawl?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","format":"uri","example":"https://example.com","description":"The URL to process"},"limit":{"type":"number","default":10,"example":10,"maximum":25,"minimum":1,"description":"Page budget — you are charged per requested page"},"exclude":{"type":"array","items":{"type":"string","maxLength":200,"minLength":1},"default":[],"maxItems":20,"description":"Skip URLs whose path+query contains one of these"},"include":{"type":"array","items":{"type":"string","maxLength":200,"minLength":1},"default":[],"maxItems":20,"description":"Only crawl URLs whose path+query contains one of these"},"timeout":{"type":"number","default":10000,"maximum":30000,"minimum":5000,"description":"Request timeout in milliseconds"},"maxChars":{"type":"number","default":8000,"maximum":50000,"minimum":500,"description":"Per-page content character cap"},"maxDepth":{"type":"number","default":2,"example":2,"maximum":3,"minimum":0,"description":"Link depth from the start URL (0 = start page only)"},"respectRobots":{"type":"boolean","default":true,"description":"Honour robots.txt (default true; disable only for sites you control)"}}},"responseSchema":{"type":"json","example":{"url":"https://example.com","pages":[{"url":"https://example.com","depth":0,"title":"Example","status":"success","content":"# Example\\n\\nWelcome...","truncated":false},{"url":"https://example.com/about","depth":1,"title":"About","status":"success","content":"# About us...","truncated":false}],"summary":{"limit":10,"failed":0,"crawled":2,"maxDepth":2,"discovered":5,"successful":2,"robotsRespected":true},"crawledAt":"2026-07-31T12:00:00.000Z","requestId":"req_crawl123"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.015","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.015/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.015","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.015","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_IvxanvLV-AorDTlNEHMjD","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.015","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls an entire website up to a configurable depth and page limit, returning structured markdown content for each discovered page","exampleAgentPrompt":"Crawl the entire docs.example.com website up to 3 levels deep, fetching up to 50 pages, and give me the content of every page in markdown so I can build a knowledge base from it.","exampleUseCases":[{"title":"Documentation site indexing for RAG","prompt":"Crawl all the pages on docs.stripe.com up to 2 levels deep — I want the full markdown content of each page so I can load it into a vector database for a support chatbot."},{"title":"Competitor website content audit","prompt":"Go through the entire marketing site at competitor.com, up to 3 levels deep and no more than 100 pages, and pull back the text content of every page so I can analyze what topics they cover."},{"title":"Internal wiki knowledge extraction","prompt":"Crawl our company wiki at wiki.internal.io starting from the homepage, up to 4 levels deep, and get me the title and full content of every page you find."}],"resultDescription":"A JSON object containing an array of crawled pages, each with its URL, depth, title, HTTP status, and full markdown content (with a truncation flag). Also includes a summary with counts of crawled, successful, failed, and discovered pages, max depth reached, robots.txt compliance status, the timestamp, and a request ID.","failureModes":["Target site blocks crawlers or returns 403/429 — affected pages will have status 'failed' in the response","Robots.txt exclusions prevent crawling certain paths when robotsRespected is true","Very large sites may hit the page limit before full discovery","Network timeouts on slow or unreliable target servers","Invalid or unreachable starting URL returns an error response","JavaScript-rendered pages may not return full content if the site requires a browser runtime"],"whenToPreferThis":"Choose this endpoint when you need to retrieve content from multiple pages across an entire site rather than a single URL. It is ideal for building knowledge bases, indexing documentation, auditing site content, or feeding website text into downstream AI pipelines. Prefer it over single-page fetch endpoints whenever you need breadth across a domain. The pay-per-call model (no monthly commitment) makes it cost-effective for sporadic or one-off full-site crawls compared to subscription-based alternatives.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-11-12T00:34:24.400Z","isFirstParty":false,"canonicalSlug":"weblens-crawl-api-e5c8cde4"}