{"uid":"cap_4CJZWZpYaA_dauhHzgIjU","slug":"vaaya-ai-crw-async-multi-page-web-crawl-80a43ef2","name":"vaaya.ai CRW — Async Multi-Page Web Crawl","description":"CRW — Start an ASYNC multi-page crawl from a seed URL, following links. Pass `url`; optional `maxPages` (1-100, default 100 — vendor bills 1 credit/page so keep it tight), `maxDepth` (default 2), `formats`, `onlyMainContent`, `renderJs`. Returns `{ id }` immediately; KEEP POLLING `crw/crawl_status` with that id until status is completed/failed. One price covers the whole job.","url":"https://vaaya.ai/api/run/crw/crawl?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"type":"object","$schema":"http://json-schema.org/draft-07/schema#","required":["url"],"properties":{"url":{"type":"string","format":"uri"},"formats":{"type":"array","items":{"type":"string"}},"maxDepth":{"type":"integer","minimum":0},"maxPages":{"type":"integer","maximum":100,"minimum":1},"renderJs":{"type":["boolean","null"]},"onlyMainContent":{"type":"boolean"}},"additionalProperties":true},"type":{"type":"string","const":"http"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"pathParams":{"type":"object"}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.1","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.1/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_0dskLcCnlwOpvScFOT3sy","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.1","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Starts an asynchronous multi-page crawl from a seed URL, following links across up to 100 pages, and returns a job ID to poll for results","exampleAgentPrompt":"Crawl the entire docs.example.com site starting from the homepage, following links up to 3 levels deep and scraping up to 50 pages — return everything as clean markdown.","exampleUseCases":[{"title":"Documentation site knowledge base build","prompt":"Crawl https://docs.myproduct.com starting from the root, follow all internal links up to 2 levels deep, scrape up to 80 pages, and give me the content as markdown so I can build a knowledge base."},{"title":"Competitor website content audit","prompt":"Crawl https://competitor.com from the homepage, follow links up to 2 levels deep with a max of 30 pages, extract only the main content as markdown — I want to audit what topics they cover."},{"title":"E-commerce product catalog extraction","prompt":"Scrape the full product catalog at https://store.example.com by crawling from the /products page, following links up to 3 levels deep, capping at 60 pages, and rendering JavaScript so I get the dynamic content."}],"resultDescription":"Returns immediately with a JSON object containing a job `id`. The agent must poll the `crw/crawl_status` endpoint with that ID until status is `completed` or `failed`. On completion, the full crawl results include scraped page content in the requested formats (markdown, HTML, JSON) for all pages visited, along with metadata per page.","failureModes":["Invalid or unreachable seed URL — crawl fails immediately with an error status","maxPages or maxDepth out of range (maxPages must be 1-100) — validation error returned","Site blocks crawlers/robots.txt disallows — crawl returns zero or few pages","Polling not performed — agent never retrieves results if it doesn't call crawl_status","Timeout on large sites — crawl may partially complete if site is very large or slow","renderJs failures on JavaScript-heavy sites with complex SPAs"],"whenToPreferThis":"Choose this endpoint when you need to extract content from multiple pages of a website in a single job, following internal links automatically. Prefer it over the single-URL scrape endpoint when the target content spans many pages. Use the URL discovery endpoint first if you only need URLs without content. Best suited for documentation sites, blogs, product catalogs, or any multi-page content extraction task where you want to set a page budget (maxPages) and depth limit.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-30T06:44:49.235Z","isFirstParty":false,"canonicalSlug":"vaaya-ai-crw-async-multi-page-web-crawl-80a43ef2"}