{"uid":"cap_WLr-4bDdlVC1tiQynwlel","slug":"olostep-web-crawler-start-crawl-fa0c5665","name":"Olostep Web Crawler - Start Crawl","description":"Starts a new crawl. You receive a `id` to track the progress. The operation may take 1-10 mins depending upon the site and depth and pages parameters.","url":"https://x402.orth.sh/olostep/v1/crawls","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"top_n":{"type":"number","description":"An optional number to only crawl the top N most relevant links on every page as per search query."},"timeout":{"type":"number","description":"End the crawl after n seconds with the pages completed until then. May take ~10s extra from provided timeout."},"max_depth":{"type":"number","description":"Maximum depth of the crawl. Useful to extract only up to n-degree of links."},"max_pages":{"type":"number","description":"Maximum number of pages to crawl. Recommended for most use cases like crawling an entire website."},"start_url":{"type":"string","description":"The starting point of the crawl."},"webhook_url":{"type":"string","description":"An optional POST request endpoint called when this crawl is completed. The body of the request will be same as the response of this [`v1/crawls/{crawl_id}`](./info#response-created) endpoint."},"exclude_urls":{"type":"array","items":{"type":"string"},"description":"URL path names in glob pattern to exclude. For example: `/careers/**`. Excluded URLs will supersede included URLs."},"include_urls":{"type":"array","items":{"type":"string"},"description":"URL path patterns to include in the crawl using glob syntax. Defaults to `/**` which includes all URLs. Use patterns like `/blog/**` to crawl specific sections (e.g., only blog pages), `/products/*.html` for product pages, or multiple patterns for different sections. Supports standard glob features like * (any characters) and ** (recursive matching)."},"search_query":{"type":"string","description":"An optional search query to find specific links and also sort the results by relevance."},"include_external":{"type":"boolean","description":"Crawl first-degree external links."},"include_subdomain":{"type":"boolean","description":"Include subdomains of the website. `false` by default."}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_fb0NlEm30-GbwH5z-_mFj","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Initiates an asynchronous multi-page web crawl from a seed URL, returning a crawl ID to track progress and results.","exampleAgentPrompt":"Crawl the entire docs.example.com website — go up to 3 levels deep, limit it to 100 pages, and only include pages under /docs/** — then ping my webhook at https://myapp.com/crawl-done when it's finished.","exampleUseCases":[{"title":"Index competitor blog for research","prompt":"Crawl all the blog posts at https://competitor.com — only pages under /blog/**, up to 50 pages and 2 levels deep — I want to analyze their content."},{"title":"Build product catalog dataset","prompt":"Crawl https://shop.example.com focusing only on /products/**  up to 200 pages, skip anything under /careers/** or /about/**, and send results to my webhook at https://myapi.com/done when finished."},{"title":"Search-relevant site crawl","prompt":"Crawl https://research-site.org with the search query 'machine learning benchmarks', limit to 30 pages and top 5 links per page, max depth 4 — I want the most relevant pages ranked by that query."}],"resultDescription":"Returns a crawl ID (and initial status) that the agent can use to poll the crawl progress endpoint or receive results via webhook. The actual crawl runs asynchronously and may take 1–10 minutes depending on site size, depth, and page count.","failureModes":["Invalid or unreachable start URL returns an error","Timeout exceeded before crawl completes, returning partial results","Max pages or depth parameters set too high causing long delays","Webhook URL unreachable, resulting in silent delivery failure","Payment failure returns 402 status and crawl is not initiated","Malformed glob patterns in include/exclude URLs cause unexpected crawl scope"],"whenToPreferThis":"Choose this endpoint when you need to crawl multiple pages of a website asynchronously, especially for large-scale content extraction, site indexing, or building datasets. It supports configurable depth, page limits, URL filtering, search-query relevance sorting, and webhook callbacks — making it ideal for long-running crawl jobs where you don't need an immediate synchronous response. Prefer this over single-page scraping endpoints when the task requires more than one page.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-13T18:40:11.432Z","isFirstParty":false}