{"uid":"cap_UNR3dUvr85zJmRPlOfB_l","slug":"olostep-web-crawler-start-crawl-c13a1959","name":"Olostep Web Crawler - Start Crawl","description":"Starts a new crawl. You receive a `id` to track the progress. The operation may take 1-10 mins depending upon the site and depth and pages parameters.","url":"https://x402.orthogonal.com/olostep/v1/crawls","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"top_n":{"type":"number","description":"An optional number to only crawl the top N most relevant links on every page as per search query."},"timeout":{"type":"number","description":"End the crawl after n seconds with the pages completed until then. May take ~10s extra from provided timeout."},"max_depth":{"type":"number","description":"Maximum depth of the crawl. Useful to extract only up to n-degree of links."},"max_pages":{"type":"number","description":"Maximum number of pages to crawl. Recommended for most use cases like crawling an entire website."},"start_url":{"type":"string","description":"The starting point of the crawl."},"webhook_url":{"type":"string","description":"An optional POST request endpoint called when this crawl is completed. The body of the request will be same as the response of this [`v1/crawls/{crawl_id}`](./info#response-created) endpoint."},"exclude_urls":{"type":"array","items":{"type":"string"},"description":"URL path names in glob pattern to exclude. For example: `/careers/**`. Excluded URLs will supersede included URLs."},"include_urls":{"type":"array","items":{"type":"string"},"description":"URL path patterns to include in the crawl using glob syntax. Defaults to `/**` which includes all URLs. Use patterns like `/blog/**` to crawl specific sections (e.g., only blog pages), `/products/*.html` for product pages, or multiple patterns for different sections. Supports standard glob features like * (any characters) and ** (recursive matching)."},"search_query":{"type":"string","description":"An optional search query to find specific links and also sort the results by relevance."},"include_external":{"type":"boolean","description":"Crawl first-degree external links."},"include_subdomain":{"type":"boolean","description":"Include subdomains of the website. `false` by default."}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_VLHXSiHQx3cRIR4fZKZT1","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Initiates a multi-page web crawl starting from a given URL, returning a crawl ID to track the async job's progress.","exampleAgentPrompt":"Start a crawl of https://docs.example.com, go up to 3 levels deep, limit it to 50 pages, only include URLs under /guides/**, and send a POST to https://myapp.com/webhook when it's done.","exampleUseCases":[{"title":"Full documentation site indexing","prompt":"Crawl all the pages on https://docs.stripe.com up to 4 levels deep, cap it at 200 pages, and only include URLs matching /docs/** — ping my webhook at https://myserver.com/crawl-done when it finishes."},{"title":"Competitor blog content gathering","prompt":"I want to scrape the blog section of https://www.hubspot.com — only crawl URLs matching /blog/**, go up to 2 levels deep, limit to 100 pages, and stop after 5 minutes if it's still running."},{"title":"Targeted product page extraction","prompt":"Start a crawl of https://www.shopify.com starting at the homepage, only include pages matching /products/*.html, crawl up to 3 levels, limit to 75 pages, and use the search query 'ecommerce tools' to prioritize the most relevant links."}],"resultDescription":"Returns a crawl ID that can be used to poll the status and retrieve results of the async crawl job. The actual crawl runs for 1–10 minutes depending on site size and configured depth/pages parameters. When a webhook_url is provided, the completed crawl results are POSTed there automatically.","failureModes":["Invalid or unreachable start_url returns an error immediately","Timeout reached before crawl completes — partial results returned","max_pages or max_depth set too low may return very limited content","Webhook URL unreachable means results must be polled manually","Rate limits or bot protection on target site may block crawl progress","Missing start_url causes validation error on submission"],"whenToPreferThis":"Choose this endpoint when you need to crawl multiple pages across a website asynchronously, especially when the full site has many pages or requires deep traversal. Prefer this over single-URL scraping endpoints when you need to index an entire site, a section of a site, or gather content matching a pattern. Ideal when you can tolerate a 1–10 minute wait and want webhook notification on completion. Use the include_urls and exclude_urls filters to target specific site sections precisely.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-13T07:02:10.895Z","isFirstParty":false}