{"uid":"cap_-QKv24gjiOrnQFy7IrmhN","slug":"oassis-web-crawler-job-based-555e29cb","name":"Oassis Web Crawler (Job-based)","description":"Follows a site's links and reads every page, as a job.","url":"https://api.oassis.dev/web/v1/crawl?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input","output"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"type":"object"},"type":{"enum":["http"],"type":"string"},"method":{"enum":["GET"],"type":"string"},"bodyType":{"enum":["json"],"type":"string"}},"additionalProperties":false},"output":{"type":"object","required":["type","example"],"properties":{"type":{"enum":["json"],"type":"string"},"example":{"type":"object"}},"additionalProperties":false}}},"responseSchema":{"type":"json","example":{"jobId":"e6af…","limit":5,"status":"queued"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.002","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.002/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.002","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_xELOeZZiEdZvoYx00_FyK","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.002","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Follows a site's links and crawls every reachable page as an asynchronous background job, returning a job ID for status polling.","exampleAgentPrompt":"Please crawl the entire site at https://docs.example.com — follow all the links and collect every page — and give me the job ID so I can check on it when it's done.","exampleUseCases":[{"title":"Full documentation site indexing","prompt":"Crawl all pages under https://docs.stripe.com — follow every internal link and grab the content from each page — and return the job ID so I can retrieve everything when it's finished."},{"title":"Competitor website content harvest","prompt":"I need to scrape every page on https://competitor.com, following all their internal links, and collect the full text. Kick off a crawl job and let me know the job ID."},{"title":"Knowledge base ingestion for RAG pipeline","prompt":"Can you crawl https://support.acme.com and follow all the links to pull in every help article? Start a background crawl job and give me the ID so I can fetch the results once it completes."}],"resultDescription":"Returns a JSON object containing a job ID (e.g. 'e6af…'), the crawl limit (number of pages), and an initial status of 'queued'. The agent can then poll a separate status endpoint using the job ID to retrieve crawled page content as the job progresses.","failureModes":["Invalid or unreachable URL results in a failed job status on polling","No URL provided in the body causes a 400 bad request","Rate limiting or payment failure (x402) returns 402 Payment Required","Crawl limit exceeded stops collection at the configured page count","Private or bot-blocked sites may return empty or partial results"],"whenToPreferThis":"Choose this endpoint when you need to recursively crawl an entire website by following links — not just scrape a single URL. It is ideal for bulk content ingestion, site indexing for RAG pipelines, or knowledge base extraction where completeness across many pages matters. Prefer this over single-page scrape endpoints when the number of target URLs is unknown upfront and you want the service to discover and collect pages automatically. The async job model makes it suitable for large sites where results won't be ready instantly.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-11T18:36:58.774Z","isFirstParty":false,"canonicalSlug":"oassis-web-crawler-job-based-555e29cb"}