{"uid":"cap_8ZQyuW2ACerCtVAq40hAh","slug":"scrapfly-web-crawl-endpoint-7976431a","name":"ScrapFly Web Crawl Endpoint","description":"69 endpoints for data, content generation, and web scraping. Pay per request via x402 on Base chain.","url":"https://scrapfly.vercel.app/api/crawl/crawl?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"url"},"maxPages":{"type":"number","description":"maxPages"}}},"responseSchema":{"type":"object","required":["success"],"properties":{"data":{"type":"object"},"success":{"type":"boolean"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.1","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.1/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_WsKq2-G69Lgc00FmbjbmR","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.1","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a given URL (and optionally multiple pages) and returns the scraped content as structured data","exampleAgentPrompt":"Can you crawl https://example.com/products for me and pull back all the page content — go through up to 5 pages?","exampleUseCases":[{"title":"Competitor product page scraping","prompt":"Scrape https://competitor.com/pricing and give me all the content from up to 3 pages so I can compare their plans to ours."},{"title":"News article content extraction","prompt":"Fetch the full content of this news article at https://techcrunch.com/2024/01/15/some-article so I can summarize it."},{"title":"E-commerce catalog crawl","prompt":"Crawl https://shop.example.com/catalog and retrieve data from up to 10 pages — I want to see all the product listings they have."}],"resultDescription":"Returns a JSON object with a boolean 'success' field and a 'data' object containing the scraped content from the target URL. The data object holds the extracted page content, which may include text, HTML, links, or structured data depending on what the page contains.","failureModes":["URL is unreachable or returns a non-200 status — success: false with no data","Target site blocks scraping (bot protection, CAPTCHA) — success: false or empty data","Invalid URL format provided — request may fail at validation","maxPages set too high causing timeout or partial results","Payment failure via x402 protocol — request not processed"],"whenToPreferThis":"Choose this endpoint when you need to programmatically retrieve and extract content from one or more pages of a website, particularly when you want pay-per-request pricing via crypto (USDC on Base) and need a simple URL-in/data-out interface. Prefer this over browser automation tools when you want a managed, hosted scraping solution without maintaining your own infrastructure.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-24T12:44:57.684Z","isFirstParty":false,"canonicalSlug":"scrapfly-web-crawl-endpoint-7976431a"}