{"uid":"cap_59rjU0yrZBnIdi2kU7uGY","slug":"scrapfly-web-crawl-endpoint-9f2610db","name":"ScrapFly Web Crawl Endpoint","description":"69 endpoints for data, content generation, and web scraping. Pay per request via x402 on Base chain.","url":"https://scravfly.vercel.app/api/crawl/crawl","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"url"},"maxPages":{"type":"number","description":"maxPages"}}},"responseSchema":{"type":"object","required":["success"],"properties":{"data":{"type":"object"},"success":{"type":"boolean"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.3","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.3/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.3","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.3","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_fC0yOpXV9aqQNZpxjr-En","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.3","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a given URL (and optionally multiple pages) and returns the scraped page content and data","exampleAgentPrompt":"Can you crawl https://example.com/products and scrape up to 5 pages of content so I can see what products they list?","exampleUseCases":[{"title":"Competitor product page scraping","prompt":"Crawl https://competitor.com/products and scrape up to 10 pages so I can see all their product listings and prices."},{"title":"News article content extraction","prompt":"Fetch the content from https://techcrunch.com/latest and grab up to 3 pages of recent articles so I can summarize what's trending."},{"title":"Research site data collection","prompt":"Scrape https://researchsite.org/papers and crawl up to 20 pages so I can collect all the paper titles and abstracts for my literature review."}],"resultDescription":"Returns a JSON object with a 'success' boolean and a 'data' object containing the scraped page content and structured data from the crawled URL(s).","failureModes":["Invalid or unreachable URL returns success: false","Rate limiting or bot detection on target site may result in empty or blocked data","maxPages exceeding available pages returns whatever pages are accessible","Network timeouts for slow-loading sites","Malformed URL input results in error response"],"whenToPreferThis":"Choose this endpoint when you need to programmatically crawl and extract content from one or more pages of a website, especially when you need multi-page crawling (via maxPages) via a pay-per-request model on Base chain. Suitable for agents that need fresh web content without maintaining their own scraping infrastructure.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T18:43:09.468Z","isFirstParty":false}