{"uid":"cap_LhXIUtRCN2oy69Ltc1is8","slug":"scrapfly-domain-crawl-3fa33402","name":"ScrapFly Domain Crawl","description":"69 endpoints for data, content generation, and web scraping. Pay per request via x402 on Base chain.","url":"https://scrapfly.vercel.app/api/crawl/domain?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["domain"],"properties":{"domain":{"type":"string","description":"domain"}}},"responseSchema":{"type":"object","required":["success"],"properties":{"data":{"type":"object"},"success":{"type":"boolean"}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.02","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.02/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Y_o2oTyXzXcrgCoqfmrht","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.02","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls and extracts content and data from an entire domain via a web scraping API, returning structured results for AI agent consumption.","exampleAgentPrompt":"Can you crawl the entire domain example.com and pull back all the content and data from it?","exampleUseCases":[{"title":"Competitor website content extraction","prompt":"Scrape the entire domain competitor.com and pull back all their product and pricing content so I can analyze what they're offering."},{"title":"Brand monitoring data collection","prompt":"Crawl all pages on mybrand.com and extract the content so I can check what information is publicly visible about us."},{"title":"Research site bulk extraction","prompt":"Can you crawl research-institute.org and collect all the text and data from their domain so I can summarize their published work?"}],"resultDescription":"Returns a JSON object with a success boolean and a data object containing the crawled content and extracted information from the specified domain.","failureModes":["Invalid or unreachable domain returns success: false","Domain blocked or rate-limited by target site returns error","Malformed domain string causes request failure","Payment failure on Base chain prevents request execution","Timeout on large domains with many pages"],"whenToPreferThis":"Choose this endpoint when you need to extract content from an entire domain (not just a single URL) and want structured data back suitable for AI processing. It is particularly useful for competitive research, site auditing, or bulk content collection. Prefer this over single-URL scrapers when the target is a full site rather than one page.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-24T12:44:11.991Z","isFirstParty":false,"canonicalSlug":"scrapfly-domain-crawl-3fa33402"}