{"uid":"cap_XUSoBJlU9FIzBERxnhone","slug":"bismuth-scraping-bfs-site-crawl-8e48f0e5","name":"Bismuth Scraping - BFS Site Crawl","description":"Playwright-powered web scraping with structured markdown, links, tables, and BFS site crawl. SSRF-protected. Part of the Bismuth utility API suite for AI agents.","url":"https://x402-scraping-api-production.up.railway.app/crawl","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Seed URL to begin crawling (http/https, max 2048 chars)"},"max_depth":{"type":"integer"},"max_pages":{"type":"integer"},"exclude_paths":{"type":"array"},"include_paths":{"type":"array"}}},"responseSchema":{},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.1","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.1/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_0gcLeojAZcroxVmo2vzWO","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.1","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a website via breadth-first search using Playwright, returning structured markdown, links, and tables for each visited page","exampleAgentPrompt":"Can you crawl the entire docs.example.com site starting from https://docs.example.com, going up to 3 levels deep and visiting at most 50 pages, and give me the content as structured markdown?","exampleUseCases":[{"title":"Documentation site indexing for RAG","prompt":"Crawl https://docs.stripe.com starting from the homepage, go up to 4 levels deep, limit it to 100 pages, and give me all the content as markdown so I can build a knowledge base from it."},{"title":"Competitor product page research","prompt":"Scrape https://www.competitor.com/products, go 2 levels deep and cap it at 30 pages — I want to see all their product listings and descriptions in structured markdown."},{"title":"Blog content archive extraction","prompt":"Can you crawl https://blog.openai.com, include only paths that start with /blog, go up to 3 levels deep with a max of 60 pages, and pull out all the article text and links?"}],"resultDescription":"A structured response containing the content of each crawled page rendered as markdown, including extracted links, tables, and page metadata. The crawl follows a breadth-first search strategy from the seed URL, respecting the specified depth and page limits, with SSRF protections applied to prevent abuse.","failureModes":["Invalid or unreachable seed URL returns an error","SSRF-protected URLs (e.g. localhost, private IP ranges) are blocked and return a rejection error","max_depth or max_pages set too high may result in very long response times or timeouts","Sites with heavy JavaScript anti-bot measures may return incomplete content","Malformed URL (exceeding 2048 chars or missing http/https scheme) returns a validation error"],"whenToPreferThis":"Choose this endpoint when you need to extract content from multiple pages of a website in a single call, not just a single page. It is ideal for building knowledge bases, indexing documentation, archiving blogs, or researching competitor sites where you need breadth of coverage. Prefer this over single-page scrapers when site structure and cross-page linking matter, or when you want to discover and collect pages automatically via BFS rather than enumerating URLs manually.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T12:31:33.702Z","isFirstParty":false}