{"uid":"cap_L-H3AEBFJmaGZ4pUEcZZ1","slug":"scrapegraphai-website-crawler-f838ac88","name":"ScrapeGraphAI Website Crawler","description":"Start a website crawl from a seed URL with configurable depth and page limits.","url":"https://x402.orthogonal.com/scrapegraphai/api/crawl","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","description":"Seed URL to crawl"},"formats":{"type":"array","description":"Output formats"},"maxDepth":{"type":"number","description":"Max crawl depth (default 2)"},"maxPages":{"type":"number","description":"Max pages (1-1000, default 50)"},"fetchConfig":{"type":"object","description":"Fetch options"},"contentTypes":{"type":"array","description":"Allowed content types"},"allowExternal":{"type":"boolean","description":"Follow external links (default false)"},"excludePatterns":{"type":"array","description":"URL patterns to exclude"},"includePatterns":{"type":"array","description":"URL patterns to include"},"maxLinksPerPage":{"type":"number","description":"Max links per page (default 10)"}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Ljg5IMn_8KyYUItqc44WB","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a website starting from a seed URL with configurable depth, page limits, URL filtering, and output format options.","exampleAgentPrompt":"Crawl the website at https://docs.example.com starting from the homepage, going up to 3 levels deep, and visiting no more than 100 pages — return the content in markdown format and skip any URLs containing '/blog/'.","exampleUseCases":[{"title":"Competitor documentation indexing","prompt":"Crawl https://docs.competitor.com up to 4 levels deep, limit it to 200 pages, and extract all the content as markdown so I can analyze their product documentation."},{"title":"E-commerce product page collection","prompt":"Spider https://shop.example.com/products starting from that URL, stay within the /products path using an include pattern, go 2 levels deep, and grab up to 500 pages so I can collect all their product listings."},{"title":"Internal knowledge base scraping","prompt":"Crawl our company wiki at https://wiki.internal.com — max depth 3, up to 150 pages, exclude anything under /archive/ — and pull back the full text content so I can feed it into a search index."}],"resultDescription":"Returns structured data from all crawled pages, including page content in the requested output formats (e.g. markdown, HTML), URLs visited, and metadata. The crawl respects depth and page count limits and optionally filters URLs using include/exclude patterns.","failureModes":["Seed URL is unreachable or returns non-200 status — crawl returns empty or error","maxPages or maxDepth set to values outside allowed range — validation error","Target site blocks crawlers (robots.txt or rate limiting) — partial or empty results","Malformed URL in input — request rejected with validation error","External link following disabled by default — pages outside seed domain silently skipped"],"whenToPreferThis":"Use this endpoint when you need to recursively collect content from multiple pages of a website rather than scraping a single URL. It is best suited for documentation indexing, site auditing, competitive intelligence gathering, or building training datasets from web content. Choose it over single-URL scrapers when you need breadth across a domain with configurable depth and filtering.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T06:33:56.561Z","isFirstParty":false}