{"uid":"cap_FlTR9erN-LbveQWe1tJEK","slug":"agishub-multi-page-web-crawler-b1041a30","name":"AGISHub Multi-Page Web Crawler","description":"Crawl multiple pages of a site, respecting link depth and domain limits. Returns async job_id; results include markdown or HTML per page. Use it to fetch and process many pages of content at once.","url":"https://api.agishub.com/paid/crawl","method":"POST","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input","output"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri","description":"Root domain URL to crawl (e.g., https://example.com)."},"limit":{"type":"integer","maximum":100,"minimum":0,"description":"Maximum pages to crawl (default 100, max 100 per call). Returns 202 with job_id for async processing. For larger sites, issue several calls: each one is priced the same, so 1.000 pages cost ten calls.","exclusiveMinimum":true},"formats":{"type":"array","items":{"enum":["markdown","html"],"type":"string"},"description":"Output formats per page (default ['markdown']). 'html' adds raw HTML."},"max_depth":{"type":"integer","maximum":10,"minimum":0,"description":"Maximum link depth from root (default 2, max 10). Depth 0 = root only, depth 1 = root + direct children.","exclusiveMinimum":true},"same_domain":{"type":"boolean","description":"Only crawl URLs on the same domain (default true). Subdomain links are excluded when false."}},"additionalProperties":false},"type":{"enum":["http"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json"],"type":"string"}},"additionalProperties":false},"output":{"type":"object","required":["type","example"],"properties":{"type":{"enum":["json"],"type":"string"},"example":{"type":"object"}},"additionalProperties":false}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.1","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.1/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_bmG37-m7yPAZAs2ftDXAK","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.1","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Asynchronously crawls multiple pages of a website up to a configurable link depth and page limit, returning markdown or HTML content per page via a job ID.","exampleAgentPrompt":"Crawl https://docs.example.com and fetch up to 50 pages going 3 levels deep, and give me all the content as markdown.","exampleUseCases":[{"title":"Documentation site ingestion for RAG","prompt":"Crawl https://docs.stripe.com up to 100 pages and 2 levels deep, returning everything as markdown, so I can index it for a retrieval system."},{"title":"Competitor website content audit","prompt":"Can you crawl https://www.competitor.com up to 40 pages, staying on the same domain and going 2 links deep, and get me the markdown for each page?"},{"title":"Blog archive extraction","prompt":"Fetch all the articles from https://blog.openai.com — crawl up to 100 pages, 3 levels deep, markdown format only, same domain only."}],"resultDescription":"Returns a 202 response with an async job_id. When complete, results include per-page records containing the page URL, markdown text, and optionally raw HTML. For sites larger than 100 pages, multiple calls can be chained using the same pricing model.","failureModes":["Invalid or unreachable root URL returns an error with no job_id","Exceeding max 100 pages per call results in a validation error","max_depth above 10 is rejected with a schema validation error","Private or bot-protected sites may return empty or blocked page content","Async job may time out for very large crawls near the 100-page limit","Non-URI formatted url parameter causes a 400 bad request"],"whenToPreferThis":"Use this endpoint when you need to bulk-extract content from many pages of a site in a single async call, especially when you need markdown output ready for LLM ingestion or RAG pipelines. It is preferable over single-page scrapers when you need multi-level link traversal across an entire domain. Choose it when you want structured per-page results without managing your own crawler infrastructure.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T00:40:45.299Z","isFirstParty":false}