{"uid":"cap_yHv18njJzdp-W-f_ZMG_Z","slug":"agishub-multi-page-site-crawler-891a123e","name":"AGISHub Multi-Page Site Crawler","description":"Crawl multiple pages of a site, respecting link depth and domain limits. Returns async job_id; results include markdown or HTML per page. Use it to fetch and process many pages of content at once.","url":"https://api.agishub.com/v1/crawl","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input","output"],"properties":{"input":{"type":"object","required":["type","method","queryParams"],"properties":{"type":{"enum":["http"],"type":"string"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri","description":"Root domain URL to crawl (e.g., https://example.com)."},"limit":{"type":"integer","maximum":100,"minimum":0,"description":"Maximum pages to crawl (default 100, max 100 per call). Returns 202 with job_id for async processing. For larger sites, issue several calls: each one is priced the same, so 1.000 pages cost ten calls.","exclusiveMinimum":true},"formats":{"type":"array","items":{"enum":["markdown","html"],"type":"string"},"description":"Output formats per page (default ['markdown']). 'html' adds raw HTML."},"max_depth":{"type":"integer","maximum":10,"minimum":0,"description":"Maximum link depth from root (default 2, max 10). Depth 0 = root only, depth 1 = root + direct children.","exclusiveMinimum":true},"same_domain":{"type":"boolean","description":"Only crawl URLs on the same domain (default true). Subdomain links are excluded when false."}},"additionalProperties":false}},"additionalProperties":false},"output":{"type":"object","required":["type","example"],"properties":{"type":{"enum":["json"],"type":"string"},"example":{"type":"object"}},"additionalProperties":false}}},"responseSchema":{"type":"json","example":{"result":"Crawl multiple pages of a site, respecting link depth and domain limits. Returns async job_id; results include markdown "}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.1","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.1/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.1","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_XDwGHkSMA0rTRq_zgSbhJ","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.1","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Asynchronously crawls multiple pages of a website up to a configurable link depth and page limit, returning markdown or HTML content per page via a job ID.","exampleAgentPrompt":"Crawl the entire docs.example.com website up to 3 levels deep, grab up to 50 pages, and return everything as markdown so I can process the content.","exampleUseCases":[{"title":"Documentation site bulk extraction","prompt":"Crawl https://docs.stripe.com up to depth 2, limit 100 pages, and return everything as markdown so I can build a knowledge base from their documentation."},{"title":"Competitor website content audit","prompt":"Scrape https://www.competitor.com following links up to 3 levels deep, max 80 pages, staying on the same domain, and give me both the markdown and raw HTML for each page."},{"title":"Blog archive ingestion","prompt":"Crawl https://blog.openai.com up to depth 2, pull up to 50 pages, and return the content as markdown so I can summarize all their recent posts."}],"resultDescription":"Returns a job_id immediately (HTTP 202) for async processing. Once complete, results include per-page content in the requested formats (markdown and/or HTML), one entry per crawled page, up to the specified page limit.","failureModes":["Invalid or unreachable root URL returns an error","Exceeding max 100 pages per call requires multiple calls","max_depth above 10 or limit above 100 rejected by schema validation","Non-public or auth-gated sites may return empty or partial results","Job ID lookup needed separately to retrieve async results after submission"],"whenToPreferThis":"Use this endpoint when you need to crawl many pages of a website at once rather than fetching a single URL. It is ideal for bulk content ingestion, documentation processing, or competitive research where you need structured markdown or HTML from an entire domain up to a configurable depth. Prefer it over single-page fetchers when the target site has multiple linked pages you need to process together.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T18:50:17.568Z","isFirstParty":false}