{"uid":"cap_ULrDdyNY-A9SGmHHsSJHC","slug":"citiview-website-crawler-f5616b3c","name":"Citiview Website Crawler","description":"Crawl a website into clean Markdown: follows same-site links from a start page (optionally under a path prefix), up to 100 pages, respecting robots.txt. Returns each page's URL, title and text. $0.005 + $0.001 per page. Add store=true (+$0.001) to get a 7-day share link instead of the data inline.","url":"https://agents.citiviewgroup.com/v1/crawl?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Start page"},"store":{"type":"boolean","description":"Return a 7-day share link (+$0.001)"},"max_pages":{"type":"integer","description":"1-100, default 20"},"path_prefix":{"type":"string","description":"Only follow links under this path, e.g. /docs/"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.025","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.025/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.025","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.025","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_YK53wzTxAi45u-gYeJdju","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.025","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a website starting from a given URL, follows same-site links up to 100 pages, and returns each page's URL, title, and cleaned Markdown text.","exampleAgentPrompt":"Can you crawl the documentation site at https://docs.example.com/api/ — only follow links under /api/, up to 50 pages — and give me all the page titles and their content as markdown?","exampleUseCases":[{"title":"Scraping documentation for RAG pipeline","prompt":"Crawl https://docs.langchain.com/docs/ and only follow links under /docs/, up to 80 pages, and return all the page text as markdown so I can feed it into my knowledge base."},{"title":"Competitor website content audit","prompt":"Can you crawl https://www.competitor.com starting from their homepage, follow up to 30 pages, and give me the text of each page so I can see what topics they cover?"},{"title":"Archiving blog content with share link","prompt":"Crawl https://blog.mysite.com and follow links under /blog/ for up to 100 pages, and store the results so I get a share link I can pass to my team — they need it for the next 7 days."}],"resultDescription":"A list of up to 100 crawled pages, each with its URL, title, and full page content converted to clean Markdown. If store=true is requested, returns a 7-day share link pointing to the data instead of returning it inline.","failureModes":["Start URL is unreachable or returns non-200 — crawl fails or returns empty results","robots.txt disallows crawling — pages are skipped or crawl is blocked","Path prefix does not match any linked pages — only the start page is returned","Max pages exceeded silently — results capped at 100","Store link expires after 7 days — downstream consumers must fetch before expiry","JavaScript-heavy SPAs may not render fully — content may be incomplete"],"whenToPreferThis":"Choose this endpoint when you need to extract textual content from multiple pages of a website in one call, especially for documentation sites, blogs, or content archives. It is ideal when you want clean Markdown output rather than raw HTML, and when following internal links automatically saves you from orchestrating individual page fetches. Prefer it over single-page scrapers when the target content spans many URLs under a shared domain or path prefix.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-15T06:38:15.370Z","isFirstParty":false,"canonicalSlug":"citiview-website-crawler-f5616b3c"}