{"uid":"cap_HH2iYyL4NDRWwqatLC1zM","slug":"agentrxi-web-crawler-47949070","name":"AgentRXI Web Crawler","description":"Crawl a website into clean Markdown: follows same-site links from a start page (optionally under a path prefix), up to 100 pages, respecting robots.txt. Returns each page's URL, title and text. $0.005 + $0.001 per page. Add store=true (+$0.001) to get a 7-day share link instead of the data inline.","url":"https://agentrxi.com/v1/crawl?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Start page"},"store":{"type":"boolean","description":"Return a 7-day share link (+$0.001)"},"max_pages":{"type":"integer","description":"1-100, default 20"},"path_prefix":{"type":"string","description":"Only follow links under this path, e.g. /docs/"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.025","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.025/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.025","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.025","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Whr61nX2ti46lTXGaTDjj","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.025","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a website starting from a given URL, following same-site links up to 100 pages, and returns each page's URL, title, and clean Markdown text","exampleAgentPrompt":"Crawl https://docs.example.com starting under the /api/ path, fetch up to 50 pages, and give me all the text as clean markdown so I can use it for RAG.","exampleUseCases":[{"title":"Index docs site for RAG","prompt":"Crawl https://docs.langchain.com starting from the root, only follow links under /docs/, and grab up to 100 pages — I need the clean markdown text of each page to build a retrieval index."},{"title":"Competitive research content pull","prompt":"Scrape https://www.competitor.com for me, following links up to 30 pages, and return each page's title and text in markdown so I can analyze what topics they cover."},{"title":"Archive site content with share link","prompt":"Crawl https://myblog.com and store the results so I get a shareable link — grab all pages you can find, up to 100, and I'll use the link to share the archive with my team."}],"resultDescription":"A list of up to 100 page objects, each containing the page URL, its title, and the full page body converted to clean Markdown text. If store=true was requested, returns a 7-day shareable link to the dataset instead of inline data.","failureModes":["Start URL is unreachable or returns non-200 status — no pages returned","robots.txt disallows crawling — pages may be skipped or crawl halted","Path prefix excludes all linked pages — only start page returned","max_pages parameter exceeds 100 — clamped or error returned","Site uses heavy JavaScript rendering — content may be incomplete or missing","Payment failure or insufficient USDC balance — 402 response, no data returned"],"whenToPreferThis":"Use this endpoint when you need to extract clean, readable text from multiple pages of a website — especially documentation sites, blogs, or wikis — rather than scraping a single URL. It is ideal for building RAG corpora, competitive research, or offline archiving when you need structured Markdown output across many linked pages without maintaining your own crawler infrastructure. Prefer it when you want to scope crawling to a specific path prefix (e.g. /docs/) and don't want to handle link-following logic yourself.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-15T06:39:28.628Z","isFirstParty":false,"canonicalSlug":"agentrxi-web-crawler-47949070"}