{"uid":"cap_ULnt92GVEtriNmnuoBtSc","slug":"dexl-agents-site-crawl-8f889977","name":"DexL Agents Site Crawl","description":"Crawl a website: fetch a page, follow the links inside the same host and return every page it reached, within a page budget you set. For reading documentation, a product catalogue or a changelog in one paid call instead of many.","url":"https://agents.dexl.io/v1/tools/site-crawl","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","description":"Absolute http(s) URL to start from"},"max_pages":{"type":"integer","default":3,"maximum":10,"minimum":1}}},"responseSchema":{"type":"json","example":{"units":1,"cached":false,"output":{}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_7KbsLuZkeMl_-ImwoouuN","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a website starting from a given URL, follows internal links, and returns the content of every page reached within a configurable page budget.","exampleAgentPrompt":"Can you crawl the docs site at https://docs.example.com starting from the homepage and fetch up to 8 pages, following only links on the same host, and give me all the content?","exampleUseCases":[{"title":"Read full documentation site at once","prompt":"Crawl https://docs.stripe.com/api and follow up to 10 internal links so I can read the full API docs in one go without fetching each page separately."},{"title":"Summarize a product changelog","prompt":"Fetch https://www.linear.app/changelog and follow up to 5 pages so I can see all recent changelog entries and summarize what's changed in the last few releases."},{"title":"Audit competitor product catalogue","prompt":"Crawl https://www.competitor.com/products — follow all internal product links, up to 8 pages — and return the content so I can see their full product lineup."}],"resultDescription":"A JSON object containing the crawl output: the content of every page reached (up to the max_pages budget), along with metadata such as whether the result was cached and how many units were consumed.","failureModes":["URL is unreachable or returns non-200 status — crawl may return partial results or an error","max_pages budget exhausted before all linked pages are visited — only pages within budget are returned","Target site blocks crawlers via robots.txt or rate limiting — may return fewer pages or empty body","Non-absolute or malformed URL provided — request rejected with validation error","Pages behind authentication or paywalls cannot be accessed — content will be missing or gated"],"whenToPreferThis":"Use this endpoint when you need to read multiple linked pages from a single host in one API call rather than making many individual fetch requests. It is ideal for ingesting documentation sites, product catalogues, changelogs, or small wikis where following internal links is essential. Prefer it over single-page fetch when you need breadth across a site within a controlled page budget.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-13T18:32:29.332Z","isFirstParty":false}