{"uid":"cap_8J0wTjeDDGxnTbBx40g58","slug":"tokenguard-web-crawl-endpoint-0f34eb9b","name":"Tokenguard Web Crawl Endpoint","description":"Crawl a site from a start URL (same-domain, breadth-first) and return each page as clean Markdown","url":"https://eltociear-tokenguard.hf.space/web/crawl","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","format":"uri","description":"Start URL"},"max_pages":{"type":"integer","description":"1-10 pages (default 5)"},"include_links":{"type":"boolean","description":"Keep hyperlinks (default true)"}}},"responseSchema":{"type":"json","example":{"ok":5,"pages":[{"url":"https://example.com/docs","depth":0,"title":"Docs","markdown":"# Docs…","word_count":800}],"start_url":"https://example.com/docs","pages_fetched":5}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.02","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.02/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_WhvtQtwH6V283AYBI3Dik","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.02","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls one or more web pages and returns their content as structured markdown, titles, and word counts for AI agent consumption","exampleAgentPrompt":"Can you crawl https://example.com/docs up to 5 pages deep and give me the full markdown content and titles of whatever you find?","exampleUseCases":[{"title":"Analyze competitor pricing pages","prompt":"Crawl our top three competitors' pricing pages and pull out all the text content as markdown so I can compare their plans side by side."},{"title":"Extract documentation for chatbot training","prompt":"Grab the full markdown content from our product docs site starting at https://docs.example.com, go about 3 levels deep, and collect all the titles and body text so I can use it to train our support bot."},{"title":"Monitor blockchain security advisories","prompt":"Crawl the last few pages of the security warnings section on the DeFi protocols we're invested in and extract all the markdown text so I can quickly scan for any new risks or updates."}],"resultDescription":"Returns a JSON object with an 'ok' count, a 'start_url', a 'pages_fetched' integer, and a 'pages' array where each entry contains the page URL, depth, title, markdown body, and word count.","failureModes":["Target URL is unreachable or returns non-200 status — pages array may be empty or partial","Crawl depth or page limit not respected if site blocks crawlers via robots.txt","Markdown extraction fails on heavily JavaScript-rendered pages — returns empty or incomplete markdown","Payment of $0.02 USDC not fulfilled — request rejected with payment-required error","Rate limiting if too many concurrent crawl requests are made"],"whenToPreferThis":"Use this endpoint when you need to fetch and extract readable text content from one or more web pages as structured markdown, especially for documentation, articles, or reference sites. Prefer it over generic search when you already know the URL and need the full page body rather than a snippet.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T12:44:59.724Z","isFirstParty":false}