{"uid":"cap_hglv6ATY7VaBxLHPZQPM6","slug":"agishub-crawl-map-dd4d5222","name":"AGISHub Crawl Map","description":"Discover all URLs reachable from a domain within a link depth limit. Returns a flat list of absolute URLs, respects robots.txt crawl delays. Use it to map a site's structure before crawling.","url":"https://api.agishub.com/paid/crawl-map","method":"POST","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input","output"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri","description":"Root domain URL to map (e.g., https://example.com)."},"limit":{"type":"integer","maximum":200,"minimum":0,"description":"Maximum URLs to return (default 100, max 200 per call). Respects robots.txt crawl-delay.","exclusiveMinimum":true},"search":{"type":"string","description":"Optional regex or plain string to filter results (case-insensitive)."},"include_subdomains":{"type":"boolean","description":"Include URLs from subdomains (default false, same domain only)."}},"additionalProperties":false},"type":{"enum":["http"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json"],"type":"string"}},"additionalProperties":false},"output":{"type":"object","required":["type","example"],"properties":{"type":{"enum":["json"],"type":"string"},"example":{"type":"object"}},"additionalProperties":false}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.05","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.05/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.05","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.05","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_gQHusWRtWfwXWYVhWCqDx","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.05","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Discovers all URLs reachable from a domain within a configurable link depth, returning a flat list of absolute URLs while respecting robots.txt crawl delays.","exampleAgentPrompt":"Can you map all the URLs reachable from https://example.com — up to 150 URLs — and only include ones that match the pattern '/blog'?","exampleUseCases":[{"title":"Pre-crawl site structure discovery","prompt":"Before I start scraping docs.mycompany.com, can you map all the URLs reachable from there — up to 200 — so I know what pages exist?"},{"title":"Blog post inventory for content audit","prompt":"Give me a list of all URLs on https://techcrunch.com that contain '/2024/' in the path, up to 100 results, so I can audit recent articles."},{"title":"Subdomain-aware full site mapping","prompt":"Map all URLs reachable from https://stripe.com including subdomains, up to 200 results, so I can understand the full scope of their web presence."}],"resultDescription":"A flat JSON list of absolute URLs reachable from the given root domain, filtered by the optional regex pattern, capped at the specified limit (default 100, max 200). The crawler respects robots.txt crawl delays during discovery.","failureModes":["Invalid or unreachable root URL returns an error","Limit of 0 or above 200 is rejected by schema validation","Regex filter syntax errors may return no results or an error","Sites blocking crawlers via robots.txt may yield fewer URLs than the limit","Network timeouts on slow or large sites may truncate results"],"whenToPreferThis":"Use this endpoint when you need to discover the full URL structure of a website before targeted scraping or content extraction. It is ideal for pre-crawl reconnaissance, sitemap generation, and finding pages matching a URL pattern. Prefer this over manual link-following or a full page-fetch endpoint when you need breadth-first link discovery across an entire domain with optional subdomain inclusion.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T00:44:39.973Z","isFirstParty":false}