{"uid":"cap_jj7QxOaGI_DoP4IIZftrt","slug":"agishub-crawl-map-930c016e","name":"AGISHub Crawl Map","description":"Discover all URLs reachable from a domain within a link depth limit. Returns a flat list of absolute URLs, respects robots.txt crawl delays. Use it to map a site's structure before crawling.","url":"https://api.agishub.com/v1/crawl-map","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input","output"],"properties":{"input":{"type":"object","required":["type","method","queryParams"],"properties":{"type":{"enum":["http"],"type":"string"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["url"],"properties":{"url":{"type":"string","format":"uri","description":"Root domain URL to map (e.g., https://example.com)."},"limit":{"type":"integer","maximum":200,"minimum":0,"description":"Maximum URLs to return (default 100, max 200 per call). Respects robots.txt crawl-delay.","exclusiveMinimum":true},"search":{"type":"string","description":"Optional regex or plain string to filter results (case-insensitive)."},"include_subdomains":{"type":"boolean","description":"Include URLs from subdomains (default false, same domain only)."}},"additionalProperties":false}},"additionalProperties":false},"output":{"type":"object","required":["type","example"],"properties":{"type":{"enum":["json"],"type":"string"},"example":{"type":"object"}},"additionalProperties":false}}},"responseSchema":{"type":"json","example":{"result":"Discover all URLs reachable from a domain within a link depth limit. Returns a flat list of absolute URLs, respects robo"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.05","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.05/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.05","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.05","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_-XfM4ispTiTDEE_MwMdb4","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.05","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Discovers and returns all URLs reachable from a root domain up to a configurable link depth, with optional regex filtering and subdomain inclusion.","exampleAgentPrompt":"Can you map out all the URLs reachable from https://example.com, up to 150 URLs, and only include ones that match '/blog'?","exampleUseCases":[{"title":"Pre-scrape site structure discovery","prompt":"Before I scrape docs.mycompany.com, can you give me a full map of all reachable URLs on that domain — up to 200 — so I know what pages exist?"},{"title":"Blog post URL enumeration","prompt":"Find all URLs on https://techcrunch.com that contain '/article' in the path, up to 100 results, so I can build a list of articles to process."},{"title":"Subdomain-inclusive site audit","prompt":"Map every URL reachable from https://acme.com, including subdomains, up to 200 URLs — I need the full picture for an SEO audit."}],"resultDescription":"A flat JSON list of absolute URLs discovered by following links from the root domain, up to the specified limit. Results respect robots.txt crawl delays and can be optionally filtered by a regex/string pattern and optionally expanded to include subdomains.","failureModes":["Invalid or unreachable root URL returns an error response","Limit of 0 or exceeding 200 returns a validation error","Robots.txt disallowing crawl may result in empty or partial results","Regex pattern syntax errors may cause filtering to fail","Sites with JavaScript-only navigation may return fewer URLs than expected"],"whenToPreferThis":"Choose this endpoint when you need to enumerate URLs across an entire domain before crawling individual pages, or when you need a sitemap-like structure without access to an XML sitemap. It is best suited for pre-crawl reconnaissance, SEO audits, or building structured link graphs. Prefer this over single-page scraping when the goal is discovery rather than content extraction, and over sitemap parsing when no sitemap.xml is available or when you need to verify actually reachable links.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T18:53:34.711Z","isFirstParty":false}