{"uid":"cap_n0SQtfvR1sCL_yiQE9NgM","slug":"17711-web-scraper-async-url-fetch-fa7086c5","name":"17711 Web Scraper – Async URL Fetch","description":"A fleet of independent, x402-payable microservices callable by AI agents.","url":"https://api.17711.xyz/registry/web-scraper/v1/web-scraper/scrape_url_async","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"The target URL to fetch, e.g. `\"https://example.com/page\"`. Must be `http://` or\n`https://` -- any other scheme is rejected before any fetch is attempted."},"callback_url":{"type":"string","description":"When set, the worker that processes this task POSTs the eventual result (a `TaskResult`\nJSON body) here, best-effort -- polling `web_scraper_get_task_status` always works too,\nregardless of whether this is set."}}},"responseSchema":{"task_id":"string"},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.005","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.005/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.005","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_0pBmYz5YU5uBoztkV1mIy","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.005","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Asynchronously fetches and scrapes a given HTTP/HTTPS URL, returning a task ID for polling or webhook delivery of the result","exampleAgentPrompt":"Asynchronously scrape the page at https://example.com/article and give me a task ID so I can check back for the result — let me know when it's ready.","exampleUseCases":[{"title":"Monitor competitor pricing pages","prompt":"Scrape https://competitor.com/pricing in the background and poll for the result so I can compare their latest prices to ours."},{"title":"Archive a news article before it disappears","prompt":"Fetch the content of https://news-site.com/breaking-story-2024 asynchronously and let me know the task ID so I can retrieve the full HTML once it's done."},{"title":"Extract product details from an e-commerce listing","prompt":"Can you scrape https://shop.example.com/product/12345 in the background and send the result to https://my-webhook.example.com/callback when it's finished?"}],"resultDescription":"Returns a JSON object containing a `task_id` string that can be used to poll the task status endpoint (`web_scraper_get_task_status`) or receive the full scraped result via the optional callback URL when processing completes.","failureModes":["Non-HTTP/HTTPS URL scheme provided — rejected before fetch attempt","Target URL is unreachable or returns an error status (captured in task result)","Task ID returned but result never arrives if callback URL is misconfigured or unreachable","Rate limiting or payment failure via x402 protocol if USDC balance is insufficient","Long-running pages may time out on the worker side"],"whenToPreferThis":"Choose this endpoint when you need to fetch a web page without blocking your workflow — ideal when the page may take time to load, when you want webhook delivery of results, or when you need to fan out many URL fetches and poll for completion. Prefer this over synchronous alternatives when scraping slow or large pages.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-15T06:35:25.061Z","isFirstParty":false}