{"uid":"cap_pU-a-c28Wp7HsEeb0tPzT","slug":"agentdiet-rag-diet-structured-web-scraper-bfb91008","name":"AgentDiet RAG-Diet Structured Web Scraper","description":"RAG-Diet Structured Web Page & Table Scraper ($0.001/page) — Extracts clean Markdown, JSON-LD, and tables from any URL.","url":"https://rag-diet.agentdiet.dev/v1/web-scrape?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET","POST"],"type":"string"},"queryParams":{"type":"object","properties":{"url":{"type":"string"},"mode":{"type":"string"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"responseSchema":{"type":"json","example":{"data":{},"status":"ok","roi_proof":{"compression_ratio_pct":91.4}}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_majDjZDvbS8MBKxvFbEQT","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches a URL and returns clean Markdown, JSON-LD, and extracted tables suitable for RAG pipelines and AI context windows.","exampleAgentPrompt":"Scrape the page at https://en.wikipedia.org/wiki/List_of_countries_by_GDP and give me the clean markdown and any tables from it so I can feed them into my RAG pipeline.","exampleUseCases":[{"title":"Extract pricing table from competitor site","prompt":"Can you scrape https://www.competitor.com/pricing and pull out the pricing table as structured data so I can compare their tiers?"},{"title":"Feed news article into RAG knowledge base","prompt":"Fetch the article at https://techcrunch.com/2024/01/15/openai-funding-round and convert it to clean markdown so I can add it to my knowledge base."},{"title":"Extract JSON-LD product data from e-commerce page","prompt":"Scrape https://www.amazon.com/dp/B09XYZ1234 and give me the JSON-LD structured data so I can pull out the product details automatically."}],"resultDescription":"Returns a JSON object with a 'data' field containing the extracted content (clean Markdown, JSON-LD, and tables), a 'status' field indicating success, and a 'roi_proof' object showing the compression ratio achieved (e.g. 91.4% reduction in token count vs raw HTML).","failureModes":["URL is unreachable or returns non-200 HTTP status — scrape fails with error","Paywalled or login-required page — returns empty or partial content","JavaScript-heavy SPA pages may not render dynamic content","Rate limiting or bot detection on target site blocks the fetch","Malformed URL input causes request rejection","USDC payment failure via x402 protocol blocks the request"],"whenToPreferThis":"Choose this endpoint when you need token-efficient, LLM-ready content from a URL — especially for RAG pipelines where clean Markdown and structured JSON-LD matter. At $0.001/page it is cost-effective for bulk document ingestion. Prefer this over generic HTTP fetchers when you need table extraction and JSON-LD parsing alongside markdown, or when you need the compression ratio metric to track context window efficiency.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-22T12:43:05.874Z","isFirstParty":false,"canonicalSlug":"agentdiet-rag-diet-structured-web-scraper-bfb91008"}