{"uid":"cap_qciETYSJhvfpoFsYHlW_a","slug":"whale-momentum-agentdiet-dev-web-distillation-with-bm25-budget-control-19db5b00","name":"Web Distillation with BM25 Budget Control","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://whale-momentum.agentdiet.dev/v1/rag-chunk)","url":"https://whale-momentum.agentdiet.dev/v1/rag-chunk?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_MuR8h_fiAdksJ_qEEYE4u","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"whale-momentum.agentdiet.dev","brandSlug":"whale-momentum-agentdiet-dev-mpp","brandBaseUrl":"https://whale-momentum.agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches and distills a URL or raw HTML into compressed Markdown, structured JSON, skeleton, or BM25-focused budget excerpts with token savings proof","exampleAgentPrompt":"Distill the page at https://stripe.com/docs/payments into clean Markdown using budget mode, focused on 'payment intents', and keep it under 500 tokens — I want to see how many tokens you saved.","exampleUseCases":[{"title":"RAG pipeline webpage compression","prompt":"Fetch and distill https://openai.com/research/gpt-4 into clean Markdown for my RAG system — I want a highly compressed version with token savings stats so I know how much I'm saving on LLM costs."},{"title":"BM25-focused budget excerpt for research","prompt":"Pull the content from https://arxiv.org/abs/2310.06825 and give me only the parts most relevant to 'chain-of-thought reasoning', keeping the output under 300 tokens using budget mode."},{"title":"Structured data extraction from HTML","prompt":"I have this raw HTML from a product page — can you distill it into structured JSON tables extracting the pricing tiers, features, and metadata? Use structured mode."}],"resultDescription":"Returns a JSON object with a 'data' field containing distilled content (Markdown text, structured JSON tables/metadata, skeleton outline, or BM25-ranked budget excerpt depending on mode), plus an 'roi_proof' object showing raw_tokens, distilled_tokens, tokens_saved, compression_ratio_pct, roi_multiplier, and llm_cost_saved_usd. Also includes optional cache hit/miss status and billing details.","failureModes":["URL is unreachable or returns non-200 — fetch failure error","HTML or URL content is empty or too short to distill","max_tokens set below 20 triggers validation error","BM25 query missing when mode=budget — may return unranked or full output","Paywalled or JavaScript-heavy pages may return incomplete content","Payment failure via x402 if USDC balance insufficient"],"whenToPreferThis":"Choose this endpoint when you need to feed webpage content into an LLM pipeline cost-efficiently, especially when token budget is a concern. It is ideal for RAG indexing, agent research tasks, or any workflow that needs clean structured or Markdown text from URLs or raw HTML. The BM25 budget mode is uniquely useful when you want only the most query-relevant content within a strict token ceiling, which no generic scraper provides. Prefer over raw scrapers when you need compression metrics and ROI proof.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-23T06:34:25.248Z","isFirstParty":false,"canonicalSlug":"whale-momentum-agentdiet-dev-web-distillation-with-bm25-budget-control-847a3290"}