{"uid":"cap_O_tMvRZe1XEA3bjGO7baF","slug":"preflight-agentdiet-dev-rag-token-diet-bm25-budget-controlled-web-13ce6e72","name":"RAG Token Diet — BM25 Budget-Controlled Web Distillation","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://preflight.agentdiet.dev/v1/rag-token-diet)","url":"https://preflight.agentdiet.dev/v1/rag-token-diet?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_PFLlHB2kx-c-nc2afj824","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"preflight.agentdiet.dev","brandSlug":"preflight-agentdiet-dev-mpp","brandBaseUrl":"https://preflight.agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches and distills a URL or raw HTML into compact, token-efficient agent-ready content using BM25-ranked budget control with configurable output modes","exampleAgentPrompt":"Distill the content at https://stripe.com/docs/payments into compact markdown, using budget mode focused on the query 'payment intents API' with a max of 800 tokens, so I can fit it into my context window without wasting tokens.","exampleUseCases":[{"title":"RAG pipeline context compression","prompt":"Fetch and distill https://docs.openai.com/api-reference into markdown mode, keeping it under 600 tokens, focused on the query 'chat completions endpoint' — I need it compact enough to pass directly into my agent's context."},{"title":"Structured data extraction from product page","prompt":"Take this raw HTML from a competitor's pricing page and distill it into structured mode so I get the tables and metadata cleanly — I want to compare their pricing tiers without wading through all the bloat."},{"title":"Skeleton outline for fast document navigation","prompt":"Distill https://arxiv.org/abs/2310.06825 in skeleton mode so I just get the outline and section headers — I want to quickly understand the paper's structure before deciding which sections to read in full."}],"resultDescription":"Returns a distilled content object (Markdown, structured JSON tables/metadata, skeleton outline, or BM25-ranked budget excerpt), accompanied by ROI proof showing raw vs. distilled token counts, compression ratio percentage, tokens saved, estimated LLM cost saved in USD, and a cache hit/miss status with latency. Billing details include the agent ID and fee charged.","failureModes":["URL unreachable or returns non-200 — distillation cannot proceed, status error returned","HTML payload too small or already minimal — compression ratio near zero, roi_proof reflects minimal savings","max_tokens set below minimum of 20 — validation error","mode=budget used without providing a query — BM25 ranking cannot focus content","Target site blocks automated fetching (bot protection) — fetch failure returned","Malformed or truncated HTML input — partial distillation or parse error"],"whenToPreferThis":"Choose this endpoint when you need to reduce token consumption before passing web content to an LLM, especially in RAG pipelines where context window size and cost matter. It is ideal when you have a specific query focus (BM25 budget mode) and a hard token ceiling. Prefer it over generic scrapers when you need built-in compression metrics and ROI proof. It also supports raw HTML input, making it useful when you've already fetched content and just need distillation without an additional HTTP fetch.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-28T00:36:21.749Z","isFirstParty":false,"canonicalSlug":"preflight-agentdiet-dev-rag-token-diet-bm25-budget-controlled-web-13ce6e72"}