{"uid":"cap_I6vPAEzmtTlaclHitLV_1","slug":"sec-intel-agentdiet-dev-rag-token-diet-url-html-distiller-with-bm25-0f58484e","name":"RAG Token Diet – URL & HTML Distiller with BM25 Budget Control","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://sec-intel.agentdiet.dev/v1/rag-token-diet)","url":"https://sec-intel.agentdiet.dev/v1/rag-token-diet?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_5KyyqD1-SnOe4y1rAmsas","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"sec-intel.agentdiet.dev","brandSlug":"sec-intel-agentdiet-dev-mpp","brandBaseUrl":"https://sec-intel.agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches a URL or accepts raw HTML and distills it into compressed Markdown, structured JSON, a skeleton outline, or a BM25-ranked budget excerpt capped by a strict token limit.","exampleAgentPrompt":"Pull the content from https://stripe.com/docs/api and distill it down to at most 500 tokens using budget mode, focused on the query 'webhook event types', so I can fit it into my prompt without blowing my context window.","exampleUseCases":[{"title":"Compress product page for RAG pipeline","prompt":"Fetch https://www.anthropic.com/product and distill it into clean markdown — I want to cut the token count as much as possible before I pass it to my LLM."},{"title":"BM25-focused excerpt from a news article","prompt":"Go to https://techcrunch.com/2024/05/01/openai-gpt5-announcement/ and pull out only the parts most relevant to 'safety evaluations', keeping it under 300 tokens in budget mode."},{"title":"Structure a large HTML doc into JSON tables","prompt":"I have a raw HTML dump of a pricing table — distill it into structured mode so I get back JSON tables and metadata I can query programmatically."}],"resultDescription":"Returns a JSON object with a 'data' field containing the distilled content (Markdown text, structured JSON tables/metadata, skeleton outline, or BM25-ranked excerpt depending on mode), plus an 'roi_proof' block showing raw token count, distilled token count, tokens saved, compression ratio percentage, and estimated LLM cost saved in USD. Also includes optional cache hit/miss status and billing details.","failureModes":["URL is unreachable or returns a non-200 status — endpoint may return an error status","HTML payload is empty or malformed — distillation may return minimal or empty data","Token budget (max_tokens) set too low — may produce a very short or truncated excerpt","BM25 query missing when mode=budget — query field required for meaningful budget excerpts","Payment failure via x402 — call rejected if USDC payment cannot be processed","Rate limiting or idempotency key collision — retry with a fresh Idempotency-Key UUID"],"whenToPreferThis":"Choose this endpoint when you need to reduce the token footprint of web content before feeding it into an LLM, especially when working with large documentation pages, HTML dumps, or any URL where you want to control the output size precisely. Prefer it over generic scrapers when you need BM25-ranked relevance filtering (budget mode) or structured JSON extraction (structured mode) rather than a raw content dump. It is purpose-built for RAG pipelines and agentic workflows where LLM context and cost are constraints.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-23T06:30:59.068Z","isFirstParty":false,"canonicalSlug":"sec-intel-agentdiet-dev-rag-token-diet-url-html-distiller-with-bm25-0f58484e"}