{"uid":"cap_YYNjk9j65DfPzszFNBQHa","slug":"token-intel-agentdiet-dev-paywall-clean-url-html-distiller-with-bm25-29788716","name":"Paywall-Clean URL & HTML Distiller with BM25 Budget Control","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://token-intel.agentdiet.dev/v1/paywall-clean-extract)","url":"https://token-intel.agentdiet.dev/v1/paywall-clean-extract?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_LpNYd_bALFh91eookO-X5","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"token-intel.agentdiet.dev","brandSlug":"token-intel-agentdiet-dev-mpp","brandBaseUrl":"https://token-intel.agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches and distills a URL or raw HTML into compressed Markdown, structured JSON, a skeleton outline, or a BM25-ranked budget excerpt, returning token-savings ROI proof.","exampleAgentPrompt":"Fetch https://stripe.com/docs/payments/checkout and distill it into clean markdown using budget mode, focusing on 'payment intent creation', keeping it under 800 tokens — and show me how many tokens you saved.","exampleUseCases":[{"title":"Token-efficient research pipeline","prompt":"Pull the content from https://techcrunch.com/2024/06/01/openai-news/ and distill it into clean markdown, keeping it under 600 tokens — I want to know how much token cost this saves me compared to feeding the raw page."},{"title":"BM25 focused excerpt for RAG","prompt":"Distill https://docs.python.org/3/library/asyncio.html in budget mode with query 'event loop run coroutine' and cap it at 500 tokens so I only get the most relevant passages."},{"title":"Structured data extraction from HTML","prompt":"I have raw HTML from a product listing page — distill it in structured mode to pull out all the tables and metadata as JSON so I can pipe it into my database."}],"resultDescription":"Returns a JSON object with a 'data' field containing the distilled content (Markdown string, structured JSON tables/metadata, skeleton outline, or BM25-ranked excerpt depending on mode), a 'roi_proof' object showing raw_tokens, distilled_tokens, tokens_saved, compression_ratio_pct, roi_multiplier, and llm_cost_saved_usd, plus optional cache hit/miss status and billing details.","failureModes":["URL is unreachable or returns non-200 → fetch error with status detail","Paywall blocks content fetch → partial or empty distillation","max_tokens set too low for meaningful content → truncated output or error","Invalid mode enum value → 400 validation error","HTML input is malformed or empty → empty or degraded distillation","BM25 query not provided when mode=budget → fallback or error","Rate limit or payment failure via x402 → 402 Payment Required"],"whenToPreferThis":"Choose this endpoint when you need to feed webpage content to an LLM but want to minimize token consumption and cost — especially useful for long documentation pages, paywalled articles, or any HTML where only the topically relevant sections matter. The BM25 budget mode is uniquely suited for RAG pipelines where you need only the highest-relevance passages within a strict token ceiling. Prefer this over generic scraping endpoints when ROI proof (tokens saved, cost saved) is important for auditing or optimization workflows.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-23T06:34:29.370Z","isFirstParty":false,"canonicalSlug":"token-intel-agentdiet-dev-paywall-clean-url-html-distiller-with-bm25-29788716"}