{"uid":"cap_7qP5qLpNsTquydBJqAbMH","slug":"arxiv-digest-agentdiet-dev-semantic-html-pruner-web-url-distiller-e19847b4","name":"Semantic HTML Pruner – Web URL Distiller","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://arxiv-digest.agentdiet.dev/v1/semantic-html-pruner)","url":"https://arxiv-digest.agentdiet.dev/v1/semantic-html-pruner?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_NFD0ke_EMlXd6yJDBrRo3","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"arxiv-digest.agentdiet.dev","brandSlug":"arxiv-digest-agentdiet-dev-mpp","brandBaseUrl":"https://arxiv-digest.agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches a webpage (or accepts raw HTML) and distills it into compact, agent-ready tokens using BM25 budget control and multiple extraction modes","exampleAgentPrompt":"Distill the content at https://arxiv.org/abs/2401.00001 into compact markdown using budget mode, focusing on 'transformer architecture' and keeping it under 500 tokens.","exampleUseCases":[{"title":"Research paper compression for LLM pipeline","prompt":"Fetch and distill https://arxiv.org/abs/2310.06825 into markdown mode so I can feed it to my summarization pipeline without blowing my token budget."},{"title":"Budget-constrained topic-focused extraction","prompt":"Pull the page at https://stripe.com/docs/payments and give me only the parts about 'refund processing' using budget mode with a max of 300 tokens."},{"title":"Structured data extraction from e-commerce page","prompt":"Distill https://example-shop.com/product/widget into structured mode so I can extract the product tables and pricing metadata as JSON."}],"resultDescription":"Returns a JSON object with a 'data' field containing the distilled content (Markdown, structured JSON, skeleton outline, or BM25-ranked excerpt depending on mode), plus an 'roi_proof' object showing raw vs distilled token counts, compression ratio percentage, tokens saved, and estimated LLM cost saved in USD. Also includes cache hit/miss status and billing details.","failureModes":["Invalid or unreachable URL returns an error status","mode=budget without a query parameter may return unfiltered content","max_tokens set below minimum (20) is rejected","Raw HTML too large may hit processing limits","Paywalled or bot-blocked pages may return empty or partial content","Missing required 'url' parameter returns a validation error"],"whenToPreferThis":"Choose this endpoint when an AI agent needs to process web content within token budget constraints, especially before passing content to an LLM to reduce inference costs. Particularly useful when you need BM25-focused extraction (budget mode) to retrieve only the most relevant passages for a specific query, or when you need structured JSON tables or a skeleton outline rather than raw HTML. Prefer over generic scrapers when token economy and ROI tracking (compression ratio, cost savings) matter.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-21T07:23:19.607Z","isFirstParty":false,"canonicalSlug":"arxiv-digest-agentdiet-dev-semantic-html-pruner-web-url-distiller-e19847b4"}