{"uid":"cap_SZDVuzsO9-5c0-liSt4Bc","slug":"arxiv-digest-agentdiet-dev-web-url-distillation-to-agent-ready-tokens-743ce045","name":"Web URL Distillation to Agent-Ready Tokens","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://arxiv-digest.agentdiet.dev/v1/pdf-extract)","url":"https://arxiv-digest.agentdiet.dev/v1/pdf-extract?utm_source=zero.xyz","method":"GET","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Ega6J3n2ofXRMXA49W83p","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"arxiv-digest.agentdiet.dev","brandSlug":"arxiv-digest-agentdiet-dev-mpp","brandBaseUrl":"https://arxiv-digest.agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches and compresses a webpage URL (or raw HTML) into compact, token-efficient representations using BM25 budget control and multiple distillation modes","exampleAgentPrompt":"Distill the content at https://en.wikipedia.org/wiki/Large_language_model into compact markdown, keeping it under 500 tokens and focusing the BM25 query on 'training data and parameters'.","exampleUseCases":[{"title":"Research paper budget extraction","prompt":"Grab the content from this arXiv paper URL https://arxiv.org/abs/2301.00001 and compress it into budget mode with a max of 300 tokens, focused on the query 'attention mechanism results'."},{"title":"Structured data table extraction","prompt":"Fetch https://stripe.com/pricing and extract it in structured mode so I get the pricing tables and metadata as JSON — I need to compare plan tiers without reading the whole page."},{"title":"Skeleton outline for navigation","prompt":"Give me a skeleton outline of https://docs.openai.com/api-reference so I can see the high-level structure of the docs without pulling in all the content."}],"resultDescription":"Returns a distilled version of the webpage in the requested mode (Markdown, structured JSON with tables/metadata, skeleton outline, or BM25 budget excerpt), along with an ROI proof object showing raw vs distilled token counts, compression ratio, tokens saved, estimated LLM cost saved, and billing details. Cache hit/miss status is also included.","failureModes":["Invalid or unreachable URL returns an error status","HTML fetch timeout for slow or blocked sites","Token budget too small (max_tokens below 20) rejected with validation error","BM25 query missing when mode=budget, may return unfiltered content or error","Payment failure via x402 protocol blocks the request","Malformed raw HTML input may produce poor distillation quality"],"whenToPreferThis":"Choose this endpoint when an AI agent needs to read a webpage but must stay within a token budget, or when you need structured extraction (tables, metadata) or a skeleton outline rather than raw HTML. It is especially useful when LLM context costs are a concern and you want a compression ROI proof. Prefer this over generic scrapers when you need BM25-focused content filtering or multiple distillation modes.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-21T07:23:31.940Z","isFirstParty":false,"canonicalSlug":"arxiv-digest-agentdiet-dev-web-url-distillation-to-agent-ready-tokens-743ce045"}