{"uid":"cap_IBz9bMX_et16xSkMfFEfe","slug":"arxiv-digest-agentdiet-dev-spa-markdown-reader-with-bm25-budget-control-3c8e335e","name":"SPA Markdown Reader with BM25 Budget Control","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://arxiv-digest.agentdiet.dev/v1/spa-markdown-reader)","url":"https://arxiv-digest.agentdiet.dev/v1/spa-markdown-reader?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_POslbVGS9TMAVx14CU1fx","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"arxiv-digest.agentdiet.dev","brandSlug":"arxiv-digest-agentdiet-dev-mpp","brandBaseUrl":"https://arxiv-digest.agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches and distills a URL or raw HTML into Markdown, structured JSON, skeleton outline, or a BM25-ranked token-budget excerpt","exampleAgentPrompt":"Fetch the page at https://arxiv.org/abs/2405.12345 and distill it into clean Markdown using budget mode, focused on 'transformer attention mechanisms', keeping it under 800 tokens.","exampleUseCases":[{"title":"Research paper distillation for LLM context","prompt":"Grab the page at https://arxiv.org/abs/2406.00001 and distill it into budget mode focused on 'diffusion model training', capped at 500 tokens — I want to feed just the relevant parts into my model."},{"title":"Structured data extraction from product page","prompt":"Fetch https://stripe.com/pricing and return it in structured mode so I can get the pricing tables and metadata as clean JSON."},{"title":"Skeleton outline of a long documentation page","prompt":"Pull down https://docs.python.org/3/library/asyncio.html and give me a skeleton outline of the whole page so I can see the section structure without reading everything."}],"resultDescription":"Returns a JSON object with a 'data' field containing the distilled content (Markdown text, structured tables/metadata, skeleton outline, or BM25-ranked excerpt depending on mode), plus an 'roi_proof' object showing raw_tokens, distilled_tokens, tokens_saved, compression_ratio_pct, roi_multiplier, and llm_cost_saved_usd, and an optional 'cache' object indicating HIT/MISS status.","failureModes":["URL is unreachable or returns non-200 HTTP response — endpoint may return an error status","HTML payload is malformed or empty — distillation may fail or return minimal output","Token budget too low (below 20) — rejected by schema validation","BM25 query missing when mode=budget — may return generic distillation without focus","Rate limiting or payment failure (x402) — call rejected without billing","SPA JavaScript-rendered content may not be fully captured if JS execution is required"],"whenToPreferThis":"Choose this endpoint when you need to compress web content to fit an LLM context window with measurable token savings, especially when relevance-focused extraction via BM25 is needed. Prefer it over generic scrapers when you want structured ROI metrics (tokens saved, cost saved) or multiple distillation modes (markdown, structured, skeleton, budget). Ideal for agentic pipelines that need to process many URLs cost-efficiently at $0.001/call.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-23T06:32:25.109Z","isFirstParty":false,"canonicalSlug":"arxiv-digest-agentdiet-dev-spa-markdown-reader-with-bm25-budget-control-3c8e335e"}