{"uid":"cap_4jIBuDWF4UHN6dxB3ophr","slug":"agentdiet-dev-agentdiet-article-extractor-bm25-budget-controlled-web-f5d3b1c2","name":"AgentDiet Article Extractor – BM25 Budget-Controlled Web Distillation","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://agentdiet.dev/v1/article-extractor)","url":"https://agentdiet.dev/v1/article-extractor?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Ld2duAvFSmDoCfDj-edgm","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"agentdiet.dev","brandSlug":"agentdiet-dev-mpp","brandBaseUrl":"https://agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches a URL or accepts raw HTML and distills it into compact Markdown, structured JSON, a skeleton outline, or a BM25-ranked token-budget excerpt — returning compression ROI metrics alongside the content.","exampleAgentPrompt":"Distill the article at https://example.com/long-article into clean Markdown using budget mode, focused on 'machine learning infrastructure', and keep it under 300 tokens — I want to see how many tokens were saved too.","exampleUseCases":[{"title":"Token-efficient research ingestion","prompt":"Fetch https://techcrunch.com/2024/05/01/openai-new-model and distill it into structured JSON tables and metadata so I can pipe it into my research pipeline without blowing my token budget."},{"title":"Topic-focused page excerpt for RAG","prompt":"Pull the content from https://docs.stripe.com/payments/accept-a-payment and extract only the parts relevant to 'webhook event handling' in budget mode, capped at 500 tokens."},{"title":"Raw HTML cleanup to Markdown","prompt":"I have a raw HTML blob from a scraped page — can you distill it into clean Markdown so my LLM can read it without all the noise? Here's the HTML: <html><body><article>...</article></body></html>"}],"resultDescription":"Returns a JSON object with: a 'data' field containing the distilled content (Markdown text, structured JSON, skeleton outline, or BM25-ranked excerpt depending on mode); an 'roi_proof' object showing raw_tokens, distilled_tokens, tokens_saved, compression_ratio_pct, roi_multiplier, and llm_cost_saved_usd; optional cache metadata (HIT/MISS, latency_ms); optional billing info; and a status string ('ok').","failureModes":["URL unreachable or returns non-200 — distillation fails with an error status","Raw HTML is empty or malformed — may produce empty or low-quality output","max_tokens set too low (below 20) — rejected by schema validation","BM25 query omitted when mode=budget — may fall back to default behavior or return error","Payment not authorized via x402 — request blocked with 402 Payment Required","Idempotency-Key reuse across different payloads — may return cached result from prior call"],"whenToPreferThis":"Choose this endpoint when you need to extract and compress web article content before feeding it to an LLM — especially when token cost is a concern. The BM25 budget mode is uniquely suited for RAG pipelines where you need only the topically relevant portion of a long page within a strict token ceiling. Prefer this over generic scraping services when you also want compression ROI metrics (tokens saved, cost saved in USD) returned alongside the content. Use it instead of full-page scrapers when your downstream LLM context window is limited.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-23T06:34:07.036Z","isFirstParty":false,"canonicalSlug":"agentdiet-dev-agentdiet-article-extractor-bm25-budget-controlled-web-f5d3b1c2"}