{"uid":"cap_0yQPvK_DYIkOUwcKv3NpJ","slug":"shopify-spy-agentdiet-dev-rag-chunk-url-html-distillation-with-bm25-8e8d3b4b","name":"RAG Chunk: URL/HTML Distillation with BM25 Budget Control","description":"Distill a URL or raw HTML payload with BM25 budget control (Live x402 v2, IETF MPP & MCP endpoint at https://shopify-spy.agentdiet.dev/v1/rag-chunk)","url":"https://shopify-spy.agentdiet.dev/v1/rag-chunk?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","minLength":1,"description":"Target webpage URL to fetch and distill (or builtin://stripe-docs)"},"html":{"type":"string","description":"Optional raw HTML string to distill directly without fetching"},"mode":{"enum":["markdown","structured","skeleton","budget"],"type":"string","default":"markdown","description":"Distillation mode"},"query":{"type":"string","description":"BM25 focus query when mode=budget"},"max_tokens":{"type":"integer","minimum":20,"description":"Strict token budget ceiling when mode=budget"}}},"responseSchema":{"type":"object","required":["status","roi_proof","data"],"properties":{"data":{"type":"object","description":"Distilled Markdown, structured JSON tables/metadata, skeleton outline, or BM25 budget excerpt."},"cache":{"type":"object","properties":{"status":{"enum":["HIT","MISS"],"type":"string"},"cache_key":{"type":"string"},"latency_ms":{"type":"number"}}},"status":{"type":"string","description":"Execution status ('ok')"},"billing":{"type":"object","properties":{"agent_id":{"type":"string"},"payment_method":{"type":"string"},"fee_charged_usd":{"type":"number"}}},"roi_proof":{"type":"object","properties":{"raw_tokens":{"type":"integer"},"tokens_saved":{"type":"integer"},"roi_multiplier":{"type":"number"},"distilled_tokens":{"type":"integer"},"llm_cost_saved_usd":{"type":"number"},"compression_ratio_pct":{"type":"number"}}}}},"example":null,"exampleRequest":null,"tags":[],"displayCostAmount":"0.001","displayCostAsset":"USD","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"mpp","network":"tempo","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_oZ1qgzEjfgcPpSeg9XEUe","protocol":"mpp","methodType":"crypto","chain":"tempo","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":"shopify-spy.agentdiet.dev","brandSlug":"shopify-spy-agentdiet-dev-mpp","brandBaseUrl":"https://shopify-spy.agentdiet.dev","brandDocsUrl":null,"whatItDoes":"Fetches and distills a URL or raw HTML into compact, agent-ready tokens using BM25-ranked budget trimming with configurable output modes.","exampleAgentPrompt":"Distill the content at https://stripe.com/docs/payments into clean markdown using BM25 budget mode, focused on the query 'payment intents', and keep it under 500 tokens.","exampleUseCases":[{"title":"RAG pipeline webpage ingestion","prompt":"Fetch https://openai.com/research/gpt-4 and distill it into structured markdown so I can feed it into my RAG pipeline — keep it compact and agent-ready."},{"title":"Token-budget Q&A grounding","prompt":"I need to answer a question about 'refund policies' using this Shopify help page: https://help.shopify.com/en/manual/orders/refunds — distill it in budget mode focused on that query, max 300 tokens."},{"title":"Structured data extraction from HTML","prompt":"Here's some raw HTML from a product listing page — can you distill it into structured JSON tables and metadata so I can extract the pricing and feature rows?"}],"resultDescription":"Returns a data object containing the distilled content (markdown, structured JSON, skeleton outline, or BM25-ranked budget excerpt depending on mode), along with ROI proof metrics (raw token count, distilled token count, tokens saved, compression ratio percentage, estimated LLM cost saved in USD, ROI multiplier), optional cache status (HIT/MISS), and billing details.","failureModes":["URL is unreachable or returns non-200 status — distillation fails with fetch error","HTML payload is malformed or empty — parser may return minimal or empty output","Token budget (max_tokens) set too low to capture meaningful content","BM25 query yields no matching chunks within budget — returns minimal excerpt","Rate limit or payment failure via x402 — returns 402 Payment Required","Invalid mode enum value — request rejected with validation error"],"whenToPreferThis":"Choose this endpoint when you need to ingest web pages or HTML into an LLM pipeline while minimizing token consumption. It is particularly valuable when you have a specific focus query and want BM25-ranked relevance filtering, or when you need to stay within a strict token budget. Prefer this over raw fetch-and-parse approaches when token cost savings and compression ratio matter, or when you want structured/skeleton output modes rather than raw HTML.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-23T06:34:11.525Z","isFirstParty":false,"canonicalSlug":"shopify-spy-agentdiet-dev-rag-chunk-url-html-distillation-with-bm25-8e8d3b4b"}