{"uid":"cap_UPHtMEyEULcqlS0UZ8xfd","slug":"kihustle-robots-parser-38c9f810","name":"KiHustle Robots Parser","description":"Kostenlose Guides, Solo-Playbooks und Artikel zu KI, Automation und Side Hustles — für Menschen, die mit echten Systemen online Einkommen aufbauen wollen. Transparent finanziert über faire Affiliate-Links.","url":"https://kihustle.tech/agents/api/v1/robots-parser?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"text":{"type":"string"}}},"responseSchema":{"type":"json","example":{"result":"processed","status":"success"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_hmFqI5Wx-Sxi2jFa5gDzk","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Parses and processes robots.txt rules or related web crawling directives submitted as text","exampleAgentPrompt":"Can you parse this robots.txt content and tell me what crawl rules it defines: 'User-agent: * Disallow: /admin/ Disallow: /private/ Allow: /'?","exampleUseCases":[{"title":"SEO audit of site crawl rules","prompt":"I have this robots.txt content from a competitor's site — can you parse it and tell me which paths are blocked from crawlers?"},{"title":"Validate robots.txt before deploying","prompt":"Here's the robots.txt I'm about to publish for my site — can you process it and confirm the syntax is valid and the rules make sense?"},{"title":"Extract disallow paths for scraping check","prompt":"Parse this robots.txt text for me and list all the disallow rules so I know which paths I shouldn't be scraping."}],"resultDescription":"Returns a JSON object with a 'result' field indicating the processed output of the submitted robots.txt or crawl directive text, along with a 'status' field confirming success. The parsed structure reflects the rules extracted from the submitted text.","failureModes":["Empty or missing 'text' field returns an error or empty result","Malformed input text may produce unexpected or minimal parsed output","Large robots.txt files may exceed input size limits","Non-robots.txt content submitted may yield meaningless parsed output"],"whenToPreferThis":"Choose this endpoint when you need a quick, low-cost programmatic way to parse or validate robots.txt content. Useful for SEO tools, web scraping agents that need to respect crawl rules, or any workflow that must interpret robots.txt directives without building a custom parser.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-22T12:41:48.093Z","isFirstParty":false,"canonicalSlug":"kihustle-robots-parser-38c9f810"}