{"uid":"cap_OgCYThq_y5n9qIYD2XQaG","slug":"kihustle-robots-txt-parser-api-67ae8340","name":"KiHustle Robots.txt Parser API","description":"Kostenlose Guides, Solo-Playbooks und Artikel zu KI, Automation und Side Hustles — für Menschen, die mit echten Systemen online Einkommen aufbauen wollen. Transparent finanziert über faire Affiliate-Links.","url":"https://kihustle.tech/nexus/api/v1/robots-parser?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"text":{"type":"string"}}},"responseSchema":{"type":"json","example":{"result":"processed","status":"success"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_m3wC56ti5zAnF8r-V5m3u","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Parses and processes robots.txt text content, returning structured results about crawler directives","exampleAgentPrompt":"Can you parse this robots.txt content for me and tell me what crawling rules and directives it contains: 'User-agent: * Disallow: /admin/ Allow: / Sitemap: https://example.com/sitemap.xml'?","exampleUseCases":[{"title":"SEO audit of site crawl rules","prompt":"I have the robots.txt content from a competitor's site — can you parse it and tell me which paths are blocked from crawlers and what sitemaps are listed? Here it is: 'User-agent: Googlebot Disallow: /private/ Allow: / Crawl-delay: 10'"},{"title":"Validate robots.txt before deployment","prompt":"Before I push this robots.txt live, can you parse it and check that the directives are correct and nothing important is accidentally blocked? Content: 'User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Sitemap: https://mysite.com/sitemap.xml'"},{"title":"Extract allowed paths for web scraper","prompt":"I'm building a scraper and need to know which URLs are allowed for my bot. Can you parse this robots.txt text and tell me what's permitted and what's off-limits: 'User-agent: Mybot Disallow: /checkout/ Disallow: /account/ Allow: /products/'"}],"resultDescription":"Returns a JSON object with a 'result' field indicating the processed output of the robots.txt parsing and a 'status' field confirming success. The parsed data includes structured information about crawler directives extracted from the provided text.","failureModes":["Empty or missing text input returns an error or empty result","Malformed or non-standard robots.txt syntax may produce incomplete parsing","Very large text inputs may time out or be truncated","Non-robots.txt text passed as input may yield unexpected or meaningless results"],"whenToPreferThis":"Use this endpoint when you need a lightweight, pay-per-call robots.txt parsing service without standing up your own parser. It is suitable for one-off checks, SEO audits, or agent workflows that occasionally need to interpret robots.txt directives without integrating a full crawling library.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-21T18:56:28.965Z","isFirstParty":false,"canonicalSlug":"kihustle-robots-txt-parser-api-b4927c34"}