{"uid":"cap_ArNm-6EAZ0R8ISwoSYmm9","slug":"apiacre-robots-sitemap-inspector-c904c46f","name":"APIAcre Robots & Sitemap Inspector","description":"Inspect a domain's robots.txt crawl rules and XML sitemap health, availability, and URL count.","url":"https://apiacre.com/v1/web/robots-sitemap","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","title":"Url","format":"uri","maxLength":2083,"minLength":1}}},"responseSchema":{"type":"json","example":{"data":{"origin":"https://example.com","robots":{"rules":0,"sitemaps":[],"available":false,"statusCode":404,"contentType":"text/html"},"sitemap":{"bytes":0,"urlCount":0,"available":false,"statusCode":404,"contentType":"text/html"}},"meta":{"cached":false,"sources":[],"warnings":[],"duration_ms":42,"next_actions":[]},"service":"web.robots-sitemap","version":"1","request_id":"018f1f54-7f38-7ba2-8dc3-5f90272d9f1a"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.025","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.025/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.025","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.025","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_IfgQcNxE8wyTgw6tOAybY","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.025","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and parses a website's robots.txt crawl rules and checks for sitemap availability at a given URL.","exampleAgentPrompt":"Can you check the robots.txt and sitemap availability for example.com — I want to know which paths are disallowed and whether there's a sitemap we can use?","exampleUseCases":[{"title":"Pre-scrape crawl rule audit","prompt":"Before I start scraping shopify.com, can you pull its robots.txt and tell me which paths are off-limits and whether there's a sitemap listed?"},{"title":"SEO sitemap discovery","prompt":"I'm doing an SEO audit for stripe.com — can you check if they have a sitemap available and what their crawl rules look like?"},{"title":"Bot access verification","prompt":"I need to verify that our new partner site allows our crawler bot — can you inspect the robots.txt for partner-portal.io and tell me which user agents are blocked?"}],"resultDescription":"Returns parsed robots.txt crawl directives including allowed and disallowed paths per user agent, crawl delay settings, and the availability and URLs of any declared sitemaps for the target domain.","failureModes":["Target domain unreachable or returns non-200 status for robots.txt","Robots.txt missing or malformed — partial or empty parse result","Sitemap declared in robots.txt but URL itself returns 404","Rate limiting or bot blocking by target domain preventing fetch","Invalid or malformed input URL"],"whenToPreferThis":"Use this endpoint when you need a structured, parsed view of a site's robots.txt and sitemap availability without writing your own HTTP fetch and parsing logic. It is ideal for pre-scrape compliance checks, SEO audits, and automated crawl planning workflows where you need machine-readable crawl rules quickly.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-13T12:52:33.166Z","isFirstParty":false}