{"uid":"cap_ZetjPefDmXBs3QNTosJkx","slug":"papacasper-check-robots-sitemap-5f6d15f7","name":"PapaCasper – Check Robots & Sitemap","description":"A hosted MCP server exposing utility tools any AI agent can call over HTTP — page-to-markdown, SEO audits, robots/sitemap checks, and more.","url":"https://papacasper.com/mcp/pay/check_robots_sitemap","method":"POST","headers":{},"bodySchema":{"type":"object","required":["url"],"properties":{"url":{"type":"string","description":"Any URL on the site to check (origin is derived from it)"}}},"responseSchema":{"type":"object"},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.01","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.01/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.01","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_yusC1JhqQJX4CRbXB9koO","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.01","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Fetches and analyzes the robots.txt and sitemap files for any given website URL, returning their contents and status.","exampleAgentPrompt":"Can you check what robots.txt and sitemap setup https://example.com has — I want to know if it blocks crawlers and whether a sitemap is present?","exampleUseCases":[{"title":"SEO audit for new client site","prompt":"Check the robots.txt and sitemap for https://myclientsite.com — I want to know if search engines are being blocked from any important sections and whether a sitemap is properly set up."},{"title":"Verify crawler access before scraping","prompt":"Before I try to scrape https://techblog.io, can you check their robots.txt to see if automated agents are disallowed, and whether they have a sitemap?"},{"title":"Competitive site crawlability check","prompt":"Look up the robots.txt and sitemap for https://competitor.com and tell me how they've configured crawler access — are there any disallow rules I should know about?"}],"resultDescription":"Returns the parsed contents of the site's robots.txt (including disallow rules, user-agent directives, and sitemap references) and the status/contents of any linked sitemaps, derived from the origin of the provided URL.","failureModes":["URL is unreachable or returns a non-200 status — robots.txt or sitemap may be reported as missing","Malformed URL input may cause parsing errors","Site may return a robots.txt that is unusually large or non-standard, leading to partial parse","Sitemap URL referenced in robots.txt may itself be inaccessible","Rate limiting or IP blocking by the target site may prevent fetching"],"whenToPreferThis":"Use this endpoint when you need a quick, hosted, pay-per-call check of a site's robots.txt and sitemap without setting up your own crawler infrastructure. Ideal for SEO audits, pre-scrape checks, and technical site health assessments where you only need robots/sitemap data rather than full page content.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T12:46:18.766Z","isFirstParty":false}