{"uid":"cap_lhxheRC8lY7Ua_mEGEWR4","slug":"crawlspur-meta-robots-robots-file-checker-895fd0a0","name":"CrawlSpur Meta-Robots Robots-File Checker","description":"Prüft am Objekt robots-datei den Befund Wert der Meta-Robots-Angabe.","url":"https://crawlspur.halowerk.com/v1/pruef/crawlspur/meta-robots/robots-datei","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"ziel":{"oneOf":[{"type":"string","maxLength":2048,"minLength":1,"description":"Öffentliche HTTP(S)-Adresse; bei DNS-Befunden Domain oder öffentliche IP."},{"type":"object","required":["adresse"],"properties":{"adresse":{"type":"string","maxLength":2048,"minLength":1},"selektor":{"type":"string","maxLength":240,"minLength":1,"description":"CSS-Selektor: genau ein Objekt, sonst erster Treffer des Objektfilters."},"vergleich":{"type":"string","maxLength":2048,"description":"Öffentliche Vergleichsadresse für Link-, Sitemap- oder Sprachbefunde."},"user_agent":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,80}$"},"dkim_selektor":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,63}$"}},"additionalProperties":false}]}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_5Oe93QCNASjFP1amtavJv","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks the meta-robots directive value found in a website's robots.txt file object","exampleAgentPrompt":"Check the meta-robots value in the robots.txt file for https://example.com and tell me what crawl directives are declared there.","exampleUseCases":[{"title":"SEO audit of robots file directives","prompt":"Can you inspect the robots.txt file at https://myshop.com and tell me what meta-robots values are set there? I want to know if any indexing restrictions are declared."},{"title":"Diagnosing unexpected de-indexing","prompt":"My site https://brand-site.de seems to have dropped from Google. Can you check the robots.txt file for any meta-robots directives that might be blocking crawlers?"},{"title":"Verifying staging site isolation","prompt":"We launched a staging environment at https://staging.company.com — can you pull the robots.txt and check what meta-robots value is set to make sure it's blocking search engines properly?"}],"resultDescription":"Returns the finding of the meta-robots directive value as declared in the target site's robots.txt file, allowing the caller to understand what crawl or indexing instructions are present in the robots file object.","failureModes":["Target URL is unreachable or returns non-200 status — endpoint may return an error or empty finding","robots.txt file does not exist on the target domain — no meta-robots value can be found","Malformed or invalid URL provided — validation error returned","CSS selector matches no element or multiple elements — first match used or error returned","Private/non-public IP provided — rejected due to public address requirement"],"whenToPreferThis":"Use this endpoint when you specifically need to extract or validate the meta-robots directive value from a website's robots.txt file as a distinct object. Prefer this over general robots.txt parsers when you need a structured finding for the robots-file object specifically (as opposed to pagination, image, or URL-parameter contexts), and when integrating into a broader SEO audit pipeline via the CrawlSpur API family.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T06:44:06.790Z","isFirstParty":false}