{"uid":"cap_KZayDg60i5OhU1Kw58Pr8","slug":"crawlspur-meta-robots-url-regel-checker-b0deec55","name":"Crawlspur Meta-Robots URL-Regel Checker","description":"Prüft am Objekt url-regel den Befund Wert der Meta-Robots-Angabe.","url":"https://crawlspur.halowerk.com/v1/pruef/crawlspur/meta-robots/url-regel","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"ziel":{"oneOf":[{"type":"string","maxLength":2048,"minLength":1,"description":"Öffentliche HTTP(S)-Adresse; bei DNS-Befunden Domain oder öffentliche IP."},{"type":"object","required":["adresse"],"properties":{"adresse":{"type":"string","maxLength":2048,"minLength":1},"selektor":{"type":"string","maxLength":240,"minLength":1,"description":"CSS-Selektor: genau ein Objekt, sonst erster Treffer des Objektfilters."},"vergleich":{"type":"string","maxLength":2048,"description":"Öffentliche Vergleichsadresse für Link-, Sitemap- oder Sprachbefunde."},"user_agent":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,80}$"},"dkim_selektor":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,63}$"}},"additionalProperties":false}]}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_1uQ1kkcsmnoWLY9RsL2Lb","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks the meta-robots tag value for a given URL rule object, returning the crawl/indexing directive found at that URL.","exampleAgentPrompt":"Can you check what the meta-robots tag value is for https://example.com/product-page — I need to know if it's set to noindex or nofollow.","exampleUseCases":[{"title":"Audit noindex status of landing page","prompt":"Check the meta-robots directive on https://shop.example.com/sale — I want to know if it's accidentally set to noindex before we launch the campaign."},{"title":"Verify crawl rules for URL pattern","prompt":"Can you inspect the meta-robots tag at https://blog.example.com/category/news and tell me exactly what value it's returning?"},{"title":"Diagnose page missing from search index","prompt":"This page https://docs.example.com/guide/setup isn't showing up in Google — can you check what the meta-robots tag says on it?"}],"resultDescription":"Returns the value of the meta-robots tag found on the inspected URL, indicating the crawl/indexing directive such as 'noindex', 'nofollow', 'index,follow', or similar robot instructions declared in the page's HTML head.","failureModes":["URL is not publicly accessible or returns a non-200 HTTP status","No meta-robots tag present on the page (empty or null result)","URL exceeds maximum length of 2048 characters","CSS selector matches no element or multiple elements (first match used)","Network timeout or DNS resolution failure for the target URL","Invalid URL format provided"],"whenToPreferThis":"Use this endpoint when you need to programmatically verify the meta-robots HTML tag value for a specific URL as part of a technical SEO audit, indexability check, or automated crawl rule validation. Prefer this over robots.txt checks when the directive is embedded in page HTML rather than the robots file, and over general page scrapers when you specifically need the structured crawl directive finding.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-16T12:47:19.961Z","isFirstParty":false}