{"uid":"cap_HxquSRf12iOWc4e4wffW_","slug":"crawlspur-url-regel-crawl-delay-check-f332b82c","name":"Crawlspur URL-Regel Crawl-Delay Check","description":"Prüft am Objekt url-regel den Befund deklarierte Crawl-Verzögerung.","url":"https://crawlspur.halowerk.com/v1/pruef/crawlspur/crawl-delay/url-regel","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"ziel":{"oneOf":[{"type":"string","maxLength":2048,"minLength":1,"description":"Öffentliche HTTP(S)-Adresse; bei DNS-Befunden Domain oder öffentliche IP."},{"type":"object","required":["adresse"],"properties":{"adresse":{"type":"string","maxLength":2048,"minLength":1},"selektor":{"type":"string","maxLength":240,"minLength":1,"description":"CSS-Selektor: genau ein Objekt, sonst erster Treffer des Objektfilters."},"vergleich":{"type":"string","maxLength":2048,"description":"Öffentliche Vergleichsadresse für Link-, Sitemap- oder Sprachbefunde."},"user_agent":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,80}$"},"dkim_selektor":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,63}$"}},"additionalProperties":false}]}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_nboMIW6RlJ9kC--_kBcEY","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks the declared crawl delay setting for a URL rule object by fetching and analyzing the relevant robots.txt directive.","exampleAgentPrompt":"Check the declared crawl delay for the URL rule at https://example.com/products/* — I want to know what crawl delay is set in its robots.txt for that URL pattern.","exampleUseCases":[{"title":"SEO audit of crawl rate limits","prompt":"Check the declared crawl delay for the URL rule at https://myshop.com/category/* and tell me what crawl delay value is configured in robots.txt for that pattern."},{"title":"Verify bot throttling before launching crawler","prompt":"Before I run my crawler, can you check if https://newssite.com/articles/ has any crawl delay declared for its URL rule in robots.txt?"},{"title":"Diagnose slow indexing of product pages","prompt":"I think Google might be slowed down by a crawl delay on our URL rules — can you check the crawl delay declared for https://store.example.com/products/ in robots.txt?"}],"resultDescription":"Returns the declared crawl delay value found in the robots.txt for the specified URL rule, indicating how many seconds the crawl delay is set to (if any), along with details about the matching rule or directive analyzed.","failureModes":["Target URL is unreachable or returns non-200 HTTP status","robots.txt file is missing or inaccessible","No crawl delay directive found for the specified URL rule","Invalid URL format provided","CSS selector matches no element or multiple elements","Request timeout on slow or unresponsive domains"],"whenToPreferThis":"Use this endpoint when you need to specifically check the crawl-delay directive for a URL rule object in a robots.txt file, particularly during SEO audits, crawler configuration, or troubleshooting indexing delays. Choose this over generic robots.txt parsers when you need a targeted per-URL-rule crawl delay check as part of a systematic crawl configuration audit.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-16T12:58:22.115Z","isFirstParty":false}