{"uid":"cap_gba0FrY7ZpLuyUHstzJ73","slug":"crawlspur-robots-txt-crawl-delay-checker-4709fb64","name":"Crawlspur robots.txt Crawl-Delay Checker","description":"Prüft am Objekt robots-datei den Befund deklarierte Crawl-Verzögerung.","url":"https://crawlspur.halowerk.com/v1/pruef/crawlspur/crawl-delay/robots-datei","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"ziel":{"oneOf":[{"type":"string","maxLength":2048,"minLength":1,"description":"Öffentliche HTTP(S)-Adresse; bei DNS-Befunden Domain oder öffentliche IP."},{"type":"object","required":["adresse"],"properties":{"adresse":{"type":"string","maxLength":2048,"minLength":1},"selektor":{"type":"string","maxLength":240,"minLength":1,"description":"CSS-Selektor: genau ein Objekt, sonst erster Treffer des Objektfilters."},"vergleich":{"type":"string","maxLength":2048,"description":"Öffentliche Vergleichsadresse für Link-, Sitemap- oder Sprachbefunde."},"user_agent":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,80}$"},"dkim_selektor":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,63}$"}},"additionalProperties":false}]}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_nEAVER-eVfELyyNgoBmV3","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks a robots.txt file to detect and extract the declared Crawl-Delay directive for a given URL or domain.","exampleAgentPrompt":"Check the robots.txt file at https://example.com and tell me what crawl-delay value is declared there.","exampleUseCases":[{"title":"SEO audit for crawl delay compliance","prompt":"I'm doing an SEO audit — can you check whether https://shop.example.com has a crawl-delay directive in its robots.txt file and what value it's set to?"},{"title":"Respecting crawl delay before scraping","prompt":"Before I start scraping https://news.example.org, check its robots.txt for any declared crawl-delay so I know how long to wait between requests."},{"title":"Monitoring competitor crawl settings","prompt":"Can you look at the robots.txt for https://competitor.example.com and extract the crawl-delay directive so I can compare it against industry norms?"}],"resultDescription":"Returns the declared crawl-delay value found in the robots.txt file for the specified target URL or domain, including the finding status (present/absent), the numeric delay value if declared, and relevant metadata about the robots.txt check.","failureModes":["Target URL unreachable or returns non-200 status — check fails with connection or HTTP error","robots.txt file not found at target domain — no crawl-delay can be extracted","Invalid or malformed URL in 'ziel' field — validation error returned","Target is a private/non-public IP or domain — rejected as invalid target","No crawl-delay directive present in robots.txt — result indicates absence rather than error"],"whenToPreferThis":"Use this endpoint when you need to programmatically determine the crawl-delay value declared in a website's robots.txt file, particularly for SEO auditing, compliance checking before web scraping, or monitoring changes in crawl policy. Prefer this over generic HTTP fetchers when you specifically need structured extraction of the crawl-delay directive with validation logic built in.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T07:00:15.407Z","isFirstParty":false}