{"uid":"cap_8I-ump5PF9hOTM6ZuDIm6","slug":"crawlspur-robots-txt-crawl-permission-checker-5d780d4a","name":"Crawlspur robots.txt Crawl Permission Checker","description":"Prüft am Objekt robots-datei den Befund Freigabe durch robots.txt.","url":"https://crawlspur.halowerk.com/v1/pruef/crawlspur/robots-erlaubt/robots-datei","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"ziel":{"oneOf":[{"type":"string","maxLength":2048,"minLength":1,"description":"Öffentliche HTTP(S)-Adresse; bei DNS-Befunden Domain oder öffentliche IP."},{"type":"object","required":["adresse"],"properties":{"adresse":{"type":"string","maxLength":2048,"minLength":1},"selektor":{"type":"string","maxLength":240,"minLength":1,"description":"CSS-Selektor: genau ein Objekt, sonst erster Treffer des Objektfilters."},"vergleich":{"type":"string","maxLength":2048,"description":"Öffentliche Vergleichsadresse für Link-, Sitemap- oder Sprachbefunde."},"user_agent":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,80}$"},"dkim_selektor":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,63}$"}},"additionalProperties":false}]}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_WVxtyZ75fdkc6gH1z0KEq","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL is allowed to be crawled according to the site's robots.txt file.","exampleAgentPrompt":"Can you check whether https://example.com/blog/my-article is allowed to be crawled according to its robots.txt file, using the user-agent Googlebot?","exampleUseCases":[{"title":"Pre-crawl robots.txt permission check","prompt":"Before I scrape https://shop.example.com/products, can you verify that the robots.txt file actually permits crawling that URL? Use user-agent 'Googlebot'."},{"title":"SEO audit for blocked pages","prompt":"I'm auditing the site https://mybusiness.com — can you check if the URL https://mybusiness.com/services is being blocked or allowed by the robots.txt file?"},{"title":"Validate crawler access before indexing","prompt":"We're about to index https://news.example.com/latest-story — please confirm that robots.txt grants crawl permission for that URL so we don't violate their crawl rules."}],"resultDescription":"Returns a finding indicating whether the target URL is permitted (allowed or disallowed) by the site's robots.txt file, based on the applicable crawl directives for the specified or default user-agent.","failureModes":["Target URL is unreachable or returns a non-200 HTTP status","robots.txt file is missing or returns an error, making permission indeterminate","Invalid or malformed URL provided in the 'ziel' field","User-agent pattern does not match any robots.txt rule, defaulting to allow","Network timeout when fetching the robots.txt file"],"whenToPreferThis":"Use this endpoint when you need to programmatically verify whether a specific URL is permitted for crawling under robots.txt rules before initiating automated access, during SEO audits, or when building compliant web scrapers. Prefer this over manual inspection when checking many URLs or integrating crawl-permission checks into automated pipelines.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-14T07:03:18.674Z","isFirstParty":false}