{"uid":"cap_jflnFQhnoLrDsWsgce-Vd","slug":"crawlspur-robots-txt-permission-check-for-parameter-url-41741ec6","name":"Crawlspur robots.txt Permission Check for Parameter URL","description":"Prüft am Objekt parameter-url den Befund Freigabe durch robots.txt.","url":"https://crawlspur.halowerk.com/v1/pruef/crawlspur/robots-erlaubt/parameter-url","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"ziel":{"oneOf":[{"type":"string","maxLength":2048,"minLength":1,"description":"Öffentliche HTTP(S)-Adresse; bei DNS-Befunden Domain oder öffentliche IP."},{"type":"object","required":["adresse"],"properties":{"adresse":{"type":"string","maxLength":2048,"minLength":1},"selektor":{"type":"string","maxLength":240,"minLength":1,"description":"CSS-Selektor: genau ein Objekt, sonst erster Treffer des Objektfilters."},"vergleich":{"type":"string","maxLength":2048,"description":"Öffentliche Vergleichsadresse für Link-, Sitemap- oder Sprachbefunde."},"user_agent":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,80}$"},"dkim_selektor":{"type":"string","pattern":"^[A-Za-z0-9_-]{1,63}$"}},"additionalProperties":false}]}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.001","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.001/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.001","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_G5ozVBgWORsdUQbis0Vdz","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.001","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Checks whether a given URL (with parameters) is allowed to be crawled according to the site's robots.txt file","exampleAgentPrompt":"Check whether the URL https://example.com/products?category=shoes&sort=price is allowed to be crawled according to robots.txt — I want to know if bots are permitted to access this parameterized page.","exampleUseCases":[{"title":"SEO audit of parameterized URLs","prompt":"I'm doing a technical SEO audit — can you check if https://shop.example.com/search?q=laptop&page=2 is permitted by robots.txt, or is it being blocked from crawlers?"},{"title":"Crawler access check before scraping","prompt":"Before I build a scraper, can you verify that https://news.example.com/articles?tag=tech&from=2024-01-01 is actually allowed by robots.txt and not excluded from crawling?"},{"title":"Diagnosing indexing issues for a URL","prompt":"My page https://blog.example.com/posts?author=jane&year=2023 isn't showing up in Google — can you check if robots.txt is blocking crawlers from accessing it?"}],"resultDescription":"Returns a finding indicating whether the specified URL (with query parameters) is permitted or disallowed by the site's robots.txt rules, including the crawl allowance status for that specific address","failureModes":["URL is not publicly accessible — endpoint cannot retrieve robots.txt","Malformed or non-HTTP(S) URL provided — validation error","robots.txt file is missing or returns non-200 status — indeterminate result","URL exceeds 2048 character maximum length","User agent pattern does not match allowed format — input validation error"],"whenToPreferThis":"Use this endpoint when you need to verify whether a specific parameterized URL is permitted by a site's robots.txt — especially useful for technical SEO auditing, pre-crawl validation, or diagnosing why a URL with query parameters might be excluded from search engine indexing. Choose this over generic HTTP checkers when you specifically need robots.txt compliance analysis for a URL including its query string parameters.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-09-16T12:51:03.525Z","isFirstParty":false}