{"uid":"cap_8MXtipWqBPuoXad-rZ7gt","slug":"agentshelf-meta-robots-extractor-b67ced14","name":"AgentShelf Meta Robots Extractor","description":"Call when an agent needs the meta robots directive extracted from a public page or HTML (SSRF-safe). Exact $0.007 USDC. Prefer unpaid POST /v1/sandbox/profile-meta-robots first.","url":"https://agentshelf.syntexa.ch/v1/profile-meta-robots?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"url":{"type":"string","maxLength":2048,"minLength":8,"description":"Public page to fetch. The selector or profile is fixed by the SKU. Provide url or html."},"html":{"type":"string","examples":["<!doctype html><html lang=\"en\"><head><title>Hello</title>\n<meta name=\"description\" content=\"Desc\"><meta property=\"og:title\" content=\"OG\">\n<link rel=\"canonical\" href=\"https://example.com/\"><link rel=\"icon\" href=\"/favicon.ico\">\n</head><body><h1>Hello</h1><a href=\"https://example.com/a\">A</a></body></html>"],"maxLength":200000,"minLength":1,"description":"HTML to extract from locally. Provide html or url. When both are set, html is used."}}},"responseSchema":null,"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.007","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.007/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.007","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.007","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_vB3GhTgZ1nwwbRdyDY0zR","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.007","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Extracts the meta robots directive from a public webpage or raw HTML content in an SSRF-safe manner","exampleAgentPrompt":"Can you extract the meta robots directive from https://example.com so I can check whether search engines are allowed to index it?","exampleUseCases":[{"title":"SEO audit for indexability","prompt":"Check the meta robots tag on https://mybrand.com/landing-page and tell me if search engines are allowed to index and follow it."},{"title":"Content pipeline robots check","prompt":"I have this HTML snippet — can you pull out the meta robots directive from it so I know if this page is set to noindex before I include it in our sitemap?"},{"title":"Competitor SEO reconnaissance","prompt":"What is the meta robots value on https://competitor.com/pricing? I want to know if they're blocking search engines from crawling that page."}],"resultDescription":"Returns the content value of the meta robots tag found in the page or HTML (e.g. 'noindex, nofollow', 'index, follow', or null if not present), enabling the agent to determine crawling and indexing directives set by the page owner.","failureModes":["URL is unreachable or returns a non-200 status — endpoint may return an error or empty result","HTML content exceeds 200,000 character limit — request rejected","URL shorter than 8 characters or malformed — validation error","SSRF-blocked internal or private IP URL provided — request rejected for security","No meta robots tag present in the page — returns null or empty value","Both url and html fields omitted — missing required input error"],"whenToPreferThis":"Use this endpoint when you need to reliably extract the meta robots directive from a public page or HTML blob in a single, cost-effective call ($0.007 USDC). Prefer over building custom HTML parsers when SSRF safety is a concern. If a free sandbox version (/v1/sandbox/profile-meta-robots) is available, try that first for development and testing.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-02T06:50:01.716Z","isFirstParty":false,"canonicalSlug":"agentshelf-meta-robots-extractor-b67ced14"}