{"uid":"cap_zGlg5mEQ-kPW2OMvcl84a","slug":"td-agent-utility-site-crawl-verification-8a5fbe37","name":"TD Agent Utility – Site Crawl Verification","description":"Independent website and deployment verification for AI agents: HTTP, DNS, TLS and Chromium screenshot evidence. Three deterministic tools, x402 V2 Base USDC.","url":"https://tdagentutility.co.uk/v1/site-crawl?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":null,"responseSchema":{"type":"json","example":{"tool":"site_crawl","pages":[],"summary":{"failed":0,"requests":0,"attempted":0,"bytes_read":0,"successful":0,"frontier_remaining":0,"requested_max_pages":0},"warnings":[],"checked_at":"2026-10-15T00:00:00Z","requested_url":"","result_status":"PASS"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.02","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"registry","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.02/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.02","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Vt41TC-XmZ6K2hNYjYmoK","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.02","costPer":"request","priority":0,"asset":null,"unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a target website and returns HTTP response data, crawl statistics, and pass/fail verification status for deployment validation.","exampleAgentPrompt":"Can you crawl https://myapp.example.com and give me an independent verification of whether the site is live and returning pages successfully — I just deployed a new version and want to confirm everything is up.","exampleUseCases":[{"title":"Post-deployment health check","prompt":"I just pushed a new release to https://staging.myproduct.io — can you crawl it and verify it's actually serving pages and not returning errors?"},{"title":"Third-party site availability audit","prompt":"I need an independent check that https://clientwebsite.com is live and accessible before I tell them their redesign is complete — can you crawl it and give me a pass/fail?"},{"title":"Monitoring a new domain launch","prompt":"We just pointed our DNS to the new server for https://newbrand.co.uk — can you verify the site is reachable and crawl a few pages to confirm it's serving content correctly?"}],"resultDescription":"A JSON object with a 'tool' field set to 'site_crawl', an array of crawled pages, a summary object containing counts of attempted, successful, and failed requests, bytes read, and frontier pages remaining, a list of warnings, the checked_at timestamp, the requested URL, and a result_status of PASS or FAIL.","failureModes":["Target URL is unreachable or returns 5xx errors — result_status will be FAIL with warnings","DNS resolution failure for the target domain — crawl fails with zero successful requests","Target URL is malformed or missing — request rejected","Site blocks crawlers via robots.txt or firewall — pages array may be empty with warnings","Timeout on slow-loading pages — partial results with failed count > 0"],"whenToPreferThis":"Choose this endpoint when you need an independent, third-party HTTP crawl-based verification of a website's liveness and deployment status — particularly after a new release, DNS change, or infrastructure update. It is ideal when a simple ping or HEAD request is not sufficient and you need evidence of actual page content being served. Prefer this over generic uptime monitors when you want deterministic pass/fail evidence with crawl statistics.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-16T12:40:52.224Z","isFirstParty":false,"canonicalSlug":"td-agent-utility-site-crawl-verification-8a5fbe37"}