{"uid":"cap_PxtvyvtEYKShBYIf02dJU","slug":"scrapeforagents-web-crawler-api-87f596c7","name":"ScrapeForAgents Web Crawler API","description":"Pay-per-call structured web data. Failed or empty runs are not charged.","url":"https://api.scrapeforagents.tech/v1/get?utm_source=zero.xyz","method":"POST","headers":{},"bodySchema":{"type":"object","properties":{"maxDepth":{"type":"integer","default":2,"maximum":10,"minimum":0,"description":"How many link levels to follow after the start URL. Zero returns only the start URL."},"maxItems":{"type":"integer","default":100,"minimum":0,"description":"Maximum output rows across the crawl. Zero removes the cap."},"startUrl":{"type":"string","description":"Public GET in IT URL to crawl. Use the job search for current vacancies or sitemap.xml for the public URL inventory."},"sameDomainOnly":{"type":"boolean","default":true,"description":"Follow only links on get-in-it.de and its www host."},"allowDuplicates":{"type":"boolean","default":false,"description":"Include a URL again when it is linked from another parent; each URL is still fetched at most once."},"ignoredExtensions":{"type":"array","default":["pdf","jpg","jpeg","png","gif","svg","webp","zip","css","js","woff","woff2"],"description":"Skip links whose paths end in these file extensions. Enter names without the dot."},"maxChildrenPerLink":{"type":"integer","default":50,"maximum":10000,"minimum":1,"description":"Maximum distinct links taken from each page, including job search results."}}},"responseSchema":{"type":"json","example":{"count":1,"items":[{"url":null,"name":null,"depth":null,"jobId":null,"jobTitle":null,"location":null,"pageType":null,"parentUrl":null}],"product":"get"}},"example":null,"exampleRequest":null,"tags":["x402"],"displayCostAmount":"0.025","displayCostAsset":"USDC","priceDynamic":false,"priceHint":null,"priceStatus":"priced","priceSource":"probe","requiresHandshake":false,"reviewCount":0,"rating":{"score":"0.00","successRate":"0.00","reviews":0,"stars":null,"state":"unrated"},"availabilityStatus":"unknown","priceObserved":null,"sessionDeposit":null,"pricing":{"kind":"static","summary":"$0.025/call","primary":{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.025","per":"call","confidence":"exact"},"accepted":[{"kind":"static","protocol":"x402","network":"base","amountUsd":"0.025","per":"call","confidence":"exact"}]},"paymentMethods":[{"uid":"pm_Akmve41oHk2CInxepo5Y6","protocol":"x402","methodType":"crypto","chain":"base","mode":"charge","costAmount":"0.025","costPer":"request","priority":0,"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","unit":"request","depositMicros":null,"planRef":null}],"brandName":null,"brandSlug":null,"brandBaseUrl":null,"brandDocsUrl":null,"whatItDoes":"Crawls a starting URL and returns structured job listing and page data up to a configurable link depth and item count.","exampleAgentPrompt":"Crawl get-in-it.de/jobs starting from the job search page, follow links up to 2 levels deep, collect up to 100 job listings, and give me back structured data with job titles, locations, and job IDs — stay on the same domain only.","exampleUseCases":[{"title":"Harvest job listings from a careers site","prompt":"Crawl the careers page at https://get-in-it.de/jobs, follow links up to 3 levels deep, collect up to 200 job postings, and return structured data with job titles, locations, and job IDs — only follow links on the same domain."},{"title":"Build a job market snapshot from a sitemap","prompt":"Start at https://get-in-it.de/sitemap.xml, follow links 1 level deep, grab up to 500 items, skip PDFs and images, and give me back a structured list of all job pages with their URLs and page types."},{"title":"Monitor new job postings on a specific job board","prompt":"Scrape the job search results at https://get-in-it.de/jobs?q=software-engineer, follow up to 2 link levels, limit to 50 results, and return structured JSON with job titles, locations, and job IDs so I can track what's new."}],"resultDescription":"Returns a JSON object with a count of items found and an array of structured records, each containing the page URL, page name, crawl depth, job ID, job title, location, page type, and parent URL. Failed or empty crawls are not billed.","failureModes":["Invalid or unreachable start URL returns an error or empty result set","maxDepth set too high may cause slow or timeout responses on large sites","Crawl blocked by site robots.txt or anti-scraping measures resulting in empty items","Non-GET-accessible URLs (login-gated pages) return no data","Exceeding maxItems limit truncates results without error notice"],"whenToPreferThis":"Choose this endpoint when you need to crawl a web page or job board multiple link levels deep and receive structured, row-oriented job data (titles, locations, job IDs) without writing your own scraper. It is especially well-suited for get-in-it.de job data extraction and supports configurable crawl depth, domain restriction, and file extension filtering. Pay-per-call with no charge on failures makes it low-risk for exploratory crawls.","instructions":null,"reviewSummary":null,"reviewSummaryHighlights":null,"reviewSummaryConcerns":null,"reviewSummaryGeneratedAt":null,"activationCount":0,"lastUsedAt":null,"lastSuccessfullyRanAt":null,"lastHealthCheckAt":"2026-10-02T04:54:10.160Z","isFirstParty":false,"canonicalSlug":"scrapeforagents-web-crawler-api-87f596c7"}