{"x402Version":2,"payTo":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","resources":[{"id":8361,"slug":"8361","resource":"https://crawl-preflight.postnov01.workers.dev/crawl-check/:domain","description":"Crawl pre-flight for a domain: will you be blocked, is content behind a paywall, and how much is actually there. Returns per-bot robots.txt verdicts for 16 AI crawlers, crawlable page count from sitemaps, detected CDN and paid-access signals (x402, TollBit), plus a fetch/skip recommendation. Call before spending requests on an unknown domain.","type":"http","x402Version":2,"accepts":[{"scheme":"exact","network":"eip155:8453","payTo":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","amount":"20000","maxTimeoutSeconds":60},{"scheme":"exact","network":"eip155:137","payTo":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","asset":"0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359","amount":"20000","maxTimeoutSeconds":60},{"scheme":"exact","network":"eip155:42161","payTo":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","asset":"0xaf88d065e77c8cC2239327C5EDb3A432268e5831","amount":"20000","maxTimeoutSeconds":60}],"outputSchema":{"bazaar":{"info":{"input":{"method":"GET","pathParams":{"domain":"sitepoint.com"},"queryParams":{},"type":"http"},"output":{"example":{"crawl_surface":{"is_estimate":true,"pages":28400,"sitemaps_found":2},"domain":"example.com","infrastructure":{"cdn":["Cloudflare"],"paid_access":[]},"reason":"No AI crawler restrictions declared.","recommendation":"fetch","robots":{"ai_crawlers_allowed":16,"ai_crawlers_blocked":[],"blocks_all_agents":false,"present":true}},"type":"json"}},"routeTemplate":"/crawl-check/:domain","schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"pathParams":{"properties":{"domain":{"description":"Bare domain to check, without scheme or path","type":"string"}},"required":["domain"],"type":"object"},"queryParams":{"properties":{},"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"}}},"tags":["bazaar"],"sourceHost":"crawl-preflight.postnov01.workers.dev","lastUpdated":"2026-08-31T09:40:26.676Z","quality":{"calls30d":2,"uniquePayers30d":1,"lastCalledAt":"2026-08-31T09:40:26.331Z"},"liveness":{},"verified":false,"featured":false}]}