{ "x402Version": 2, "id": 35921, "slug": "35921", "resource": "https://tanod.dev/v1/robots", "description": "sitepeek (Tanod) robots.txt check: whether a crawler may fetch a URL under RFC 9309, with the matching rule, crawl delay and sitemaps. Read-only.", "type": "http", "serviceName": "Tanod sitepeek", "iconUrl": "https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1791364949/bazaar-icons/d8143c9bc952f5cd.png", "resourceType": "Beauty & Personal Care", "accepts": [ { "scheme": "exact", "network": "eip155:8453", "payTo": "0x593857A4a4F619543ea12394137C3004ce841720", "asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", "amount": "1000", "amountUsd": 0.001, "maxTimeoutSeconds": 300 }, { "scheme": "exact", "network": "eip155:137", "payTo": "0x593857A4a4F619543ea12394137C3004ce841720", "asset": "0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359", "amount": "1000", "amountUsd": 0.001, "maxTimeoutSeconds": 300 } ], "outputSchema": { "properties": { "example": { "properties": { "allowed": { "type": "boolean" }, "crawl_delay": { "type": [ "number", "string", "null" ] }, "fetch": { "enum": [ "ok", "unavailable", "unreachable" ], "type": "string" }, "group": { "type": [ "string", "null" ] }, "http_status": { "type": [ "integer", "null" ] }, "matched_rule": { "properties": { "line": { "type": "integer" }, "pattern": { "type": "string" }, "type": { "enum": [ "allow", "disallow" ], "type": "string" } }, "type": [ "object", "null" ] }, "path": { "type": "string" }, "product_token": { "type": "string" }, "reason": { "type": "string" }, "robots_url": { "type": [ "string", "null" ] }, "sitemaps": { "items": { "type": "string" }, "type": "array" }, "sitemaps_total": { "type": "integer" }, "stats": { "type": [ "object", "null" ] }, "truncated": { "type": "boolean" }, "untrusted_content": { "type": "boolean" }, "url": { "type": "string" }, "user_agent": { "type": "string" } }, "required": [ "url", "robots_url", "user_agent", "product_token", "path", "http_status", "fetch", "allowed", "matched_rule", "group", "crawl_delay", "sitemaps", "sitemaps_total", "truncated", "stats", "reason", "untrusted_content" ], "type": "object" }, "type": { "type": "string" } }, "required": [ "type" ], "type": "object" }, "extensions": { "bazaar": { "info": { "input": { "body": { "url": "https://www.google.com/search", "user_agent": "Googlebot" }, "bodyType": "json", "method": "POST", "type": "http" }, "output": { "example": { "allowed": false, "crawl_delay": null, "fetch": "ok", "group": "*", "http_status": 200, "matched_rule": { "line": 3, "pattern": "/search", "type": "disallow" }, "path": "/search", "product_token": "googlebot", "reason": "most specific matching rule: disallow /search", "robots_url": "https://www.google.com/robots.txt", "sitemaps": [ "https://www.google.com/sitemap.xml" ], "sitemaps_total": 1, "stats": { "groups": 5, "ignored_lines": 0, "lines": 264, "rules": 245 }, "truncated": false, "untrusted_content": true, "url": "https://www.google.com/search", "user_agent": "Googlebot" }, "type": "json" } }, "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "properties": { "input": { "additionalProperties": false, "properties": { "body": { "additionalProperties": false, "properties": { "url": { "description": "Public http(s) URL to test; /robots.txt is fetched and the URL's path is matched against it.", "maxLength": 2048, "minLength": 1, "type": "string" }, "user_agent": { "default": "*", "description": "Product token (e.g. Googlebot) or a full User-Agent string; '*' (default) = any crawler.", "maxLength": 200, "minLength": 1, "type": "string" } }, "required": [ "url" ], "type": "object" }, "bodyType": { "enum": [ "json", "form-data", "text" ], "type": "string" }, "method": { "enum": [ "POST" ], "type": "string" }, "type": { "const": "http", "type": "string" } }, "required": [ "type", "method", "bodyType", "body" ], "type": "object" }, "output": { "properties": { "example": { "properties": { "allowed": { "type": "boolean" }, "crawl_delay": { "type": [ "number", "string", "null" ] }, "fetch": { "enum": [ "ok", "unavailable", "unreachable" ], "type": "string" }, "group": { "type": [ "string", "null" ] }, "http_status": { "type": [ "integer", "null" ] }, "matched_rule": { "properties": { "line": { "type": "integer" }, "pattern": { "type": "string" }, "type": { "enum": [ "allow", "disallow" ], "type": "string" } }, "type": [ "object", "null" ] }, "path": { "type": "string" }, "product_token": { "type": "string" }, "reason": { "type": "string" }, "robots_url": { "type": [ "string", "null" ] }, "sitemaps": { "items": { "type": "string" }, "type": "array" }, "sitemaps_total": { "type": "integer" }, "stats": { "type": [ "object", "null" ] }, "truncated": { "type": "boolean" }, "untrusted_content": { "type": "boolean" }, "url": { "type": "string" }, "user_agent": { "type": "string" } }, "required": [ "url", "robots_url", "user_agent", "product_token", "path", "http_status", "fetch", "allowed", "matched_rule", "group", "crawl_delay", "sitemaps", "sitemaps_total", "truncated", "stats", "reason", "untrusted_content" ], "type": "object" }, "type": { "type": "string" } }, "required": [ "type" ], "type": "object" } }, "required": [ "input" ], "type": "object" } }, "builder-code": { "info": { "a": "bc_h7pxv14u" }, "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { "a": { "description": "App builder code", "pattern": "^[a-z0-9_]{1,32}$", "type": "string" }, "s": { "description": "Service builder codes", "items": { "pattern": "^[a-z0-9_]{1,32}$", "type": "string" }, "maxItems": 11, "type": "array" }, "w": { "description": "Wallet builder code", "pattern": "^[a-z0-9_]{1,32}$", "type": "string" } }, "type": "object" } } }, "tags": [ "sitepeek", "render", "markdown", "pdf", "ocr", "bazaar", "builder-code" ], "sourceHost": "tanod.dev", "lastUpdated": "2026-10-08T22:46:00.651Z", "quality": { "calls30d": 2, "uniquePayers30d": 2, "lastCalledAt": "2026-10-08T22:46:00.119Z" }, "metadata": { "serviceName": "Tanod sitepeek", "iconUrl": "https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1791364949/bazaar-icons/d8143c9bc952f5cd.png", "extensions": { "bazaar": { "info": { "input": { "body": { "url": "https://www.google.com/search", "user_agent": "Googlebot" }, "bodyType": "json", "method": "POST", "type": "http" }, "output": { "example": { "allowed": false, "crawl_delay": null, "fetch": "ok", "group": "*", "http_status": 200, "matched_rule": { "line": 3, "pattern": "/search", "type": "disallow" }, "path": "/search", "product_token": "googlebot", "reason": "most specific matching rule: disallow /search", "robots_url": "https://www.google.com/robots.txt", "sitemaps": [ "https://www.google.com/sitemap.xml" ], "sitemaps_total": 1, "stats": { "groups": 5, "ignored_lines": 0, "lines": 264, "rules": 245 }, "truncated": false, "untrusted_content": true, "url": "https://www.google.com/search", "user_agent": "Googlebot" }, "type": "json" } }, "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "properties": { "input": { "additionalProperties": false, "properties": { "body": { "additionalProperties": false, "properties": { "url": { "description": "Public http(s) URL to test; /robots.txt is fetched and the URL's path is matched against it.", "maxLength": 2048, "minLength": 1, "type": "string" }, "user_agent": { "default": "*", "description": "Product token (e.g. Googlebot) or a full User-Agent string; '*' (default) = any crawler.", "maxLength": 200, "minLength": 1, "type": "string" } }, "required": [ "url" ], "type": "object" }, "bodyType": { "enum": [ "json", "form-data", "text" ], "type": "string" }, "method": { "enum": [ "POST" ], "type": "string" }, "type": { "const": "http", "type": "string" } }, "required": [ "type", "method", "bodyType", "body" ], "type": "object" }, "output": { "properties": { "example": { "properties": { "allowed": { "type": "boolean" }, "crawl_delay": { "type": [ "number", "string", "null" ] }, "fetch": { "enum": [ "ok", "unavailable", "unreachable" ], "type": "string" }, "group": { "type": [ "string", "null" ] }, "http_status": { "type": [ "integer", "null" ] }, "matched_rule": { "properties": { "line": { "type": "integer" }, "pattern": { "type": "string" }, "type": { "enum": [ "allow", "disallow" ], "type": "string" } }, "type": [ "object", "null" ] }, "path": { "type": "string" }, "product_token": { "type": "string" }, "reason": { "type": "string" }, "robots_url": { "type": [ "string", "null" ] }, "sitemaps": { "items": { "type": "string" }, "type": "array" }, "sitemaps_total": { "type": "integer" }, "stats": { "type": [ "object", "null" ] }, "truncated": { "type": "boolean" }, "untrusted_content": { "type": "boolean" }, "url": { "type": "string" }, "user_agent": { "type": "string" } }, "required": [ "url", "robots_url", "user_agent", "product_token", "path", "http_status", "fetch", "allowed", "matched_rule", "group", "crawl_delay", "sitemaps", "sitemaps_total", "truncated", "stats", "reason", "untrusted_content" ], "type": "object" }, "type": { "type": "string" } }, "required": [ "type" ], "type": "object" } }, "required": [ "input" ], "type": "object" } }, "builder-code": { "info": { "a": "bc_h7pxv14u" }, "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { "a": { "description": "App builder code", "pattern": "^[a-z0-9_]{1,32}$", "type": "string" }, "s": { "description": "Service builder codes", "items": { "pattern": "^[a-z0-9_]{1,32}$", "type": "string" }, "maxItems": 11, "type": "array" }, "w": { "description": "Wallet builder code", "pattern": "^[a-z0-9_]{1,32}$", "type": "string" } }, "type": "object" } } } }, "liveness": { "reliability": null }, "verified": false, "featured": false, "provider": { "host": "tanod.dev", "manifest_name": "tanod.dev", "source": "cdp-mirror", "source_manifest_url": "https://api.cdp.coinbase.com/platform/v2/x402/discovery/resources", "submitted_at": "2026-10-14T01:15:01.050Z" }, "exampleResponse": { "example": { "allowed": true, "crawl_delay": 0, "fetch": "ok", "group": "string", "http_status": 0, "matched_rule": { "line": 0, "pattern": "string", "type": "allow" }, "path": "string", "product_token": "string", "reason": "string", "robots_url": "string", "sitemaps": [ "string" ], "sitemaps_total": 0, "stats": {}, "truncated": true, "untrusted_content": true, "url": "string", "user_agent": "string" }, "type": "string" } }