{
  "x402Version": 2,
  "id": 29496,
  "slug": "29496",
  "resource": "https://netintel.dev/robots-txt/analyze",
  "description": "Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.",
  "type": "http",
  "serviceName": "NetIntel",
  "iconUrl": "https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1784214098/bazaar-icons/7d2bf437631b2e4e.png",
  "resourceType": "Beauty & Personal Care",
  "accepts": [
    {
      "scheme": "exact",
      "network": "eip155:8453",
      "payTo": "0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df",
      "asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 300
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "8FVeybWnB4dYrxTuPK7WzFZNAiEcFVaDLFnKpRwVGr5s",
      "asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
      "amount": "10000",
      "amountUsd": 0.01,
      "maxTimeoutSeconds": 300
    }
  ],
  "outputSchema": {
    "properties": {
      "example": {
        "properties": {
          "domain": {
            "type": "string"
          },
          "findings": {
            "items": {
              "type": "object"
            },
            "type": "array"
          },
          "found": {
            "type": "boolean"
          },
          "grade": {
            "type": "string"
          },
          "path_check": {
            "type": "object"
          },
          "raw_content_preview": {
            "type": "string"
          },
          "robots_url": {
            "type": "string"
          },
          "rules": {
            "items": {
              "type": "object"
            },
            "type": "array"
          },
          "score": {
            "type": "number"
          },
          "sitemaps": {
            "items": {
              "type": "string"
            },
            "type": "array"
          },
          "status_code": {
            "type": "number"
          }
        },
        "type": "object"
      },
      "type": {
        "type": "string"
      }
    },
    "required": [
      "type"
    ],
    "type": "object"
  },
  "extensions": {
    "bazaar": {
      "info": {
        "input": {
          "method": "GET",
          "queryParams": {
            "domain": "example.com"
          },
          "type": "http"
        },
        "output": {
          "example": {
            "domain": "example.com",
            "findings": [],
            "found": true,
            "grade": "B",
            "path_check": {
              "allowed": false,
              "matched_rule": "Disallow: /private/",
              "path": "/private/page",
              "rule_type": "disallow",
              "user_agent": "*"
            },
            "raw_content_preview": "User-agent: *\nDisallow: /private/\n...",
            "robots_url": "https://example.com/robots.txt",
            "rules": [
              {
                "allow": [
                  "/public/"
                ],
                "crawl_delay": null,
                "disallow": [
                  "/private/",
                  "/admin/"
                ],
                "user_agent": "*"
              }
            ],
            "score": 85,
            "sitemaps": [
              "https://example.com/sitemap.xml"
            ],
            "status_code": 200
          },
          "type": "json"
        }
      },
      "schema": {
        "$schema": "https://json-schema.org/draft/2020-12/schema",
        "properties": {
          "input": {
            "additionalProperties": false,
            "properties": {
              "method": {
                "enum": [
                  "GET"
                ],
                "type": "string"
              },
              "queryParams": {
                "properties": {
                  "domain": {
                    "description": "Domain name to fetch robots.txt from (e.g. example.com)",
                    "type": "string"
                  },
                  "path": {
                    "description": "URL path to check permission for (e.g. /api/data)",
                    "type": "string"
                  },
                  "user_agent": {
                    "description": "Bot name to check rules for (default: *)",
                    "type": "string"
                  }
                },
                "required": [
                  "domain"
                ],
                "type": "object"
              },
              "type": {
                "const": "http",
                "type": "string"
              }
            },
            "required": [
              "type",
              "method"
            ],
            "type": "object"
          },
          "output": {
            "properties": {
              "example": {
                "properties": {
                  "domain": {
                    "type": "string"
                  },
                  "findings": {
                    "items": {
                      "type": "object"
                    },
                    "type": "array"
                  },
                  "found": {
                    "type": "boolean"
                  },
                  "grade": {
                    "type": "string"
                  },
                  "path_check": {
                    "type": "object"
                  },
                  "raw_content_preview": {
                    "type": "string"
                  },
                  "robots_url": {
                    "type": "string"
                  },
                  "rules": {
                    "items": {
                      "type": "object"
                    },
                    "type": "array"
                  },
                  "score": {
                    "type": "number"
                  },
                  "sitemaps": {
                    "items": {
                      "type": "string"
                    },
                    "type": "array"
                  },
                  "status_code": {
                    "type": "number"
                  }
                },
                "type": "object"
              },
              "type": {
                "type": "string"
              }
            },
            "required": [
              "type"
            ],
            "type": "object"
          }
        },
        "required": [
          "input"
        ],
        "type": "object"
      }
    }
  },
  "tags": [
    "robots.txt",
    "crawl rules",
    "seo",
    "web intelligence",
    "scraping",
    "bazaar"
  ],
  "sourceHost": "netintel.dev",
  "lastUpdated": "2026-10-11T14:25:13.815Z",
  "quality": {
    "calls30d": 5,
    "uniquePayers30d": 4,
    "lastCalledAt": "2026-10-11T14:25:13.575Z"
  },
  "metadata": {
    "serviceName": "NetIntel",
    "iconUrl": "https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1784214098/bazaar-icons/7d2bf437631b2e4e.png",
    "extensions": {
      "bazaar": {
        "info": {
          "input": {
            "method": "GET",
            "queryParams": {
              "domain": "example.com"
            },
            "type": "http"
          },
          "output": {
            "example": {
              "domain": "example.com",
              "findings": [],
              "found": true,
              "grade": "B",
              "path_check": {
                "allowed": false,
                "matched_rule": "Disallow: /private/",
                "path": "/private/page",
                "rule_type": "disallow",
                "user_agent": "*"
              },
              "raw_content_preview": "User-agent: *\nDisallow: /private/\n...",
              "robots_url": "https://example.com/robots.txt",
              "rules": [
                {
                  "allow": [
                    "/public/"
                  ],
                  "crawl_delay": null,
                  "disallow": [
                    "/private/",
                    "/admin/"
                  ],
                  "user_agent": "*"
                }
              ],
              "score": 85,
              "sitemaps": [
                "https://example.com/sitemap.xml"
              ],
              "status_code": 200
            },
            "type": "json"
          }
        },
        "schema": {
          "$schema": "https://json-schema.org/draft/2020-12/schema",
          "properties": {
            "input": {
              "additionalProperties": false,
              "properties": {
                "method": {
                  "enum": [
                    "GET"
                  ],
                  "type": "string"
                },
                "queryParams": {
                  "properties": {
                    "domain": {
                      "description": "Domain name to fetch robots.txt from (e.g. example.com)",
                      "type": "string"
                    },
                    "path": {
                      "description": "URL path to check permission for (e.g. /api/data)",
                      "type": "string"
                    },
                    "user_agent": {
                      "description": "Bot name to check rules for (default: *)",
                      "type": "string"
                    }
                  },
                  "required": [
                    "domain"
                  ],
                  "type": "object"
                },
                "type": {
                  "const": "http",
                  "type": "string"
                }
              },
              "required": [
                "type",
                "method"
              ],
              "type": "object"
            },
            "output": {
              "properties": {
                "example": {
                  "properties": {
                    "domain": {
                      "type": "string"
                    },
                    "findings": {
                      "items": {
                        "type": "object"
                      },
                      "type": "array"
                    },
                    "found": {
                      "type": "boolean"
                    },
                    "grade": {
                      "type": "string"
                    },
                    "path_check": {
                      "type": "object"
                    },
                    "raw_content_preview": {
                      "type": "string"
                    },
                    "robots_url": {
                      "type": "string"
                    },
                    "rules": {
                      "items": {
                        "type": "object"
                      },
                      "type": "array"
                    },
                    "score": {
                      "type": "number"
                    },
                    "sitemaps": {
                      "items": {
                        "type": "string"
                      },
                      "type": "array"
                    },
                    "status_code": {
                      "type": "number"
                    }
                  },
                  "type": "object"
                },
                "type": {
                  "type": "string"
                }
              },
              "required": [
                "type"
              ],
              "type": "object"
            }
          },
          "required": [
            "input"
          ],
          "type": "object"
        }
      }
    }
  },
  "liveness": {
    "reliability": null
  },
  "verified": false,
  "featured": false,
  "provider": {
    "host": "netintel.dev",
    "manifest_name": "netintel.dev",
    "source": "cdp-mirror",
    "source_manifest_url": "https://api.cdp.coinbase.com/platform/v2/x402/discovery/resources",
    "submitted_at": "2026-10-14T01:15:01.050Z"
  },
  "exampleResponse": {
    "domain": "example.com",
    "findings": [],
    "found": true,
    "grade": "B",
    "path_check": {
      "allowed": false,
      "matched_rule": "Disallow: /private/",
      "path": "/private/page",
      "rule_type": "disallow",
      "user_agent": "*"
    },
    "raw_content_preview": "User-agent: *\nDisallow: /private/\n...",
    "robots_url": "https://example.com/robots.txt",
    "rules": [
      {
        "allow": [
          "/public/"
        ],
        "crawl_delay": null,
        "disallow": [
          "/private/",
          "/admin/"
        ],
        "user_agent": "*"
      }
    ],
    "score": 85,
    "sitemaps": [
      "https://example.com/sitemap.xml"
    ],
    "status_code": 200
  }
}