{
  "x402Version": 2,
  "id": 754,
  "slug": "754",
  "resource": "https://minifetch.com/api/v1/x402/preflight/url-check",
  "description": "Cheap robots.txt preflight before you pay for other Minifetch endpoints: returns whether the Minifetch user agent may fetch a target URL (allow/deny) plus its crawl-delay. Blocked or setting a crawl delay? https://minifetch.com/tutorials/unblock-minifetch . Checks the URL as given — does not follow redirects. Supports GET & POST",
  "type": "http",
  "accepts": [
    {
      "scheme": "exact",
      "network": "eip155:8453",
      "payTo": "0xffA9155A0e4B3d8d42351Da226cC1feEe449686f",
      "asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
      "amount": "1000",
      "maxTimeoutSeconds": 300
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "46vMcwuC4sK11sB3gkLhyA7J7GEwfkhn5rFyDtihBwqe",
      "asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
      "amount": "1000",
      "maxTimeoutSeconds": 300
    }
  ],
  "outputSchema": {
    "bazaar": {
      "info": {
        "input": {
          "method": "GET",
          "queryParams": {
            "url": "https://github.com"
          },
          "type": "http"
        },
        "output": {
          "example": {
            "results": [
              {
                "data": {
                  "allowed": true,
                  "crawlDelay": 1,
                  "message": "Access allowed by robots.txt",
                  "minifetchCache": {
                    "cachedAt": "2026-08-16T18:22:10.000Z",
                    "expiresAt": "2026-08-17T18:22:10.000Z",
                    "hit": false
                  },
                  "proxy": {
                    "proxyType": "minifetch",
                    "userAgent": "minifetch/1.0 (+https://minifetch.com/site-owner-faq)"
                  },
                  "url": "https://github.com"
                }
              }
            ],
            "success": true
          },
          "type": "json"
        }
      },
      "schema": {
        "$schema": "https://json-schema.org/draft/2020-12/schema",
        "properties": {
          "input": {
            "additionalProperties": false,
            "properties": {
              "method": {
                "enum": [
                  "GET"
                ],
                "type": "string"
              },
              "queryParams": {
                "properties": {
                  "fresh": {
                    "description": "Optional. If true, bypasses the 24h robots.txt cache and re-fetches robots.txt for this check. Use after editing your robots.txt or crawl-delay (or adding/removing a Minifetch block) to confirm the change took effect.",
                    "type": "boolean"
                  },
                  "url": {
                    "description": "The URL to check against robots.txt",
                    "type": "string"
                  }
                },
                "required": [
                  "url"
                ],
                "type": "object"
              },
              "type": {
                "const": "http",
                "type": "string"
              }
            },
            "required": [
              "type",
              "method"
            ],
            "type": "object"
          },
          "output": {
            "properties": {
              "example": {
                "properties": {
                  "results": {
                    "items": {
                      "properties": {
                        "data": {
                          "properties": {
                            "allowed": {
                              "description": "Whether the minifetch user agent may fetch this URL per robots.txt",
                              "type": "boolean"
                            },
                            "crawlDelay": {
                              "description": "Crawl-delay (seconds) for the minifetch UA; omitted when blocked or the domain is invalid",
                              "type": "number"
                            },
                            "message": {
                              "description": "e.g. 'Access allowed by robots.txt', 'URL is blocked by robots.txt', or 'robots.txt not found, defaulting to allowed'",
                              "type": "string"
                            },
                            "minifetchCache": {
                              "description": "robots.txt cache state (~24h TTL; pass fresh=true to bypass)",
                              "properties": {
                                "cachedAt": {
                                  "type": "string"
                                },
                                "expiresAt": {
                                  "type": "string"
                                },
                                "hit": {
                                  "type": "boolean"
                                }
                              },
                              "type": "object"
                            },
                            "proxy": {
                              "description": "Fetch provenance — always minifetch UA here",
                              "properties": {
                                "proxyType": {
                                  "enum": [
                                    "minifetch"
                                  ],
                                  "type": "string"
                                },
                                "userAgent": {
                                  "type": "string"
                                }
                              },
                              "type": "object"
                            },
                            "url": {
                              "description": "The URL checked, exactly as given — redirects are not followed",
                              "type": "string"
                            }
                          },
                          "required": [
                            "url",
                            "allowed",
                            "message",
                            "proxy"
                          ],
                          "type": "object"
                        }
                      },
                      "type": "object"
                    },
                    "type": "array"
                  },
                  "success": {
                    "type": "boolean"
                  },
                  "tip": {
                    "description": "Present only when allowed=false: how site owners can unblock the minifetch user agent",
                    "properties": {
                      "message": {
                        "type": "string"
                      },
                      "url": {
                        "type": "string"
                      }
                    },
                    "type": "object"
                  }
                },
                "type": "object"
              },
              "type": {
                "type": "string"
              }
            },
            "required": [
              "type"
            ],
            "type": "object"
          }
        },
        "required": [
          "input"
        ],
        "type": "object"
      }
    }
  },
  "tags": [
    "bazaar"
  ],
  "sourceHost": "minifetch.com",
  "lastUpdated": "2026-09-17T06:25:58.711Z",
  "quality": {
    "calls30d": 23,
    "uniquePayers30d": 11,
    "lastCalledAt": "2026-09-17T06:25:58.35Z"
  },
  "liveness": {},
  "verified": false,
  "featured": false,
  "provider": {
    "host": "minifetch.com",
    "manifest_name": "minifetch.com",
    "source": "cdp-mirror",
    "source_manifest_url": "https://api.cdp.coinbase.com/platform/v2/x402/discovery/resources",
    "submitted_at": "2026-09-18T22:00:31.484Z"
  }
}