{
  "x402Version": 2,
  "id": 26443,
  "slug": "26443",
  "resource": "https://minifetch.com/api/v1/x402/preflight/url-check",
  "description": "Cheap robots.txt preflight check before you call the Minifetch /extract and /run/seo-page-audit endpoints: returns whether Minifetch may fetch a target URL (allow/deny) plus its crawl-delay. If blocked and the site is not yours try our external proxy routes. If blocked and the site is yours or to set a custom crawl-delay read https://minifetch.com/tutorials/unblock-minifetch Checks the URL as given — does not follow redirects. Supports GET & POST",
  "type": "http",
  "serviceName": "Minifetch Technical SEO API",
  "iconUrl": "https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1781035850/bazaar-icons/d02845f9304a54ee.png",
  "resourceType": "Travel",
  "accepts": [
    {
      "scheme": "exact",
      "network": "eip155:8453",
      "payTo": "0xffA9155A0e4B3d8d42351Da226cC1feEe449686f",
      "asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
      "amount": "1000",
      "amountUsd": 0.001,
      "maxTimeoutSeconds": 300
    },
    {
      "scheme": "exact",
      "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
      "payTo": "46vMcwuC4sK11sB3gkLhyA7J7GEwfkhn5rFyDtihBwqe",
      "asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
      "amount": "1000",
      "amountUsd": 0.001,
      "maxTimeoutSeconds": 300
    }
  ],
  "outputSchema": {
    "properties": {
      "example": {
        "properties": {
          "results": {
            "items": {
              "properties": {
                "data": {
                  "properties": {
                    "allowed": {
                      "description": "Whether the minifetch user agent may fetch this URL per robots.txt",
                      "type": "boolean"
                    },
                    "crawlDelay": {
                      "description": "Crawl-delay (seconds) for the minifetch UA; omitted when blocked or the domain is invalid",
                      "type": "number"
                    },
                    "message": {
                      "description": "e.g. 'Access allowed by robots.txt', 'URL is blocked by robots.txt', or 'robots.txt not found, defaulting to allowed'",
                      "type": "string"
                    },
                    "minifetchCache": {
                      "description": "robots.txt cache state (~24h TTL; pass fresh=true to bypass)",
                      "properties": {
                        "cachedAt": {
                          "type": "string"
                        },
                        "expiresAt": {
                          "type": "string"
                        },
                        "hit": {
                          "type": "boolean"
                        }
                      },
                      "type": "object"
                    },
                    "proxy": {
                      "description": "Fetch provenance — always minifetch UA here",
                      "properties": {
                        "proxyType": {
                          "enum": [
                            "minifetch"
                          ],
                          "type": "string"
                        },
                        "userAgent": {
                          "type": "string"
                        }
                      },
                      "type": "object"
                    },
                    "url": {
                      "description": "The URL checked, exactly as given — redirects are not followed",
                      "type": "string"
                    }
                  },
                  "required": [
                    "url",
                    "allowed",
                    "message",
                    "proxy"
                  ],
                  "type": "object"
                }
              },
              "type": "object"
            },
            "type": "array"
          },
          "success": {
            "type": "boolean"
          },
          "tip": {
            "description": "Present only when allowed=false: how to retry via the external proxy routes, and how site owners can unblock the minifetch user agent",
            "properties": {
              "message": {
                "type": "string"
              },
              "url": {
                "type": "string"
              }
            },
            "type": "object"
          }
        },
        "type": "object"
      },
      "type": {
        "type": "string"
      }
    },
    "required": [
      "type"
    ],
    "type": "object"
  },
  "extensions": {
    "bazaar": {
      "info": {
        "input": {
          "method": "GET",
          "queryParams": {
            "url": "https://github.com"
          },
          "type": "http"
        },
        "output": {
          "example": {
            "results": [
              {
                "data": {
                  "allowed": true,
                  "crawlDelay": 1,
                  "message": "Access allowed by robots.txt",
                  "minifetchCache": {
                    "cachedAt": "2026-08-16T18:22:10.000Z",
                    "expiresAt": "2026-08-17T18:22:10.000Z",
                    "hit": false
                  },
                  "proxy": {
                    "proxyType": "minifetch",
                    "userAgent": "minifetch/1.0 (+https://minifetch.com/site-owner-faq)"
                  },
                  "url": "https://github.com"
                }
              }
            ],
            "success": true
          },
          "type": "json"
        }
      },
      "schema": {
        "$schema": "https://json-schema.org/draft/2020-12/schema",
        "properties": {
          "input": {
            "additionalProperties": false,
            "properties": {
              "method": {
                "enum": [
                  "GET"
                ],
                "type": "string"
              },
              "queryParams": {
                "properties": {
                  "fresh": {
                    "description": "Optional. If true, bypasses the 24h robots.txt cache and re-fetches robots.txt for this check. Use after editing your robots.txt or crawl-delay (or adding/removing a Minifetch block) to confirm the change took effect.",
                    "type": "boolean"
                  },
                  "url": {
                    "description": "The target URL to check against robots.txt",
                    "type": "string"
                  }
                },
                "required": [
                  "url"
                ],
                "type": "object"
              },
              "type": {
                "const": "http",
                "type": "string"
              }
            },
            "required": [
              "type",
              "method"
            ],
            "type": "object"
          },
          "output": {
            "properties": {
              "example": {
                "properties": {
                  "results": {
                    "items": {
                      "properties": {
                        "data": {
                          "properties": {
                            "allowed": {
                              "description": "Whether the minifetch user agent may fetch this URL per robots.txt",
                              "type": "boolean"
                            },
                            "crawlDelay": {
                              "description": "Crawl-delay (seconds) for the minifetch UA; omitted when blocked or the domain is invalid",
                              "type": "number"
                            },
                            "message": {
                              "description": "e.g. 'Access allowed by robots.txt', 'URL is blocked by robots.txt', or 'robots.txt not found, defaulting to allowed'",
                              "type": "string"
                            },
                            "minifetchCache": {
                              "description": "robots.txt cache state (~24h TTL; pass fresh=true to bypass)",
                              "properties": {
                                "cachedAt": {
                                  "type": "string"
                                },
                                "expiresAt": {
                                  "type": "string"
                                },
                                "hit": {
                                  "type": "boolean"
                                }
                              },
                              "type": "object"
                            },
                            "proxy": {
                              "description": "Fetch provenance — always minifetch UA here",
                              "properties": {
                                "proxyType": {
                                  "enum": [
                                    "minifetch"
                                  ],
                                  "type": "string"
                                },
                                "userAgent": {
                                  "type": "string"
                                }
                              },
                              "type": "object"
                            },
                            "url": {
                              "description": "The URL checked, exactly as given — redirects are not followed",
                              "type": "string"
                            }
                          },
                          "required": [
                            "url",
                            "allowed",
                            "message",
                            "proxy"
                          ],
                          "type": "object"
                        }
                      },
                      "type": "object"
                    },
                    "type": "array"
                  },
                  "success": {
                    "type": "boolean"
                  },
                  "tip": {
                    "description": "Present only when allowed=false: how to retry via the external proxy routes, and how site owners can unblock the minifetch user agent",
                    "properties": {
                      "message": {
                        "type": "string"
                      },
                      "url": {
                        "type": "string"
                      }
                    },
                    "type": "object"
                  }
                },
                "type": "object"
              },
              "type": {
                "type": "string"
              }
            },
            "required": [
              "type"
            ],
            "type": "object"
          }
        },
        "required": [
          "input"
        ],
        "type": "object"
      }
    }
  },
  "tags": [
    "seo",
    "robots-txt",
    "preflight",
    "crawlability",
    "indexability",
    "bazaar"
  ],
  "sourceHost": "minifetch.com",
  "lastUpdated": "2026-10-13T21:00:56.202Z",
  "quality": {
    "calls30d": 45,
    "uniquePayers30d": 14,
    "lastCalledAt": "2026-10-13T21:00:55.84Z"
  },
  "metadata": {
    "serviceName": "Minifetch Technical SEO API",
    "iconUrl": "https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1781035850/bazaar-icons/d02845f9304a54ee.png",
    "extensions": {
      "bazaar": {
        "info": {
          "input": {
            "method": "GET",
            "queryParams": {
              "url": "https://github.com"
            },
            "type": "http"
          },
          "output": {
            "example": {
              "results": [
                {
                  "data": {
                    "allowed": true,
                    "crawlDelay": 1,
                    "message": "Access allowed by robots.txt",
                    "minifetchCache": {
                      "cachedAt": "2026-08-16T18:22:10.000Z",
                      "expiresAt": "2026-08-17T18:22:10.000Z",
                      "hit": false
                    },
                    "proxy": {
                      "proxyType": "minifetch",
                      "userAgent": "minifetch/1.0 (+https://minifetch.com/site-owner-faq)"
                    },
                    "url": "https://github.com"
                  }
                }
              ],
              "success": true
            },
            "type": "json"
          }
        },
        "schema": {
          "$schema": "https://json-schema.org/draft/2020-12/schema",
          "properties": {
            "input": {
              "additionalProperties": false,
              "properties": {
                "method": {
                  "enum": [
                    "GET"
                  ],
                  "type": "string"
                },
                "queryParams": {
                  "properties": {
                    "fresh": {
                      "description": "Optional. If true, bypasses the 24h robots.txt cache and re-fetches robots.txt for this check. Use after editing your robots.txt or crawl-delay (or adding/removing a Minifetch block) to confirm the change took effect.",
                      "type": "boolean"
                    },
                    "url": {
                      "description": "The target URL to check against robots.txt",
                      "type": "string"
                    }
                  },
                  "required": [
                    "url"
                  ],
                  "type": "object"
                },
                "type": {
                  "const": "http",
                  "type": "string"
                }
              },
              "required": [
                "type",
                "method"
              ],
              "type": "object"
            },
            "output": {
              "properties": {
                "example": {
                  "properties": {
                    "results": {
                      "items": {
                        "properties": {
                          "data": {
                            "properties": {
                              "allowed": {
                                "description": "Whether the minifetch user agent may fetch this URL per robots.txt",
                                "type": "boolean"
                              },
                              "crawlDelay": {
                                "description": "Crawl-delay (seconds) for the minifetch UA; omitted when blocked or the domain is invalid",
                                "type": "number"
                              },
                              "message": {
                                "description": "e.g. 'Access allowed by robots.txt', 'URL is blocked by robots.txt', or 'robots.txt not found, defaulting to allowed'",
                                "type": "string"
                              },
                              "minifetchCache": {
                                "description": "robots.txt cache state (~24h TTL; pass fresh=true to bypass)",
                                "properties": {
                                  "cachedAt": {
                                    "type": "string"
                                  },
                                  "expiresAt": {
                                    "type": "string"
                                  },
                                  "hit": {
                                    "type": "boolean"
                                  }
                                },
                                "type": "object"
                              },
                              "proxy": {
                                "description": "Fetch provenance — always minifetch UA here",
                                "properties": {
                                  "proxyType": {
                                    "enum": [
                                      "minifetch"
                                    ],
                                    "type": "string"
                                  },
                                  "userAgent": {
                                    "type": "string"
                                  }
                                },
                                "type": "object"
                              },
                              "url": {
                                "description": "The URL checked, exactly as given — redirects are not followed",
                                "type": "string"
                              }
                            },
                            "required": [
                              "url",
                              "allowed",
                              "message",
                              "proxy"
                            ],
                            "type": "object"
                          }
                        },
                        "type": "object"
                      },
                      "type": "array"
                    },
                    "success": {
                      "type": "boolean"
                    },
                    "tip": {
                      "description": "Present only when allowed=false: how to retry via the external proxy routes, and how site owners can unblock the minifetch user agent",
                      "properties": {
                        "message": {
                          "type": "string"
                        },
                        "url": {
                          "type": "string"
                        }
                      },
                      "type": "object"
                    }
                  },
                  "type": "object"
                },
                "type": {
                  "type": "string"
                }
              },
              "required": [
                "type"
              ],
              "type": "object"
            }
          },
          "required": [
            "input"
          ],
          "type": "object"
        }
      }
    }
  },
  "liveness": {
    "reliability": null
  },
  "verified": false,
  "featured": false,
  "provider": {
    "host": "minifetch.com",
    "manifest_name": "minifetch.com",
    "source": "cdp-mirror",
    "source_manifest_url": "https://api.cdp.coinbase.com/platform/v2/x402/discovery/resources",
    "submitted_at": "2026-10-14T01:15:01.050Z"
  },
  "exampleResponse": {
    "example": {
      "results": [
        {
          "data": {
            "allowed": true,
            "crawlDelay": 0,
            "message": "string",
            "minifetchCache": {
              "cachedAt": "string",
              "expiresAt": "string",
              "hit": true
            },
            "proxy": {
              "proxyType": "minifetch",
              "userAgent": "string"
            },
            "url": "string"
          }
        }
      ],
      "success": true,
      "tip": {
        "message": "string",
        "url": "string"
      }
    },
    "type": "string"
  }
}