Cheap robots
Cheap robots.txt preflight before you pay for other Minifetch endpoints: returns whether the Minifetch user agent may fetch a target URL (allow/deny) plus its crawl-delay. Blocked or setting a crawl delay? https://minifetch.com/tutorials/unblock-minifetch . Checks the URL as given — does not follow redirects. Supports GET & POST
1000 (raw units)
price
23
calls / 30d
11
unique payers
2026-09-17
updated
Provider
minifetch.com · discovered, not yet claimed by its owner
Payment (x402 accepts[])
[
{
"scheme": "exact",
"network": "eip155:8453",
"payTo": "0xffA9155A0e4B3d8d42351Da226cC1feEe449686f",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"amount": "1000",
"maxTimeoutSeconds": 300
},
{
"scheme": "exact",
"network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
"payTo": "46vMcwuC4sK11sB3gkLhyA7J7GEwfkhn5rFyDtihBwqe",
"asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
"amount": "1000",
"maxTimeoutSeconds": 300
}
]Output schema
{
"bazaar": {
"info": {
"input": {
"method": "GET",
"queryParams": {
"url": "https://github.com"
},
"type": "http"
},
"output": {
"example": {
"results": [
{
"data": {
"allowed": true,
"crawlDelay": 1,
"message": "Access allowed by robots.txt",
"minifetchCache": {
"cachedAt": "2026-08-16T18:22:10.000Z",
"expiresAt": "2026-08-17T18:22:10.000Z",
"hit": false
},
"proxy": {
"proxyType": "minifetch",
"userAgent": "minifetch/1.0 (+https://minifetch.com/site-owner-faq)"
},
"url": "https://github.com"
}
}
],
"success": true
},
"type": "json"
}
},
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"input": {
"additionalProperties": false,
"properties": {
"method": {
"enum": [
"GET"
],
"type": "string"
},
"queryParams": {
"properties": {
"fresh": {
"description": "Optional. If true, bypasses the 24h robots.txt cache and re-fetches robots.txt for this check. Use after editing your robots.txt or crawl-delay (or adding/removing a Minifetch block) to confirm the change took effect.",
"type": "boolean"
},
"url": {
"description": "The URL to check against robots.txt",
"type": "string"
}
},
"required": [
"url"
],
"type": "object"
},
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type",
"method"
],
"type": "object"
},
"output": {
"properties": {
"example": {
"properties": {
"results": {
"items": {
"properties": {
"data": {
"properties": {
"allowed": {
"description": "Whether the minifetch user agent may fetch this URL per robots.txt",
"type": "boolean"
},
"crawlDelay": {
"description": "Crawl-delay (seconds) for the minifetch UA; omitted when blocked or the domain is invalid",
"type": "number"
},
"message": {
"description": "e.g. 'Access allowed by robots.txt', 'URL is blocked by robots.txt', or 'robots.txt not found, defaulting to allowed'",
"type": "string"
},
"minifetchCache": {
"description": "robots.txt cache state (~24h TTL; pass fresh=true to bypass)",
"properties": {
"cachedAt": {
"type": "string"
},
"expiresAt": {
"type": "string"
},
"hit": {
"type": "boolean"
}
},
"type": "object"
},
"proxy": {
"description": "Fetch provenance — always minifetch UA here",
"properties": {
"proxyType": {
"enum": [
"minifetch"
],
"type": "string"
},
"userAgent": {
"type": "string"
}
},
"type": "object"
},
"url": {
"description": "The URL checked, exactly as given — redirects are not followed",
"type": "string"
}
},
"required": [
"url",
"allowed",
"message",
"proxy"
],
"type": "object"
}
},
"type": "object"
},
"type": "array"
},
"success": {
"type": "boolean"
},
"tip": {
"description": "Present only when allowed=false: how site owners can unblock the minifetch user agent",
"properties": {
"message": {
"type": "string"
},
"url": {
"type": "string"
}
},
"type": "object"
}
},
"type": "object"
},
"type": {
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
},
"required": [
"input"
],
"type": "object"
}
}
}Use it
curl
curl "https://minifetch.com/api/v1/x402/preflight/url-check" # -> 402 Payment Required, accepts[] lists how to pay # retry with a PAYMENT-SIGNATURE (or PAYMENT header) once paid
JavaScript
const res = await fetch("https://minifetch.com/api/v1/x402/preflight/url-check");
if (res.status === 402) {
const { accepts } = await res.json();
// pay one of accepts[] via an x402 client, then retry with the payment header
}Python
import httpx
res = httpx.get("https://minifetch.com/api/v1/x402/preflight/url-check")
if res.status_code == 402:
accepts = res.json()["accepts"]
# pay one of accepts[] via an x402 client, then retry with the payment headerMachine-readable
Everything on this page is also available as clean JSON at /resources/754.json, and this resource appears in /discovery/resources and /discovery/search.