Crawl a website: fetch a page, follow the links inside the same host and return every page
Crawl a website: fetch a page, follow the links inside the same host and return every page it reached, within a page budget you set. For reading documentation, a product catalogue or a changelog in one paid call instead of many.
10000 (raw units)
price
30
calls / 30d
2
unique payers
2026-09-14
updated
Provider
agents.dexl.io · discovered, not yet claimed by its owner
Payment (x402 accepts[])
[
{
"scheme": "upto",
"network": "eip155:8453",
"payTo": "0x820D61a720963fab63aB6e0aeF4D1779Eb42F6BB",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"amount": "10000",
"maxTimeoutSeconds": 300
},
{
"scheme": "exact",
"network": "eip155:8453",
"payTo": "0x820D61a720963fab63aB6e0aeF4D1779Eb42F6BB",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"amount": "10000",
"maxTimeoutSeconds": 300
},
{
"scheme": "upto",
"network": "eip155:137",
"payTo": "0x820D61a720963fab63aB6e0aeF4D1779Eb42F6BB",
"asset": "0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359",
"amount": "10000",
"maxTimeoutSeconds": 300
},
{
"scheme": "exact",
"network": "eip155:137",
"payTo": "0x820D61a720963fab63aB6e0aeF4D1779Eb42F6BB",
"asset": "0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359",
"amount": "10000",
"maxTimeoutSeconds": 300
},
{
"scheme": "upto",
"network": "eip155:42161",
"payTo": "0x820D61a720963fab63aB6e0aeF4D1779Eb42F6BB",
"asset": "0xaf88d065e77c8cC2239327C5EDb3A432268e5831",
"amount": "10000",
"maxTimeoutSeconds": 300
},
{
"scheme": "exact",
"network": "eip155:42161",
"payTo": "0x820D61a720963fab63aB6e0aeF4D1779Eb42F6BB",
"asset": "0xaf88d065e77c8cC2239327C5EDb3A432268e5831",
"amount": "10000",
"maxTimeoutSeconds": 300
},
{
"scheme": "exact",
"network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
"payTo": "78XC8ziNQoGHpKDkGWhd4FtzHVPd5QXq8scZjWaYQcjC",
"asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
"amount": "10000",
"maxTimeoutSeconds": 300
}
]Output schema
{
"bazaar": {
"info": {
"input": {
"body": {
"maxPages": 3,
"url": "https://example.com"
},
"bodyType": "json",
"method": "POST",
"type": "http"
},
"output": {
"example": {
"cached": false,
"output": {},
"units": 1
},
"type": "json"
}
},
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"input": {
"additionalProperties": false,
"properties": {
"body": {
"properties": {
"max_pages": {
"default": 3,
"maximum": 10,
"minimum": 1,
"type": "integer"
},
"url": {
"description": "Absolute http(s) URL to start from",
"type": "string"
}
},
"required": [
"url"
],
"type": "object"
},
"bodyType": {
"enum": [
"json",
"form-data",
"text"
],
"type": "string"
},
"method": {
"enum": [
"POST"
],
"type": "string"
},
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type",
"method",
"bodyType",
"body"
],
"type": "object"
},
"output": {
"properties": {
"example": {
"type": "object"
},
"type": {
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
},
"required": [
"input"
],
"type": "object"
}
}
}Use it
curl
curl "https://agents.dexl.io/v1/tools/site-crawl" # -> 402 Payment Required, accepts[] lists how to pay # retry with a PAYMENT-SIGNATURE (or PAYMENT header) once paid
JavaScript
const res = await fetch("https://agents.dexl.io/v1/tools/site-crawl");
if (res.status === 402) {
const { accepts } = await res.json();
// pay one of accepts[] via an x402 client, then retry with the payment header
}Python
import httpx
res = httpx.get("https://agents.dexl.io/v1/tools/site-crawl")
if res.status_code == 402:
accepts = res.json()["accepts"]
# pay one of accepts[] via an x402 client, then retry with the payment headerMachine-readable
Everything on this page is also available as clean JSON at /resources/7011.json, and this resource appears in /discovery/resources and /discovery/search.