Discover all URLs reachable from a domain within a link depth limit
Discover all URLs reachable from a domain within a link depth limit. Returns a flat list of absolute URLs, respects robots.txt crawl delays. Use it to map a site's structure before crawling.
50000 (raw units)
price
2
calls / 30d
1
unique payers
2026-09-08
updated
Provider
api.agishub.com · discovered, not yet claimed by its owner
Payment (x402 accepts[])
[
{
"scheme": "exact",
"network": "eip155:8453",
"payTo": "0x23Fc725e8EAa1c58900C6507Ba89be99a6DFc75d",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"amount": "50000",
"maxTimeoutSeconds": 300
}
]Output schema
{
"bazaar": {
"info": {
"input": {
"body": {
"limit": 5,
"url": "https://example.com"
},
"bodyType": "json",
"method": "POST",
"type": "http"
},
"output": {
"example": {
"result": "Discover all URLs reachable from a domain within a link depth limit. Returns a flat list of absolute URLs, respects robo"
},
"type": "json"
}
},
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"input": {
"additionalProperties": false,
"properties": {
"body": {
"additionalProperties": false,
"properties": {
"include_subdomains": {
"description": "Include URLs from subdomains (default false, same domain only).",
"type": "boolean"
},
"limit": {
"description": "Maximum URLs to return (default 100, max 200 per call). Respects robots.txt crawl-delay.",
"exclusiveMinimum": true,
"maximum": 200,
"minimum": 0,
"type": "integer"
},
"search": {
"description": "Optional regex or plain string to filter results (case-insensitive).",
"type": "string"
},
"url": {
"description": "Root domain URL to map (e.g., https://example.com).",
"format": "uri",
"type": "string"
}
},
"required": [
"url"
],
"type": "object"
},
"bodyType": {
"enum": [
"json"
],
"type": "string"
},
"method": {
"enum": [
"POST"
],
"type": "string"
},
"type": {
"enum": [
"http"
],
"type": "string"
}
},
"required": [
"type",
"method",
"bodyType",
"body"
],
"type": "object"
},
"output": {
"additionalProperties": false,
"properties": {
"example": {
"type": "object"
},
"type": {
"enum": [
"json"
],
"type": "string"
}
},
"required": [
"type",
"example"
],
"type": "object"
}
},
"required": [
"input",
"output"
],
"type": "object"
}
}
}Use it
curl
curl "https://api.agishub.com/paid/crawl-map" # -> 402 Payment Required, accepts[] lists how to pay # retry with a PAYMENT-SIGNATURE (or PAYMENT header) once paid
JavaScript
const res = await fetch("https://api.agishub.com/paid/crawl-map");
if (res.status === 402) {
const { accepts } = await res.json();
// pay one of accepts[] via an x402 client, then retry with the payment header
}Python
import httpx
res = httpx.get("https://api.agishub.com/paid/crawl-map")
if res.status_code == 402:
accepts = res.json()["accepts"]
# pay one of accepts[] via an x402 client, then retry with the payment headerMachine-readable
Everything on this page is also available as clean JSON at /resources/12920.json, and this resource appears in /discovery/resources and /discovery/search.