Lints and validates a caller-supplied robots
Lints and validates a caller-supplied robots.txt against RFC 9309 and reports every check, not only the failures: group structure, product token charset, path patterns, special characters, other records, size limit. Given a user agent and URLs it recomputes the allow/disallow decision per URL, naming the winning rule, its octet length and the tiebreak that applied. Where the RFC is silent or contradicts itself, the reading used is stated. Makes no outbound request.
50000 (raw units)
price
1
calls / 30d
1
unique payers
2026-09-16
updated
Provider
api.ozdreamtools.de · discovered, not yet claimed by its owner
Payment (x402 accepts[])
[
{
"scheme": "exact",
"network": "eip155:8453",
"payTo": "0x4Dd5E59475A55f1C2c0816469600663cb74cC1Dc",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"amount": "50000",
"maxTimeoutSeconds": 300
}
]Output schema
{
"bazaar": {
"info": {
"input": {
"body": {
"robots_txt": "User-Agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\nSitemap: https://example.de/sitemap.xml\n\nUser-Agent: foobot\nDisallow: /\nAllow: /example/page.html\n",
"urls": [
"/example/page.html",
"/example/other"
],
"user_agent": "foobot"
},
"bodyType": "json",
"method": "POST",
"type": "http"
},
"output": {
"example": {
"applies_to": {
"match": "exact",
"merged_from": 1,
"rules": 2,
"user_agent": "foobot"
},
"checks": [
{
"findings": 0,
"key": "product_token_charset",
"rfc": "§2.2.1",
"status": "ok"
}
],
"decisions": [
{
"decision": "allowed",
"normalized": "/example/page.html",
"octets": 18,
"reason": "most_specific_match",
"rule": "allow: /example/page.html",
"rule_type": "allow",
"tiebreak": null,
"url": "/example/page.html"
}
],
"findings": [
{
"check": "path_leading_slash",
"detail": "…",
"key": "path_starts_with_wildcard",
"line": 2,
"rfc": "§2.2 (ABNF) vs §5.1",
"severity": "warning"
}
],
"groups": [
{
"line": 8,
"rules": [
{
"effect": "matching",
"line": 9,
"pattern": "/example/page.html",
"type": "allow"
}
],
"user_agents": [
"foobot"
]
}
],
"input": {
"bytes": 180,
"groups": 2,
"lines": 10,
"other_records": 1,
"rules": 5
},
"interpretations": [
{
"choice": "…",
"rfc": "§2.2.2",
"topic": "octet_counting",
"why": "…"
}
],
"limits": {
"bytes": 180,
"rfc_parsing_limit_bytes": 512000,
"within_limit": true
},
"linted_at": "2026-07-26T09:00:00.000Z",
"note": "…",
"other_records": [
{
"key": "sitemap",
"line": 5,
"standard": false,
"value": "https://example.de/sitemap.xml"
}
],
"summary": {
"errors": 0,
"infos": 0,
"warnings": 1
}
},
"type": "json"
}
},
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"input": {
"additionalProperties": false,
"properties": {
"body": {
"properties": {
"robots_txt": {
"description": "Contents of the robots.txt to check",
"type": "string"
},
"urls": {
"description": "URLs or paths whose access decision should be recomputed, at most 50",
"items": {
"type": "string"
},
"type": "array"
},
"user_agent": {
"description": "Product token to evaluate for, e.g. 'GPTBot'. A full User-Agent header is resolved to the token it contains. Required when 'urls' is given.",
"type": "string"
}
},
"required": [
"robots_txt"
]
},
"bodyType": {
"enum": [
"json",
"form-data",
"text"
],
"type": "string"
},
"method": {
"enum": [
"POST"
],
"type": "string"
},
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type",
"method",
"bodyType",
"body"
],
"type": "object"
},
"output": {
"properties": {
"example": {
"type": "object"
},
"type": {
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
},
"required": [
"input"
],
"type": "object"
}
}
}Use it
curl
curl "https://api.ozdreamtools.de/api/robots/lint" # -> 402 Payment Required, accepts[] lists how to pay # retry with a PAYMENT-SIGNATURE (or PAYMENT header) once paid
JavaScript
const res = await fetch("https://api.ozdreamtools.de/api/robots/lint");
if (res.status === 402) {
const { accepts } = await res.json();
// pay one of accepts[] via an x402 client, then retry with the payment header
}Python
import httpx
res = httpx.get("https://api.ozdreamtools.de/api/robots/lint")
if res.status_code == 402:
accepts = res.json()["accepts"]
# pay one of accepts[] via an x402 client, then retry with the payment headerMachine-readable
Everything on this page is also available as clean JSON at /resources/3603.json, and this resource appears in /discovery/resources and /discovery/search.