From 1ab57e3ccdbda7a9057d6a999619498ba2ebb365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Sat, 5 Sep 2026 17:44:24 +0000
Subject: [PATCH 1/3] Add JSON Schema for bot definitions

---
 README.md                   |  15 ++
 well-known-bots.json        |   7 -
 well-known-bots.schema.json | 379 ++++++++++++++++++++++++++++++++++++
 3 files changed, 394 insertions(+), 7 deletions(-)
 create mode 100644 well-known-bots.schema.json

diff --git a/README.md b/README.md
index dfe8f92..2d79e72 100644
--- a/README.md
+++ b/README.md
@@ -40,6 +40,20 @@ To add a new bot to the list, you need to edit the `well-known-bots.json` file a
 5. **Run validation** to ensure your entry is correct: `node validate.js --check`
 6. **Submit a pull request** with your changes
 
+### JSON Schema
+
+The [`well-known-bots.schema.json`](well-known-bots.schema.json) file describes
+the intended structure of `well-known-bots.json` for editors and external JSON
+Schema validators. The repository's built-in `validate.js` script is still the
+source of truth for checks that JSON Schema cannot express, such as compiling
+JavaScript regular expressions and testing `instances` against those patterns.
+
+You can validate the JSON file against the schema with Ajv:
+
+```bash
+npx --package ajv-cli --package ajv-formats ajv validate --spec=draft2020 -c ajv-formats -s well-known-bots.schema.json -d well-known-bots.json --all-errors
+```
+
 ### Bot Entry Structure
 
 Each entry in the JSON represents a specific bot or crawler and includes the following fields:
@@ -60,6 +74,7 @@ Each entry in the JSON represents a specific bot or crawler and includes the fol
   - **`accepted`** (array): User-Agent strings that should match the pattern
   - **`rejected`** (array): User-Agent strings that should not match
 - **`aliases`** (array): Alternative identifiers for the bot used in other data sources
+- **`description`** (string): Free-form human-readable notes about the bot
 - **`addition_date`** (string): Date the bot was added in YYYY/MM/DD format
 
 ### Available Categories
diff --git a/well-known-bots.json b/well-known-bots.json
index c27761a..4a5fb4e 100644
--- a/well-known-bots.json
+++ b/well-known-bots.json
@@ -2278,7 +2278,6 @@
       "forbidden": []
     },
     "addition_date": "2011/06/21",
-    "url": "",
     "verification": [],
     "instances": {
       "accepted": [
@@ -4273,9 +4272,6 @@
       "forbidden": []
     },
     "url": "http://www.archive.org/details/archive.org_bot",
-    "depends_on": [
-      "heritrix"
-    ],
     "verification": [],
     "instances": {
       "accepted": [
@@ -8933,9 +8929,6 @@
       "forbidden": []
     },
     "addition_date": "2018/10/14",
-    "depends_on": [
-      "libwww-perl"
-    ],
     "verification": [],
     "instances": {
       "accepted": [
diff --git a/well-known-bots.schema.json b/well-known-bots.schema.json
new file mode 100644
index 0000000..126267b
--- /dev/null
+++ b/well-known-bots.schema.json
@@ -0,0 +1,379 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/arcjet/well-known-bots/main/well-known-bots.schema.json",
+  "title": "Well Known Bots",
+  "description": "Schema for the well-known-bots.json bot definition list.",
+  "type": "array",
+  "items": {
+    "$ref": "#/$defs/bot"
+  },
+  "minItems": 1,
+  "uniqueItems": true,
+  "$defs": {
+    "bot": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "categories",
+        "pattern",
+        "verification"
+      ],
+      "properties": {
+        "id": {
+          "description": "Unique kebab-case bot identifier.",
+          "type": "string",
+          "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"
+        },
+        "categories": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/category"
+          },
+          "minItems": 1,
+          "uniqueItems": true
+        },
+        "pattern": {
+          "$ref": "#/$defs/userAgentPattern"
+        },
+        "verification": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/verification"
+          },
+          "uniqueItems": true
+        },
+        "url": {
+          "description": "Documentation URL for the bot.",
+          "$ref": "#/$defs/httpUrl"
+        },
+        "instances": {
+          "$ref": "#/$defs/instances"
+        },
+        "aliases": {
+          "description": "Alternative identifiers used by other data sources.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "minItems": 1,
+          "uniqueItems": true
+        },
+        "description": {
+          "description": "Free-form human-readable notes about the bot.",
+          "type": "string",
+          "minLength": 1
+        },
+        "addition_date": {
+          "description": "Date the bot was added, in YYYY/MM/DD format.",
+          "$ref": "#/$defs/slashDate"
+        }
+      }
+    },
+    "httpUrl": {
+      "type": "string",
+      "format": "uri",
+      "pattern": "^https?://"
+    },
+    "slashDate": {
+      "type": "string",
+      "pattern": "^(?:(?:19|20)\\d{2}/(?:(?:0[13578]|1[02])/(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)/(?:0[1-9]|[12]\\d|30)|02/(?:0[1-9]|1\\d|2[0-8]))|(?:19(?:0[48]|[2468][048]|[13579][26])|2000|20(?:0[48]|[2468][048]|[13579][26]))/02/29)$"
+    },
+    "category": {
+      "type": "string",
+      "enum": [
+        "academic",
+        "advertising",
+        "ai",
+        "amazon",
+        "apple",
+        "archive",
+        "feedfetcher",
+        "google",
+        "meta",
+        "microsoft",
+        "monitor",
+        "optimizer",
+        "preview",
+        "programmatic",
+        "search-engine",
+        "slack",
+        "social",
+        "tool",
+        "unknown",
+        "vercel",
+        "webhook",
+        "yahoo"
+      ]
+    },
+    "userAgentPattern": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "accepted",
+        "forbidden"
+      ],
+      "properties": {
+        "accepted": {
+          "description": "JavaScript regular expressions that must all match the User-Agent.",
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/regexString"
+          },
+          "minItems": 1,
+          "uniqueItems": true
+        },
+        "forbidden": {
+          "description": "JavaScript regular expressions that disqualify the User-Agent when any match.",
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/regexString"
+          },
+          "uniqueItems": true
+        }
+      }
+    },
+    "regexString": {
+      "type": "string",
+      "minLength": 1,
+      "format": "regex"
+    },
+    "instances": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "accepted",
+        "rejected"
+      ],
+      "properties": {
+        "accepted": {
+          "description": "User-Agent examples that must match every accepted pattern and no forbidden pattern.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "uniqueItems": true
+        },
+        "rejected": {
+          "description": "User-Agent examples that must not be accepted by this definition.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "uniqueItems": true
+        }
+      }
+    },
+    "verification": {
+      "oneOf": [
+        {
+          "$ref": "#/$defs/dnsVerification"
+        },
+        {
+          "$ref": "#/$defs/cidrVerification"
+        },
+        {
+          "$ref": "#/$defs/staticIpVerification"
+        },
+        {
+          "$ref": "#/$defs/sourceIpVerification"
+        }
+      ]
+    },
+    "dnsVerification": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "type",
+        "masks"
+      ],
+      "properties": {
+        "type": {
+          "const": "dns"
+        },
+        "masks": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "minItems": 1,
+          "uniqueItems": true
+        }
+      }
+    },
+    "cidrVerification": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "type",
+        "sources"
+      ],
+      "properties": {
+        "type": {
+          "const": "cidr"
+        },
+        "sources": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/cidrSource"
+          },
+          "minItems": 1,
+          "uniqueItems": true
+        }
+      }
+    },
+    "staticIpVerification": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "type",
+        "ips"
+      ],
+      "properties": {
+        "type": {
+          "const": "ip"
+        },
+        "ips": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/ipAddressOrCidr"
+          },
+          "minItems": 1,
+          "uniqueItems": true
+        }
+      }
+    },
+    "sourceIpVerification": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "type",
+        "sources"
+      ],
+      "properties": {
+        "type": {
+          "const": "ip"
+        },
+        "sources": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/ipSource"
+          },
+          "minItems": 1,
+          "uniqueItems": true
+        }
+      }
+    },
+    "cidrSource": {
+      "oneOf": [
+        {
+          "$ref": "#/$defs/httpJsonSource"
+        },
+        {
+          "$ref": "#/$defs/httpCsvSource"
+        },
+        {
+          "$ref": "#/$defs/httpTextSource"
+        }
+      ]
+    },
+    "ipSource": {
+      "oneOf": [
+        {
+          "$ref": "#/$defs/httpJsonSource"
+        },
+        {
+          "$ref": "#/$defs/httpTextSource"
+        }
+      ]
+    },
+    "httpJsonSource": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "type",
+        "url",
+        "selector"
+      ],
+      "properties": {
+        "type": {
+          "const": "http-json"
+        },
+        "url": {
+          "$ref": "#/$defs/httpUrl"
+        },
+        "selector": {
+          "type": "string",
+          "minLength": 1,
+          "not": {
+            "pattern": "\\\\\""
+          }
+        }
+      }
+    },
+    "httpCsvSource": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "type",
+        "url"
+      ],
+      "properties": {
+        "type": {
+          "const": "http-csv"
+        },
+        "url": {
+          "$ref": "#/$defs/httpUrl"
+        }
+      }
+    },
+    "httpTextSource": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "type",
+        "url"
+      ],
+      "properties": {
+        "type": {
+          "const": "http-text"
+        },
+        "url": {
+          "$ref": "#/$defs/httpUrl"
+        }
+      }
+    },
+    "ipAddress": {
+      "type": "string",
+      "anyOf": [
+        {
+          "format": "ipv4"
+        },
+        {
+          "format": "ipv6"
+        }
+      ]
+    },
+    "ipAddressOrCidr": {
+      "type": "string",
+      "anyOf": [
+        {
+          "format": "ipv4"
+        },
+        {
+          "format": "ipv6"
+        },
+        {
+          "pattern": "^(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)/(?:[0-9]|[12]\\d|3[0-2])$"
+        },
+        {
+          "pattern": "^[0-9A-Fa-f:.]+/(?:[0-9]|[1-9]\\d|1[01]\\d|12[0-8])$"
+        }
+      ]
+    }
+  }
+}

From fdaff7165268bd82da4f0edab192491e859a9bc4 Mon Sep 17 00:00:00 2001
From: David Mytton <david@mytton.net>
Date: Wed, 9 Sep 2026 08:41:54 +0100
Subject: [PATCH 2/3] fix: validate bot schema and CIDR ranges in CI
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reject malformed IPv4 and IPv6 CIDR ranges, including invalid prefix lengths, while supporting compressed IPv6 and embedded IPv4 addresses. Cross-check the schema patterns against Node's IP parser and run schema validation in CI.

Builds on Viktor Szépe's JSON Schema contribution in arcjet/well-known-bots#72; the original commit and authorship are preserved.

Co-authored-by: Codex <codex@openai.com>
---
 .github/workflows/ci-validation.yml | 11 ++++-
 AGENTS.md                           | 15 +++---
 README.md                           |  7 ++-
 schema.test.js                      | 72 +++++++++++++++++++++++++++++
 well-known-bots.schema.json         | 13 +++++-
 5 files changed, 107 insertions(+), 11 deletions(-)
 create mode 100644 schema.test.js

diff --git a/.github/workflows/ci-validation.yml b/.github/workflows/ci-validation.yml
index 0e0161f..0746e3f 100644
--- a/.github/workflows/ci-validation.yml
+++ b/.github/workflows/ci-validation.yml
@@ -1,6 +1,9 @@
 name: CI validation
 
-on: [push, pull_request]
+on: [push, pull_request, merge_group]
+
+permissions:
+  contents: read
 
 jobs:
   build:
@@ -11,3 +14,9 @@ jobs:
         with:
           node-version: 20
       - run: node validate.js --check
+      - run: node --test schema.test.js
+      - name: Validate bot definitions against the JSON Schema
+        run: >-
+          npx --yes --package ajv-cli@5.0.0 --package ajv-formats@3.0.1
+          ajv validate --spec=draft2020 -c ajv-formats
+          -s well-known-bots.schema.json -d well-known-bots.json --all-errors
diff --git a/AGENTS.md b/AGENTS.md
index 46fe27b..750ac4e 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -16,11 +16,12 @@ This repository maintains a curated list of well-known bots, crawlers, validator
 
 ### No Package Manager Required
 
-This is a **data repository** with NO package.json or dependencies. It uses only Node.js built-in modules.
+This is a **data repository** with no package.json or project installation. The built-in validator and CIDR tests use only Node.js built-in modules; CI also runs the pinned Ajv command documented in README.md.
 
-- ❌ **DO NOT** run `npm install`, `npm init`, or any package manager commands
+- ❌ **DO NOT** run `npm install` or `npm init`
 - ❌ **DO NOT** create a package.json file
 - ✅ **DO** use Node.js directly to run scripts (e.g., `node validate.js --check`)
+- ✅ **DO** run the documented pinned `npx` command for JSON Schema validation
 
 ### Validation Script (Critical)
 
@@ -77,8 +78,8 @@ When adding or modifying bot entries, refer to the [README.md](README.md) for:
 
 The repository uses GitHub Actions for validation:
 
-- **Trigger**: Runs on every push and pull request
-- **What it does**: Executes `node validate.js --check`
+- **Trigger**: Runs on every push, pull request, and merge group
+- **What it does**: Runs `node validate.js --check`, `node --test schema.test.js`, and the pinned Ajv schema validation command from README.md
 - **Node version**: 20.x
 - **Location**: `.github/workflows/ci-validation.yml`
 
@@ -139,14 +140,14 @@ node validate.js --check
 
 ## Testing Your Changes
 
-Since this is a data repository with no test suite, validation IS the testing:
+Run the built-in validation, CIDR regression tests, and the pinned Ajv schema validation command from README.md:
 
 ```bash
 # Run validation - this is your test suite
 node validate.js --check
 
-# Expected output: (nothing) with exit code 0
-# If there are errors, they will be printed to stderr
+# The built-in validator exits with code 0 and no output on success.
+node --test schema.test.js
 ```
 
 ## Tips for Agents
diff --git a/README.md b/README.md
index 2d79e72..0c2927c 100644
--- a/README.md
+++ b/README.md
@@ -51,9 +51,14 @@ JavaScript regular expressions and testing `instances` against those patterns.
 You can validate the JSON file against the schema with Ajv:
 
 ```bash
-npx --package ajv-cli --package ajv-formats ajv validate --spec=draft2020 -c ajv-formats -s well-known-bots.schema.json -d well-known-bots.json --all-errors
+npx --yes --package ajv-cli@5.0.0 --package ajv-formats@3.0.1 ajv validate --spec=draft2020 -c ajv-formats -s well-known-bots.schema.json -d well-known-bots.json --all-errors
 ```
 
+CI runs both validators and the CIDR regression tests (`node --test schema.test.js`).
+Ajv is downloaded on demand for schema validation; no project installation or
+`package.json` is needed. Static IP lists accept IPv4 and IPv6 addresses and CIDR
+ranges, including IPv6 ranges with an embedded IPv4 address.
+
 ### Bot Entry Structure
 
 Each entry in the JSON represents a specific bot or crawler and includes the following fields:
diff --git a/schema.test.js b/schema.test.js
new file mode 100644
index 0000000..0f8e999
--- /dev/null
+++ b/schema.test.js
@@ -0,0 +1,72 @@
+const assert = require("node:assert/strict");
+const { isIP } = require("node:net");
+const { test } = require("node:test");
+const schema = require("./well-known-bots.schema.json");
+
+const patterns = {
+  4: new RegExp(schema.$defs.ipv4Cidr.pattern, "u"),
+  6: new RegExp(schema.$defs.ipv6Cidr.pattern, "u"),
+};
+
+// Check the schema's CIDR patterns against Node's independent address parser.
+function checkCidr(value, expected) {
+  const parts = value.split("/");
+  const family = isIP(parts[0]);
+  const valid = parts.length === 2 && family !== 0 &&
+    !parts[0].includes("%") && /^(0|[1-9][0-9]*)(?![\s\S])/u.test(parts[1]) &&
+    Number(parts[1]) <= (family === 4 ? 32 : 128);
+  assert.equal(valid, expected, `Node parser: ${JSON.stringify(value)}`);
+  assert.equal(
+    patterns[4].test(value) || patterns[6].test(value),
+    expected,
+    `Schema: ${JSON.stringify(value)}`,
+  );
+}
+
+test("CIDRs accept valid addresses and prefix boundaries", () => {
+  for (const address of ["0.0.0.0", "192.0.2.1", "255.255.255.255"]) {
+    for (let prefix = 0; prefix <= 32; prefix++) {
+      checkCidr(`${address}/${prefix}`, true);
+    }
+  }
+  for (const address of [
+    "::", "::1", "2001:DB8::", "2001:db8:1:2:3:4:5:6",
+    "::ffff:192.0.2.1", "1:2:3:4:5:6:192.0.2.1",
+  ]) {
+    for (let prefix = 0; prefix <= 128; prefix++) {
+      checkCidr(`${address}/${prefix}`, true);
+    }
+  }
+});
+
+test("IPv6 compression accepts every position and rejects too many groups", () => {
+  for (const embedded of [false, true]) {
+    const groups = embedded ? 6 : 8;
+    for (let left = 0; left <= groups + 1; left++) {
+      for (let right = 0; right <= groups + 1; right++) {
+        const before = Array(left).fill("abcd").join(":");
+        const after = [
+          ...Array(right).fill("1234"),
+          ...(embedded ? ["192.0.2.1"] : []),
+        ].join(":");
+        checkCidr(`${before}::${after}/64`, left + right < groups);
+      }
+    }
+  }
+});
+
+test("CIDRs reject malformed addresses and prefixes", () => {
+  for (const value of [
+    "deadbeef/64", "::::/128", "1.2.3.4/64",
+    "192.0.2.1/33", "192.0.2.256/24", "192.00.2.1/24",
+    "2001:db8::/129", "2001:db8::/-1", "2001:db8::/01",
+    "2001:db8::/1.5", "2001:db8::/+64", "2001:db8::/",
+    "2001:db8::/64/64", "1:2:3:4:5:6:7/64", "1:2:3:4:5:6:7:8:9/64",
+    "2001::db8::1/64", "2001:db8:12345::/64", "2001:db8:xyz::/64",
+    "::ffff:256.1.2.3/64", "::ffff:192.00.2.1/64", "fe80::1%eth0/64",
+    " 2001:db8::/64", "2001:db8::/64 ", "2001:db8::/64\n",
+    "192.0.2.1/24\n", "2001:db8::", "192.0.2.1",
+  ]) {
+    checkCidr(value, false);
+  }
+});
diff --git a/well-known-bots.schema.json b/well-known-bots.schema.json
index 126267b..503c476 100644
--- a/well-known-bots.schema.json
+++ b/well-known-bots.schema.json
@@ -368,12 +368,21 @@
           "format": "ipv6"
         },
         {
-          "pattern": "^(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)/(?:[0-9]|[12]\\d|3[0-2])$"
+          "$ref": "#/$defs/ipv4Cidr"
         },
         {
-          "pattern": "^[0-9A-Fa-f:.]+/(?:[0-9]|[1-9]\\d|1[01]\\d|12[0-8])$"
+          "$ref": "#/$defs/ipv6Cidr"
         }
       ]
+    },
+    "ipv4Cidr": {
+      "type": "string",
+      "pattern": "^(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)/(?:[0-9]|[12]\\d|3[0-2])(?![\\s\\S])"
+    },
+    "ipv6Cidr": {
+      "$comment": "The alternatives cover eight explicit hextets, :: compression with zero through seven leading hextets, and embedded IPv4 (which occupies two hextets). A compressed address must omit at least one hextet. Scoped addresses are not network ranges.",
+      "type": "string",
+      "pattern": "^(?:(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,6})?|[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,5})?|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,4})?|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,3})?|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,2})?|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,1})?|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,0})?|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::|(?:[0-9A-Fa-f]{1,4}:){6}(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)|::(?:[0-9A-Fa-f]{1,4}:){0,5}(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)|[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,4}(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,3}(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,2}(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,1}(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)|[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,0}(?:(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d))/(?:[0-9]|[1-9]\\d|1[01]\\d|12[0-8])(?![\\s\\S])"
     }
   }
 }

From d7a77a7e46649e433d825b6d8a00c7735f7e1158 Mon Sep 17 00:00:00 2001
From: David Mytton <david@mytton.net>
Date: Wed, 9 Sep 2026 08:50:40 +0100
Subject: [PATCH 3/3] fix: lock schema validation dependencies in CI

Replace on-demand npx execution with an isolated tools/schema package and a committed dependency lockfile. Install with npm ci --ignore-scripts and invoke Ajv directly, removing ajv-cli and its extra dependencies.

Use releases older than seven days and cover the validator exit status for malformed bot definitions and CIDR ranges.

Co-authored-by: Codex <codex@openai.com>
---
 .github/workflows/ci-validation.yml |  9 ++--
 AGENTS.md                           | 28 +++++-----
 README.md                           | 22 +++++---
 tools/schema/package-lock.json      | 84 +++++++++++++++++++++++++++++
 tools/schema/package.json           |  8 +++
 tools/schema/validate.cjs           | 18 +++++++
 tools/schema/validate.test.cjs      | 56 +++++++++++++++++++
 7 files changed, 202 insertions(+), 23 deletions(-)
 create mode 100644 tools/schema/package-lock.json
 create mode 100644 tools/schema/package.json
 create mode 100644 tools/schema/validate.cjs
 create mode 100644 tools/schema/validate.test.cjs

diff --git a/.github/workflows/ci-validation.yml b/.github/workflows/ci-validation.yml
index 0746e3f..8cb267b 100644
--- a/.github/workflows/ci-validation.yml
+++ b/.github/workflows/ci-validation.yml
@@ -14,9 +14,8 @@ jobs:
         with:
           node-version: 20
       - run: node validate.js --check
-      - run: node --test schema.test.js
+      - name: Install schema validation dependencies
+        run: npm ci --prefix tools/schema --ignore-scripts
+      - run: node --test schema.test.js tools/schema/validate.test.cjs
       - name: Validate bot definitions against the JSON Schema
-        run: >-
-          npx --yes --package ajv-cli@5.0.0 --package ajv-formats@3.0.1
-          ajv validate --spec=draft2020 -c ajv-formats
-          -s well-known-bots.schema.json -d well-known-bots.json --all-errors
+        run: node tools/schema/validate.cjs
diff --git a/AGENTS.md b/AGENTS.md
index 750ac4e..e9b33c3 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -14,14 +14,19 @@ This repository maintains a curated list of well-known bots, crawlers, validator
 
 ## Working with This Repository
 
-### No Package Manager Required
+### Validation Dependencies
 
-This is a **data repository** with no package.json or project installation. The built-in validator and CIDR tests use only Node.js built-in modules; CI also runs the pinned Ajv command documented in README.md.
+This is a **data repository** with no root package.json or project installation.
+The built-in validator and CIDR tests use only Node.js built-in modules. JSON
+Schema validation uses the isolated `tools/schema/` package.
 
-- ❌ **DO NOT** run `npm install` or `npm init`
-- ❌ **DO NOT** create a package.json file
-- ✅ **DO** use Node.js directly to run scripts (e.g., `node validate.js --check`)
-- ✅ **DO** run the documented pinned `npx` command for JSON Schema validation
+- Do not create a root package.json or install dependencies at the repository root.
+- Install validation dependencies with `npm ci --prefix tools/schema --ignore-scripts`.
+- Run schema validation with `node tools/schema/validate.cjs`.
+- Keep validation dependency versions exact and commit the generated lockfile.
+  For an intentional update, run `npm install --prefix tools/schema --ignore-scripts
+  --save-exact <package>@<version>`, review the lockfile, and run
+  `npm audit --prefix tools/schema` plus all validation checks.
 
 ### Validation Script (Critical)
 
@@ -79,7 +84,7 @@ When adding or modifying bot entries, refer to the [README.md](README.md) for:
 The repository uses GitHub Actions for validation:
 
 - **Trigger**: Runs on every push, pull request, and merge group
-- **What it does**: Runs `node validate.js --check`, `node --test schema.test.js`, and the pinned Ajv schema validation command from README.md
+- **What it does**: Installs the locked schema dependencies with scripts disabled, then runs `node validate.js --check`, `node --test schema.test.js tools/schema/validate.test.cjs`, and `node tools/schema/validate.cjs`
 - **Node version**: 20.x
 - **Location**: `.github/workflows/ci-validation.yml`
 
@@ -140,14 +145,13 @@ node validate.js --check
 
 ## Testing Your Changes
 
-Run the built-in validation, CIDR regression tests, and the pinned Ajv schema validation command from README.md:
+Run the built-in validation, regression tests, and JSON Schema validation:
 
 ```bash
-# Run validation - this is your test suite
+npm ci --prefix tools/schema --ignore-scripts
 node validate.js --check
-
-# The built-in validator exits with code 0 and no output on success.
-node --test schema.test.js
+node --test schema.test.js tools/schema/validate.test.cjs
+node tools/schema/validate.cjs
 ```
 
 ## Tips for Agents
diff --git a/README.md b/README.md
index 0c2927c..809f0a6 100644
--- a/README.md
+++ b/README.md
@@ -48,16 +48,26 @@ Schema validators. The repository's built-in `validate.js` script is still the
 source of truth for checks that JSON Schema cannot express, such as compiling
 JavaScript regular expressions and testing `instances` against those patterns.
 
-You can validate the JSON file against the schema with Ajv:
+Install the locked validation dependencies, then validate the JSON file with Ajv:
 
 ```bash
-npx --yes --package ajv-cli@5.0.0 --package ajv-formats@3.0.1 ajv validate --spec=draft2020 -c ajv-formats -s well-known-bots.schema.json -d well-known-bots.json --all-errors
+npm ci --prefix tools/schema --ignore-scripts
+node tools/schema/validate.cjs
 ```
 
-CI runs both validators and the CIDR regression tests (`node --test schema.test.js`).
-Ajv is downloaded on demand for schema validation; no project installation or
-`package.json` is needed. Static IP lists accept IPv4 and IPv6 addresses and CIDR
-ranges, including IPv6 ranges with an embedded IPv4 address.
+CI runs both validators and the regression tests:
+
+```bash
+node --test schema.test.js tools/schema/validate.test.cjs
+```
+
+Validation dependencies are isolated in `tools/schema/`. Its lockfile fixes the
+full dependency tree and package integrity hashes; installation disables lifecycle
+scripts. Review dependency and lockfile updates together. To validate a different
+JSON file, pass its path to `node tools/schema/validate.cjs`.
+
+Static IP lists accept IPv4 and IPv6 addresses and CIDR ranges, including IPv6
+ranges with an embedded IPv4 address.
 
 ### Bot Entry Structure
 
diff --git a/tools/schema/package-lock.json b/tools/schema/package-lock.json
new file mode 100644
index 0000000..d06c2cf
--- /dev/null
+++ b/tools/schema/package-lock.json
@@ -0,0 +1,84 @@
+{
+  "name": "well-known-bots-schema-validation",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "well-known-bots-schema-validation",
+      "dependencies": {
+        "ajv": "8.20.0",
+        "ajv-formats": "3.0.1"
+      }
+    },
+    "node_modules/ajv": {
+      "version": "8.20.0",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz",
+      "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==",
+      "license": "MIT",
+      "dependencies": {
+        "fast-deep-equal": "^3.1.3",
+        "fast-uri": "^3.0.1",
+        "json-schema-traverse": "^1.0.0",
+        "require-from-string": "^2.0.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
+      }
+    },
+    "node_modules/ajv-formats": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz",
+      "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==",
+      "license": "MIT",
+      "dependencies": {
+        "ajv": "^8.0.0"
+      },
+      "peerDependencies": {
+        "ajv": "^8.0.0"
+      },
+      "peerDependenciesMeta": {
+        "ajv": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "license": "MIT"
+    },
+    "node_modules/fast-uri": {
+      "version": "3.1.6",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz",
+      "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fastify"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fastify"
+        }
+      ],
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/json-schema-traverse": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+      "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+      "license": "MIT"
+    },
+    "node_modules/require-from-string": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
+      "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    }
+  }
+}
diff --git a/tools/schema/package.json b/tools/schema/package.json
new file mode 100644
index 0000000..4ab4041
--- /dev/null
+++ b/tools/schema/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "well-known-bots-schema-validation",
+  "private": true,
+  "dependencies": {
+    "ajv": "8.20.0",
+    "ajv-formats": "3.0.1"
+  }
+}
diff --git a/tools/schema/validate.cjs b/tools/schema/validate.cjs
new file mode 100644
index 0000000..5b6eaef
--- /dev/null
+++ b/tools/schema/validate.cjs
@@ -0,0 +1,18 @@
+const { readFileSync } = require("node:fs");
+const { basename, resolve } = require("node:path");
+const Ajv = require("ajv/dist/2020");
+const addFormats = require("ajv-formats");
+
+const schema = require("../../well-known-bots.schema.json");
+const dataPath = process.argv[2] ?? resolve(__dirname, "../../well-known-bots.json");
+const data = JSON.parse(readFileSync(dataPath, "utf8"));
+const ajv = new Ajv({ allErrors: true, strict: true });
+addFormats(ajv);
+const validate = ajv.compile(schema);
+
+if (validate(data)) {
+  console.log(`${basename(dataPath)} valid`);
+} else {
+  console.error(ajv.errorsText(validate.errors, { separator: "\n" }));
+  process.exitCode = 1;
+}
diff --git a/tools/schema/validate.test.cjs b/tools/schema/validate.test.cjs
new file mode 100644
index 0000000..82e4bf4
--- /dev/null
+++ b/tools/schema/validate.test.cjs
@@ -0,0 +1,56 @@
+const assert = require("node:assert/strict");
+const { spawnSync } = require("node:child_process");
+const { mkdtempSync, rmSync, writeFileSync } = require("node:fs");
+const { tmpdir } = require("node:os");
+const { join } = require("node:path");
+const { test } = require("node:test");
+
+const script = join(__dirname, "validate.cjs");
+
+function validateFixture(t, changes) {
+  const directory = mkdtempSync(join(tmpdir(), "bot-schema-test-"));
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  const file = join(directory, "bots.json");
+  const bot = {
+    id: "test-bot",
+    categories: ["monitor"],
+    pattern: { accepted: ["TestBot"], forbidden: [] },
+    verification: [],
+    ...changes,
+  };
+  writeFileSync(file, JSON.stringify([bot]));
+  return spawnSync(process.execPath, [script, file], { encoding: "utf8" });
+}
+
+test("validates the repository data independently of the working directory", () => {
+  const result = spawnSync(process.execPath, [script], {
+    cwd: tmpdir(),
+    encoding: "utf8",
+  });
+  assert.equal(result.status, 0, result.stderr);
+  assert.match(result.stdout, /well-known-bots\.json valid/);
+});
+
+test("rejects structural schema violations", (t) => {
+  const result = validateFixture(t, { categories: ["not-a-category"] });
+  assert.equal(result.status, 1, result.stderr);
+  assert.match(result.stderr, /categories/);
+});
+
+test("enforces the registered IP address formats", (t) => {
+  const result = validateFixture(t, {
+    verification: [{ type: "ip", ips: ["999.999.999.999"] }],
+  });
+  assert.equal(result.status, 1, result.stderr);
+  assert.match(result.stderr, /verification/);
+});
+
+test("rejects the malformed CIDRs from the review through the full schema", (t) => {
+  for (const ip of ["deadbeef/64", "::::/128", "1.2.3.4/64"]) {
+    const result = validateFixture(t, {
+      verification: [{ type: "ip", ips: [ip] }],
+    });
+    assert.equal(result.status, 1, `${ip}: ${result.stderr}`);
+    assert.match(result.stderr, /verification/);
+  }
+});