From f322d030b199a5cabeaea243ed0bad710db30f0d Mon Sep 17 00:00:00 2001 From: David Mytton Date: Wed, 9 Sep 2026 09:27:47 +0100 Subject: [PATCH] refactor: run TypeScript validators on Node 26 Convert the bot and JSON Schema validators to TypeScript and execute them directly with Node 26. Treat parsed JSON as unknown and narrow it through runtime checks. Add strict type checking with a locked compiler and Node 26 types, and preserve check, generate, and schema-validation behavior with regression coverage. Co-authored-by: Codex --- .github/workflows/ci-validation.yml | 10 +- AGENTS.md | 43 +-- README.md | 17 +- tools/schema/package-lock.json | 396 ++++++++++++++++++++++++++++ tools/schema/package.json | 8 + tools/schema/tsconfig.json | 14 + tools/schema/validate.cjs | 18 -- tools/schema/validate.test.cjs | 2 +- tools/schema/validate.ts | 18 ++ validate.test.cjs | 63 +++++ validate.js => validate.ts | 96 ++++--- 11 files changed, 602 insertions(+), 83 deletions(-) create mode 100644 tools/schema/tsconfig.json delete mode 100644 tools/schema/validate.cjs create mode 100644 tools/schema/validate.ts create mode 100644 validate.test.cjs rename validate.js => validate.ts (81%) diff --git a/.github/workflows/ci-validation.yml b/.github/workflows/ci-validation.yml index 88b27a4..d52d8bd 100644 --- a/.github/workflows/ci-validation.yml +++ b/.github/workflows/ci-validation.yml @@ -12,10 +12,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - node-version: 20 - - run: node validate.js --check + node-version: 26 + - run: node validate.ts --check - name: Install schema validation dependencies run: npm ci --prefix tools/schema --ignore-scripts - - run: node --test schema.test.js tools/schema/validate.test.cjs + - name: Type-check validators + run: npm run --prefix tools/schema typecheck + - run: node --test schema.test.js validate.test.cjs tools/schema/validate.test.cjs - name: Validate bot definitions against the JSON Schema - run: node tools/schema/validate.cjs + run: node tools/schema/validate.ts diff --git a/AGENTS.md b/AGENTS.md index e9b33c3..75549d9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ This repository maintains a curated list of well-known bots, crawlers, validator ### Key Files - **`well-known-bots.json`**: Main data file (~13,653 lines) containing bot definitions -- **`validate.js`**: Validation and formatting script - THE MOST IMPORTANT TOOL +- **`validate.ts`**: Validation and formatting script - THE MOST IMPORTANT TOOL - **`.github/workflows/ci-validation.yml`**: CI workflow that runs validation on every push/PR ## Working with This Repository @@ -18,11 +18,13 @@ This repository maintains a curated list of well-known bots, crawlers, validator This is a **data repository** with no root package.json or project installation. The built-in validator and CIDR tests use only Node.js built-in modules. JSON -Schema validation uses the isolated `tools/schema/` package. +Schema validation and strict type checking use the isolated `tools/schema/` package. +Use Node.js 26 to execute the TypeScript validators directly without a build step. - Do not create a root package.json or install dependencies at the repository root. - Install validation dependencies with `npm ci --prefix tools/schema --ignore-scripts`. -- Run schema validation with `node tools/schema/validate.cjs`. +- Type-check both validators with `npm run --prefix tools/schema typecheck`. +- Run schema validation with `node tools/schema/validate.ts`. - Keep validation dependency versions exact and commit the generated lockfile. For an intentional update, run `npm install --prefix tools/schema --ignore-scripts --save-exact @`, review the lockfile, and run @@ -30,16 +32,16 @@ Schema validation uses the isolated `tools/schema/` package. ### Validation Script (Critical) -The `validate.js` script is your primary tool. It has TWO modes: +The `validate.ts` script is your primary tool. It has TWO modes: -1. **Check mode**: `node validate.js --check` +1. **Check mode**: `node validate.ts --check` - Validates JSON formatting (2 spaces, proper newlines) - Validates all required fields exist and have correct types - Validates regex patterns compile correctly - Validates instances match/don't match their patterns - **ALWAYS run this before committing any changes** -2. **Generate mode**: `node validate.js --generate` +2. **Generate mode**: `node validate.ts --generate` - Automatically reformats the JSON file with correct formatting - Use this if formatting is incorrect - **IMPORTANT**: Only use when you need to fix formatting @@ -69,23 +71,23 @@ When adding or modifying bot entries, refer to the [README.md](README.md) for: #### Adding a New Bot 1. Edit `well-known-bots.json` to add your bot entry (see README.md for structure) -2. Validate your changes: `node validate.js --check` -3. If formatting is wrong, auto-fix it: `node validate.js --generate` -4. Validate again to ensure correctness: `node validate.js --check` +2. Validate your changes: `node validate.ts --check` +3. If formatting is wrong, auto-fix it: `node validate.ts --generate` +4. Validate again to ensure correctness: `node validate.ts --check` #### Modifying an Existing Bot 1. Find the bot entry in `well-known-bots.json` 2. Make your changes -3. Always validate: `node validate.js --check` +3. Always validate: `node validate.ts --check` ### CI/CD Pipeline The repository uses GitHub Actions for validation: - **Trigger**: Runs on every push, pull request, and merge group -- **What it does**: Installs the locked schema dependencies with scripts disabled, then runs `node validate.js --check`, `node --test schema.test.js tools/schema/validate.test.cjs`, and `node tools/schema/validate.cjs` -- **Node version**: 20.x +- **What it does**: Installs the locked schema dependencies with scripts disabled, then type-checks both validators and runs `node validate.ts --check`, `node --test schema.test.js validate.test.cjs tools/schema/validate.test.cjs`, and `node tools/schema/validate.ts` +- **Node version**: 26.x - **Location**: `.github/workflows/ci-validation.yml` All PRs must pass validation before merging. @@ -98,8 +100,8 @@ All PRs must pass validation before merging. **Solution**: ```bash -node validate.js --generate -node validate.js --check +node validate.ts --generate +node validate.ts --check ``` ### Error: "Item is missing required `X` field" @@ -131,7 +133,7 @@ node validate.js --check 1. **JSON Editing**: - Use 2-space indentation - Keep the JSON structure consistent with existing entries - - Let `node validate.js --generate` handle formatting if unsure + - Let `node validate.ts --generate` handle formatting if unsure 2. **Regex Patterns**: - Test patterns before adding them @@ -149,9 +151,10 @@ Run the built-in validation, regression tests, and JSON Schema validation: ```bash npm ci --prefix tools/schema --ignore-scripts -node validate.js --check -node --test schema.test.js tools/schema/validate.test.cjs -node tools/schema/validate.cjs +npm run --prefix tools/schema typecheck +node validate.ts --check +node --test schema.test.js validate.test.cjs tools/schema/validate.test.cjs +node tools/schema/validate.ts ``` ## Tips for Agents @@ -160,7 +163,7 @@ node tools/schema/validate.cjs 2. **Don't create build artifacts** - this is a data-only repo 3. **Read the README.md** for context on the project's purpose 4. **Check existing entries** for examples when adding new bots -5. **Use `node validate.js --generate`** to fix formatting issues automatically +5. **Use `node validate.ts --generate`** to fix formatting issues automatically 6. **Focus on data quality** - this file is consumed by other projects ## Example Bot Entry @@ -184,5 +187,5 @@ For detailed examples with verification methods, see the [Verification Methods]( ## Need Help? - Check existing bot entries in `well-known-bots.json` for examples -- Review `validate.js` to understand validation rules +- Review `validate.ts` to understand validation rules - See the README.md for project background and usage diff --git a/README.md b/README.md index 809f0a6..9b6f656 100644 --- a/README.md +++ b/README.md @@ -31,20 +31,22 @@ block custom bots. ## Adding a New Bot +Use Node.js 26 to run the TypeScript validators directly. + To add a new bot to the list, you need to edit the `well-known-bots.json` file and add a new entry. Follow these steps: 1. **Create a new bot entry** with the required fields (see structure below) 2. **Add User-Agent pattern(s)** that identify the bot 3. **Add verification method(s)** if the bot provider supports verification 4. **Add example instances** to validate your patterns work correctly -5. **Run validation** to ensure your entry is correct: `node validate.js --check` +5. **Run validation** to ensure your entry is correct: `node validate.ts --check` 6. **Submit a pull request** with your changes ### JSON Schema The [`well-known-bots.schema.json`](well-known-bots.schema.json) file describes the intended structure of `well-known-bots.json` for editors and external JSON -Schema validators. The repository's built-in `validate.js` script is still the +Schema validators. The repository's built-in `validate.ts` script is still the source of truth for checks that JSON Schema cannot express, such as compiling JavaScript regular expressions and testing `instances` against those patterns. @@ -52,19 +54,22 @@ Install the locked validation dependencies, then validate the JSON file with Ajv ```bash npm ci --prefix tools/schema --ignore-scripts -node tools/schema/validate.cjs +npm run --prefix tools/schema typecheck +node tools/schema/validate.ts ``` CI runs both validators and the regression tests: ```bash -node --test schema.test.js tools/schema/validate.test.cjs +node --test schema.test.js validate.test.cjs tools/schema/validate.test.cjs ``` -Validation dependencies are isolated in `tools/schema/`. Its lockfile fixes the +Node runs the validators using built-in type stripping; the separate `typecheck` +command checks their types without generating JavaScript. Validation and +type-checking dependencies are isolated in `tools/schema/`. Its lockfile fixes the full dependency tree and package integrity hashes; installation disables lifecycle scripts. Review dependency and lockfile updates together. To validate a different -JSON file, pass its path to `node tools/schema/validate.cjs`. +JSON file, pass its path to `node tools/schema/validate.ts`. Static IP lists accept IPv4 and IPv6 addresses and CIDR ranges, including IPv6 ranges with an embedded IPv4 address. diff --git a/tools/schema/package-lock.json b/tools/schema/package-lock.json index d06c2cf..337091f 100644 --- a/tools/schema/package-lock.json +++ b/tools/schema/package-lock.json @@ -8,6 +8,360 @@ "dependencies": { "ajv": "8.20.0", "ajv-formats": "3.0.1" + }, + "devDependencies": { + "@types/node": "26.4.1", + "typescript": "7.0.2" + } + }, + "node_modules/@types/node": { + "version": "26.4.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz", + "integrity": "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" } }, "node_modules/ajv": { @@ -79,6 +433,48 @@ "engines": { "node": ">=0.10.0" } + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" } } } diff --git a/tools/schema/package.json b/tools/schema/package.json index 4ab4041..dc0173f 100644 --- a/tools/schema/package.json +++ b/tools/schema/package.json @@ -4,5 +4,13 @@ "dependencies": { "ajv": "8.20.0", "ajv-formats": "3.0.1" + }, + "type": "module", + "scripts": { + "typecheck": "tsc --project tsconfig.json" + }, + "devDependencies": { + "@types/node": "26.4.1", + "typescript": "7.0.2" } } diff --git a/tools/schema/tsconfig.json b/tools/schema/tsconfig.json new file mode 100644 index 0000000..7912684 --- /dev/null +++ b/tools/schema/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "preserve", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "verbatimModuleSyntax": true, + "erasableSyntaxOnly": true, + "strict": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["../../validate.ts", "validate.ts"] +} diff --git a/tools/schema/validate.cjs b/tools/schema/validate.cjs deleted file mode 100644 index 5b6eaef..0000000 --- a/tools/schema/validate.cjs +++ /dev/null @@ -1,18 +0,0 @@ -const { readFileSync } = require("node:fs"); -const { basename, resolve } = require("node:path"); -const Ajv = require("ajv/dist/2020"); -const addFormats = require("ajv-formats"); - -const schema = require("../../well-known-bots.schema.json"); -const dataPath = process.argv[2] ?? resolve(__dirname, "../../well-known-bots.json"); -const data = JSON.parse(readFileSync(dataPath, "utf8")); -const ajv = new Ajv({ allErrors: true, strict: true }); -addFormats(ajv); -const validate = ajv.compile(schema); - -if (validate(data)) { - console.log(`${basename(dataPath)} valid`); -} else { - console.error(ajv.errorsText(validate.errors, { separator: "\n" })); - process.exitCode = 1; -} diff --git a/tools/schema/validate.test.cjs b/tools/schema/validate.test.cjs index 82e4bf4..2794f5f 100644 --- a/tools/schema/validate.test.cjs +++ b/tools/schema/validate.test.cjs @@ -5,7 +5,7 @@ const { tmpdir } = require("node:os"); const { join } = require("node:path"); const { test } = require("node:test"); -const script = join(__dirname, "validate.cjs"); +const script = join(__dirname, "validate.ts"); function validateFixture(t, changes) { const directory = mkdtempSync(join(tmpdir(), "bot-schema-test-")); diff --git a/tools/schema/validate.ts b/tools/schema/validate.ts new file mode 100644 index 0000000..5e20d78 --- /dev/null +++ b/tools/schema/validate.ts @@ -0,0 +1,18 @@ +import { readFileSync } from "node:fs"; +import { basename, resolve } from "node:path"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import addFormats from "ajv-formats"; +import schema from "../../well-known-bots.schema.json" with { type: "json" }; + +const dataPath = process.argv[2] ?? resolve(import.meta.dirname, "../../well-known-bots.json"); +const data: unknown = JSON.parse(readFileSync(dataPath, "utf8")); +const ajv = new Ajv2020({ allErrors: true, strict: true }); +addFormats(ajv); +const validate = ajv.compile(schema); + +if (validate(data)) { + console.log(`${basename(dataPath)} valid`); +} else { + console.error(ajv.errorsText(validate.errors, { separator: "\n" })); + process.exitCode = 1; +} diff --git a/validate.test.cjs b/validate.test.cjs new file mode 100644 index 0000000..7548116 --- /dev/null +++ b/validate.test.cjs @@ -0,0 +1,63 @@ +const assert = require("node:assert/strict"); +const { spawnSync } = require("node:child_process"); +const { copyFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } = require("node:fs"); +const { tmpdir } = require("node:os"); +const { join } = require("node:path"); +const { test } = require("node:test"); + +function fixture(t, changes = {}) { + const directory = mkdtempSync(join(tmpdir(), "bot-validator-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const script = join(directory, "validate.ts"); + const file = join(directory, "well-known-bots.json"); + copyFileSync(join(__dirname, "validate.ts"), script); + const bots = [{ + id: "test-bot", + categories: ["monitor"], + pattern: { accepted: ["TestBot"], forbidden: ["Preview"] }, + verification: [], + instances: { accepted: ["TestBot/1"], rejected: ["TestBot Preview"] }, + ...changes, + }]; + writeFileSync(file, JSON.stringify(bots, null, 2) + "\n"); + return { + file, + bots, + run: (mode) => spawnSync(process.execPath, [script, mode], { + cwd: tmpdir(), + encoding: "utf8", + }), + }; +} + +test("check mode validates patterns without modifying the data", (t) => { + const data = fixture(t); + const original = readFileSync(data.file, "utf8"); + const result = data.run("--check"); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(data.file, "utf8"), original); +}); + +test("check mode rejects pattern and verification errors", (t) => { + for (const changes of [ + { instances: { accepted: ["OtherBot"], rejected: [] } }, + { instances: { accepted: [], rejected: ["TestBot/1"] } }, + { pattern: { accepted: ["["], forbidden: [] } }, + { verification: [null] }, + { verification: [{ type: "cidr", sources: [null] }] }, + ]) { + const result = fixture(t, changes).run("--check"); + assert.equal(result.status, 1, result.stderr); + assert.notEqual(result.stderr, ""); + } +}); + +test("generate mode normalizes formatting and preserves bot definitions", (t) => { + const data = fixture(t); + writeFileSync(data.file, JSON.stringify(data.bots)); + assert.equal(data.run("--check").status, 1); + const result = data.run("--generate"); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(data.file, "utf8"), JSON.stringify(data.bots, null, 2) + "\n"); + assert.equal(data.run("--check").status, 0); +}); diff --git a/validate.js b/validate.ts similarity index 81% rename from validate.js rename to validate.ts index 077c80d..5e9f804 100644 --- a/validate.js +++ b/validate.ts @@ -1,8 +1,8 @@ /** * This file is used for checking and updating the format of the JSON file. * - * You can check the format via `node format.js --check` and regenerate the - * file with the correct formatting using `node format.js --generate`. + * You can check the format via `node validate.ts --check` and regenerate the + * file with the correct formatting using `node validate.ts --generate`. * * The formatting logic uses `JSON.stringify` with 2 spaces, which will keep * separating commas on the same line as any closing character. This technique @@ -10,16 +10,34 @@ * such as VSCode. */ -const fs = require("fs"); -const path = require("path"); +import * as fs from "node:fs"; +import * as path from "node:path"; -const jsonFilePath = path.join(__dirname, "well-known-bots.json"); +const jsonFilePath = path.join(import.meta.dirname, "well-known-bots.json"); const original = fs.readFileSync(jsonFilePath, "utf-8"); -const updated = JSON.stringify(JSON.parse(original), null, 2) + '\n'; +const data: unknown = JSON.parse(original); +const updated = JSON.stringify(data, null, 2) + '\n'; -function validateJsonSelector(selector, item, verify, source) { +function isArray(value: unknown): value is unknown[] { + return Array.isArray(value); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !isArray(value); +} + +function validatePatterns(patterns: unknown[], kind: "accepted" | "forbidden", item: unknown): asserts patterns is string[] { + for (const pat of patterns) { + if (typeof pat !== "string") { + console.error(`Pattern (${kind}) entry was not a string:`, item, pat); + process.exit(1); + } + } +} + +function validateJsonSelector(selector: string, item: unknown, verify: unknown, source: unknown): void { if (selector.includes('\\"')) { console.error("JSON selector contains escaped quotes; use JSON string escaping only:", item, verify, source); process.exit(1); @@ -31,41 +49,39 @@ if (process.argv[2] === "--generate") { process.exit(0); } else if (process.argv[2] === "--check") { if (updated !== original) { - console.error("JSON file format is wrong. Run `node format.js --generate` to update."); + console.error("JSON file format is wrong. Run `node validate.ts --generate` to update."); console.error("Format must be 2 spaces, with newlines for objects and arrays, and separating commas on the line with the previous closing character."); process.exit(1); } - for (const item of JSON.parse(original)) { + if (!isArray(data)) { + console.error("Bot definitions must be an array:", data); + process.exit(1); + } + for (const item of data) { + if (!isRecord(item)) { + console.error("Bot entry must be an object:", item); + process.exit(1); + } if (typeof item.id !== "string") { console.error("Item is missing required `id` string field:", item); process.exit(1); } - if (typeof item.pattern !== "object" || item.pattern === null || Array.isArray(item.pattern)) { + if (!isRecord(item.pattern)) { console.error("Item is missing required pattern object with accepted and forbidden arrays:", item); process.exit(1); } - if (!Array.isArray(item.pattern.accepted)) { + if (!isArray(item.pattern.accepted)) { console.error("Item pattern.accepted is missing or is not an array:", item); process.exit(1); } - for (const pat of item.pattern.accepted) { - if (typeof pat !== "string") { - console.error("Pattern (accepted) entry was not a string:", item, pat); - process.exit(1); - } - } - if (!Array.isArray(item.pattern.forbidden)) { + validatePatterns(item.pattern.accepted, "accepted", item); + if (!isArray(item.pattern.forbidden)) { console.error("Item pattern.forbidden is missing or is not an array:", item); process.exit(1); } - for (const pat of item.pattern.forbidden) { - if (typeof pat !== "string") { - console.error("Pattern (forbidden) entry was not a string:", item, pat); - process.exit(1); - } - } - if (!Array.isArray(item.categories)) { + validatePatterns(item.pattern.forbidden, "forbidden", item); + if (!isArray(item.categories)) { console.error("Item is missing required `categories` array field:", item); process.exit(1); } @@ -78,17 +94,25 @@ if (process.argv[2] === "--generate") { console.error("Item has wrong type specified for `url` string field:", item); process.exit(1); } - if (!Array.isArray(item.verification)) { + if (!isArray(item.verification)) { console.error("Item is missing required `verification` array field:", item); process.exit(1); } for (const verify of item.verification) { + if (!isRecord(verify)) { + console.error("Verification entry must be an object:", item, verify); + process.exit(1); + } if (verify.type === "cidr") { - if (!Array.isArray(verify.sources)) { + if (!isArray(verify.sources)) { console.error("Item cidr validation entry is missing required `sources` array field:", item, verify); process.exit(1); } for (const source of verify.sources) { + if (!isRecord(source)) { + console.error("Verification source must be an object:", item, verify, source); + process.exit(1); + } if (source.type !== "http-json" && source.type !== "http-csv" && source.type !== "http-text") { console.error("Cidr source `type` must be a valid type (currently `http-json`, `http-csv`, and `http-text` are supported)", item, verify, source); process.exit(1); @@ -108,7 +132,7 @@ if (process.argv[2] === "--generate") { } } } else if (verify.type === "dns") { - if (!Array.isArray(verify.masks)) { + if (!isArray(verify.masks)) { console.error("Item dns validation entry is missing required `masks` array field:", item, verify); process.exit(1); } @@ -127,7 +151,7 @@ if (process.argv[2] === "--generate") { if (verify.ips) { // Static IP list - if (!Array.isArray(verify.ips)) { + if (!isArray(verify.ips)) { console.error("Item IP validation `ips` field must be an array:", item, verify); process.exit(1); } @@ -139,11 +163,15 @@ if (process.argv[2] === "--generate") { } } else if (verify.sources) { // Remote IP sources - if (!Array.isArray(verify.sources)) { + if (!isArray(verify.sources)) { console.error("Item IP validation entry is missing required `sources` array field:", item, verify); process.exit(1); } for (const source of verify.sources) { + if (!isRecord(source)) { + console.error("Verification source must be an object:", item, verify, source); + process.exit(1); + } if (source.type !== "http-json" && source.type !== "http-text") { console.error("IP source `type` must be a valid type (`http-json` or `http-text` are supported)", item, verify, source); process.exit(1); @@ -173,7 +201,7 @@ if (process.argv[2] === "--generate") { } } if (typeof item.aliases !== "undefined") { - if (!Array.isArray(item.aliases)) { + if (!isArray(item.aliases)) { console.error("Item has wrong type specified for `aliases` array field:", item); process.exit(1); } @@ -187,18 +215,18 @@ if (process.argv[2] === "--generate") { // TODO: Check `addition_date` is defined properly // TODO: Check or remove `depends_on` field if (typeof item.instances !== "undefined") { - if (typeof item.instances !== "object" || item.instances === null || Array.isArray(item.instances)) { + if (!isRecord(item.instances)) { console.error( "Item has wrong type specified for instances, it must be an object with accepted and rejected arrays:", item ); process.exit(1); } - if (!Array.isArray(item.instances.accepted)) { + if (!isArray(item.instances.accepted)) { console.error("Item instances.accepted is missing or is not an array:", item); process.exit(1); } - if (!Array.isArray(item.instances.rejected)) { + if (!isArray(item.instances.rejected)) { console.error("Item instances.rejected is missing or is not an array:", item); process.exit(1); }