Harden fuzzing runtime and evaluation

.github/workflows/dockerhub_release.yml

@@ -10,17 +10,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12'
 
     - name: Install the latest version of uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v6
 
     - name: login to dockerhub
       uses: docker/login-action@v3

.github/workflows/e2e_tests.yml

@@ -8,32 +8,29 @@ permissions:
 jobs:
   setup-and-test:
     runs-on: ubuntu-latest
+    env:
+      TZ: UTC
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
-    - name: Setup timezone
-      uses: zcong1993/setup-timezone@master
-      with:
-        timezone: UTC
 
     - name: Set up Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: '20'
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12'
 
     - name: Install the latest version of uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v6
 
     - name: Install Python dependencies
       run: uv sync
 
     - name: Run e2e tests in parallel
-      run: |
-        uv run pytest tests/e2e/ -n auto --dist loadfile --verbose --cov=. --cov-report html
+      run: uv run pytest tests/e2e/ -n auto --dist loadfile --verbose

.github/workflows/githubcr_release.yml

@@ -10,17 +10,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12'
 
     - name: Install the latest version of uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v6
 
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v3

.github/workflows/integration_tests.yml

@@ -9,31 +9,28 @@ jobs:
   integration-tests:
     name: Run GraphQLer Integration Tests
     runs-on: ubuntu-latest
+    env:
+      TZ: UTC
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
-    - name: Setup timezone
-      uses: zcong1993/setup-timezone@master
-      with:
-        timezone: UTC
 
     - name: Set up Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: '20'
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12'
 
     - name: Install the latest version of uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v6
 
     - name: Install Python dependencies
       run: uv sync
 
-    - name: Run integration tests (CLI modes + subscription support)
-      run: |
-        uv run pytest tests/integration/ --exitfirst --verbose --failed-first --cov=. --cov-report html
+    - name: Run integration tests
+      run: uv run pytest tests/integration/ --exitfirst --verbose --failed-first

.github/workflows/lint.yml

@@ -6,6 +6,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: astral-sh/ruff-action@v1
+      - uses: astral-sh/ruff-action@v3
         with:
             changed-files: "true"

.github/workflows/pypi_release.yml

@@ -13,17 +13,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12'
 
     - name: Install the latest version of uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v6
 
     - name: Publish to PyPI
       env:

.github/workflows/type_check.yml

@@ -10,7 +10,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Install the latest version of uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v6
 
     - name: Install Python dependencies
       run: uv sync --extra mcp

.github/workflows/unit_tests.yml

@@ -6,21 +6,19 @@ jobs:
     name: Run GraphQLer Unit Tests
     runs-on: ubuntu-latest
 
+    env:
+      TZ: UTC
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
-    - name: Setup timezone
-      uses: zcong1993/setup-timezone@master
-      with:
-        timezone: UTC
 
     - name: Set up Python 3.12
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: 3.12
 
     - name: Install the latest version of uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v6
 
     - name: Install Python dependencies
       run: uv sync --extra mcp
@@ -31,6 +29,5 @@ jobs:
           sudo apt install -y $(grep -o ^[^#][[:alnum:]-]* packages.list)
         fi
 
-    - name: Test with pytest
-      run: |
-        uv run pytest tests/unit/ --exitfirst --verbose --failed-first --cov=. --cov-report html
+    - name: Test with pytest and enforce coverage
+      run: uv run pytest tests/unit/ --exitfirst --verbose --failed-first --cov=graphqler --cov-branch --cov-report=term-missing --cov-report=xml --cov-fail-under=50

README.md

@@ -181,6 +181,26 @@ python -m graphqler --mode fuzz --url <URL> --path <SAVE_PATH>
 
 While fuzzing, statistics related to the GraphQL API and any ongoing request counts are logged in the console. Any request return codes are written to `<SAVE_PATH>/stats.txt`. All logs during fuzzing are kept in `<SAVE_PATH>/logs/fuzzer.log`. The log file will tell you exactly which requests are sent to which endpoints, and what the response was. This can be used for further result analysis. If IDOR chains were generated during compile, the fuzzer automatically tests them and writes detection results to `<SAVE_PATH>/detections/`.
 
+Compilation writes `<SAVE_PATH>/manifest.json` with the artifact schema version, target endpoint, compile phase, and SHA-256 hashes. Fuzzing validates this contract before loading YAML, so incomplete, stale, endpoint-mismatched, or modified artifacts fail with a specific error instead of failing later during graph execution.
+
+Interrupted fuzz runs can continue from their latest atomic `serialized/stats.json` and `serialized/objects_bucket.json` checkpoints:
+
+```sh
+python -m graphqler --mode fuzz --url <URL> --path <SAVE_PATH> --resume
+```
+
+For broader authorization testing, provide named identities and enable differential replay. GraphQLer replays likely private queries and mutations with the exact same payload under anonymous and alternate profiles, compares returned fields, and also checks subscription event exposure when subscriptions are enabled:
+
+```sh
+python -m graphqler --mode fuzz --url <URL> --path <SAVE_PATH> \
+  --auth primary='Bearer <USER_A_TOKEN>' \
+  --auth user-b='Bearer <USER_B_TOKEN>' \
+  --auth admin='Bearer <ADMIN_TOKEN>' \
+  --authorization-differential --subscriptions
+```
+
+Differential replay is opt-in because it adds requests and can repeat mutation side effects. Anonymous exact-data matches are confirmed findings; alternate authenticated-profile responses are potential findings unless an ownership-aware IDOR chain confirms them.
+
 ### IDOR Checking mode
 
 ```sh
@@ -192,7 +212,7 @@ python -m graphqler --mode idor --url <URL> --path <SAVE_PATH>
 
 [Insecure direct object reference (IDOR)](https://portswigger.net/web-security/access-control/idor) detection works via multi-profile chain replay. During **compile**, `--idor-auth` enables generation of IDOR candidate chains: endpoints that create or expose user-scoped objects are identified via heuristics (and optionally an LLM classifier), then split into primary-profile steps (authenticated user) and secondary-profile steps (attacker token). These chains are saved to `compiled/chains/idor.yml`.
 
-During **fuzz**, the `IDORChainDetector` executes each IDOR chain — the primary profile creates or retrieves the object, then the secondary profile attempts to access it. Any data returned to the secondary profile is flagged as a potential IDOR vulnerability and written to `<SAVE_PATH>/detections/IDOR/<endpoint>/`.
+During **fuzz**, the `IDORChainDetector` executes each IDOR chain — the primary profile creates or retrieves the object, then the secondary profile attempts to access it. Any data returned to the secondary profile is flagged as a potential IDOR vulnerability and written to `<SAVE_PATH>/detections/IDOR_CHAIN/<endpoint>/`.
 
 The standalone **idor** mode re-executes only the IDOR chains without running regular fuzzing. This is useful for targeted re-testing after fixing an issue, or when you only want to check access-control without the overhead of a full fuzz run.
 
@@ -236,6 +256,10 @@ There are also variables that can be modified with the `--config` flag as a TOML
 | SKIP_NODES | Nodes to skip (query or mutation names) | List | [] |
 | DISABLE_MUTATIONS | Only generate and run Query chains — all Mutation nodes are excluded from chain generation and fuzzing. Can also be set via `--disable-mutations` CLI flag. | Boolean | False |
 | IDOR_SECONDARY_AUTH | Secondary (attacker) authentication token for IDOR chain detection (e.g. `"Bearer token2"`). If not set, the IDOR chain phase is skipped. | String | None |
+| AUTHORIZATION_DIFFERENTIAL | Replay likely private operations under anonymous and alternate profiles. Opt-in because it adds requests and repeats mutation payloads. | Boolean | False |
+| PROFILES | Named runtime profiles with auth tokens, headers, or variables; CLI equivalent is repeated `--auth name=token`. | Object | `{}` |
+| SKIP_SUBSCRIPTIONS | Disable WebSocket subscription execution. Set false or pass `--subscriptions` to enable it. | Boolean | True |
+| RESUME | Continue an interrupted fuzz run from atomic checkpoints. CLI equivalent is `--resume`. | Boolean | False |
 
 ## AI Features
 

benchmark/README.md

@@ -151,6 +151,28 @@ uv run python benchmark/benchmark_llm_chains.py \
 
 ---
 
+### `benchmark_detector_accuracy.py` — Vulnerability detector accuracy
+
+Measures detector precision, recall, and F1 against the versioned labels in
+`ground_truth/detectors.yml`. Expected detector/node/level tuples are positives;
+any additional finding in the same completed runs is treated as a negative-control
+false positive. This prevents integration tests that only prove one vulnerable
+example from being reported as scanner-wide accuracy.
+
+The results root must contain the E2E output directories named by the corpus:
+
+```bash
+uv run python benchmark/benchmark_detector_accuracy.py \
+  --results-root . \
+  --output benchmark/detector_accuracy_results.json
+```
+
+The JSON report includes aggregate and per-detector TP/FP/FN, precision, recall,
+F1, and the exact unmatched findings. Update labels only with a reproducible API
+revision and retained run output.
+
+---
+
 ## Directory structure
 
 ```
@@ -159,10 +181,12 @@ benchmark/
 ├── benchmark_oob.py                  # Objects-bucket-only baseline
 ├── benchmark_ablation.py             # 4-config ablation study
 ├── benchmark_inference_accuracy.py   # Dependency inference precision/recall
+├── benchmark_detector_accuracy.py    # Detector precision/recall/F1 corpus
 ├── benchmark_llm_chains.py           # LLM vs heuristic chain generation
 ├── ground_truth/
 │   ├── countries.yml
 │   ├── rick_and_morty.yml
-│   └── graphql_zero.yml
-└── readme.md
+│   ├── graphql_zero.yml
+│   └── detectors.yml
+└── README.md
 ```