diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0bfb411ec7c..30ad90a6f38 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -203,10 +203,20 @@ jobs: # The docker leg runs vitest directly instead of through # test:integration:sandbox:docker: that script would rebuild the image # the step above just built. + # external-context-mem0-write, external-context-auto-recall, + # context-compress-interactive and qwen-serve-channel-workers are + # quarantined to the nightly isolated matrix below as a precaution. + # Their #10272 startup stalls were the goal-runtime wait bug fixed + # by #10290, not a platform defect, and they have not stalled on + # any platform since. The nightly canaries run on + # ubuntu-latest — which never reproduced the stall — so they keep + # functional coverage on hosted Linux but give no signal about the + # former stall platforms; restoring these suites to the push lanes + # requires verifying them on those runners first. if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then - npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}' + npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --exclude '**/interactive/external-context-mem0-write.test.ts' --exclude '**/interactive/external-context-auto-recall.test.ts' --exclude '**/interactive/context-compress-interactive.test.ts' --exclude '**/cli/qwen-serve-channel-workers.test.ts' --shard='${{ matrix.shard }}' else - npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}' + npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --exclude '**/interactive/external-context-mem0-write.test.ts' --exclude '**/interactive/external-context-auto-recall.test.ts' --exclude '**/interactive/context-compress-interactive.test.ts' --exclude '**/cli/qwen-serve-channel-workers.test.ts' --shard='${{ matrix.shard }}' fi # The sandbox build retags the same image name every run, so on a @@ -275,7 +285,7 @@ jobs: OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}' OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}' OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}' - run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --exclude "**/channel-plugin.test.ts" --shard="${{ matrix.shard }}"' + run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --exclude "**/channel-plugin.test.ts" --exclude "**/interactive/external-context-mem0-write.test.ts" --exclude "**/interactive/external-context-auto-recall.test.ts" --exclude "**/interactive/context-compress-interactive.test.ts" --exclude "**/cli/qwen-serve-channel-workers.test.ts" --shard="${{ matrix.shard }}"' isolated-nightly: name: '${{ matrix.label }} (nightly)' @@ -291,6 +301,16 @@ jobs: test_file: 'interactive/cron-interactive.test.ts' - label: 'channel-plugin E2E' test_file: 'channel-plugin.test.ts' + # Quarantined from the push lanes as a precaution; see the + # excludes and rationale in the push jobs above. + - label: 'external-context-mem0-write E2E' + test_file: 'interactive/external-context-mem0-write.test.ts' + - label: 'external-context-auto-recall E2E' + test_file: 'interactive/external-context-auto-recall.test.ts' + - label: 'context-compress-interactive E2E' + test_file: 'interactive/context-compress-interactive.test.ts' + - label: 'qwen-serve-channel-workers E2E' + test_file: 'cli/qwen-serve-channel-workers.test.ts' steps: - name: 'Checkout' uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 diff --git a/scripts/tests/e2e-workflow.test.js b/scripts/tests/e2e-workflow.test.js index 4331ada05d5..c4d7a3b09a2 100644 --- a/scripts/tests/e2e-workflow.test.js +++ b/scripts/tests/e2e-workflow.test.js @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { readFileSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; import { parse } from 'yaml'; @@ -32,4 +32,70 @@ describe('e2e workflow', () => { expect(group).toContain('github.event_name'); expect(group).toContain('github.head_ref || github.ref_name'); }); + + it('quarantines the nightly-isolated suites from every push lane', () => { + // cron-interactive is timing-flaky, and external-context-mem0-write, + // external-context-auto-recall, context-compress-interactive and + // qwen-serve-channel-workers are quarantined to the nightly isolated + // matrix as a precaution after the #10272 startup stalls (since traced + // to the goal-runtime wait bug fixed by #10290). Every nightly canary + // must be excluded from every push lane, and every push-lane exclusion + // must keep its nightly canary — dropping either side silently loses + // coverage. + const nightlyFiles = yml.jobs['isolated-nightly'].strategy.matrix.include + .map((entry) => entry.test_file) + .sort(); + // Anchor the oracle to the filesystem: a renamed suite would otherwise + // match no exclude (vitest ignores a non-matching --exclude silently) + // and no canary file while this test compared e2e.yml strings only. + for (const file of nightlyFiles) { + expect(existsSync(`integration-tests/${file}`), file).toBe(true); + } + const countExcludes = (run) => { + const counts = new Map(); + for (const match of run.matchAll(/--exclude ['"]([^'"]+)['"]/g)) { + const file = match[1].replace(/^\*\*\//, ''); + counts.set(file, (counts.get(file) ?? 0) + 1); + } + return counts; + }; + // Discover push lanes instead of listing job names: every job with a + // `Run E2E tests` step is one, so a future lane that forgets the + // excludes fails here instead of stalling on the quarantined suites. + let lanes = 0; + for (const [jobName, job] of Object.entries(yml.jobs)) { + const step = job.steps.find((s) => s.name === 'Run E2E tests'); + if (!step) continue; + lanes += 1; + // One vitest invocation per matrix leg; counting them separately + // catches an exclude moved between legs, which a whole-step count + // cancels out. + const invocations = step.run + .split('\n') + .filter((line) => !line.trim().startsWith('#')) + .filter((line) => /vitest run|test:integration/.test(line)); + expect(invocations.length, jobName).toBeGreaterThan(0); + for (const invocation of invocations) { + const counts = countExcludes(invocation); + expect([...counts.keys()].sort(), `${jobName}: ${invocation}`).toEqual( + nightlyFiles, + ); + for (const file of nightlyFiles) { + expect(counts.get(file), `${jobName}: ${file}`).toBe(1); + } + } + } + expect(lanes).toBeGreaterThan(0); + }); + + it('runs the nightly canary step on its matrix file', () => { + // The quarantine test above pins the canary matrix against the push + // lanes but never reads the canary run step: if that step stopped + // consuming matrix.test_file, every canary would silently run nothing + // while continue-on-error painted the jobs green. + const canaryStep = yml.jobs['isolated-nightly'].steps.find( + (step) => step.name === 'Run ${{ matrix.label }} tests', + ); + expect(canaryStep.run).toContain('${{ matrix.test_file }}'); + }); });