From b1a940c0a70fda3c7d942af505796b6bb8156483 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 3 Aug 2026 07:20:46 +0000 Subject: [PATCH 1/3] test(ci): type-check and run the custom-observability probes (#867) The package's four probes ran only when someone ran them by hand. It has no typecheck script, so `pnpm typecheck:all` skipped it silently, and javascript-ci's example step runs `pnpm -F vitest-examples test`, which does not reach it. It gets a tsconfig matching the openai-realtime-demo example, a typecheck script, and its own CI step. The step carries no fork guard, unlike the example step below it: the probes run in-process against an InMemorySpanExporter with no network and no LLM, so a fork PR that breaks the auto-init contract should go red rather than skip. Turning the type check on surfaced a real drift the ticket predicted but did not name. The example pinned @opentelemetry/sdk-trace-base ^1.30.0 while the library depends on 2.7.1, so it handed a 1.x SimpleSpanProcessor to a 2.x setupScenarioTracing. It worked at runtime and did not typecheck. The example is documentation, so it now pins what the library pins. The bigger find is that test-no-auto-init proved nothing. It compared `trace.getTracerProvider().constructor.name` before and after the import, but that call returns the same ProxyTracerProvider instance whether or not anything is registered, so both reads are the string "ProxyTracerProvider" forever. The probe the issue describes as guarding a regression that has shipped before could not observe that regression: with a provider force-registered on import it still printed PASS and exited 0. Registration swaps the proxy's delegate, so the probe now reads the delegate, and asserts the delegate starts as NoopTracerProvider so it cannot go vacuous again from the other end. Evidence per acceptance criterion: - AC1: annotating a tracer as `number` in test-scenario-only.ts turns typecheck:all red; reverting returns it green. - AC3: registering a NodeTracerProvider on import exits 1 with "Provider changed from NoopTracerProvider to NodeTracerProvider". The same mutation against the old probe exited 0 and printed PASS. - AC4: all four probes pass with OPENAI_API_KEY and LANGWATCH_API_KEY unset. typecheck:all, lint:all and lint:lib clean; 1079 library tests unchanged. Closes #867 Co-Authored-By: Claude Opus 4.8 --- .github/workflows/javascript-ci.yml | 10 ++ .../custom-observability/package.json | 11 +- .../custom-observability/test-no-auto-init.ts | 47 ++++++-- .../custom-observability/tsconfig.json | 28 +++++ javascript/pnpm-lock.yaml | 105 ++---------------- 5 files changed, 96 insertions(+), 105 deletions(-) create mode 100644 javascript/examples/custom-observability/tsconfig.json diff --git a/.github/workflows/javascript-ci.yml b/.github/workflows/javascript-ci.yml index 49c515b01..7ef98001f 100644 --- a/.github/workflows/javascript-ci.yml +++ b/.github/workflows/javascript-ci.yml @@ -101,6 +101,16 @@ jobs: run: pnpm run test:ci working-directory: javascript + # The custom-observability probes assert that importing @langwatch/scenario + # does not auto-initialize OpenTelemetry, and that scenarioOnly filtering + # works both from setupScenarioTracing() and from a scenario.config.mjs. + # They run in-process against an InMemorySpanExporter with no network and + # no LLM, so unlike the step below they need no secrets and carry no fork + # guard: a fork PR that breaks the auto-init contract should go red. + - name: Test (Custom observability probes) + run: pnpm -F custom-observability-example test:all + working-directory: javascript + # Examples — skipped for Dependabot PRs and fork PRs since they don't have access to repo secrets - name: Test (Examples) if: github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) diff --git a/javascript/examples/custom-observability/package.json b/javascript/examples/custom-observability/package.json index cd50ca95a..892a8cfd6 100644 --- a/javascript/examples/custom-observability/package.json +++ b/javascript/examples/custom-observability/package.json @@ -7,13 +7,18 @@ "test:scenario-only": "tsx test-scenario-only.ts", "test:custom-scopes": "tsx test-custom-scopes.ts", "test:config-file": "tsx test-config-file.ts", - "test:all": "tsx test-no-auto-init.ts && tsx test-scenario-only.ts && tsx test-custom-scopes.ts && tsx test-config-file.ts" + "test:all": "tsx test-no-auto-init.ts && tsx test-scenario-only.ts && tsx test-custom-scopes.ts && tsx test-config-file.ts", + "typecheck": "tsc --noEmit" }, "dependencies": { "@langwatch/scenario": "workspace:*", "@opentelemetry/api": "^1.9.0", - "@opentelemetry/sdk-trace-base": "^1.30.0", - "@opentelemetry/sdk-trace-node": "^1.30.0", + "@opentelemetry/sdk-trace-base": "2.7.1", + "@opentelemetry/sdk-trace-node": "2.7.1", "tsx": "^4.19.0" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "typescript": "^5.9.3" } } diff --git a/javascript/examples/custom-observability/test-no-auto-init.ts b/javascript/examples/custom-observability/test-no-auto-init.ts index ee19dd803..ee2f0a56c 100644 --- a/javascript/examples/custom-observability/test-no-auto-init.ts +++ b/javascript/examples/custom-observability/test-no-auto-init.ts @@ -3,24 +3,55 @@ * * Before this fix, just importing the module with LANGWATCH_API_KEY set would * trigger setupObservability() and instrument all HTTP requests, middleware, etc. + * + * What this reads, and why it is not the provider itself: `trace.getTracerProvider()` + * always hands back the SAME ProxyTracerProvider instance, registered or not. Comparing + * that object, or its constructor name, before and after the import compares a value + * that cannot change, so the check printed PASS even with the regression present. + * Registration swaps the proxy's DELEGATE, from NoopTracerProvider to a real provider, + * so the delegate is what carries the signal. */ import { trace } from "@opentelemetry/api"; +/** + * Name of the provider the global proxy currently delegates to. + * + * `getDelegate` is not on the public TracerProvider type. The fallback reports + * the proxy itself on an API version that does not expose it, which makes the + * start-state assertion below fail loudly rather than quietly comparing two + * constants again. + */ +const delegateName = (): string => { + const provider = trace.getTracerProvider() as { getDelegate?: () => object }; + return (provider.getDelegate ? provider.getDelegate() : provider).constructor + .name; +}; + // Check the provider BEFORE importing scenario -const providerBefore = trace.getTracerProvider(); -const providerNameBefore = providerBefore.constructor.name; +const delegateBefore = delegateName(); // Dynamically import scenario to test the side-effect const scenario = await import("@langwatch/scenario"); +void scenario; // Check the provider AFTER importing scenario -const providerAfter = trace.getTracerProvider(); -const providerNameAfter = providerAfter.constructor.name; +const delegateAfter = delegateName(); -console.log(`Provider before import: ${providerNameBefore}`); -console.log(`Provider after import: ${providerNameAfter}`); +console.log(`Provider before import: ${delegateBefore}`); +console.log(`Provider after import: ${delegateAfter}`); + +if (delegateBefore !== "NoopTracerProvider") { + // Guards the guard. If anything registered a provider before this point, the + // comparison below proves nothing, and a check that cannot fail is worse than + // no check at all because it reads as coverage. + console.error( + `\nFAIL: expected no tracer provider registered at start, found ${delegateBefore}` + ); + console.error(" From that starting state this test proves nothing."); + process.exit(1); +} -if (providerNameBefore === providerNameAfter) { +if (delegateBefore === delegateAfter) { console.log( "\nPASS: Importing @langwatch/scenario did NOT auto-initialize OpenTelemetry" ); @@ -30,7 +61,7 @@ if (providerNameBefore === providerNameAfter) { "\nFAIL: Importing @langwatch/scenario auto-initialized OpenTelemetry!" ); console.error( - ` Provider changed from ${providerNameBefore} to ${providerNameAfter}` + ` Provider changed from ${delegateBefore} to ${delegateAfter}` ); process.exit(1); } diff --git a/javascript/examples/custom-observability/tsconfig.json b/javascript/examples/custom-observability/tsconfig.json new file mode 100644 index 000000000..a1da7511f --- /dev/null +++ b/javascript/examples/custom-observability/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": [ + "ES2023" + ], + "moduleResolution": "bundler", + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "isolatedModules": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "allowJs": true + }, + "include": [ + "**/*.ts", + "**/*.mjs" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/javascript/pnpm-lock.yaml b/javascript/pnpm-lock.yaml index 1c0c7d80e..3cd258b86 100644 --- a/javascript/pnpm-lock.yaml +++ b/javascript/pnpm-lock.yaml @@ -176,14 +176,21 @@ importers: specifier: ^1.9.0 version: 1.9.1 '@opentelemetry/sdk-trace-base': - specifier: ^1.30.0 - version: 1.30.1(@opentelemetry/api@1.9.1) + specifier: 2.7.1 + version: 2.7.1(@opentelemetry/api@1.9.1) '@opentelemetry/sdk-trace-node': - specifier: ^1.30.0 - version: 1.30.1(@opentelemetry/api@1.9.1) + specifier: 2.7.1 + version: 2.7.1(@opentelemetry/api@1.9.1) tsx: specifier: ^4.19.0 version: 4.22.3 + devDependencies: + '@types/node': + specifier: ^24.10.1 + version: 24.12.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 examples/openai-realtime-demo: dependencies: @@ -1103,12 +1110,6 @@ packages: peerDependencies: '@opentelemetry/api': ^1.9.0 - '@opentelemetry/context-async-hooks@1.30.1': - resolution: {integrity: sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/context-async-hooks@2.7.1': resolution: {integrity: sha512-OPFBYuXEn1E4ja3Y6eeA7O+ZnLBNcXTV5Cgsn1VaqBZ6hC5FnpZPLBNme1LJY8ZtF4aOujPKFoeWN4ik487KuQ==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1126,12 +1127,6 @@ packages: resolution: {integrity: sha512-B42kO3zIMVbJ+wj5nlSkDvLF8cJY+7wDKLomHp10GL00nvUnhY67UQ/soZQgKR4dvPf8zTKbcONDsOiJLyRuXw==} engines: {node: ^18.19.0 || >=20.6.0} - '@opentelemetry/core@1.30.1': - resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/core@2.1.0': resolution: {integrity: sha512-RMEtHsxJs/GiHHxYT58IY57UXAQTuUnZVco6ymDEqTNlJKTimM4qPUPVe8InNFyBjhHBEAx4k3Q8LtNayBsbUQ==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1270,36 +1265,18 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/propagator-b3@1.30.1': - resolution: {integrity: sha512-oATwWWDIJzybAZ4pO76ATN5N6FFbOA1otibAVlS8v90B4S1wClnhRUk7K+2CHAwN1JKYuj4jh/lpCEG5BAqFuQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-b3@2.7.1': resolution: {integrity: sha512-RJid6E2CKyeGfKBzXKF21ejabGMHypFkPAh3qZ+NvI+SGjuIye79t3PmiqcDgtRzdKH6ynXzbfslQ8DfpRUg2A==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@1.30.1': - resolution: {integrity: sha512-Pj/BfnYEKIOImirH76M4hDaBSx6HyZ2CXUqk+Kj02m6BB80c/yo4BdWkn/1gDFfU+YPY+bPR2U0DKBfdxCKwmg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@2.7.1': resolution: {integrity: sha512-KMjVBHzP4N60bOzxja76M1F1hZZ43lGPga5ix+mkv9+kk1nx9SbkxSvJsMbuVUxdPQmsPTqGShmhN8ulrMOg6Q==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/resources@1.30.1': - resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/resources@2.1.0': resolution: {integrity: sha512-1CJjf3LCvoefUOgegxi8h6r4B/wLSzInyhGP2UmIBYNlo4Qk5CZ73e1eEyWmfXvFtm1ybkmfb2DqWvspsYLrWw==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1354,12 +1331,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' - '@opentelemetry/sdk-trace-base@1.30.1': - resolution: {integrity: sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/sdk-trace-base@2.7.1': resolution: {integrity: sha512-NAYIlsF8MPUsKqJMiDQJTMPOmlbawC1Iz/omMLygZ1C9am8fTKYjTaI+OZM+WTY3t3Glo0wnOg/6/pac6RGPPw==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1372,12 +1343,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' - '@opentelemetry/sdk-trace-node@1.30.1': - resolution: {integrity: sha512-cBjYOINt1JxXdpw1e5MlHmFRc5fgj4GW/86vsKFxJCJ8AL4PdVtYH41gWwl4qd4uQjqEL1oJVrXkSy5cnduAnQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/sdk-trace-node@2.7.1': resolution: {integrity: sha512-pCpQxU68lV+I9s9svqMyVu5iHdDDUnqUpSxqwyCU8A9ejEsSnMPCbearwsUO4yk08ZJzAIUCFuReMdVQvHrdvg==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1390,10 +1355,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/semantic-conventions@1.28.0': - resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} - engines: {node: '>=14'} - '@opentelemetry/semantic-conventions@1.41.1': resolution: {integrity: sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==} engines: {node: '>=14'} @@ -6326,10 +6287,6 @@ snapshots: '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.1) yaml: 2.9.0 - '@opentelemetry/context-async-hooks@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/context-async-hooks@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6348,11 +6305,6 @@ snapshots: - '@opentelemetry/api' optional: true - '@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/core@2.1.0(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6551,32 +6503,16 @@ snapshots: '@opentelemetry/sdk-metrics': 2.8.0(@opentelemetry/api@1.9.1) '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-b3@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-b3@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-jaeger@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-jaeger@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.1) - '@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/resources@2.1.0(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6661,13 +6597,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6682,16 +6611,6 @@ snapshots: '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) '@opentelemetry/semantic-conventions': 1.41.1 - '@opentelemetry/sdk-trace-node@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/context-async-hooks': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-b3': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-jaeger': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.1) - semver: 7.8.1 - '@opentelemetry/sdk-trace-node@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6706,8 +6625,6 @@ snapshots: '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.1) optional: true - '@opentelemetry/semantic-conventions@1.28.0': {} - '@opentelemetry/semantic-conventions@1.41.1': {} '@oxc-project/types@0.137.0': {} From fa76d1a3e659247a53726ad7bbc43bb0233b5c2f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 5 Aug 2026 08:29:23 +0000 Subject: [PATCH 2/3] test(observability): make the scenario-only probe fail when the thing it probes breaks test-scenario-only.ts printed PASS and exited 0 unconditionally, so wiring test:all into CI made a non-asserting script a required gate: a broken run() or tracing setup would still have gone green. The other three probes already exit non-zero on their own failure conditions; this one did not. Turn its observations into gates: the scenario must succeed, scenario spans must exist, and both deliberately-created noise spans must reach the collector, since otherwise "no noise was exported" would be true for a reason unrelated to filtering. The exporter is not reachable from the probe, so the filtering claim is bound where it can be observed: scenarioOnly's selected scope must be the scope the scenario spans are actually emitted under, and must not also select the noise scope. Renaming the instrumentation scope would otherwise turn the filter into a drop-everything rule with nothing to catch it. Co-Authored-By: Claude Opus 4.8 --- .../test-scenario-only.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/javascript/examples/custom-observability/test-scenario-only.ts b/javascript/examples/custom-observability/test-scenario-only.ts index e10db1bb6..0d57eea2e 100644 --- a/javascript/examples/custom-observability/test-scenario-only.ts +++ b/javascript/examples/custom-observability/test-scenario-only.ts @@ -128,6 +128,59 @@ if (noiseSpans.length > 0) { // SimpleSpanProcessor, ALL spans get collected. The filtering happens when spans // are exported to LangWatch. This test verifies that scenario spans ARE created // and the noise spans are separate -- the LangWatchTraceExporter would filter them. + +if (!result.success) { + console.error(`\nFAIL: Scenario did not succeed: ${result.reasoning}`); + process.exit(1); +} + +if (scenarioSpans.length === 0) { + console.error("\nFAIL: No @langwatch/scenario spans were collected"); + process.exit(1); +} + +// The two spans created at step 3 must survive to here. If they do not, the +// collector is dropping spans and "no noise reached the exporter" would be true +// for a reason that has nothing to do with filtering. +if (noiseSpans.length !== 2) { + console.error( + `\nFAIL: expected the 2 http-server noise spans to be collected, found ${noiseSpans.length}` + ); + process.exit(1); +} + +// The exporter is not reachable from here, so the closest observable claim is +// that the scope scenarioOnly selects on is the scope the scenario spans are +// actually emitted under. Renaming the instrumentation scope would silently +// turn the filter into a drop-everything rule, and only this binding catches it. +const scenarioOnlyScopes = scenarioOnly.flatMap((filter) => + "include" in filter + ? (filter.include.instrumentationScopeName ?? []).flatMap((match) => + match.equals ? [match.equals] : [] + ) + : [] +); + +const unselected = scenarioSpans.filter( + (span) => !scenarioOnlyScopes.includes(getScopeName(span)) +); + +if (unselected.length > 0) { + console.error( + `\nFAIL: scenarioOnly selects ${JSON.stringify(scenarioOnlyScopes)}, but ` + + `${unselected.length} scenario span(s) are emitted under ` + + `${JSON.stringify([...new Set(unselected.map(getScopeName))])}` + ); + process.exit(1); +} + +if (scenarioOnlyScopes.some((scope) => noiseSpans.some((span) => getScopeName(span) === scope))) { + console.error( + "\nFAIL: scenarioOnly also selects the http-server noise scope, so it would not filter it out" + ); + process.exit(1); +} + console.log( "\nPASS: Scenario runs correctly with custom observability config" ); From 7171c385521942ae4173f05bb03ee70d271208a1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 5 Aug 2026 08:58:59 +0000 Subject: [PATCH 3/3] test(tracing): prove the trace filters actually drop spans, not just their shape filters.test.ts asserted the rule objects deep-equal a literal. Every one of those assertions still passes if the exporter ignores `filters` entirely, so nothing showed that a noise span is dropped, which is the only thing the filters exist to do. LangWatchTraceExporter applies its filters in export() before delegating to the OTLP exporter it extends, so spying on the parent makes the drop observable with no network call and no live endpoint. Two cases: scenarioOnly forwards the scenario scope and drops http-server and next.js, and withCustomScopes forwards the named scope alongside it. Mutation-checked: inverting scenarioOnly's include to exclude, and widening it to admit the noise scope, each fail two of the new cases. Co-Authored-By: Claude Opus 4.8 --- .../src/tracing/__tests__/filters.test.ts | 80 ++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/javascript/src/tracing/__tests__/filters.test.ts b/javascript/src/tracing/__tests__/filters.test.ts index bc43d91b8..85dbdb248 100644 --- a/javascript/src/tracing/__tests__/filters.test.ts +++ b/javascript/src/tracing/__tests__/filters.test.ts @@ -1,5 +1,48 @@ -import { describe, it, expect } from "vitest"; -import { scenarioOnly, withCustomScopes } from "../filters"; +import { describe, it, expect, vi, afterEach } from "vitest"; +import { LangWatchTraceExporter } from "langwatch/observability"; +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { scenarioOnly, withCustomScopes, type TraceFilter } from "../filters"; + +/** + * A span carrying only what the exporter's filter reads. The filter matches on + * the instrumentation scope and the span name, so the rest of ReadableSpan is + * irrelevant here and faking it keeps the test free of an SDK setup. + */ +const spanInScope = (instrumentationScopeName: string, name: string) => + ({ + name, + instrumentationScope: { name: instrumentationScopeName }, + }) as unknown as ReadableSpan; + +/** + * The scopes that survive `filters` and reach the transport. + * + * LangWatchTraceExporter applies its filters in `export()` and then delegates + * to the OTLP exporter it extends, so spying on the parent is what makes the + * drop observable without a network call or a live endpoint. + */ +const exportedScopes = ( + filters: TraceFilter[], + spans: ReadableSpan[], +): string[] => { + const parent = Object.getPrototypeOf(LangWatchTraceExporter.prototype); + const forwarded: string[] = []; + vi.spyOn(parent, "export").mockImplementation((( + batch: ReadableSpan[], + done: (result: { code: number }) => void, + ) => { + forwarded.push(...batch.map((span) => span.instrumentationScope.name)); + done({ code: 0 }); + }) as never); + + new LangWatchTraceExporter({ + endpoint: "http://127.0.0.1:1/v1/traces", + apiKey: "sk-lw-test", + filters, + }).export(spans, () => undefined); + + return forwarded; +}; describe("filters", () => { describe("scenarioOnly", () => { @@ -72,4 +115,37 @@ describe("filters", () => { expect(a).not.toBe(b); }); }); + + // The suite above pins the shape of the rules. Shape is not behaviour: every + // assertion there still passes if the exporter ignores `filters` entirely, so + // nothing yet showed that a noise span is actually dropped. + describe("when the rules are handed to the exporter that consumes them", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe("given scenarioOnly", () => { + it("forwards scenario spans and drops the other scopes", () => { + const forwarded = exportedScopes(scenarioOnly, [ + spanInScope("@langwatch/scenario", "Scenario Turn"), + spanInScope("http-server", "GET /api/health"), + spanInScope("next.js", "middleware"), + ]); + + expect(forwarded).toEqual(["@langwatch/scenario"]); + }); + }); + + describe("given withCustomScopes", () => { + it("forwards the named scopes alongside scenario and drops the rest", () => { + const forwarded = exportedScopes(withCustomScopes("my-database"), [ + spanInScope("@langwatch/scenario", "Scenario Turn"), + spanInScope("my-database", "SELECT 1"), + spanInScope("http-server", "GET /api/health"), + ]); + + expect(forwarded).toEqual(["@langwatch/scenario", "my-database"]); + }); + }); + }); });