test(ci): type-check and run the custom-observability probes (#867)

.github/workflows/javascript-ci.yml

@@ -101,6 +101,16 @@ jobs:
         run: pnpm run test:ci
         working-directory: javascript
 
+      # The custom-observability probes assert that importing @langwatch/scenario
+      # does not auto-initialize OpenTelemetry, and that scenarioOnly filtering
+      # works both from setupScenarioTracing() and from a scenario.config.mjs.
+      # They run in-process against an InMemorySpanExporter with no network and
+      # no LLM, so unlike the step below they need no secrets and carry no fork
+      # guard: a fork PR that breaks the auto-init contract should go red.
+      - name: Test (Custom observability probes)
+        run: pnpm -F custom-observability-example test:all
+        working-directory: javascript
+
       # Examples — skipped for Dependabot PRs and fork PRs since they don't have access to repo secrets
       - name: Test (Examples)
         if: github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)

javascript/examples/custom-observability/package.json

@@ -7,13 +7,18 @@
     "test:scenario-only": "tsx test-scenario-only.ts",
     "test:custom-scopes": "tsx test-custom-scopes.ts",
     "test:config-file": "tsx test-config-file.ts",
-    "test:all": "tsx test-no-auto-init.ts && tsx test-scenario-only.ts && tsx test-custom-scopes.ts && tsx test-config-file.ts"
+    "test:all": "tsx test-no-auto-init.ts && tsx test-scenario-only.ts && tsx test-custom-scopes.ts && tsx test-config-file.ts",
+    "typecheck": "tsc --noEmit"
   },
   "dependencies": {
     "@langwatch/scenario": "workspace:*",
     "@opentelemetry/api": "^1.9.0",
-    "@opentelemetry/sdk-trace-base": "^1.30.0",
-    "@opentelemetry/sdk-trace-node": "^1.30.0",
+    "@opentelemetry/sdk-trace-base": "2.7.1",
+    "@opentelemetry/sdk-trace-node": "2.7.1",
     "tsx": "^4.19.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.10.1",
+    "typescript": "^5.9.3"
   }
 }

javascript/examples/custom-observability/test-no-auto-init.ts

@@ -3,24 +3,55 @@
  *
  * Before this fix, just importing the module with LANGWATCH_API_KEY set would
  * trigger setupObservability() and instrument all HTTP requests, middleware, etc.
+ *
+ * What this reads, and why it is not the provider itself: `trace.getTracerProvider()`
+ * always hands back the SAME ProxyTracerProvider instance, registered or not. Comparing
+ * that object, or its constructor name, before and after the import compares a value
+ * that cannot change, so the check printed PASS even with the regression present.
+ * Registration swaps the proxy's DELEGATE, from NoopTracerProvider to a real provider,
+ * so the delegate is what carries the signal.
  */
 import { trace } from "@opentelemetry/api";
 
+/**
+ * Name of the provider the global proxy currently delegates to.
+ *
+ * `getDelegate` is not on the public TracerProvider type. The fallback reports
+ * the proxy itself on an API version that does not expose it, which makes the
+ * start-state assertion below fail loudly rather than quietly comparing two
+ * constants again.
+ */
+const delegateName = (): string => {
+  const provider = trace.getTracerProvider() as { getDelegate?: () => object };
+  return (provider.getDelegate ? provider.getDelegate() : provider).constructor
+    .name;
+};
+
 // Check the provider BEFORE importing scenario
-const providerBefore = trace.getTracerProvider();
-const providerNameBefore = providerBefore.constructor.name;
+const delegateBefore = delegateName();
 
 // Dynamically import scenario to test the side-effect
 const scenario = await import("@langwatch/scenario");
+void scenario;
 
 // Check the provider AFTER importing scenario
-const providerAfter = trace.getTracerProvider();
-const providerNameAfter = providerAfter.constructor.name;
+const delegateAfter = delegateName();
 
-console.log(`Provider before import: ${providerNameBefore}`);
-console.log(`Provider after import:  ${providerNameAfter}`);
+console.log(`Provider before import: ${delegateBefore}`);
+console.log(`Provider after import:  ${delegateAfter}`);
+
+if (delegateBefore !== "NoopTracerProvider") {
+  // Guards the guard. If anything registered a provider before this point, the
+  // comparison below proves nothing, and a check that cannot fail is worse than
+  // no check at all because it reads as coverage.
+  console.error(
+    `\nFAIL: expected no tracer provider registered at start, found ${delegateBefore}`
+  );
+  console.error("   From that starting state this test proves nothing.");
+  process.exit(1);
+}
 
-if (providerNameBefore === providerNameAfter) {
+if (delegateBefore === delegateAfter) {
   console.log(
     "\nPASS: Importing @langwatch/scenario did NOT auto-initialize OpenTelemetry"
   );
@@ -30,7 +61,7 @@ if (providerNameBefore === providerNameAfter) {
     "\nFAIL: Importing @langwatch/scenario auto-initialized OpenTelemetry!"
   );
   console.error(
-    `   Provider changed from ${providerNameBefore} to ${providerNameAfter}`
+    `   Provider changed from ${delegateBefore} to ${delegateAfter}`
   );
   process.exit(1);
 }

javascript/examples/custom-observability/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "lib": [
+      "ES2023"
+    ],
+    "moduleResolution": "bundler",
+    "strict": true,
+    "noImplicitAny": true,
+    "strictNullChecks": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "forceConsistentCasingInFileNames": true,
+    "skipLibCheck": true,
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "allowJs": true
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.mjs"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}