diff --git a/.github/workflows/javascript-ci.yml b/.github/workflows/javascript-ci.yml index 49c515b01..7ef98001f 100644 --- a/.github/workflows/javascript-ci.yml +++ b/.github/workflows/javascript-ci.yml @@ -101,6 +101,16 @@ jobs: run: pnpm run test:ci working-directory: javascript + # The custom-observability probes assert that importing @langwatch/scenario + # does not auto-initialize OpenTelemetry, and that scenarioOnly filtering + # works both from setupScenarioTracing() and from a scenario.config.mjs. + # They run in-process against an InMemorySpanExporter with no network and + # no LLM, so unlike the step below they need no secrets and carry no fork + # guard: a fork PR that breaks the auto-init contract should go red. + - name: Test (Custom observability probes) + run: pnpm -F custom-observability-example test:all + working-directory: javascript + # Examples — skipped for Dependabot PRs and fork PRs since they don't have access to repo secrets - name: Test (Examples) if: github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) diff --git a/javascript/examples/custom-observability/package.json b/javascript/examples/custom-observability/package.json index cd50ca95a..892a8cfd6 100644 --- a/javascript/examples/custom-observability/package.json +++ b/javascript/examples/custom-observability/package.json @@ -7,13 +7,18 @@ "test:scenario-only": "tsx test-scenario-only.ts", "test:custom-scopes": "tsx test-custom-scopes.ts", "test:config-file": "tsx test-config-file.ts", - "test:all": "tsx test-no-auto-init.ts && tsx test-scenario-only.ts && tsx test-custom-scopes.ts && tsx test-config-file.ts" + "test:all": "tsx test-no-auto-init.ts && tsx test-scenario-only.ts && tsx test-custom-scopes.ts && tsx test-config-file.ts", + "typecheck": "tsc --noEmit" }, "dependencies": { "@langwatch/scenario": "workspace:*", "@opentelemetry/api": "^1.9.0", - "@opentelemetry/sdk-trace-base": "^1.30.0", - "@opentelemetry/sdk-trace-node": "^1.30.0", + "@opentelemetry/sdk-trace-base": "2.7.1", + "@opentelemetry/sdk-trace-node": "2.7.1", "tsx": "^4.19.0" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "typescript": "^5.9.3" } } diff --git a/javascript/examples/custom-observability/test-no-auto-init.ts b/javascript/examples/custom-observability/test-no-auto-init.ts index ee19dd803..ee2f0a56c 100644 --- a/javascript/examples/custom-observability/test-no-auto-init.ts +++ b/javascript/examples/custom-observability/test-no-auto-init.ts @@ -3,24 +3,55 @@ * * Before this fix, just importing the module with LANGWATCH_API_KEY set would * trigger setupObservability() and instrument all HTTP requests, middleware, etc. + * + * What this reads, and why it is not the provider itself: `trace.getTracerProvider()` + * always hands back the SAME ProxyTracerProvider instance, registered or not. Comparing + * that object, or its constructor name, before and after the import compares a value + * that cannot change, so the check printed PASS even with the regression present. + * Registration swaps the proxy's DELEGATE, from NoopTracerProvider to a real provider, + * so the delegate is what carries the signal. */ import { trace } from "@opentelemetry/api"; +/** + * Name of the provider the global proxy currently delegates to. + * + * `getDelegate` is not on the public TracerProvider type. The fallback reports + * the proxy itself on an API version that does not expose it, which makes the + * start-state assertion below fail loudly rather than quietly comparing two + * constants again. + */ +const delegateName = (): string => { + const provider = trace.getTracerProvider() as { getDelegate?: () => object }; + return (provider.getDelegate ? provider.getDelegate() : provider).constructor + .name; +}; + // Check the provider BEFORE importing scenario -const providerBefore = trace.getTracerProvider(); -const providerNameBefore = providerBefore.constructor.name; +const delegateBefore = delegateName(); // Dynamically import scenario to test the side-effect const scenario = await import("@langwatch/scenario"); +void scenario; // Check the provider AFTER importing scenario -const providerAfter = trace.getTracerProvider(); -const providerNameAfter = providerAfter.constructor.name; +const delegateAfter = delegateName(); -console.log(`Provider before import: ${providerNameBefore}`); -console.log(`Provider after import: ${providerNameAfter}`); +console.log(`Provider before import: ${delegateBefore}`); +console.log(`Provider after import: ${delegateAfter}`); + +if (delegateBefore !== "NoopTracerProvider") { + // Guards the guard. If anything registered a provider before this point, the + // comparison below proves nothing, and a check that cannot fail is worse than + // no check at all because it reads as coverage. + console.error( + `\nFAIL: expected no tracer provider registered at start, found ${delegateBefore}` + ); + console.error(" From that starting state this test proves nothing."); + process.exit(1); +} -if (providerNameBefore === providerNameAfter) { +if (delegateBefore === delegateAfter) { console.log( "\nPASS: Importing @langwatch/scenario did NOT auto-initialize OpenTelemetry" ); @@ -30,7 +61,7 @@ if (providerNameBefore === providerNameAfter) { "\nFAIL: Importing @langwatch/scenario auto-initialized OpenTelemetry!" ); console.error( - ` Provider changed from ${providerNameBefore} to ${providerNameAfter}` + ` Provider changed from ${delegateBefore} to ${delegateAfter}` ); process.exit(1); } diff --git a/javascript/examples/custom-observability/test-scenario-only.ts b/javascript/examples/custom-observability/test-scenario-only.ts index e10db1bb6..0d57eea2e 100644 --- a/javascript/examples/custom-observability/test-scenario-only.ts +++ b/javascript/examples/custom-observability/test-scenario-only.ts @@ -128,6 +128,59 @@ if (noiseSpans.length > 0) { // SimpleSpanProcessor, ALL spans get collected. The filtering happens when spans // are exported to LangWatch. This test verifies that scenario spans ARE created // and the noise spans are separate -- the LangWatchTraceExporter would filter them. + +if (!result.success) { + console.error(`\nFAIL: Scenario did not succeed: ${result.reasoning}`); + process.exit(1); +} + +if (scenarioSpans.length === 0) { + console.error("\nFAIL: No @langwatch/scenario spans were collected"); + process.exit(1); +} + +// The two spans created at step 3 must survive to here. If they do not, the +// collector is dropping spans and "no noise reached the exporter" would be true +// for a reason that has nothing to do with filtering. +if (noiseSpans.length !== 2) { + console.error( + `\nFAIL: expected the 2 http-server noise spans to be collected, found ${noiseSpans.length}` + ); + process.exit(1); +} + +// The exporter is not reachable from here, so the closest observable claim is +// that the scope scenarioOnly selects on is the scope the scenario spans are +// actually emitted under. Renaming the instrumentation scope would silently +// turn the filter into a drop-everything rule, and only this binding catches it. +const scenarioOnlyScopes = scenarioOnly.flatMap((filter) => + "include" in filter + ? (filter.include.instrumentationScopeName ?? []).flatMap((match) => + match.equals ? [match.equals] : [] + ) + : [] +); + +const unselected = scenarioSpans.filter( + (span) => !scenarioOnlyScopes.includes(getScopeName(span)) +); + +if (unselected.length > 0) { + console.error( + `\nFAIL: scenarioOnly selects ${JSON.stringify(scenarioOnlyScopes)}, but ` + + `${unselected.length} scenario span(s) are emitted under ` + + `${JSON.stringify([...new Set(unselected.map(getScopeName))])}` + ); + process.exit(1); +} + +if (scenarioOnlyScopes.some((scope) => noiseSpans.some((span) => getScopeName(span) === scope))) { + console.error( + "\nFAIL: scenarioOnly also selects the http-server noise scope, so it would not filter it out" + ); + process.exit(1); +} + console.log( "\nPASS: Scenario runs correctly with custom observability config" ); diff --git a/javascript/examples/custom-observability/tsconfig.json b/javascript/examples/custom-observability/tsconfig.json new file mode 100644 index 000000000..a1da7511f --- /dev/null +++ b/javascript/examples/custom-observability/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": [ + "ES2023" + ], + "moduleResolution": "bundler", + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "isolatedModules": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "allowJs": true + }, + "include": [ + "**/*.ts", + "**/*.mjs" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/javascript/pnpm-lock.yaml b/javascript/pnpm-lock.yaml index 1c0c7d80e..3cd258b86 100644 --- a/javascript/pnpm-lock.yaml +++ b/javascript/pnpm-lock.yaml @@ -176,14 +176,21 @@ importers: specifier: ^1.9.0 version: 1.9.1 '@opentelemetry/sdk-trace-base': - specifier: ^1.30.0 - version: 1.30.1(@opentelemetry/api@1.9.1) + specifier: 2.7.1 + version: 2.7.1(@opentelemetry/api@1.9.1) '@opentelemetry/sdk-trace-node': - specifier: ^1.30.0 - version: 1.30.1(@opentelemetry/api@1.9.1) + specifier: 2.7.1 + version: 2.7.1(@opentelemetry/api@1.9.1) tsx: specifier: ^4.19.0 version: 4.22.3 + devDependencies: + '@types/node': + specifier: ^24.10.1 + version: 24.12.4 + typescript: + specifier: ^5.9.3 + version: 5.9.3 examples/openai-realtime-demo: dependencies: @@ -1103,12 +1110,6 @@ packages: peerDependencies: '@opentelemetry/api': ^1.9.0 - '@opentelemetry/context-async-hooks@1.30.1': - resolution: {integrity: sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/context-async-hooks@2.7.1': resolution: {integrity: sha512-OPFBYuXEn1E4ja3Y6eeA7O+ZnLBNcXTV5Cgsn1VaqBZ6hC5FnpZPLBNme1LJY8ZtF4aOujPKFoeWN4ik487KuQ==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1126,12 +1127,6 @@ packages: resolution: {integrity: sha512-B42kO3zIMVbJ+wj5nlSkDvLF8cJY+7wDKLomHp10GL00nvUnhY67UQ/soZQgKR4dvPf8zTKbcONDsOiJLyRuXw==} engines: {node: ^18.19.0 || >=20.6.0} - '@opentelemetry/core@1.30.1': - resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/core@2.1.0': resolution: {integrity: sha512-RMEtHsxJs/GiHHxYT58IY57UXAQTuUnZVco6ymDEqTNlJKTimM4qPUPVe8InNFyBjhHBEAx4k3Q8LtNayBsbUQ==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1270,36 +1265,18 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/propagator-b3@1.30.1': - resolution: {integrity: sha512-oATwWWDIJzybAZ4pO76ATN5N6FFbOA1otibAVlS8v90B4S1wClnhRUk7K+2CHAwN1JKYuj4jh/lpCEG5BAqFuQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-b3@2.7.1': resolution: {integrity: sha512-RJid6E2CKyeGfKBzXKF21ejabGMHypFkPAh3qZ+NvI+SGjuIye79t3PmiqcDgtRzdKH6ynXzbfslQ8DfpRUg2A==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@1.30.1': - resolution: {integrity: sha512-Pj/BfnYEKIOImirH76M4hDaBSx6HyZ2CXUqk+Kj02m6BB80c/yo4BdWkn/1gDFfU+YPY+bPR2U0DKBfdxCKwmg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@2.7.1': resolution: {integrity: sha512-KMjVBHzP4N60bOzxja76M1F1hZZ43lGPga5ix+mkv9+kk1nx9SbkxSvJsMbuVUxdPQmsPTqGShmhN8ulrMOg6Q==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/resources@1.30.1': - resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/resources@2.1.0': resolution: {integrity: sha512-1CJjf3LCvoefUOgegxi8h6r4B/wLSzInyhGP2UmIBYNlo4Qk5CZ73e1eEyWmfXvFtm1ybkmfb2DqWvspsYLrWw==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1354,12 +1331,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' - '@opentelemetry/sdk-trace-base@1.30.1': - resolution: {integrity: sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/sdk-trace-base@2.7.1': resolution: {integrity: sha512-NAYIlsF8MPUsKqJMiDQJTMPOmlbawC1Iz/omMLygZ1C9am8fTKYjTaI+OZM+WTY3t3Glo0wnOg/6/pac6RGPPw==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1372,12 +1343,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' - '@opentelemetry/sdk-trace-node@1.30.1': - resolution: {integrity: sha512-cBjYOINt1JxXdpw1e5MlHmFRc5fgj4GW/86vsKFxJCJ8AL4PdVtYH41gWwl4qd4uQjqEL1oJVrXkSy5cnduAnQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/sdk-trace-node@2.7.1': resolution: {integrity: sha512-pCpQxU68lV+I9s9svqMyVu5iHdDDUnqUpSxqwyCU8A9ejEsSnMPCbearwsUO4yk08ZJzAIUCFuReMdVQvHrdvg==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1390,10 +1355,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/semantic-conventions@1.28.0': - resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} - engines: {node: '>=14'} - '@opentelemetry/semantic-conventions@1.41.1': resolution: {integrity: sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==} engines: {node: '>=14'} @@ -6326,10 +6287,6 @@ snapshots: '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.1) yaml: 2.9.0 - '@opentelemetry/context-async-hooks@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/context-async-hooks@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6348,11 +6305,6 @@ snapshots: - '@opentelemetry/api' optional: true - '@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/core@2.1.0(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6551,32 +6503,16 @@ snapshots: '@opentelemetry/sdk-metrics': 2.8.0(@opentelemetry/api@1.9.1) '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-b3@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-b3@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-jaeger@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-jaeger@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 '@opentelemetry/core': 2.7.1(@opentelemetry/api@1.9.1) - '@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/resources@2.1.0(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6661,13 +6597,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6682,16 +6611,6 @@ snapshots: '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) '@opentelemetry/semantic-conventions': 1.41.1 - '@opentelemetry/sdk-trace-node@1.30.1(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/context-async-hooks': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-b3': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-jaeger': 1.30.1(@opentelemetry/api@1.9.1) - '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.1) - semver: 7.8.1 - '@opentelemetry/sdk-trace-node@2.7.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -6706,8 +6625,6 @@ snapshots: '@opentelemetry/sdk-trace-base': 2.7.1(@opentelemetry/api@1.9.1) optional: true - '@opentelemetry/semantic-conventions@1.28.0': {} - '@opentelemetry/semantic-conventions@1.41.1': {} '@oxc-project/types@0.137.0': {} diff --git a/javascript/src/tracing/__tests__/filters.test.ts b/javascript/src/tracing/__tests__/filters.test.ts index bc43d91b8..85dbdb248 100644 --- a/javascript/src/tracing/__tests__/filters.test.ts +++ b/javascript/src/tracing/__tests__/filters.test.ts @@ -1,5 +1,48 @@ -import { describe, it, expect } from "vitest"; -import { scenarioOnly, withCustomScopes } from "../filters"; +import { describe, it, expect, vi, afterEach } from "vitest"; +import { LangWatchTraceExporter } from "langwatch/observability"; +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { scenarioOnly, withCustomScopes, type TraceFilter } from "../filters"; + +/** + * A span carrying only what the exporter's filter reads. The filter matches on + * the instrumentation scope and the span name, so the rest of ReadableSpan is + * irrelevant here and faking it keeps the test free of an SDK setup. + */ +const spanInScope = (instrumentationScopeName: string, name: string) => + ({ + name, + instrumentationScope: { name: instrumentationScopeName }, + }) as unknown as ReadableSpan; + +/** + * The scopes that survive `filters` and reach the transport. + * + * LangWatchTraceExporter applies its filters in `export()` and then delegates + * to the OTLP exporter it extends, so spying on the parent is what makes the + * drop observable without a network call or a live endpoint. + */ +const exportedScopes = ( + filters: TraceFilter[], + spans: ReadableSpan[], +): string[] => { + const parent = Object.getPrototypeOf(LangWatchTraceExporter.prototype); + const forwarded: string[] = []; + vi.spyOn(parent, "export").mockImplementation((( + batch: ReadableSpan[], + done: (result: { code: number }) => void, + ) => { + forwarded.push(...batch.map((span) => span.instrumentationScope.name)); + done({ code: 0 }); + }) as never); + + new LangWatchTraceExporter({ + endpoint: "http://127.0.0.1:1/v1/traces", + apiKey: "sk-lw-test", + filters, + }).export(spans, () => undefined); + + return forwarded; +}; describe("filters", () => { describe("scenarioOnly", () => { @@ -72,4 +115,37 @@ describe("filters", () => { expect(a).not.toBe(b); }); }); + + // The suite above pins the shape of the rules. Shape is not behaviour: every + // assertion there still passes if the exporter ignores `filters` entirely, so + // nothing yet showed that a noise span is actually dropped. + describe("when the rules are handed to the exporter that consumes them", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe("given scenarioOnly", () => { + it("forwards scenario spans and drops the other scopes", () => { + const forwarded = exportedScopes(scenarioOnly, [ + spanInScope("@langwatch/scenario", "Scenario Turn"), + spanInScope("http-server", "GET /api/health"), + spanInScope("next.js", "middleware"), + ]); + + expect(forwarded).toEqual(["@langwatch/scenario"]); + }); + }); + + describe("given withCustomScopes", () => { + it("forwards the named scopes alongside scenario and drops the rest", () => { + const forwarded = exportedScopes(withCustomScopes("my-database"), [ + spanInScope("@langwatch/scenario", "Scenario Turn"), + spanInScope("my-database", "SELECT 1"), + spanInScope("http-server", "GET /api/health"), + ]); + + expect(forwarded).toEqual(["@langwatch/scenario", "my-database"]); + }); + }); + }); });