feat(cli): emit telemetry for the validate action

packages/@aws-cdk-testing/cli-integ/tests/telemetry-integ-tests/cdk-validate-telemetry.integtest.ts

@@ -0,0 +1,53 @@
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import { integTest, withSpecificFixture } from '../../lib';
+
+integTest(
+  'cdk validate emits ONLINE_VALIDATE telemetry event with violation counters',
+  withSpecificFixture('validate-app', async (fixture) => {
+    const telemetryFile = path.join(fixture.integTestDir, `telemetry-validate-${Date.now()}.json`);
+
+    // --no-online keeps the run deterministic; onlineViolations is asserted to be 0.
+    const output = await fixture.cdk(
+      ['--unstable=validate', 'validate', fixture.fullStackName('validate'), '--no-online', `--telemetry-file=${telemetryFile}`],
+      {
+        verboseLevel: 3, // trace mode
+        allowErrExit: true, // violations make validate exit non-zero
+      },
+    );
+
+    // The endpoint sink POSTs the whole event batch to the real telemetry
+    // endpoint, which validates it against a request schema. This passes only
+    // once the backend accepts the ONLINE_VALIDATE event type.
+    expect(output).toContain('Telemetry Sent Successfully');
+
+    const json = fs.readJSONSync(telemetryFile);
+    const validateEvent = json.find((e: any) => e.event?.eventType === 'ONLINE_VALIDATE');
+    expect(validateEvent).toBeDefined();
+    expect(validateEvent.event.state).toEqual('SUCCEEDED');
+
+    // The app's single S3 bucket makes SecurityPlugin report one violation each
+    // of fatal/error/warning/cost-optimization severity, plus one construct
+    // annotation warning. The plugin failure is what would have failed a deploy.
+    expect(validateEvent.counters).toEqual(
+      expect.objectContaining({
+        'offlineViolations:fatal': 1,
+        'offlineViolations:error': 1,
+        'offlineViolations:warning': 2,
+        'onlineViolations': 0,
+        'offlineWouldFailDeploy': 1,
+      }),
+    );
+
+    // The plugin's non-standard 'cost-optimization' severity is reported under
+    // a library-version-dependent key ('offlineViolations:cost-optimization'
+    // on older aws-cdk-lib, 'offlineViolations:custom' on newer), so assert
+    // the total offline violation count instead of that key.
+    const totalOfflineViolations = Object.entries(validateEvent.counters)
+      .filter(([key]) => key.startsWith('offlineViolations:'))
+      .reduce((acc, [, value]) => acc + Number(value), 0);
+    expect(totalOfflineViolations).toEqual(5);
+
+    fs.unlinkSync(telemetryFile);
+  }),
+);

packages/@aws-cdk/toolkit-lib/docs/message-registry.md

@@ -157,6 +157,8 @@ Please let us know by [opening an issue](https://github.com/aws/aws-cdk-cli/issu
 | `CDK_TOOLKIT_E9600` | Policy validation failed | `error` | {@link ValidateResult} |
 | `CDK_TOOLKIT_I9601` | No policy validation report found | `info` | n/a |
 | `CDK_TOOLKIT_W9602` | Online validation could not be completed for a stack | `warn` | n/a |
+| `CDK_TOOLKIT_I9603` | Online validation is starting | `trace` | {@link StackSelectionDetails} |
+| `CDK_TOOLKIT_I9604` | Online validation has finished. Provides online validation timing and validation outcome counters. | `trace` | {@link OnlineValidationResult} |
 | `CDK_TOOLKIT_I0100` | Notices decoration (the header or footer of a list of notices) | `info` | n/a |
 | `CDK_TOOLKIT_W0101` | A notice that is marked as a warning | `warn` | n/a |
 | `CDK_TOOLKIT_E0101` | A notice that is marked as an error | `error` | n/a |

packages/@aws-cdk/toolkit-lib/lib/actions/validate/index.ts

@@ -41,6 +41,18 @@ export interface ValidateResult {
    * Reports from each validation plugin
    */
   readonly pluginReports: PluginReportJson[];
+
+  /**
+   * The subset of `pluginReports` produced by online (CloudFormation change
+   * set) validation, as opposed to offline sources: policy validation plugins
+   * and construct annotations, both read from the cloud assembly.
+   *
+   * Contains the same object references as `pluginReports`. An empty array
+   * means online validation ran and found no problems.
+   *
+   * @default - online validation was skipped
+   */
+  readonly onlineReports?: PluginReportJson[];
 }
 
 export type { PolicyValidationReportJson, PolicyValidationReportConclusion, PluginReportJson };

packages/@aws-cdk/toolkit-lib/lib/api/io/private/messages.ts

@@ -27,6 +27,7 @@ import type {
   ContextProviderMessageSource,
   Duration,
   ErrorPayload,
+  OnlineValidationResult,
   Operation,
   SingleStack,
   StackAndAssemblyData,
@@ -558,6 +559,18 @@ export const IO = {
     description: 'Online validation could not be completed for a stack',
   }),
 
+  CDK_TOOLKIT_I9603: make.trace<StackSelectionDetails>({
+    code: 'CDK_TOOLKIT_I9603',
+    description: 'Online validation is starting',
+    interface: 'StackSelectionDetails',
+  }),
+
+  CDK_TOOLKIT_I9604: make.trace<OnlineValidationResult>({
+    code: 'CDK_TOOLKIT_I9604',
+    description: 'Online validation has finished. Provides online validation timing and validation outcome counters.',
+    interface: 'OnlineValidationResult',
+  }),
+
   // Notices
   CDK_TOOLKIT_I0100: make.info({
     code: 'CDK_TOOLKIT_I0100',
@@ -729,4 +742,9 @@ export const SPAN = {
     start: IO.CDK_TOOLKIT_I5400,
     end: IO.CDK_TOOLKIT_I5410,
   },
+  ONLINE_VALIDATE: {
+    name: 'Online validation',
+    start: IO.CDK_TOOLKIT_I9603,
+    end: IO.CDK_TOOLKIT_I9604,
+  },
 } satisfies Record<string, SpanDefinition<any, any>>;

packages/@aws-cdk/toolkit-lib/lib/api/work-graph/work-graph.ts

@@ -1,7 +1,7 @@
 import type { WorkNode, StackNode, AssetBuildNode, AssetPublishNode, MarkerNode } from './work-graph-types';
 import { DeploymentState } from './work-graph-types';
 import { ToolkitError } from '../../toolkit/toolkit-error';
-import { parallelPromises } from '../../util';
+import { parallelPromises, sum } from '../../util';
 import type { IoHelper } from '../io/private';
 export type Concurrency = number | Record<WorkNode['type'], number>;
 
@@ -416,14 +416,6 @@ export interface WorkGraphActions {
   marker: (markerNode: MarkerNode) => Promise<void>;
 }
 
-function sum(xs: number[]) {
-  let ret = 0;
-  for (const x of xs) {
-    ret += x;
-  }
-  return ret;
-}
-
 function retainOnly<A>(xs: A[], pred: (x: A) => boolean) {
   xs.splice(0, xs.length, ...xs.filter(pred));
 }

packages/@aws-cdk/toolkit-lib/lib/payloads/types.ts

@@ -100,6 +100,24 @@ export interface Operation extends Duration {
   readonly error?: Error;
 }
 
+/**
+ * The result of the online validation phase of the `validate` action.
+ *
+ * Offline validation is performed during synthesis and its timing is captured
+ * by the synth event, so this only times the online (CloudFormation change set)
+ * validation phase. The `counters` summarize the outcome of the whole validate
+ * run (offline and online violation counts, and whether offline validation
+ * would have failed a deploy).
+ */
+export interface OnlineValidationResult extends Duration {
+  /**
+   * Counters describing the outcome of the validate run.
+   *
+   * @default - no counters
+   */
+  readonly counters?: Record<string, number>;
+}
+
 /**
  * Generic payload of a simple yes/no question.
  *

packages/@aws-cdk/toolkit-lib/lib/toolkit/private/count-assembly-results.ts

@@ -1,6 +1,7 @@
 import type * as cxapi from '@aws-cdk/cloud-assembly-api';
 import { SynthesisMessageLevel } from '@aws-cdk/cloud-assembly-api';
 import type { IMessageSpan } from '../../api/io/private/span';
+import { sum } from '../../util';
 
 export function countAssemblyResults(span: IMessageSpan<any>, assembly: cxapi.CloudAssembly) {
   const stacksRecursively = assembly.stacksRecursively;
@@ -21,10 +22,6 @@ export function countAssemblyResults(span: IMessageSpan<any>, assembly: cxapi.Cl
   }
 }
 
-function sum(xs: number[]) {
-  return xs.reduce((a, b) => a + b, 0);
-}
-
 /**
  * Well-known and agreed-upon value between aws-cdk-lib and the toolkit
  *

packages/@aws-cdk/toolkit-lib/lib/toolkit/private/count-validation-results.ts

@@ -0,0 +1,32 @@
+import { wouldFailDeploy } from './validation-report';
+import type { ValidateResult } from '../../actions/validate';
+import type { IMessageSpan } from '../../api/io/private/span';
+import { sum } from '../../util';
+
+/**
+ * Add counters describing the outcome of a validate run to the given span
+ *
+ * Offline violations (policy plugin reports and construct annotations read
+ * from the cloud assembly) are counted per severity. `offlineWouldFailDeploy`
+ * records whether the offline reports fail `wouldFailDeploy` at the default
+ * 'error' threshold; a deploy run with `--strict` or `--ignore-errors` moves
+ * that threshold, so this counter approximates the default deploy behavior.
+ *
+ * Online reports are identified by reference via `onlineReports`, not by
+ * plugin name: `pluginName` is a plugin-supplied string, so an offline
+ * policy plugin may carry any name.
+ */
+export function countValidationResults(span: IMessageSpan<any>, result: Pick<ValidateResult, 'pluginReports' | 'onlineReports'>) {
+  const online = result.onlineReports ?? [];
+  const onlineSet = new Set(online);
+  const offline = result.pluginReports.filter((r) => !onlineSet.has(r));
+
+  for (const report of offline) {
+    for (const violation of report.violations) {
+      span.incCounter(`offlineViolations:${violation.severity}`);
+    }
+  }
+
+  span.incCounter('onlineViolations', sum(online.map((r) => r.violations.length)));
+  span.incCounter('offlineWouldFailDeploy', wouldFailDeploy(offline, 'error') ? 1 : 0);
+}

packages/@aws-cdk/toolkit-lib/lib/toolkit/private/validation-report.ts

@@ -88,26 +88,36 @@ export async function throwIfValidationFailures(
   const result: ValidateResult = { conclusion, pluginReports };
   await ioHelper.notify(hostMessageFromValidation(process.cwd(), result));
 
+  if (!wouldFailDeploy(pluginReports, failAt)) {
+    return;
+  }
+
+  if (failAt === 'warn') {
+    const error = AssemblyError.withStacks('Synthesis finished with warnings (--strict mode)', stacks.stackArtifacts);
+    error.attachSynthesisErrorCode('StrictAnnotationWarnings');
+    throw error;
+  }
+
+  const error = AssemblyError.withStacks('Synthesis finished with errors', stacks.stackArtifacts);
+  error.attachSynthesisErrorCode('AnnotationErrors');
+  throw error;
+}
+
+/**
+ * Whether the given validation reports make a deploy-like action fail at the given severity threshold
+ *
+ * This is the exact predicate applied by `throwIfValidationFailures`.
+ */
+export function wouldFailDeploy(pluginReports: PluginReportJson[], failAt: MinimumSeverity): boolean {
   switch (failAt) {
     case 'error':
-      if (conclusion === 'failure') {
-        const error = AssemblyError.withStacks('Synthesis finished with errors', stacks.stackArtifacts);
-        error.attachSynthesisErrorCode('AnnotationErrors');
-        throw error;
-      }
-      break;
+      return combineConclusions(pluginReports) === 'failure';
     case 'warn':
-      // if we're failing at 'warn', then both warnings and errors cause failure, so the initial conclusion is correct
-      if (conclusion === 'failure' || hasWarnings(pluginReports)) {
-        const error = AssemblyError.withStacks('Synthesis finished with warnings (--strict mode)', stacks.stackArtifacts);
-        error.attachSynthesisErrorCode('StrictAnnotationWarnings');
-        throw error;
-      }
-
-      break;
+      // if we're failing at 'warn', then both warnings and errors cause failure
+      return combineConclusions(pluginReports) === 'failure' || hasWarnings(pluginReports);
     case 'none':
       // if we're not failing at all, then the conclusion is always success
-      break;
+      return false;
   }
 }
 

packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts

@@ -116,6 +116,7 @@ import { formatErrorMessage, formatExpressStabilizationWarning, formatTime, obsc
 import { pLimit } from '../util/concurrency';
 import { createIgnoreMatcher } from '../util/glob-matcher';
 import { promiseWithResolvers } from '../util/promises';
+import { countValidationResults } from './private/count-validation-results';
 import { combineConclusions, obtainUnifiedValidationReport, throwIfValidationFailures } from './private/validation-report';
 
 export interface ToolkitOptions {
@@ -693,14 +694,27 @@ export class Toolkit extends CloudAssemblySourceBuilder {
 
     const reports = await obtainUnifiedValidationReport(assembly, stacks);
 
-    // Online validation: submit templates to CloudFormation for early validation
-    if (options.online ?? true) {
-      const deployments = await this.deploymentsForAction('validate');
+    // Online validation: submit templates to CloudFormation for early validation.
+    //
+    // Offline validation (policy plugins and construct annotations) is performed
+    // during synthesis, so its timing is already captured by the synth event.
+    // Online validation is the only phase not otherwise measured, so we wrap it
+    // in its own span and emit an ONLINE_VALIDATE telemetry event carrying the
+    // online duration plus counters that summarize the whole validate outcome
+    // (including `offlineWouldFailDeploy`).
+    const onlineSpan = await ioHelper.span(SPAN.ONLINE_VALIDATE).begin({ stacks: selectStacks });
+    let onlineReports: PluginReportJson[] | undefined;
+    try {
+      if (options.online ?? true) {
+        const deployments = await this.deploymentsForAction('validate');
 
-      const onlineReport = await this.validateOnline(ioHelper, stacks, deployments);
-      if (onlineReport) {
-        reports.push(onlineReport);
+        const onlineReport = await this.validateOnline(ioHelper, stacks, deployments);
+        onlineReports = onlineReport ? [onlineReport] : [];
+        reports.push(...onlineReports);
       }
+    } finally {
+      countValidationResults(onlineSpan, { pluginReports: reports, onlineReports });
+      await onlineSpan.end({});
     }
 
     const hasAnyViolations = reports.some(report => report.violations && report.violations.length > 0);
@@ -709,6 +723,7 @@ export class Toolkit extends CloudAssemblySourceBuilder {
       conclusion: combineConclusions(reports),
       title: undefined,
       pluginReports: reports,
+      onlineReports,
     };
 
     if (!hasAnyViolations) {

packages/@aws-cdk/toolkit-lib/lib/util/arrays.ts

@@ -12,6 +12,13 @@ export function flatten<T>(xs: T[][]): T[] {
   return Array.prototype.concat.apply([], xs);
 }
 
+/**
+ * Sum a list of numbers
+ */
+export function sum(xs: number[]): number {
+  return xs.reduce((a, b) => a + b, 0);
+}
+
 /**
  * Partition a collection by removing and returning all elements that match a predicate
  *