Fix ${cmake.testProgram} launch from the Run and Debug view (#4574)

CHANGELOG.md

@@ -29,6 +29,7 @@ Improvements:
 - Make the CMake Cache Editor ("Edit Cache (UI)") aware of cache variables defined by the active CMake configure preset: a new "Source" column shows where each variable comes from (project/user preset vs. the CMake cache) and whether it still matches the preset, and edits to preset-backed variables are now persisted as an override in `CMakeUserPresets.json` (and reconfigured with it) instead of being silently reverted by the preset on the next configure. [#4884](https://github.com/microsoft/vscode-cmake-tools/pull/4884) [@BilboBeutlin89](https://github.com/BilboBeutlin89)
 
 Bug Fixes:
+- Fix launch configurations that use `${cmake.testProgram}` (and the related `${cmake.testArgs}`, `${cmake.testWorkingDirectory}` and `${cmake.testEnvironment}` placeholders) failing with `program '${cmake.testProgram}' does not exist` when started from the Run and Debug view (F5) instead of the Test Explorer. Such a launch now prompts you to pick one of the discovered CTest tests and resolves the placeholders for it, so the same configuration works from both entry points. [#4574](https://github.com/microsoft/vscode-cmake-tools/issues/4574)
 - Fix disabled tests (for example GoogleTest `DISABLED_` tests, which CTest reports as "Not Run (Disabled)") being shown in the Test Explorer as failed ("failed with completion status 'Disabled'") and dragging their group's status down. Such tests are intentionally not executed, so they are now reported as skipped and no longer taint the run result. [#4267](https://github.com/microsoft/vscode-cmake-tools/issues/4267)
 - Fix CMake syntax highlighting treating escaped `$<` sequences in quoted regex strings as generator expressions. [#5028](https://github.com/microsoft/vscode-cmake-tools/issues/5028)
 - Fix the Test Explorer reporting a test that crashed or timed out (for example one killed by a segmentation fault) as "failed with exit code 0". CTest records the cause in its `Exit Code` measurement (such as `SEGFAULT` or `Timeout`) while `Exit Value` stays 0, so the failure message now names that status instead of the misleading exit value. [#5043](https://github.com/microsoft/vscode-cmake-tools/issues/5043)

docs/debug-launch.md

@@ -312,7 +312,7 @@ You can substitute the value of any variable in the CMake cache by adding a `com
 You can also construct launch.json configurations that allow you to debug tests in the Test Explorer.
 
 > **Note:**
-> These launch.json configurations are to be used specifically from the UI of the Test Explorer. 
+> These configurations are primarily intended to be used from the Test Explorer UI, where the specific test being debugged is already known. You can also start them directly from the Run and Debug view (F5): because that entry point has no associated test, CMake Tools will prompt you to pick one of the discovered CTest tests and resolve the `cmake.test*` placeholders for it. If you only want these configurations to be launched from the Test Explorer, add `"presentation": { "hidden": true }` (as shown below) so they don't appear in the Run and Debug dropdown.
 
 The easiest way to do this is to construct the debug configuration using `cmake.testProgram` for the `program` field, `cmake.testArgs` for 
 the `args` field, `cmake.testWorkingDirectory` for the `cwd` field, and `cmake.testEnvironment` for the `environment` field.

src/cmakeProject.ts

@@ -3109,6 +3109,16 @@ export class CMakeProject {
         return this.ctest(false, undefined, [testName], undefined, buildTargets.length ? buildTargets : undefined);
     }
 
+    /**
+     * Resolve the `${cmake.test*}` placeholders for a launch configuration that was started from the
+     * Run and Debug view (F5). Delegates to the CTest controller, which prompts the user to select a
+     * test when needed. Returns the substituted configuration, or `undefined` to abort the launch.
+     */
+    async resolveCTestLaunchConfiguration(config: vscode.DebugConfiguration): Promise<vscode.DebugConfiguration | undefined> {
+        const drv = await this.getCMakeDriverInstance();
+        return this.cTestController.resolveLaunchConfigurationForTest(config, drv);
+    }
+
     async debugCTest(testName: string): Promise<vscode.DebugSession | null> {
         const drv = await this.getCMakeDriverInstance();
         if (!drv) {

src/ctest.ts

@@ -1924,7 +1924,7 @@ export class CTestDriver implements vscode.Disposable {
         if (this.tests) {
             for (const test of this.tests.tests) {
                 if (test.name === testName) {
-                    return test.command[0];
+                    return test.command?.[0] ?? '';
                 }
             }
         } else if (this.legacyTests) {
@@ -1952,7 +1952,7 @@ export class CTestDriver implements vscode.Disposable {
         if (this.tests) {
             for (const test of this.tests.tests) {
                 if (test.name === testName) {
-                    return test.command.slice(1);
+                    return test.command?.slice(1) ?? [];
                 }
             }
         }
@@ -2078,6 +2078,95 @@ export class CTestDriver implements vscode.Disposable {
         return allConfigItems;
     }
 
+    /**
+     * The `${cmake.testProgram}`, `${cmake.testArgs}`, `${cmake.testWorkingDirectory}` and
+     * `${cmake.testEnvironment}` placeholders are not real VS Code variables; they are only
+     * meaningful for a specific CTest test. This returns true if the given launch configuration
+     * references any of them.
+     */
+    public static configReferencesTestPlaceholders(config: vscode.DebugConfiguration): boolean {
+        const marker = '${cmake.test';
+        const scan = (value: unknown): boolean => {
+            if (typeof value === 'string') {
+                return value.includes(marker);
+            }
+            if (Array.isArray(value)) {
+                return value.some(scan);
+            }
+            if (value && typeof value === 'object') {
+                return Object.values(value as Record<string, unknown>).some(scan);
+            }
+            return false;
+        };
+        return scan(config);
+    }
+
+    /**
+     * Substitute the `${cmake.test*}` placeholders in a launch configuration for the given test.
+     */
+    private applyTestPlaceholderSubstitutions(config: vscode.DebugConfiguration, testName: string): vscode.DebugConfiguration {
+        config = this.replaceAllInObject<vscode.DebugConfiguration>(config, '${cmake.testProgram}', this.testProgram(testName));
+        config = this.replaceAllInObject<vscode.DebugConfiguration>(config, '${cmake.testWorkingDirectory}', this.testWorkingDirectory(testName));
+
+        // Replace cmake.testArgs wrapped in quotes, like `"${command:cmake.testArgs}"`, without any spaces in between,
+        // since we need to replace the quotes as well.
+        config = this.replaceArrayItems(config, '${cmake.testArgs}', this.testArgs(testName)) as vscode.DebugConfiguration;
+
+        // Replace cmake.testEnvironment with the test's ENVIRONMENT property as an array of { name, value } objects.
+        const testEnv = this.testEnvironment(testName);
+        const testEnvArray = Object.entries(testEnv).map(([name, value]) => ({ name, value }));
+        config = this.replaceValueInObject<vscode.DebugConfiguration>(config, '${cmake.testEnvironment}', testEnvArray);
+        return config;
+    }
+
+    /**
+     * Resolve the `${cmake.test*}` placeholders in a launch configuration that is started directly
+     * from the Run and Debug view (i.e., via F5) rather than from the Test Explorer. Because that
+     * entry point has no associated test, the user is prompted to pick one of the discovered CTest
+     * tests. Returns the substituted configuration, or `undefined` to abort the launch (no tests
+     * were found, or the user cancelled the pick).
+     */
+    public async resolveLaunchConfigurationForTest(config: vscode.DebugConfiguration, driver: CMakeDriver | null): Promise<vscode.DebugConfiguration | undefined> {
+        if (!CTestDriver.configReferencesTestPlaceholders(config)) {
+            return config;
+        }
+
+        // Refresh from the driver so that this.tests carries the full CTest information (in
+        // particular each test's `command`, which is what ${cmake.testProgram} resolves to). The
+        // Test Explorer may have only populated a lighter-weight view, so we cannot rely on
+        // getTestNames() alone having usable program paths.
+        if (driver) {
+            await this.refreshTests(driver);
+        }
+
+        const testNames = this.getTestNames();
+        log.debug(localize('ctest.launch.discovered.tests', 'CTest launch resolver discovered {0} test(s).', String(testNames?.length ?? 0)));
+        if (testNames === undefined || testNames.length === 0) {
+            void vscode.window.showErrorMessage(localize('ctest.launch.no.tests', 'This launch configuration resolves to a specific CTest test, but no tests were found. Configure and build your project so tests are discovered, then try again (or start debugging a test from the Test Explorer).'));
+            return undefined;
+        }
+
+        let testName: string | undefined;
+        if (testNames.length === 1) {
+            testName = testNames[0];
+        } else {
+            testName = await vscode.window.showQuickPick(testNames.sort(), { placeHolder: localize('ctest.launch.pick.test', 'Select the CTest test to debug') });
+        }
+
+        if (testName === undefined) {
+            return undefined;
+        }
+
+        const program = this.testProgram(testName);
+        log.debug(localize('ctest.launch.resolved.program', 'CTest launch resolver resolved test \'{0}\' to program \'{1}\'.', testName, program));
+        if (!program) {
+            void vscode.window.showErrorMessage(localize('ctest.launch.no.program', 'Could not determine the executable for CTest test \'{0}\'. Build the test\'s target, then try again.', testName));
+            return undefined;
+        }
+
+        return this.applyTestPlaceholderSubstitutions(config, testName);
+    }
+
     private async debugCTestImpl(workspaceFolder: vscode.WorkspaceFolder, testName: string, cancellation: vscode.CancellationToken, preSelectedConfig?: ConfigItem): Promise<void> {
         const magicValue = sessionNum++;
         let chosenConfig: ConfigItem | undefined = preSelectedConfig;
@@ -2109,17 +2198,7 @@ export class CTestDriver implements vscode.Disposable {
 
         // Commands can't be used to replace array (i.e., args); and both test program and test args requires folder and
         // test name as parameters, which means one launch config for each test. So replacing them here is a better way.
-        chosenConfig.config = this.replaceAllInObject<vscode.DebugConfiguration>(chosenConfig.config, '${cmake.testProgram}', this.testProgram(testName));
-        chosenConfig.config = this.replaceAllInObject<vscode.DebugConfiguration>(chosenConfig.config, '${cmake.testWorkingDirectory}', this.testWorkingDirectory(testName));
-
-        // Replace cmake.testArgs wrapped in quotes, like `"${command:cmake.testArgs}"`, without any spaces in between,
-        // since we need to replace the quotes as well.
-        chosenConfig.config = this.replaceArrayItems(chosenConfig.config, '${cmake.testArgs}', this.testArgs(testName)) as vscode.DebugConfiguration;
-
-        // Replace cmake.testEnvironment with the test's ENVIRONMENT property as an array of { name, value } objects.
-        const testEnv = this.testEnvironment(testName);
-        const testEnvArray = Object.entries(testEnv).map(([name, value]) => ({ name, value }));
-        chosenConfig.config = this.replaceValueInObject<vscode.DebugConfiguration>(chosenConfig.config, '${cmake.testEnvironment}', testEnvArray);
+        chosenConfig.config = this.applyTestPlaceholderSubstitutions(chosenConfig.config, testName);
 
         // Identify the session we started
         chosenConfig.config[magicKey] = magicValue;
@@ -2450,3 +2529,41 @@ export function deIntegrateTestExplorer(): void {
         testExplorer = undefined;
     }
 }
+
+/**
+ * Resolves the `${cmake.test*}` placeholders in third-party debugger launch configurations (e.g.,
+ * cppdbg, cppvsdbg, lldb) when they are started directly from the Run and Debug view (F5), so users
+ * are not required to launch such configurations exclusively from the Test Explorer. When a
+ * placeholder is present but no test context exists, the user is prompted to choose one of the
+ * discovered CTest tests. Configurations that do not reference these placeholders (including those
+ * already substituted by the Test Explorer flow) are returned unchanged.
+ */
+export class CTestLaunchConfigurationProvider implements vscode.DebugConfigurationProvider {
+    constructor(private readonly projectController: ProjectController) {}
+
+    async resolveDebugConfiguration(folder: vscode.WorkspaceFolder | undefined, debugConfiguration: vscode.DebugConfiguration, _token?: vscode.CancellationToken): Promise<vscode.DebugConfiguration | undefined | null> {
+        const referencesPlaceholders = CTestDriver.configReferencesTestPlaceholders(debugConfiguration);
+        if (!referencesPlaceholders) {
+            return debugConfiguration;
+        }
+        log.debug(localize('ctest.launch.resolve.invoked', 'CTest launch resolver invoked for debug type \'{0}\'.', debugConfiguration.type));
+
+        try {
+            const project = folder
+                ? await this.projectController.getProjectForFolder(folder.uri.fsPath)
+                : this.projectController.getActiveCMakeProject();
+
+            if (!project) {
+                void vscode.window.showErrorMessage(localize('ctest.launch.no.project', 'Cannot resolve the CTest program for this launch configuration because no CMake project is associated with it.'));
+                return undefined;
+            }
+
+            return await project.resolveCTestLaunchConfiguration(debugConfiguration);
+        } catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            log.error(localize('ctest.launch.resolve.failed', 'Failed to resolve the CTest launch configuration: {0}', message), e instanceof Error ? (e.stack ?? '') : '');
+            void vscode.window.showErrorMessage(localize('ctest.launch.resolve.failed', 'Failed to resolve the CTest launch configuration: {0}', message));
+            return undefined;
+        }
+    }
+}

src/extension.ts

@@ -50,7 +50,7 @@ import { DebugAdapterNamedPipeServerDescriptorFactory } from '@cmt/debug/cmakeDe
 import { getCMakeExecutableInformation } from '@cmt/cmakeExecutable';
 import { DebuggerInformation, getDebuggerPipeName } from '@cmt/debug/cmakeDebugger/debuggerConfigureDriver';
 import { DebugConfigurationProvider, DynamicDebugConfigurationProvider } from '@cmt/debug/cmakeDebugger/debugConfigurationProvider';
-import { deIntegrateTestExplorer } from "@cmt/ctest";
+import { deIntegrateTestExplorer, CTestLaunchConfigurationProvider } from "@cmt/ctest";
 import collections from '@cmt/diagnostics/collections';
 import { LanguageServiceData } from './languageServices/languageServiceData';
 import { CMakeListsModifier } from './cmakeListsModifier';
@@ -2694,6 +2694,14 @@ async function setup(context: vscode.ExtensionContext, progress?: ProgressHandle
             vscode.DebugConfigurationProviderTriggerKind.Dynamic)
     );
 
+    // Resolve the ${cmake.test*} placeholders when a test's launch configuration is started directly
+    // from the Run and Debug view (F5), rather than only from the Test Explorer (issue #4574).
+    const ctestLaunchResolver = new CTestLaunchConfigurationProvider(ext.projectController);
+    for (const debugType of ["cppdbg", "cppvsdbg", "lldb", "lldb-dap", "gdb"]) {
+        context.subscriptions.push(vscode.debug.registerDebugConfigurationProvider(debugType, ctestLaunchResolver));
+    }
+    log.debug(localize('registered.ctest.launch.resolver', 'Registered CTest launch configuration resolver for external debugger types.'));
+
     // List of functions that will be bound commands
     const funs: (keyof ExtensionManager)[] = [
         'activeFolderName',