Add end to end test devoted to ctest

.vscode/launch.json

@@ -105,6 +105,7 @@
             ],
             "sourceMaps": true,
             "outFiles": [
+                "${workspaceFolder}/dist/**/*.js",
                 "${workspaceFolder}/out/*",
                 "${workspaceFolder}/out/src/**",
                 "${workspaceFolder}/out/test/*",
@@ -118,6 +119,32 @@
                 "TEST_FILTER": ".*"
             },
         },
+        {
+            "name": "Run single-root-ctest Tests",
+            "type": "extensionHost",
+            "request": "launch",
+            "runtimeExecutable": "${execPath}",
+            "args": [
+                "${workspaceFolder}/test/end-to-end-tests/single-root-ctest/project-folder",
+                "--extensionDevelopmentPath=${workspaceFolder}",
+                "--extensionTestsPath=${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/index"
+            ],
+            "sourceMaps": true,
+            "outFiles": [
+                "${workspaceFolder}/dist/**/*.js",
+                "${workspaceFolder}/out/*",
+                "${workspaceFolder}/out/src/**",
+                "${workspaceFolder}/out/test/*",
+                "${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/*",
+                "${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/test/*"
+            ],
+            "preLaunchTask": "Pretest",
+            "env": {
+                "CMT_TESTING": "1",
+                "CMT_QUIET_CONSOLE": "1",
+                "TEST_FILTER": ".*"
+            },
+        },
         {
             "name": "Run multi-root-UI Tests",
             "type": "extensionHost",

package.json

@@ -3698,6 +3698,7 @@
     "integrationTests": "yarn run pretest && node ./out/test/integration-tests/runTest.js",
     "endToEndTestsSuccessfulBuild": "yarn run pretest && node ./out/test/end-to-end-tests/successful-build/runTest.js",
     "endToEndTestsSingleRoot": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-UI/runTest.js",
+    "endToEndTestsSingleRootCTest": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-ctest/runTest.js",
     "endToEndTestsMultiRoot": "yarn run pretest && node ./out/test/end-to-end-tests/multi-root-UI/runTest.js",
     "backendTests": "node ./node_modules/mocha/bin/_mocha -u tdd --timeout 999999 --colors -r ts-node/register -r tsconfig-paths/register ./test/unit-tests/backend/**/*.test.ts",
     "build-product-icon-font": "yarn --cwd ./tools/product-icon-font-generator/ install && yarn --cwd ./tools/product-icon-font-generator/ build && node ./tools/product-icon-font-generator/dist/index.js --source-directory ./res/product-icons/ --output-directory ./res/ --woff2"

src/cmakeProject.ts

@@ -2318,7 +2318,7 @@ export class CMakeProject {
     async ctest(fromWorkflow: boolean = false): Promise<number> {
         const drv = await this.preTest(fromWorkflow);
         const retc = await this.cTestController.runCTest(drv);
-        return (retc) ? 0 : -1;
+        return (retc === 0) ? 0 : -1;
     }
 
     async cpack(fromWorkflow: boolean = false): Promise<number> {

src/ctest.ts

@@ -710,11 +710,6 @@ export class CTestDriver implements vscode.Disposable {
             return -1;
         }
 
-        if (util.isTestMode()) {
-            // ProjectController can't be initialized in test mode, so we don't have a usable test explorer
-            return 0;
-        }
-
         const initializedTestExplorer = this.ensureTestExplorerInitialized();
         const sourceDir = util.platformNormalizePath(driver.sourceDir);
         const testExplorerRoot = initializedTestExplorer.items.get(sourceDir);

test/end-to-end-tests/single-root-ctest/index.ts

@@ -0,0 +1,54 @@
+// eslint-disable-next-line import/no-unassigned-import
+import 'module-alias/register';
+
+import * as path from 'path';
+import * as Mocha from 'mocha';
+import * as glob from 'glob';
+import { Logger } from '@cmt/logging';
+
+export function run(): Promise<void> {
+    // Create the mocha test
+    const mocha = new Mocha({
+        ui: 'tdd',
+        color: true
+    });
+
+    const testsRoot = __dirname;
+
+    return new Promise((c, e) => {
+        glob('**/*.test.js', { cwd: testsRoot }, (err, files) => {
+            if (err) {
+                return e(err);
+            }
+
+            // Add files to the test suite
+            const regex = process.env.TEST_FILTER ? new RegExp(process.env.TEST_FILTER) : /.*/;
+            files.forEach(f => {
+                if (regex.test(f)) {
+                    mocha.addFile(path.resolve(testsRoot, f));
+                }
+            });
+
+            try {
+                // Run the mocha test
+                mocha.timeout(100000);
+
+                // Log the name of each test before it starts.
+                const beforeEach: Mocha.Func = function (this: Mocha.Context, done: Mocha.Done) {
+                    Logger.logTestName(this.currentTest?.parent?.title, this.currentTest?.title);
+                    done();
+                };
+                mocha.rootHooks({beforeEach});
+                mocha.run(failures => {
+                    if (failures > 0) {
+                        e(new Error(`${failures} tests failed.`));
+                    } else {
+                        c();
+                    }
+                });
+            } catch (err) {
+                e(err);
+            }
+        });
+    });
+}

test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "cmake.buildDirectory": "${workspaceFolder}/build",
+    "cmake.useCMakePresets": "always",
+    "cmake.configureOnOpen": false,
+    "cmake.ctest.allowParallelJobs": false
+}

test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt

@@ -0,0 +1,55 @@
+cmake_minimum_required(VERSION 3.6.0)
+project(TestCTestProcess VERSION 0.1.0)
+
+add_library(TestUtils SHARED)
+target_sources(TestUtils PRIVATE test_utils.cpp)
+target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED
+                                                           ON)
+
+set(TESTS_DIR
+    ""
+    CACHE STRING "Directory where the files generated by tests will be written")
+# Will generate a file get_test_dir.h with the content of get_test_dir.h.in It's
+# goal is to provide a function that returns the directory where the tests will
+# be executed
+configure_file("get_test_dir.h.in" "get_test_dir.h" @ONLY)
+add_library(GetTestDir INTERFACE)
+target_include_directories(GetTestDir INTERFACE ${CMAKE_CURRENT_BINARY_DIR})
+
+# Adds an executable that will generate an output file by concatenating the
+# content of each file in the test directory
+add_executable(GenerateOutputFile generate_output_file.cpp)
+target_link_libraries(GenerateOutputFile PRIVATE GetTestDir)
+set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17
+                                                    CXX_STANDARD_REQUIRED ON)
+
+enable_testing()
+
+# The generation of json readable output file must occur after each test run
+# Declares a fixture that will be executed after each test run
+add_test(NAME "Generate_Output_File" COMMAND GenerateOutputFile)
+set_tests_properties("Generate_Output_File" PROPERTIES FIXTURES_CLEANUP GENOUT)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+include(RegisterTest)
+
+set(TESTS_OUTPUT_FILES
+    ""
+    CACHE STRING "Output files generated by tests")
+set(TESTS_NAMES
+    ""
+    CACHE STRING "Names of the tests")
+set(TESTS_SUCCESS
+    ""
+    CACHE BOOL "Success of the tests")
+
+register_tests(
+  TEST_DIRECTORY
+  ${TESTS_DIR}
+  TEST_NAME_LIST
+  ${TESTS_NAMES}
+  TEST_OUTPUT_FILE_LIST
+  ${TESTS_OUTPUT_FILES}
+  TEST_SUCCESS_LIST
+  ${TESTS_SUCCESS})

test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json

@@ -0,0 +1,44 @@
+{
+  "version": 2,
+  "configurePresets": [
+    {
+      "name": "2Successes",
+      "description": "Sets generator, build and install directory, vcpkg",
+      "generator": "Ninja",
+      "binaryDir": "${workspaceFolder}/build",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "TESTS_DIR": "/tmp/vscode-cmake-tools-tests",
+        "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt",
+        "TESTS_NAMES": "Suite1.TestA;Suite2.TestB",
+        "TESTS_SUCCESS": "true;true"
+      }
+    },
+    {
+      "name": "2Successes1Failure",
+      "description": "Sets generator, build and install directory, vcpkg",
+      "generator": "Ninja",
+      "binaryDir": "${workspaceFolder}/build",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "TESTS_DIR": "/tmp/vscode-cmake-tools-tests",
+        "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt",
+        "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC",
+        "TESTS_SUCCESS": "true;false;true"
+      }
+    },
+    {
+      "name": "3Failures",
+      "description": "Sets generator, build and install directory, vcpkg",
+      "generator": "Ninja",
+      "binaryDir": "${workspaceFolder}/build",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "TESTS_DIR": "/tmp/vscode-cmake-tools-tests",
+        "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt",
+        "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC",
+        "TESTS_SUCCESS": "false;false;false"
+      }
+    }
+  ]
+}

test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake

@@ -0,0 +1,154 @@
+#--------------------------------------------------------------------
+# Generate the source file of the test in argument
+# 
+# Parameters:
+#  output_file_path: path to the file, the test will write
+#  test_success: true if the test end successfully. False otherwise.
+# Returns:
+#  test_source: name of the source file that will be generated
+#--------------------------------------------------------------------
+function(generate_test_source_file output_file_path test_success)
+  get_filename_component(output_file_name ${output_file_path} NAME_WE)
+  # Declare variables used in the template file (test.cpp.in)
+  set(test_filename ${output_file_path})
+  set(success ${test_success})
+  # Generate test source file
+  set(test_source "${output_file_name}.cpp")
+  configure_file(test.cpp.in ${test_source} @ONLY)
+  set(test_source "${test_source}" PARENT_SCOPE)
+endfunction()
+
+#--------------------------------------------------------------------
+# Build the name of the test executable from the name of the test source file
+#
+# The name of the executable will be the one of the test source file without '_'
+# and in CamelCase.
+#
+# Parameters:
+#   test_source: name of the test source file
+# Returns:
+#   test_exe: name of the test executable
+#--------------------------------------------------------------------
+function(build_test_exe_name test_source)
+  get_filename_component(test_name ${test_source} NAME_WE)
+  # Replace _ with ; so that the name becomes a list of sub-words
+  string(REPLACE "_" ";" splitted_test_name ${test_name})
+  set(test_exe)
+  # For each of the sub-word, extract the first letter
+  # from the rest of the word (radical)
+  foreach(word ${splitted_test_name})
+    string(SUBSTRING ${word} 0 1 first_letter)
+    string(SUBSTRING ${word} 1 -1 radical)
+    # Turns first sub-word letter into upper case
+    string(TOUPPER ${first_letter} up_first_letter)
+    # Concat uppercase first letter and radical
+    set(test_exe "${test_exe}${up_first_letter}${radical}")
+  endforeach()
+  # Returns test_exe
+  set(test_exe ${test_exe} PARENT_SCOPE)
+endfunction()
+
+#--------------------------------------------------------------------
+# Create and register a test
+#
+# Usage:
+#   register_test(TEST_DIR <dir> TEST_NAME <name> TEST_OUTPUT_FILE <path> TEST_SUCCESS <success>)
+# Parameters:
+#   TEST_DIR: path to the directory where the test will write its output file
+#   TEST_NAME: name of the test
+#   TEST_OUTPUT_FILE: name of the file the test should generate
+#   TEST_SUCCESS: whether or not the test should end successfully
+#--------------------------------------------------------------------
+function(register_test)
+  set(options)
+  set(oneValueArgs "TEST_DIR;TEST_NAME;TEST_OUTPUT_FILE;TEST_SUCCESS")
+  ### PARSING ARGUMENTS
+  cmake_parse_arguments(register_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+  if(DEFINED register_test_KEYWORDS_MISSING_VALUES)
+    message(
+      FATAL_ERROR
+        "In the call to register_test function, the keywords ${register_test_KEYWORDS_MISSING_VALUES} are awaiting for at least one value"
+    )
+  endif()
+  if(DEFINED register_test_UNPARSED_ARGUMENTS)
+    message(
+      FATAL_ERROR
+        "Following arguments are unknown to register_test function: ${register_test_UNPARSED_ARGUMENTS}"
+    )
+  endif()
+  if(NOT DEFINED register_test_TEST_DIR)
+    message(FATAL_ERROR "The function register_test is awaiting for TEST_DIR keyword")
+  endif()
+  if(NOT DEFINED register_test_TEST_NAME)
+    message(FATAL_ERROR "The function register_test is awaiting for TEST_NAME keyword")
+  endif()
+  if(NOT DEFINED register_test_TEST_OUTPUT_FILE)
+    message(FATAL_ERROR "The function register_test is awaiting for TEST_OUTPUT_FILE keyword")
+  endif()
+  if(NOT DEFINED register_test_TEST_SUCCESS)
+    message(FATAL_ERROR "The function register_test is awaiting for TEST_SUCCESS keyword")
+  endif()
+
+  set(test_output_file_path "${register_test_TEST_DIR}/${register_test_TEST_OUTPUT_FILE}")
+  message(STATUS "Creating test named ${register_test_TEST_NAME} with result stored in ${test_output_file_path} returning as success: ${register_test_TEST_SUCCESS}")
+  ### GENERATE TEST
+  generate_test_source_file(${test_output_file_path} ${register_test_TEST_SUCCESS}) # => returns test_source
+  build_test_exe_name(${test_source}) # => returns test_exe
+  message(STATUS "--> Creating test executable ${test_exe} with source ${test_source}")
+  add_executable(${test_exe} ${test_source})
+  target_link_libraries(${test_exe} PRIVATE TestUtils GetTestDir)
+  add_test(NAME "${register_test_TEST_NAME}" COMMAND "${test_exe}")
+  set_tests_properties("${register_test_TEST_NAME}" PROPERTIES FIXTURES_REQUIRED GENOUT)
+endfunction()
+
+#--------------------------------------------------------------------
+# Create and register tests in arguments
+#
+# Usage:
+#   register_tests(TEST_DIRECTORY <dir> TEST_NAME_LIST <names> TEST_OUTPUT_FILE_LIST <paths> TEST_SUCCESS_LIST <successes>)
+# Parameters:
+#   TEST_DIRECTORY: path to the directory where the tests will write their output files
+#   TEST_NAME_LIST: list of test names
+#   TEST_OUTPUT_FILE_LIST: list of file names the tests should generate
+#   TEST_SUCCESS_LIST: list of boolean values indicating whether or not the tests should end successfully
+#--------------------------------------------------------------------
+function(register_tests)
+  set(options)
+  set(oneValueArgs "TEST_DIRECTORY")
+  set(multiValueArgs "TEST_NAME_LIST;TEST_OUTPUT_FILE_LIST;TEST_SUCCESS_LIST")
+  ### PARSING ARGUMENTS
+  cmake_parse_arguments(register_tests "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+  if(DEFINED register_tests_KEYWORDS_MISSING_VALUES)
+    message(
+      FATAL_ERROR
+        "In the call to register_tests function, the keywords ${register_tests_KEYWORDS_MISSING_VALUES} are awaiting for at least one value"
+    )
+  endif()
+  if(DEFINED register_tests_UNPARSED_ARGUMENTS)
+    message(
+      FATAL_ERROR
+        "Following arguments are unknown to register_tests function: ${register_tests_UNPARSED_ARGUMENTS}"
+    )
+  endif()
+  if(NOT DEFINED register_tests_TEST_DIRECTORY)
+    message(FATAL_ERROR "The function register_tests is awaiting for TEST_DIRECTORY keyword")
+  endif()
+  if(NOT DEFINED register_tests_TEST_NAME_LIST)
+    message(FATAL_ERROR "The function register_tests is awaiting for TEST_NAME_LIST keyword")
+  endif()
+  if(NOT DEFINED register_tests_TEST_OUTPUT_FILE_LIST)
+    message(FATAL_ERROR "The function register_tests is awaiting for TEST_OUTPUT_FILE_LIST keyword")
+  endif()
+  if(NOT DEFINED register_tests_TEST_SUCCESS_LIST)
+    message(FATAL_ERROR "The function register_tests is awaiting for TEST_SUCCESS_LIST keyword")
+  endif()
+
+  list(LENGTH register_tests_TEST_NAME_LIST NB_TESTS)
+  math(EXPR MAX_INDEX "${NB_TESTS}-1")
+  foreach(test_index RANGE ${MAX_INDEX})
+      list(GET register_tests_TEST_OUTPUT_FILE_LIST ${test_index} test_output)
+      list(GET register_tests_TEST_NAME_LIST ${test_index} test_name)
+      list(GET register_tests_TEST_SUCCESS_LIST ${test_index} test_success)
+      register_test(TEST_DIR ${register_tests_TEST_DIRECTORY} TEST_NAME ${test_name} TEST_OUTPUT_FILE ${test_output} TEST_SUCCESS ${test_success})
+  endforeach()
+endfunction()