fix unicode characters like Umlaute üäÜÖßéè

Author: henderkes

CMakeLists.txt

@@ -17,7 +17,7 @@ endif()
 
 set(VERSION_MAJOR 1)
 set(VERSION_MINOR 6)
-set(VERSION_PATCH 2)
+set(VERSION_PATCH 3)
 set(VERSION_TWEAK 0)
 
 set(VERSION_RC "${CMAKE_CURRENT_BINARY_DIR}/version.rc")

header/Defines.h

@@ -29,6 +29,8 @@ inline void Message(const char* format, ...)
 inline void Info(const char* format, ...)
 {
 #ifdef _DEBUG
+    const HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
+    [[maybe_unused]] const auto success = SetConsoleTextAttribute(hConsole, 0); // white
     va_list args;
     va_start(args, format);
     vprintf(format, args);
@@ -39,8 +41,8 @@ inline void Info(const char* format, ...)
 inline void Warning(const char* format, ...)
 {
 #ifdef _DEBUG
-    static HANDLE hConsole = GetStdHandle(STD_ERROR_HANDLE);
-    [[maybe_unused]] static auto success = SetConsoleTextAttribute(hConsole, 6);
+    const HANDLE hConsole = GetStdHandle(STD_ERROR_HANDLE);
+    [[maybe_unused]] const auto success = SetConsoleTextAttribute(hConsole, 6); // yellow
     va_list args;
     va_start(args, format);
     vfprintf(stderr, format, args);

header/Utils.h

@@ -12,4 +12,17 @@ namespace utils
             vec.erase(found);
         }
     }
+
+    inline std::wstring utf8_to_wstring(const std::string& utf8str) {
+        if (utf8str.empty()) return {};
+
+        // Calculate the number of wide characters needed for the conversion
+        const int count = MultiByteToWideChar(CP_UTF8, 0, utf8str.c_str(), -1, nullptr, 0);
+        if (count == 0) throw std::runtime_error("Failed to convert UTF-8 to UTF-16");
+
+        std::wstring wstr(count, 0);
+        MultiByteToWideChar(CP_UTF8, 0, utf8str.c_str(), -1, wstr.data(), count);
+
+        return wstr;
+    }
 }

modules/ModfileLoader.ixx

@@ -15,7 +15,7 @@ import <filesystem>;
 import ModfileLoader.TpfReader;
 
 export class ModfileLoader {
-    std::string file_name;
+    std::filesystem::path file_name;
     const std::string TPF_PASSWORD{
         0x73, 0x2A, 0x63, 0x7D, 0x5F, 0x0A, static_cast<char>(0xA6), static_cast<char>(0xBD),
         0x7D, 0x65, 0x7E, 0x67, 0x61, 0x2A, 0x7F, 0x7F,
@@ -26,7 +26,7 @@ export class ModfileLoader {
     };
 
 public:
-    ModfileLoader(const std::string& fileName);
+    ModfileLoader(const std::filesystem::path& fileName);
 
     std::vector<TexEntry> GetContents();
 
@@ -39,15 +39,15 @@ private:
     void LoadEntries(libzippp::ZipArchive& archive, std::vector<TexEntry>& entries);
 };
 
-ModfileLoader::ModfileLoader(const std::string& fileName)
+ModfileLoader::ModfileLoader(const std::filesystem::path& fileName)
 {
-    file_name = std::filesystem::absolute(fileName).string();
+    file_name = std::filesystem::absolute(fileName);
 }
 
 std::vector<TexEntry> ModfileLoader::GetContents()
 {
     try {
-        return file_name.ends_with(".tpf") ? GetTpfContents() : GetFileContents();
+        return file_name.wstring().ends_with(L".tpf") ? GetTpfContents() : GetFileContents();
     }
     catch (const std::exception&) {
         Warning("Failed to open mod file: %s\n", file_name.c_str());
@@ -81,7 +81,7 @@ std::vector<TexEntry> ModfileLoader::GetFileContents()
 {
     std::vector<TexEntry> entries;
 
-    libzippp::ZipArchive zip_archive(file_name);
+    libzippp::ZipArchive zip_archive(file_name.string());
     zip_archive.open();
     LoadEntries(zip_archive, entries);
     zip_archive.close();

modules/ModfileLoader_TpfReader.ixx

@@ -1,12 +1,13 @@
 export module ModfileLoader.TpfReader;
 
+import <filesystem>;
 import <string>;
 import <fstream>;
 import <vector>;
 
 export class TpfReader {
 public:
-    TpfReader(const std::string& path)
+    TpfReader(const std::filesystem::path& path)
     {
         file_stream = std::ifstream(path, std::ios::binary);
         if (!file_stream.seekg(0, std::ios::end).good() || !file_stream.seekg(0, std::ios::beg).good()) {

modules/TextureClient.ixx

@@ -164,7 +164,7 @@ gsl::owner<TextureFileStruct*> AddFile(TexEntry& entry, const bool compress, con
     return texture_file_struct;
 }
 
-std::vector<gsl::owner<TextureFileStruct*>> ProcessModfile(const std::string& modfile, const std::filesystem::path& dll_path, const bool compress)
+std::vector<gsl::owner<TextureFileStruct*>> ProcessModfile(const std::filesystem::path& modfile, const std::filesystem::path& dll_path, const bool compress)
 {
     const auto hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
     if (FAILED(hr)) return {};
@@ -192,25 +192,29 @@ std::vector<gsl::owner<TextureFileStruct*>> ProcessModfile(const std::string& mo
 
 void TextureClient::LoadModsFromFile(const char* source)
 {
-    static std::vector<std::string> loaded_modfiles{};
+    static std::vector<std::filesystem::path> loaded_modfiles{};
     Message("Initialize: searching in %s\n", source);
 
-    std::ifstream file(source);
+    std::locale::global(std::locale(""));
+    std::ifstream file(source, std::ios::binary);
     if (!file.is_open()) {
         Warning("LoadModsFromFile: failed to open modlist.txt for reading; aborting!!!");
         return;
     }
     Message("Initialize: found modlist.txt. Reading\n");
     std::string line;
-    std::vector<std::string> modfiles;
+    std::vector<std::filesystem::path> modfiles;
     while (std::getline(file, line)) {
         // Remove newline character
         line.erase(std::ranges::remove(line, '\r').begin(), line.end());
         line.erase(std::ranges::remove(line, '\n').begin(), line.end());
 
-        if (!std::ranges::contains(loaded_modfiles, line)) {
-            modfiles.push_back(line);
-            loaded_modfiles.push_back(line);
+        const auto wline = utils::utf8_to_wstring(line);
+        const auto fsline = std::filesystem::path(wline);
+
+        if (!std::ranges::contains(loaded_modfiles, fsline)) {
+            modfiles.push_back(fsline);
+            loaded_modfiles.push_back(fsline);
         }
     }
     auto files_size = 0u;