diff --git a/CMakeLists.txt b/CMakeLists.txt index 60096a8d..f1bc4a4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,9 @@ option(LIBLCF_ENABLE_BENCHMARKS "Whether to build the benchmarks (default: OFF)" option(LIBLCF_ENABLE_INSTALL "Whether to add an install target (default: ON)" ${LIBLCF_MAIN_PROJECT}) set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Override CMAKE_DEBUG_POSTFIX.") +# Dependencies provided by CMake Presets +list(APPEND CMAKE_PREFIX_PATH "${LIBLCF_PREFIX_PATH_APPEND}") + # C++17 is required set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -348,10 +351,23 @@ target_link_libraries(lcf inih::inih) # icu set(LCF_SUPPORT_ICU 0) if(LIBLCF_WITH_ICU) - find_package(ICU COMPONENTS i18n uc data REQUIRED) - target_link_libraries(lcf ICU::i18n ICU::uc ICU::data) - list(APPEND LIBLCF_DEPS "icu-i18n") - set(LCF_SUPPORT_ICU 1) + find_package(ICU COMPONENTS i18n uc data) + if(ICU_FOUND) + target_link_libraries(lcf ICU::i18n ICU::uc ICU::data) + list(APPEND LIBLCF_DEPS "icu-i18n") + set(LCF_SUPPORT_ICU 1) + else() + if(NOT WIN32) + message(FATAL_ERROR "ICU not found. Use LCF_SUPPORT_ICU=0 to disable ICU support (not recommended).") + endif() + + message(STATUS "ICU not found. Using the system library.") + message(STATUS "This build will only work on Windows 10 Version 1903 (May 2019) and newer!") + + target_link_libraries(lcf icu) + set(LCF_SUPPORT_ICU 2) + list(APPEND LIBLCF_DEPS "icu") + endif() endif() # expat diff --git a/CMakePresets.json b/CMakePresets.json index 45aa431f..b7719ea9 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -49,7 +49,7 @@ "name": "linux-parent", "toolchainFile": "${sourceDir}/builds/cmake/LinuxToolchain.cmake", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/linux-static" + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/linux-static" }, "hidden": true, "inherits": "base-user" @@ -79,34 +79,34 @@ ] }, { - "name": "windows-x86-parent", + "name": "windows-parent", "cacheVariables": { - "VCPKG_TARGET_TRIPLET": "x86-windows-static" + "VCPKG_TARGET_TRIPLET": "$env{VSCMD_ARG_TGT_ARCH}-windows-static" }, "inherits": "win-user", "hidden": true }, { - "name": "windows-x86-debug", - "displayName": "Windows (x86) (Debug)", + "name": "windows-debug", + "displayName": "Windows (Debug)", "inherits": [ - "windows-x86-parent", + "windows-parent", "type-debug" ] }, { - "name": "windows-x86-relwithdebinfo", - "displayName": "Windows (x86) (RelWithDebInfo)", + "name": "windows-relwithdebinfo", + "displayName": "Windows (RelWithDebInfo)", "inherits": [ - "windows-x86-parent", + "windows-parent", "type-relwithdebinfo" ] }, { - "name": "windows-x86-release", - "displayName": "Windows (x86) (Release)", + "name": "windows-release", + "displayName": "Windows (Release)", "inherits": [ - "windows-x86-parent", + "windows-parent", "type-release" ] }, @@ -144,39 +144,6 @@ "type-release" ] }, - { - "name": "windows-x64-parent", - "architecture": "x64", - "cacheVariables": { - "VCPKG_TARGET_TRIPLET": "x64-windows-static" - }, - "inherits": "win-user", - "hidden": true - }, - { - "name": "windows-x64-debug", - "displayName": "Windows (x64) (Debug)", - "inherits": [ - "windows-x64-parent", - "type-debug" - ] - }, - { - "name": "windows-x64-relwithdebinfo", - "displayName": "Windows (x64) (RelWithDebInfo)", - "inherits": [ - "windows-x64-parent", - "type-relwithdebinfo" - ] - }, - { - "name": "windows-x64-release", - "displayName": "Windows (x64) (Release)", - "inherits": [ - "windows-x64-parent", - "type-release" - ] - }, { "name": "windows-x64-vs2022-parent", "generator": "Visual Studio 17 2022", @@ -214,7 +181,7 @@ { "name": "macos-parent", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/osx", + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/osx", "CMAKE_OSX_DEPLOYMENT_TARGET": "10.9" }, "condition": { @@ -254,7 +221,7 @@ "toolchainFile": "$env{EASYRPG_BUILDSCRIPTS}/emscripten/emsdk-portable/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake", "cacheVariables": { "LIBLCF_ENABLE_TOOLS": "OFF", - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/emscripten", + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/emscripten", "CMAKE_FIND_ROOT_PATH": "$env{EASYRPG_BUILDSCRIPTS}/emscripten" }, "hidden": true, @@ -288,7 +255,8 @@ "name": "3ds-parent", "toolchainFile": "$env{DEVKITPRO}/cmake/3DS.cmake", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/3ds" + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/3ds", + "LIBLCF_ENABLE_TOOLS": "OFF" }, "inherits": "dkp-user", "hidden": true @@ -321,7 +289,8 @@ "name": "switch-parent", "toolchainFile": "$env{DEVKITPRO}/cmake/Switch.cmake", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/switch" + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/switch", + "LIBLCF_ENABLE_TOOLS": "OFF" }, "inherits": "dkp-user", "hidden": true @@ -354,7 +323,8 @@ "name": "wii-parent", "toolchainFile": "$env{DEVKITPRO}/cmake/Wii.cmake", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/wii" + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/wii", + "LIBLCF_ENABLE_TOOLS": "OFF" }, "inherits": "dkp-user", "hidden": true @@ -383,6 +353,40 @@ "type-release" ] }, + { + "name": "wiiu-parent", + "toolchainFile": "$env{DEVKITPRO}/cmake/WiiU.cmake", + "cacheVariables": { + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/wiiu", + "LIBLCF_ENABLE_TOOLS": "OFF" + }, + "inherits": "dkp-user", + "hidden": true + }, + { + "name": "wiiu-debug", + "displayName": "Nintendo WiiU (Debug)", + "inherits": [ + "wiiu-parent", + "type-debug" + ] + }, + { + "name": "wiiu-relwithdebinfo", + "displayName": "Nintendo WiiU (RelWithDebInfo)", + "inherits": [ + "wiiu-parent", + "type-relwithdebinfo" + ] + }, + { + "name": "wiiu-release", + "displayName": "Nintendo WiiU (Release)", + "inherits": [ + "wiiu-parent", + "type-release" + ] + }, { "name": "psvita-parent", "toolchainFile": "$env{EASYRPG_BUILDSCRIPTS}/vita/vitasdk/share/vita.toolchain.cmake", @@ -575,16 +579,16 @@ "configurePreset": "linux-release" }, { - "name": "windows-x86-debug", - "configurePreset": "windows-x86-debug" + "name": "windows-debug", + "configurePreset": "windows-debug" }, { - "name": "windows-x86-relwithdebinfo", - "configurePreset": "windows-x86-relwithdebinfo" + "name": "windows-relwithdebinfo", + "configurePreset": "windows-relwithdebinfo" }, { - "name": "windows-x86-release", - "configurePreset": "windows-x86-release" + "name": "windows-release", + "configurePreset": "windows-release" }, { "name": "windows-x86-vs2022-debug", @@ -598,18 +602,6 @@ "name": "windows-x86-vs2022-release", "configurePreset": "windows-x86-vs2022-release" }, - { - "name": "windows-x64-debug", - "configurePreset": "windows-x64-debug" - }, - { - "name": "windows-x64-relwithdebinfo", - "configurePreset": "windows-x64-relwithdebinfo" - }, - { - "name": "windows-x64-release", - "configurePreset": "windows-x64-release" - }, { "name": "windows-x64-vs2022-debug", "configurePreset": "windows-x64-vs2022-debug" @@ -682,6 +674,18 @@ "name": "wii-release", "configurePreset": "wii-release" }, + { + "name": "wiiu-debug", + "configurePreset": "wiiu-debug" + }, + { + "name": "wiiu-relwithdebinfo", + "configurePreset": "wiiu-relwithdebinfo" + }, + { + "name": "wiiu-release", + "configurePreset": "wiiu-release" + }, { "name": "psvita-debug", "configurePreset": "psvita-debug" diff --git a/builds/cmake/CMakePresets.json.template b/builds/cmake/CMakePresets.json.template index 916d537b..39d0c51b 100644 --- a/builds/cmake/CMakePresets.json.template +++ b/builds/cmake/CMakePresets.json.template @@ -18,14 +18,14 @@ "displayName": "Linux", "toolchainFile": "${sourceDir}/builds/cmake/LinuxToolchain.cmake", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/linux-static" + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/linux-static" } }, { - "name": "windows-x86", - "displayName": "Windows (x86)", + "name": "windows", + "displayName": "Windows", "cacheVariables": { - "VCPKG_TARGET_TRIPLET": "x86-windows-static" + "VCPKG_TARGET_TRIPLET": "$env{VSCMD_ARG_TGT_ARCH}-windows-static" }, "inherits": "win-user" }, @@ -39,15 +39,6 @@ }, "inherits": "win-user" }, - { - "name": "windows-x64", - "displayName": "Windows (x64)", - "architecture": "x64", - "cacheVariables": { - "VCPKG_TARGET_TRIPLET": "x64-windows-static" - }, - "inherits": "win-user" - }, { "name": "windows-x64-vs2022", "displayName": "Windows (x64) using Visual Studio 2022", @@ -62,7 +53,7 @@ "name": "macos", "displayName": "macOS", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/osx", + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/osx", "CMAKE_OSX_DEPLOYMENT_TARGET": "10.9" }, "condition": { @@ -77,7 +68,7 @@ "toolchainFile": "$env{EASYRPG_BUILDSCRIPTS}/emscripten/emsdk-portable/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake", "cacheVariables": { "LIBLCF_ENABLE_TOOLS": "OFF", - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/emscripten", + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/emscripten", "CMAKE_FIND_ROOT_PATH": "$env{EASYRPG_BUILDSCRIPTS}/emscripten" } }, @@ -86,7 +77,8 @@ "displayName": "Nintendo 3DS", "toolchainFile": "$env{DEVKITPRO}/cmake/3DS.cmake", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/3ds" + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/3ds", + "LIBLCF_ENABLE_TOOLS": "OFF" }, "inherits": "dkp-user" }, @@ -95,7 +87,8 @@ "displayName": "Nintendo Switch", "toolchainFile": "$env{DEVKITPRO}/cmake/Switch.cmake", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/switch" + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/switch", + "LIBLCF_ENABLE_TOOLS": "OFF" }, "inherits": "dkp-user" }, @@ -104,7 +97,18 @@ "displayName": "Nintendo Wii", "toolchainFile": "$env{DEVKITPRO}/cmake/Wii.cmake", "cacheVariables": { - "CMAKE_PREFIX_PATH": "$env{EASYRPG_BUILDSCRIPTS}/wii" + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/wii", + "LIBLCF_ENABLE_TOOLS": "OFF" + }, + "inherits": "dkp-user" + }, + { + "name": "wiiu", + "displayName": "Nintendo WiiU", + "toolchainFile": "$env{DEVKITPRO}/cmake/WiiU.cmake", + "cacheVariables": { + "LIBLCF_PREFIX_PATH_APPEND": "$env{EASYRPG_BUILDSCRIPTS}/wiiu", + "LIBLCF_ENABLE_TOOLS": "OFF" }, "inherits": "dkp-user" }, diff --git a/builds/cmake/CMakePresetsBase.json b/builds/cmake/CMakePresetsBase.json index 15551a52..ab187f76 100644 --- a/builds/cmake/CMakePresetsBase.json +++ b/builds/cmake/CMakePresetsBase.json @@ -17,7 +17,8 @@ "displayName": "build Debug", "hidden": true, "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_CONFIGURATION_TYPES": "Debug" } }, { @@ -25,7 +26,8 @@ "displayName": "build RelWithDebInfo", "hidden": true, "cacheVariables": { - "CMAKE_BUILD_TYPE": "ReleaseWithDebInfo" + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "CMAKE_CONFIGURATION_TYPES": "RelWithDebInfo" } }, { @@ -33,7 +35,8 @@ "displayName": "build release", "hidden": true, "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_CONFIGURATION_TYPES": "Release" } }, { diff --git a/builds/cmake/liblcf-config.cmake.in b/builds/cmake/liblcf-config.cmake.in index b595bc1c..729f0173 100644 --- a/builds/cmake/liblcf-config.cmake.in +++ b/builds/cmake/liblcf-config.cmake.in @@ -7,7 +7,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") find_dependency(inih REQUIRED) -if(@LCF_SUPPORT_ICU@) +if(@LCF_SUPPORT_ICU@ EQUAL 1) find_dependency(ICU COMPONENTS i18n uc data REQUIRED) endif() diff --git a/src/encoder.cpp b/src/encoder.cpp index d968b5bd..c9000585 100644 --- a/src/encoder.cpp +++ b/src/encoder.cpp @@ -14,9 +14,14 @@ #include #include -#if LCF_SUPPORT_ICU +#if LCF_SUPPORT_ICU == 1 # include # include +#elif LCF_SUPPORT_ICU == 2 +# ifndef _WIN32 +# error "icu.h only supported on Windows" +# endif +# include #else # include #endif diff --git a/src/reader_util.cpp b/src/reader_util.cpp index 1bea1ebf..8067fc76 100644 --- a/src/reader_util.cpp +++ b/src/reader_util.cpp @@ -10,12 +10,16 @@ #include "lcf/config.h" #include "lcf/scope_guard.h" -#if LCF_SUPPORT_ICU +#if LCF_SUPPORT_ICU == 1 # include # include # include -# include -# include +# include +#elif LCF_SUPPORT_ICU == 2 +# ifndef _WIN32 +# error "icu.h only supported on Windows" +# endif +# include #endif #ifdef _WIN32 @@ -268,27 +272,74 @@ std::string ReaderUtil::Recode(StringView str_to_encode, StringView source_encod } std::string ReaderUtil::Normalize(StringView str) { + if (str.empty()) { + return {}; + } + #if LCF_SUPPORT_ICU - icu::UnicodeString uni = icu::UnicodeString(str.data(), str.length(), "utf-8").toLower(icu::Locale::getRoot()); UErrorCode err = U_ZERO_ERROR; - std::string res; - const icu::Normalizer2* norm = icu::Normalizer2::getNFKCInstance(err); + + auto log_warning = [err, &str](const char* func_name) { + Log::Error("%s failed while normalizing \"%s\": %s", func_name, std::string(str).c_str(), u_errorName(err)); + return std::string(str); + }; + + std::vector uni(str.length() + 1); // including \0 + int32_t uni_length; // length in utf-16 + u_strFromUTF8Lenient(uni.data(), uni.size(), &uni_length, str.data(), str.length(), &err); + if (U_FAILURE(err)) { + return log_warning("u_strFromUTF8Lenient"); + } + + uni_length = u_strToLower(uni.data(), uni.size(), uni.data(), uni_length, "", &err); + if (U_FAILURE(err)) { + return log_warning("u_strToLower"); + } + + std::vector res; + int res_capac = uni.size() * 4 + 1; // a codepoint in utf-8 is at most 4 bytes + res.resize(res_capac); + + const UNormalizer2* norm = unorm2_getNFKCInstance(&err); if (U_FAILURE(err)) { static bool err_reported = false; if (!err_reported) { - Log::Warning("Normalizer2::getNFKCInstance failed (%s). \"nrm\" is probably missing in the ICU data file. Unicode normalization will not work!", u_errorName(err)); + Log::Error("Normalizer2::getNFKCInstance failed (%s). \"nrm\" is probably missing in the ICU data file. Unicode normalization will not work!", u_errorName(err)); err_reported = true; } - uni.toUTF8String(res); - return res; + err = U_ZERO_ERROR; + + // error handling: return the lowercased string + u_strToUTF8(res.data(), res_capac, &uni_length, uni.data(), uni_length, &err); + if (U_FAILURE(err)) { + return log_warning("u_strToUTF8 (1)"); + } + + return std::string(res.data(), uni_length); } - icu::UnicodeString f = norm->normalize(uni, err); + + std::vector uni_norm(uni_length * 2 + 1); // * 2 for cases where the normalization is larger than the input + auto uni_norm_length = unorm2_normalize(norm, uni.data(), uni_length, uni_norm.data(), uni_norm.size(), &err); + if (U_FAILURE(err)) { - uni.toUTF8String(res); + log_warning("unorm2_normalize"); + + err = U_ZERO_ERROR; + + // error handling: return the lowercased string + u_strToUTF8(res.data(), res_capac, &uni_length, uni.data(), uni_length, &err); + if (U_FAILURE(err)) { + return log_warning("u_strToUTF8 (2)"); + } } else { - f.toUTF8String(res); + // success: return the lowercased and normalized string + u_strToUTF8(res.data(), res_capac, &uni_length, uni_norm.data(), uni_norm_length, &err); + if (U_FAILURE(err)) { + return log_warning("u_strToUTF8 (3)"); + } } - return res; + + return std::string(res.data(), uni_length); #else auto result = std::string(str); std::transform(result.begin(), result.end(), result.begin(), tolower);