utf : Full utf8,16,32 support in file input. Better char support in cmd.

CMakeLists.conan.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required (VERSION 3.10)
 project(conan-setup NONE)
 
-execute_process(COMMAND conan install ${CMAKE_CURRENT_SOURCE_DIR} --build missing -s build_type=Debug --settings compiler.runtime=MT)
+execute_process(COMMAND conan install ${CMAKE_CURRENT_SOURCE_DIR} --build missing -s build_type=Debug --settings compiler.runtime=MTd)
 execute_process(COMMAND conan install ${CMAKE_CURRENT_SOURCE_DIR} --build missing -s build_type=Release --settings compiler.runtime=MT)

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_policy(SET CMP0091 NEW)
 include(${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.conan.txt)
 cmake_minimum_required (VERSION 3.15)
-project(wsay VERSION 1.3.0 LANGUAGES CXX)
+project(wsay VERSION 1.3.1 LANGUAGES CXX)
 
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
@@ -46,8 +46,8 @@ function(set_compile_options REQUIRED_ARG)
 	elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
 	  # using Intel C++
 	elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
-		target_compile_definitions(${REQUIRED_ARG} PUBLIC NOMINMAX)
-		target_compile_options(${REQUIRED_ARG} PRIVATE /Zc:__cplusplus /Zc:alignedNew /permissive- /W4 /WX)
+		target_compile_definitions(${REQUIRED_ARG} PUBLIC NOMINMAX _UNICODE)
+		target_compile_options(${REQUIRED_ARG} PRIVATE /Zc:__cplusplus /Zc:alignedNew /permissive- /W4 /WX /utf-8)
 		set_target_properties(${REQUIRED_ARG} PROPERTIES MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 	endif()
 endfunction()
@@ -89,6 +89,7 @@ target_include_directories(${PROJECT_NAME} PRIVATE  ${LIB_INCLUDE_DIR})
 set_compile_options(${PROJECT_NAME})
 add_definitions(-DWSAY_VERSION="${PROJECT_VERSION}")
 target_link_libraries(${PROJECT_NAME} PRIVATE ${LIB_NAME})
+target_compile_definitions(${PROJECT_NAME} PRIVATE -DWIN32_LEAN_AND_MEAN -DWIN32_EXTRA_LEAN -DVC_EXTRALEAN)
 
 
 ##
@@ -143,4 +144,34 @@ ExternalProject_Add(fea_utils_ext
 set_target_properties(ns_getopt_ext wil_ext fea_utils_ext PROPERTIES FOLDER ${DEPENDENCY_FOLDER})
 add_dependencies(${PROJECT_NAME} ns_getopt_ext fea_utils_ext)
 add_dependencies(${GUITOOL_NAME} fea_utils_ext)
-add_dependencies(${LIB_NAME} wil_ext fea_utils_ext)
+add_dependencies(${LIB_NAME} wil_ext fea_utils_ext)
+
+
+# Tests
+option(BUILD_TESTING "Build and run tests." Off)
+if (${BUILD_TESTING})
+	enable_testing()
+
+	set(TEST_NAME ${PROJECT_NAME}_tests)
+	file(GLOB_RECURSE TEST_SOURCES "tests/*.cpp" "tests/*.c" "tests/*.hpp" "tests/*.h" "tests/*.tpp")
+	add_executable(${TEST_NAME} ${TEST_SOURCES})
+
+	find_package(GTest CONFIG REQUIRED)
+	target_include_directories(${TEST_NAME} PRIVATE  ${LIB_INCLUDE_DIR})
+	target_link_libraries(${TEST_NAME} PRIVATE ${LIB_NAME} GTest::GTest)
+
+	gtest_discover_tests(${TEST_NAME})
+	add_dependencies(${TEST_NAME} ${PROJECT_NAME})
+
+	set_compile_options(${TEST_NAME})
+
+	##
+	# Copy test data on build.
+	##
+	set(DATA_IN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tests/data)
+	set(DATA_OUT_DIR ${BINARY_OUT_DIR}/tests_data)
+	add_custom_command(TARGET ${TEST_NAME} POST_BUILD
+		COMMAND ${CMAKE_COMMAND} -E make_directory ${DATA_OUT_DIR}
+		COMMAND ${CMAKE_COMMAND} -E copy_directory ${DATA_IN_DIR} ${DATA_OUT_DIR}
+	)
+endif()

libsrc/wsay.cpp

@@ -114,19 +114,14 @@ bool parse_text_file(
 		return false;
 	}
 
-	std::vector<std::wstring> sentences;
-	if (!fea::wopen_text_file(path, sentences)) {
+	std::ifstream ifs{ path };
+	if (!ifs.is_open()) {
 		return false;
 	}
 
-	for (const std::wstring& str : sentences) {
-		if (str.empty()) {
-			continue;
-		}
+	std::u32string text = fea::read_with_bom(ifs);
 
-		out_text += str;
-		out_text += '\n';
-	}
+	out_text = fea::utf32_to_utf16_w(text);
 
 	if (out_text.empty()) {
 		fprintf(stderr,

src_cmd/main.cpp

@@ -1,13 +1,24 @@
-#include <filesystem>
+#include <fea_utils/fea_utils.hpp>
+#include <filesystem>
 #include <iostream>
 #include <ns_getopt/ns_getopt.h>
 #include <string>
+#include <windows.h>
 #include <wsay/wsay.hpp>
 
 const std::wstring exit_cmd = L"!exit";
 const std::wstring shutup_cmd = L"!stop";
 
+// cmd only : Character support will be ok-ish for now.
+// TODO : abstract char type in ns_getopt and use wmain.
+//		Once I do that, we can also set :
+//		_setmode( _fileno( stdin ), _O_U8TEXT );
+//		and everything should be gucci.
 int main(int argc, char** argv) {
+	// Doesn't really do much for us, but set it anyways to make sure user cmd
+	// is in utf8 mode.
+	SetConsoleCP(CP_UTF8);
+
 	wsy::voice voice;
 
 	bool interactive_mode = false;
@@ -18,8 +29,8 @@ int main(int argc, char** argv) {
 			// clang-format
 			{ "\"sentence\"", opt::type::raw_arg,
 					[&](std::string_view f) {
-						// TEMP HACK, TODO : Support wide strings in ns_getopt.
-						speech_text = { f.begin(), f.end() };
+						std::string t = { f.begin(), f.end() };
+						speech_text = fea::current_codepage_to_utf16_w(t);
 						return true;
 					},
 					"Sentence to say. You can use speech xml." },

tests/data/languages.txt

@@ -0,0 +1,20 @@
+nán
+
+<?xml version="1.0" encoding="UTF-8"?>
+<speak version="1.1" xmlns="http://www.w3.org/2001/10/synthesis"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="http://www.w3.org/2001/10/synthesis
+                 http://www.w3.org/TR/speech-synthesis11/synthesis.xsd"
+       xml:lang="sk-SK">
+	<s xml:lang="en-US">
+		<voice name="David">
+			This is in English: naan
+		</voice>
+	</s>
+	<s xml:lang="sk-SK">
+		<voice name="Filip">
+			Toto je po Slovensky: nán
+		</voice>
+	</s>
+
+</speak>

tests/main.cpp

@@ -0,0 +1,29 @@
+#include <fea_utils/fea_utils.hpp>
+#include <gtest/gtest.h>
+#include <wsay/wsay.hpp>
+
+namespace {
+std::filesystem::path exe_path;
+
+TEST(libwsay, unicode) {
+	std::filesystem::path testfiles_dir = exe_path / "tests_data/";
+	for (const std::filesystem::path& filepath :
+			std::filesystem::directory_iterator(testfiles_dir)) {
+
+		printf("%s\n", filepath.string().c_str());
+
+		std::wstring text;
+		wsy::parse_text_file(filepath, text);
+
+		// wprintf(L"%s\n", text.c_str());
+	}
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+	exe_path = fea::executable_dir(argv[0]);
+
+	::testing::InitGoogleTest(&argc, argv);
+	return RUN_ALL_TESTS();
+}