feat: fix various unicode text issues

CMakeLists.txt

@@ -162,6 +162,9 @@ search_lib_recursive_ext(DEPENDENCY_LUASYSTEM_LIBRARY luasystem)
 set(DEPENDENCY_MATHUTIL_INCLUDE ${CMAKE_CURRENT_LIST_DIR}/external_libs/mathutil/include CACHE PATH "Path to include directory.")
 search_lib_recursive_ext(DEPENDENCY_MATHUTIL_LIBRARY mathutil)
 
+set(DEPENDENCY_UTIL_UNICODE_INCLUDE ${CMAKE_CURRENT_LIST_DIR}/external_libs/util_unicode/include CACHE PATH "Path to include directory.")
+search_lib_recursive_ext(DEPENDENCY_UTIL_UNICODE_LIBRARY util_unicode)
+
 set(DEPENDENCY_SHAREDUTILS_INCLUDE ${CMAKE_CURRENT_LIST_DIR}/external_libs/sharedutils/include CACHE PATH "Path to include directory.")
 search_lib_recursive_ext(DEPENDENCY_SHAREDUTILS_LIBRARY sharedutils)
 search_lib_recursive(DEPENDENCY_SHAREDUTILS_LIBRARY_STATIC external_libs/sharedutils-static/${BINARY_PRAGMA_DIR} sharedutils-static STATIC)
@@ -1043,6 +1046,7 @@ add_dependencies(iglfw vfilesystem)
 add_dependencies(prosper vfilesystem iglfw util_image util_udm muparserx lz4)
 
 add_dependencies(cmaterialsystem materialsystem util_image prosper VTFLib13)
+add_dependencies(util_formatted_text util_unicode)
 add_dependencies(wgui cmaterialsystem util_formatted_text)
 
 add_dependencies(util_sound vfilesystem mpaheader ogg vorbis vorbisfile)
@@ -1056,7 +1060,7 @@ if(WIN32)
 add_dependencies(util_zip 7zpp)
 endif()
 
-add_dependencies(shared materialsystem util_zip wms_shared util_pragma_doc luasystem networkmanager muparserx libz2 bzip2 clip2tri-static clipper-static poly2tri-static util_udm panima util_bsp util_smdmodel util_source_script rectangle_bin_pack Recast Detour util_pad noise-static noiseutils-static ogg vorbis vorbisfile cppbezierfit spdlog fmt)
+add_dependencies(shared materialsystem util_zip wms_shared util_pragma_doc luasystem networkmanager muparserx libz2 bzip2 clip2tri-static clipper-static poly2tri-static util_udm panima util_bsp util_smdmodel util_source_script rectangle_bin_pack Recast Detour util_pad noise-static noiseutils-static ogg vorbis vorbisfile cppbezierfit spdlog fmt util_unicode)
 add_dependencies(ishared util_udm shared)
 add_dependencies(iclient ishared client)
 add_dependencies(iserver ishared server)
@@ -1143,6 +1147,7 @@ pragma_install(util_image "${BINARY_OUTPUT_DIR}")
 pragma_install(util_sound "${BINARY_OUTPUT_DIR}")
 pragma_install(vfilesystem "${BINARY_OUTPUT_DIR}")
 pragma_install(wgui "${BINARY_OUTPUT_DIR}")
+pragma_install(util_unicode "${BINARY_OUTPUT_DIR}")
 
 message("Custom install targets: ${PRAGMA_INSTALL_CUSTOM_TARGETS}")
 add_custom_target(pragma-install
@@ -1265,7 +1270,7 @@ pragma_install_lib("${DEPENDENCY_SPDLOG_LIBRARY}" "${BINARY_OUTPUT_DIR}")
 pragma_install_lib("${DEPENDENCY_MPAHEADER_LIBRARY}" "${BINARY_OUTPUT_DIR}")
 pragma_install_lib("${DEPENDENCY_VKV_PARSER_LIBRARY}" "${BINARY_OUTPUT_DIR}")
 if(WIN32)
-pragma_install_lib_c("${DEPENDENCY_ZLIB_LIBRARY}" zlib1 "${BINARY_OUTPUT_DIR}")
+	pragma_install_lib_c("${DEPENDENCY_ZLIB_LIBRARY}" zlib1 "${BINARY_OUTPUT_DIR}")
 else()
     pragma_install_lib("${DEPENDENCY_ZLIB_LIBRARY}" "${BINARY_OUTPUT_DIR}")
 endif()
@@ -1282,6 +1287,8 @@ function(pragma_install_file FILE_PATH INSTALL_PATH)
 	)
 endfunction(pragma_install_file)
 
+pragma_install_file("${DEPENDENCY_ICU_ICUUC_BINARY}" "${BINARY_OUTPUT_DIR}")
+pragma_install_file("${DEPENDENCY_ICU_ICUDT_BINARY}" "${BINARY_OUTPUT_DIR}")
 
 function(pragma_install_program FILE_PATH INSTALL_PATH)
     if("${FILE_PATH}" STREQUAL "")

build_scripts/build.py

@@ -348,6 +348,34 @@ def execscript(filepath):
 cp("zconf.h","../")
 os.chdir("../..")
 
+########## icu ##########
+# Download
+os.chdir(deps_dir)
+icu_root = os.getcwd() +"/icu"
+if not Path(icu_root).is_dir():
+	print_msg("icu not found. Downloading...")
+	mkpath(icu_root)
+	os.chdir(icu_root)
+	base_url = "https://github.com/unicode-org/icu/releases/download/release-75-1/"
+	if platform == "win32":
+		http_extract(base_url +"icu4c-75_1-Win64-MSVC2022.zip")
+	else:
+		http_extract(base_url +"icu4c-75_1-Ubuntu22.04-x64.tgz",format="tar.gz")
+if platform == "win32":
+	cmake_args += [
+		"-DDEPENDENCY_ICU_INCLUDE=" +icu_root +"/include/",
+		"-DDEPENDENCY_ICU_ICUUC_LIBRARY=" +icu_root +"/lib64/icuuc.lib",
+		"-DDEPENDENCY_ICU_ICUUC_BINARY=" +icu_root +"/bin64/icuuc75.dll",
+		"-DDEPENDENCY_ICU_ICUDT_BINARY=" +icu_root +"/bin64/icudt75.dll"
+	]
+else:
+	cmake_args += [
+		"-DDEPENDENCY_ICU_INCLUDE=" +icu_root +"/icu/usr/local/include/",
+		"-DDEPENDENCY_ICU_ICUUC_LIBRARY=" +icu_root +"/icu/usr/local/lib/libicuuc.so",
+		"-DDEPENDENCY_ICU_ICUUC_BINARY=" +icu_root +"/icu/usr/local/lib/libicuuc.so",
+		"-DDEPENDENCY_ICU_ICUDT_BINARY=" +icu_root +"/icu/usr/local/lib/libicudata.so"
+	]
+
 ########## boost ##########
 # Download
 os.chdir(deps_dir)

build_scripts/scripts/external_libs.py

@@ -17,9 +17,9 @@
 get_submodule("networkmanager","https://github.com/Silverlan/networkmanager.git","68d2f5c")
 get_submodule("panima","https://github.com/Silverlan/panima.git","30132a1e4d74bc6b5a6e52d9905ea881a4ae8bdf")
 get_submodule("prosper","https://github.com/Silverlan/prosper.git","40b52a18ea02ab611e602f84c172ecd3afd2af19")
-get_submodule("sharedutils","https://github.com/Silverlan/sharedutils.git","616eb852416df1c2e81d70c6f2fce1e513f2d1c3")
+get_submodule("sharedutils","https://github.com/Silverlan/sharedutils.git","98407d6c07515010571228c97fa5481e03d59fbf")
 get_submodule("util_bsp","https://github.com/Silverlan/util_bsp.git","3c11053")
-get_submodule("util_formatted_text","https://github.com/Silverlan/util_formatted_text.git","3bd9e2de37d0cb14bf8228fde5e25c97698e927b")
+get_submodule("util_formatted_text","https://github.com/Silverlan/util_formatted_text.git","5b441caacec34c3bdc89b6f529e07387f2fadb7b")
 get_submodule("util_image","https://github.com/Silverlan/util_image.git","7a66f93")
 get_submodule("util_pad","https://github.com/Silverlan/util_pad.git","7a827f8")
 get_submodule("util_pragma_doc","https://github.com/Silverlan/util_pragma_doc.git","6a4a089")
@@ -33,6 +33,7 @@
 get_submodule("util_vmf","https://github.com/Silverlan/util_vmf.git","cdba99d")
 get_submodule("util_zip","https://github.com/Silverlan/util_zip.git","166f108587962b59391a37e14dcfe1a51f81a81b")
 get_submodule("vfilesystem","https://github.com/Silverlan/vfilesystem.git","ba2db128eec7a23cb65c67b396d77f2d9a93e948")
-get_submodule("wgui","https://github.com/Silverlan/wgui.git","a42566e9b84e8e31d2c55c73b7178d37adeacda4")
+get_submodule("wgui","https://github.com/Silverlan/wgui.git","6b3556e2b47f5cf5671af19d52efc808dea101cf")
+get_submodule("util_unicode","https://github.com/Silverlan/util_unicode.git","0953ecb20f91e6d27c02cd51f6f970e55f6d4282")
 
 os.chdir(curDir)

core/client/CMakeLists.txt

@@ -35,6 +35,7 @@ link_external_library(util_udm)
 link_external_library(panima)
 link_external_library(spdlog)
 link_external_library(fmt)
+link_external_library(util_unicode)
 if(CONFIG_ENABLE_VTF_SUPPORT)
 	link_external_library(VTFLib)
 endif()

core/client/include/pragma/c_engine.h

@@ -14,7 +14,6 @@
 #include "pragma/rendering/c_render_context.hpp"
 #include "pragma/rendering/c_sci_gpu_timer_manager.hpp"
 #include "pragma/input/c_keybind.h"
-#include "pragma/util/font_set.hpp"
 #include <sharedutils/util_clock.hpp>
 #include <unordered_map>
 
@@ -34,6 +33,7 @@ namespace prosper {
 };
 struct InputBindingLayer;
 struct CoreInputBindingLayer;
+struct FontSet;
 #pragma warning(push)
 #pragma warning(disable : 4251)
 class DLLCLIENT CEngine : public Engine, public pragma::RenderContext {
@@ -275,7 +275,7 @@ class DLLCLIENT CEngine : public Engine, public pragma::RenderContext {
 	std::vector<CallbackHandle> m_gpuProfileHandlers = {};
 
 	std::string m_defaultFontSet;
-	std::unordered_map<std::string, FontSet> m_fontSets;
+	std::unordered_map<std::string, std::unique_ptr<FontSet>> m_fontSets;
 	float m_rawInputJoystickMagnitude = 0.f;
 	std::unordered_map<GLFW::Key, GLFW::KeyState> m_joystickKeyStates;
 

core/client/include/pragma/gui/wiconsole.hpp

@@ -9,7 +9,6 @@
 #define __WICONSOLE_HPP__
 
 #include "pragma/clientdefinitions.h"
-#include <sharedutils/util_utf8.hpp>
 #include <wgui/wibase.h>
 #include <queue>
 #include <string_view>
@@ -18,6 +17,9 @@ class WITextEntry;
 class WICommandLineEntry;
 class WIFrame;
 class WISnapArea;
+namespace util {
+	class Utf8String;
+};
 class DLLCLIENT WIConsole : public WIBase {
   public:
 	static WIConsole *Open();

core/client/include/pragma/gui/wiframe.h

@@ -10,7 +10,6 @@
 
 #include "pragma/clientdefinitions.h"
 #include "pragma/gui/witransformable.h"
-#include <sharedutils/util_utf8.hpp>
 
 class DLLCLIENT WIFrame : public WITransformable {
   public:
@@ -21,7 +20,7 @@ class DLLCLIENT WIFrame : public WITransformable {
 	WIBase *GetContents();
 	using WITransformable::SetSize;
 	void SetTitle(std::string title);
-	util::Utf8String GetTitle() const;
+	const util::Utf8String &GetTitle() const;
 	void SetCloseButtonEnabled(bool b);
 	void SetDetachButtonEnabled(bool b);
 	virtual util::EventReply MouseCallback(GLFW::MouseButton button, GLFW::KeyState state, GLFW::Modifier mods) override;

core/client/include/pragma/lua/libraries/c_lengine.h

@@ -10,6 +10,7 @@
 
 #include "pragma/clientdefinitions.h"
 #include <pragma/lua/ldefinitions.h>
+#include <pragma/util/font_set.hpp>
 
 class FontInfo;
 class LFile;

core/client/src/c_engine.cpp

@@ -65,6 +65,7 @@
 #include <pragma/asset/util_asset.hpp>
 #include <prosper_window.hpp>
 #include <fsys/ifile.hpp>
+#include <pragma/util/font_set.hpp>
 #ifdef _WIN32
 
 #include <dwmapi.h>
@@ -873,7 +874,7 @@ const FontSet &CEngine::GetDefaultFontSet() const
 const FontSet *CEngine::FindFontSet(const std::string &name) const
 {
 	auto it = m_fontSets.find(name);
-	return (it != m_fontSets.end()) ? &it->second : nullptr;
+	return (it != m_fontSets.end()) ? it->second.get() : nullptr;
 }
 void CEngine::LoadFontSets()
 {
@@ -911,7 +912,7 @@ void CEngine::LoadFontSets()
 							fontSet.fileData.push_back(fileData);
 						}
 					}
-					m_fontSets[dir] = std::move(fontSet);
+					m_fontSets[dir] = std::make_unique<FontSet>(std::move(fontSet));
 				}
 			}
 		}

core/client/src/debug/c_debug_gui_cursor_element.cpp

@@ -14,6 +14,7 @@
 #include <wgui/types/wiroot.h>
 #include <prosper_window.hpp>
 #include <fsys/ifile.hpp>
+#include <util_unicode.hpp>
 
 extern DLLCLIENT CEngine *c_engine;
 extern DLLCLIENT ClientState *client;

core/client/src/debug/c_debugoverlay.cpp

@@ -20,6 +20,7 @@
 #include <prosper_command_buffer.hpp>
 #include <prosper_descriptor_set_group.hpp>
 #include <pragma/math/intersection.h>
+#include <util_unicode.hpp>
 
 extern DLLCLIENT CEngine *c_engine;
 extern DLLCLIENT ClientState *client;

core/client/src/gui/mainmenu/wimainmenu_base.cpp

@@ -305,8 +305,8 @@ void WIMainMenuElement::OnCursorExited() { Deselect(); }
 void WIMainMenuElement::SetText(std::string &text)
 {
 	if(m_hText.IsValid()) {
-		auto upperText = text;
-		util::utf8_string_to_upper(upperText);
+		util::Utf8String upperText {text};
+		upperText.toUpper();
 		WIText *pText = static_cast<WIText *>(m_hText.get());
 		pText->SetText(upperText);
 		pText->SizeToContents();

core/client/src/gui/wicommandlineentry.cpp

@@ -11,6 +11,7 @@
 #include <wgui/types/wicontextmenu.hpp>
 #include <wgui/types/witext.h>
 #include <mathutil/umath_geometry.hpp>
+#include <util_unicode.hpp>
 
 LINK_WGUI_TO_CLASS(WICommandLineEntry, WICommandLineEntry);
 

core/client/src/gui/wiconsole.cpp

@@ -19,6 +19,7 @@
 #include <pragma/engine_info.hpp>
 #include <pragma/lua/lua_error_handling.hpp>
 #include <pragma/localization.h>
+#include <util_unicode.hpp>
 
 extern DLLCLIENT CEngine *c_engine;
 extern DLLCLIENT ClientState *client;

core/client/src/gui/wifps.cpp

@@ -8,6 +8,7 @@
 #include "stdafx_client.h"
 #include "pragma/gui/wifps.h"
 #include <wgui/types/witext.h>
+#include <util_unicode.hpp>
 
 extern DLLCLIENT CEngine *c_engine;
 extern DLLCLIENT ClientState *client;

core/client/src/gui/wiframe.cpp

@@ -13,6 +13,7 @@
 #include <wgui/types/wirect.h>
 #include <mathutil/umath.h>
 #include <prosper_window.hpp>
+#include <util_unicode.hpp>
 
 LINK_WGUI_TO_CLASS(WIFrame, WIFrame);
 
@@ -182,9 +183,11 @@ void WIFrame::SetTitle(std::string title)
 	pText->SizeToContents();
 	pText->SetVisible(!title.empty());
 }
-util::Utf8String WIFrame::GetTitle() const
+const util::Utf8String &WIFrame::GetTitle() const
 {
-	if(!m_hTitle.IsValid())
-		return "";
+	if(!m_hTitle.IsValid()) {
+		static util::Utf8String emptyString {};
+		return emptyString;
+	}
 	return static_cast<const WIText *>(m_hTitle.get())->GetText();
 }

core/client/src/gui/winetgraph.cpp

@@ -9,6 +9,7 @@
 #include "pragma/gui/winetgraph.h"
 #include <wgui/types/witext.h>
 #include <pragma/networking/netmessages.h>
+#include <util_unicode.hpp>
 
 extern DLLCLIENT CEngine *c_engine;
 extern DLLCLIENT ClientState *client;

core/client/src/gui/wiprogressbar.cpp

@@ -12,6 +12,7 @@
 #include <sharedutils/util.h>
 #include <mathutil/umath.h>
 #include <sharedutils/property/util_property_color.hpp>
+#include <util_unicode.hpp>
 
 LINK_WGUI_TO_CLASS(WIProgressBar, WIProgressBar);
 

core/client/src/gui/witable.cpp

@@ -14,6 +14,7 @@
 #include <wgui/types/wiscrollbar.h>
 #include "pragma/gui/wiscrollcontainer.h"
 #include <prosper_window.hpp>
+#include <util_unicode.hpp>
 
 LINK_WGUI_TO_CLASS(WITable, WITable);
 LINK_WGUI_TO_CLASS(WITableRow, WITableRow);

core/client/src/lua/c_library.cpp

@@ -74,6 +74,7 @@
 #include <fsys/ifile.hpp>
 #include <wgui/types/witooltip.h>
 #include <wgui/types/wiroot.h>
+#include <util_unicode.hpp>
 
 extern DLLCLIENT CGame *c_game;
 extern DLLCLIENT ClientState *client;

core/client/src/lua/classes/c_lwibase.cpp

@@ -50,6 +50,7 @@
 #include <prosper_window.hpp>
 #include <luabind/copy_policy.hpp>
 #include <pragma/debug/intel_vtune.hpp>
+#include <util_unicode.hpp>
 
 extern DLLCLIENT CEngine *c_engine;
 extern DLLCLIENT CGame *c_game;

core/client/src/lua/libraries/c_lengine.cpp

@@ -50,7 +50,7 @@ std::shared_ptr<const FontInfo> Lua::engine::create_font(lua_State *l, const std
 		size += *fontFileData->fontSizeAdjustment;
 	FontInfo::FontSettings settings {};
 	settings.fontSize = size;
-	settings.requiredChars = fontSet->requiredChars;
+	settings.requiredChars = fontSet->requiredChars ? std::make_unique<util::Utf8String>(*fontSet->requiredChars) : std::unique_ptr<util::Utf8String> {};
 	return FontManager::LoadFont(identifier.c_str(), fontFileData->fileName, settings, reload);
 }
 std::shared_ptr<const FontInfo> Lua::engine::create_font(lua_State *l, const std::string &identifier, const std::string &fontSetName, FontSetFlag features, uint32_t size) { return create_font(l, identifier, fontSetName, features, size, false); }
@@ -90,15 +90,15 @@ Vector2i Lua::engine::get_text_size(lua_State *l, const std::string &text, const
 		return {0, 0};
 	int w = 0;
 	int h = 0;
-	FontManager::GetTextSize(text.c_str(), 0u, info.get(), &w, &h);
+	FontManager::GetTextSize(text, 0u, info.get(), &w, &h);
 	return Vector2i {w, h};
 }
 
 Vector2i Lua::engine::get_text_size(lua_State *l, const std::string &text, const FontInfo &font)
 {
 	int w = 0;
 	int h = 0;
-	FontManager::GetTextSize(text.c_str(), 0u, &font, &w, &h);
+	FontManager::GetTextSize(text, 0u, &font, &w, &h);
 	return Vector2i {w, h};
 }
 

core/client/src/lua/libraries/c_lgui.cpp

@@ -22,6 +22,7 @@
 #include <pragma/lua/classes/ldef_vector.h>
 #include <prosper_window.hpp>
 #include <prosper_render_pass.hpp>
+#include <util_unicode.hpp>
 
 extern DLLCLIENT CEngine *c_engine;
 extern DLLCLIENT ClientState *client;

core/server/CMakeLists.txt

@@ -11,6 +11,7 @@ link_external_library(recast)
 link_external_library(detour)
 link_external_library(spdlog)
 link_external_library(fmt)
+link_external_library(util_unicode)
 if(WIN32)
 	add_external_library(lua)
 endif()

core/shared/CMakeLists.txt

@@ -26,6 +26,7 @@ link_external_library(panima)
 link_external_library(cppbezierfit)
 link_external_library(spdlog)
 link_external_library(fmt)
+link_external_library(util_unicode)
 
 add_external_library(luajit)
 add_external_library(luabind)

core/shared/include/pragma/localization.h

@@ -13,7 +13,7 @@
 #include <vector>
 #include <optional>
 #include <unordered_map>
-#include <sharedutils/util_utf8.hpp>
+#include <util_unicode.hpp>
 
 class Locale;
 struct DLLNETWORK Localization {