FSUI: Remove font glyph ranges and add exclude ranges

The original glyph ranges are legacy and aren't used

.github/workflows/scripts/common/update_base_translation.sh

@@ -9,6 +9,5 @@ set -e
 "$SCRIPTDIR/../../../../tools/retry.sh" sudo apt-get -y install qt6-l10n-tools python3
 
 "$SCRIPTDIR/../../../../tools/generate_fullscreen_ui_translation_strings.py"
-"$SCRIPTDIR/../../../../pcsx2-qt/Translations/update_glyph_ranges.py"
 "$SCRIPTDIR/../../../../tools/generate_update_fa_glyph_ranges.py"
 PATH=/usr/lib/qt6/bin:$PATH "$SCRIPTDIR/../../../../pcsx2-qt/Translations/update_base_translation.sh"

pcsx2-gsrunner/Main.cpp

@@ -103,7 +103,7 @@ bool GSRunner::InitializeConfig()
 	if (!VMManager::PerformEarlyHardwareChecks(&error))
 		return false;
 
-	ImGuiManager::SetFontPathAndRange(Path::Combine(EmuFolders::Resources, "fonts" FS_OSPATH_SEPARATOR_STR "Roboto-Regular.ttf"), {});
+	ImGuiManager::SetFontPath(Path::Combine(EmuFolders::Resources, "fonts" FS_OSPATH_SEPARATOR_STR "Roboto-Regular.ttf"));
 
 	// don't provide an ini path, or bother loading. we'll store everything in memory.
 	MemorySettingsInterface& si = s_settings_interface;

pcsx2-qt/Translations/update_glyph_ranges.py

@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import re
-import xml.etree.ElementTree as ET
-
-languages_to_update = [
-    "ja-JP",
-    "ko-KR",
-    "zh-CN",
-    "zh-TW"
-]
-
-src_path = os.path.join(os.path.dirname(__file__), "..", "Translations.cpp")
-ts_dir = os.path.join(os.path.dirname(__file__))
-
-def parse_xml(path):
-    tree = ET.parse(path)
-    root = tree.getroot()
-    translations = ""
-    for node in root.findall("context/message/translation"):
-        if node.text:
-            translations += node.text
-
-    ords = list(set([ord(ch) for ch in translations if ord(ch) >= 0x2000]))
-    if len(ords) == 0:
-        return ""
-
-    # Try to organize it into ranges
-    ords.sort()
-    ord_pairs = []
-    start_ord = None
-    last_ord = None
-    for nord in ords:
-        if start_ord is not None and nord == (last_ord + 1):
-            last_ord = nord
-            continue
-        if start_ord is not None:
-            ord_pairs.append(start_ord)
-            ord_pairs.append(last_ord)
-        start_ord = nord
-        last_ord = nord
-
-    if start_ord is not None:
-        ord_pairs.append(start_ord)
-        ord_pairs.append(last_ord)
-
-    chars = "".join([chr(ch) for ch in ord_pairs])
-    return chars
-
-def update_src_file(ts_file, chars):
-    ts_name = os.path.basename(ts_file)
-    pattern = re.compile('(// auto update.*' + ts_name + '.*\n[^"]+")[^"]*(".*)')
-    with open(src_path, "r", encoding="utf-8") as f:
-        original = f.read()
-        update = pattern.sub("\\1" + chars + "\\2", original)
-    if original != update:
-        with open(src_path, "w", encoding="utf-8") as f:
-            f.write(update)
-        print(f"Updated character list for {ts_file}.")
-    else:
-        print(f"Character list is unchanged for {ts_file}.")
-
-if __name__ == "__main__":
-    for language in languages_to_update:
-        ts_file = os.path.join(ts_dir, f"pcsx2-qt_{language}.ts")
-        chars = parse_xml(ts_file)
-        print(f"{language}: {len(chars)} character(s) detected.")
-        update_src_file(ts_file, chars)

pcsx2/ImGui/ImGuiManager.cpp

@@ -68,7 +68,6 @@ namespace ImGuiManager
 static float s_global_scale = 1.0f;
 
 static std::string s_font_path;
-static std::vector<ImWchar> s_font_range;
 
 static ImFont* s_standard_font;
 static ImFont* s_fixed_font;
@@ -101,13 +100,12 @@ static bool s_scale_changed = false;
 
 static std::array<ImGuiManager::SoftwareCursor, InputManager::MAX_SOFTWARE_CURSORS> s_software_cursors = {};
 
-void ImGuiManager::SetFontPathAndRange(std::string path, std::vector<u16> range)
+void ImGuiManager::SetFontPath(std::string path)
 {
-	if (s_font_path == path && s_font_range == range)
+	if (s_font_path == path)
 		return;
 
 	s_font_path = std::move(path);
-	s_font_range = std::move(range);
 	s_standard_font_data = {};
 
 	if (ImGui::GetCurrentContext())
@@ -449,31 +447,16 @@ void ImGuiManager::UnloadFontData()
 
 ImFont* ImGuiManager::AddTextFont()
 {
-	static const ImWchar default_ranges[] = {
+	// Exclude FA and PF ranges
-		// Basic Latin + Latin Supplement + Central European diacritics
+	// clang-format off
-		0x0020,
+	static constexpr ImWchar range_exclude_icons[] = { 0x2198,0x2199,0x219e,0x21a7,0x21b0,0x21b3,0x21ba,0x21c3,0x21ce,0x21d4,0x21dc,0x21dd,0x21e0,0x21e3,0x21e6,0x21e8,0x21f3,0x21f3,0x21f7,0x21fb,0x2206,0x2208,0x221a,0x221a,0x227a,0x227d,0x22bf,0x22c8,0x2349,0x2349,0x235a,0x2361,0x2364,0x2367,0x237a,0x237f,0x23b2,0x23b5,0x23cc,0x23cc,0x23f4,0x23f7,0x2427,0x243a,0x243d,0x243d,0x2443,0x2443,0x2460,0x246b,0x248f,0x248f,0x24f5,0x24ff,0x2605,0x2605,0x2699,0x2699,0x278a,0x278e,0xff21,0xff3a,0x0,0x0 };
-		0x017F,
+	// clang-format on
-
-		// Cyrillic + Cyrillic Supplement
-		0x0400,
-		0x052F,
-
-		// Cyrillic Extended-A
-		0x2DE0,
-		0x2DFF,
-
-		// Cyrillic Extended-B
-		0xA640,
-		0xA69F,
-
-		0,
-	};
 
 	ImFontConfig cfg;
 	cfg.FontDataOwnedByAtlas = false;
+	cfg.GlyphExcludeRanges = range_exclude_icons;
 	return ImGui::GetIO().Fonts->AddFontFromMemoryTTF(
-		s_standard_font_data.data(), static_cast<int>(s_standard_font_data.size()), FONT_BASE_SIZE, &cfg,
+		s_standard_font_data.data(), static_cast<int>(s_standard_font_data.size()), FONT_BASE_SIZE, &cfg, nullptr);
-		s_font_range.empty() ? default_ranges : s_font_range.data());
 }
 
 ImFont* ImGuiManager::AddFixedFont()
@@ -486,11 +469,7 @@ ImFont* ImGuiManager::AddFixedFont()
 
 bool ImGuiManager::AddIconFonts()
 {
-	// clang-format off
+	// Load FontAwesome after to avoid aliased codepoints overriding promptfont
-	static constexpr ImWchar range_fa[] = { 0xe06f,0xe06f,0xe097,0xe097,0xe2ca,0xe2ca,0xe494,0xe494,0xe4bb,0xe4bb,0xe4cf,0xe4cf,0xe51f,0xe51f,0xf001,0xf002,0xf005,0xf005,0xf007,0xf007,0xf009,0xf00a,0xf00c,0xf00d,0xf011,0xf011,0xf013,0xf013,0xf017,0xf017,0xf019,0xf019,0xf01e,0xf01e,0xf022,0xf023,0xf025,0xf028,0xf02b,0xf02b,0xf02e,0xf02e,0xf030,0xf030,0xf037,0xf037,0xf03a,0xf03a,0xf03d,0xf03e,0xf043,0xf043,0xf047,0xf047,0xf04b,0xf04c,0xf04e,0xf04e,0xf050,0xf050,0xf052,0xf052,0xf05a,0xf05a,0xf05e,0xf05e,0xf063,0xf063,0xf066,0xf066,0xf06a,0xf06a,0xf06e,0xf06e,0xf071,0xf071,0xf075,0xf075,0xf077,0xf078,0xf07b,0xf07c,0xf080,0xf080,0xf084,0xf084,0xf08e,0xf08e,0xf091,0xf091,0xf0a0,0xf0a0,0xf0ac,0xf0ad,0xf0b0,0xf0b0,0xf0c0,0xf0c0,0xf0c5,0xf0c5,0xf0c7,0xf0c8,0xf0cb,0xf0cb,0xf0dc,0xf0dc,0xf0e2,0xf0e2,0xf0eb,0xf0ec,0xf0f3,0xf0f3,0xf0fe,0xf0fe,0xf110,0xf110,0xf11b,0xf11c,0xf120,0xf121,0xf125,0xf125,0xf129,0xf129,0xf140,0xf140,0xf14a,0xf14a,0xf14c,0xf14c,0xf15b,0xf15b,0xf15d,0xf15d,0xf185,0xf185,0xf187,0xf188,0xf191,0xf192,0xf1b3,0xf1b3,0xf1c0,0xf1c0,0xf1da,0xf1da,0xf1de,0xf1de,0xf1e6,0xf1e6,0xf1ea,0xf1eb,0xf1f8,0xf1f8,0xf1fb,0xf1fc,0xf21e,0xf21e,0xf245,0xf245,0xf26c,0xf26c,0xf279,0xf279,0xf2a8,0xf2a8,0xf2bd,0xf2bd,0xf2d3,0xf2d3,0xf2db,0xf2db,0xf2f2,0xf2f2,0xf302,0xf303,0xf31e,0xf31e,0xf360,0xf360,0xf3a5,0xf3a5,0xf3c1,0xf3c1,0xf462,0xf462,0xf466,0xf466,0xf49e,0xf49e,0xf4e2,0xf4e2,0xf51f,0xf51f,0xf530,0xf530,0xf54c,0xf54c,0xf552,0xf553,0xf5a2,0xf5a2,0xf5a5,0xf5a5,0xf5bc,0xf5bc,0xf5c7,0xf5c7,0xf624,0xf625,0xf62a,0xf62a,0xf65d,0xf65e,0xf6a9,0xf6a9,0xf70e,0xf70e,0xf756,0xf756,0xf780,0xf780,0xf794,0xf794,0xf7d8,0xf7d8,0xf815,0xf815,0xf84c,0xf84c,0xf87c,0xf87c,0xf8cc,0xf8cc,0x0,0x0 };
-	static constexpr ImWchar range_pf[] = { 0x2198,0x2199,0x219e,0x21a7,0x21b0,0x21b3,0x21ba,0x21c3,0x21ce,0x21ce,0x21d0,0x21d4,0x21dc,0x21dd,0x21e0,0x21e3,0x21e6,0x21e8,0x21f3,0x21f3,0x21f7,0x21f8,0x21fa,0x21fb,0x2206,0x2208,0x221a,0x221a,0x227a,0x227d,0x22bf,0x22c8,0x2349,0x2349,0x235a,0x235e,0x2360,0x2361,0x2364,0x2367,0x237a,0x237d,0x237f,0x237f,0x23b2,0x23b5,0x23cc,0x23cc,0x23f4,0x23f7,0x2427,0x243a,0x243d,0x243d,0x2443,0x2443,0x2460,0x246b,0x248f,0x248f,0x24f5,0x24fd,0x24ff,0x24ff,0x2605,0x2605,0x2699,0x2699,0x278a,0x278e,0xe000,0xe001,0xff21,0xff3a,0x0,0x0 };
-	// clang-format on
-
 	{
 		ImFontConfig cfg;
 		cfg.MergeMode = true;
@@ -500,22 +479,26 @@ bool ImGuiManager::AddIconFonts()
 		cfg.FontDataOwnedByAtlas = false;
 
 		if (!ImGui::GetIO().Fonts->AddFontFromMemoryTTF(
-				s_icon_fa_font_data.data(), static_cast<int>(s_icon_fa_font_data.size()), FONT_BASE_SIZE * 0.75f, &cfg, range_fa))
+				s_icon_pf_font_data.data(), static_cast<int>(s_icon_pf_font_data.size()), FONT_BASE_SIZE * 1.2f, &cfg, nullptr))
 		{
 			return false;
 		}
 	}
 
 	{
+		// Exclude any characters outside the BMP PUA plane
+		static constexpr ImWchar range_exclude_non_bmp[] = {0x1, 0xdfff, 0xf900, 0xffff, 0x0, 0x0};
+
 		ImFontConfig cfg;
 		cfg.MergeMode = true;
 		cfg.PixelSnapH = true;
 		cfg.GlyphMinAdvanceX = FONT_BASE_SIZE;
 		cfg.GlyphMaxAdvanceX = FONT_BASE_SIZE;
+		cfg.GlyphExcludeRanges = range_exclude_non_bmp;
 		cfg.FontDataOwnedByAtlas = false;
 
 		if (!ImGui::GetIO().Fonts->AddFontFromMemoryTTF(
-				s_icon_pf_font_data.data(), static_cast<int>(s_icon_pf_font_data.size()), FONT_BASE_SIZE * 1.2f, &cfg, range_pf))
+				s_icon_fa_font_data.data(), static_cast<int>(s_icon_fa_font_data.size()), FONT_BASE_SIZE * 0.75f, &cfg, nullptr))
 		{
 			return false;
 		}

pcsx2/ImGui/ImGuiManager.h

@@ -17,7 +17,7 @@ enum class InputLayout : u8;
 namespace ImGuiManager
 {
 	/// Sets the path to the font to use. Empty string means to use the default.
-	void SetFontPathAndRange(std::string path, std::vector<u16> range);
+	void SetFontPath(std::string path);
 
 	/// Initializes ImGui, creates fonts, etc.
 	bool Initialize();

tools/generate_update_fa_glyph_ranges.py

@@ -32,7 +32,9 @@ all_source_files = list(functools.reduce(lambda prev, src_dir: prev + glob.glob(
     glob.glob(os.path.join(src_dir, "**", "*.h"), recursive=True) + \
     glob.glob(os.path.join(src_dir, "**", "*.inl"), recursive=True), src_dirs, []))
 
-tokens = set()
+# All FA tokens are within a Unicode private area
+# PF, however, needs to replace predefined Unicode characters
+fa_tokens = set()
 pf_tokens = set()
 for filename in all_source_files:
     data = None
@@ -42,11 +44,11 @@ for filename in all_source_files:
         except:
             continue
 
-    tokens = tokens.union(set(re.findall("(ICON_FA_[a-zA-Z0-9_]+)", data)))
+    fa_tokens = fa_tokens.union(set(re.findall("(ICON_FA_[a-zA-Z0-9_]+)", data)))
     pf_tokens = pf_tokens.union(set(re.findall("(ICON_PF_[a-zA-Z0-9_]+)", data)))
 
-print("{}/{} tokens found.".format(len(tokens), len(pf_tokens)))
+print("{}/{} tokens found.".format(len(fa_tokens), len(pf_tokens)))
-if len(tokens) == 0 and len(pf_tokens) == 0:
+if len(pf_tokens) == 0:
     sys.exit(0)
 
 def decode_encoding(value):
@@ -58,42 +60,63 @@ def decode_encoding(value):
 
     return bytes(value, 'utf-8')
 
-u8_encodings = {}
+u8_encodings_fa = {}
 with open(fa_file, "r") as f:
     for line in f.readlines():
         match = re.match("#define (ICON_FA_[^ ]+) \"([^\"]+)\"", line)
         if match is None:
             continue
-        u8_encodings[match[1]] = decode_encoding(match[2])
+        u8_encodings_fa[match[1]] = decode_encoding(match[2])
+u8_encodings_pf = {}
 with open(pf_file, "r") as f:
     for line in f.readlines():
         match = re.match("#define (ICON_PF_[^ ]+) \"([^\"]+)\"", line)
         if match is None:
             continue
-        u8_encodings[match[1]] = decode_encoding(match[2])
+        u8_encodings_pf[match[1]] = decode_encoding(match[2])
 
-out_pattern = "(static constexpr ImWchar range_fa\[\] = \{)[0-9A-Z_a-z, \n]+(\};)"
+# PF also uses the Unicode private area, check for conflicts with FA
-out_pf_pattern = "(static constexpr ImWchar range_pf\[\] = \{)[0-9A-Z_a-z, \n]+(\};)"
+cf_tokens_all = {}
+for pf_token in u8_encodings_pf.keys():
+    for fa_token in u8_encodings_fa.keys():
+        if u8_encodings_pf[pf_token] == u8_encodings_fa[fa_token]:
+            cf_tokens_all[pf_token] = fa_token
 
-def get_pairs(tokens):
+cf_tokens_used = []
+for token in pf_tokens:
+    if token in cf_tokens_all:
+        cf_tokens_used.append(token)
+
+print("{} font conflicts found, of which we use {} of them.".format(len(cf_tokens_all), len(cf_tokens_used)))
+if len(cf_tokens_used) > 0:
+    raise NotImplementedError("A used PF token conflicts with a FA token, generating exclude ranges is not implemented")
+
+out_ex_pattern = r"(static constexpr ImWchar range_exclude_icons\[\] = \{)[0-9A-Z_a-z, \n]+(\};)"
+
+def get_pairs(tokens, limit_pairs=-1):
     codepoints = list()
     for token in tokens:
-        u8_bytes = u8_encodings[token]
+        u8_bytes = u8_encodings_pf[token]
         u8 = str(u8_bytes, "utf-8")
         u16 = u8.encode("utf-16le")
         if len(u16) > 2:
             raise ValueError("{} {} too long".format(u8_bytes, token))
 
         codepoint = int.from_bytes(u16, byteorder="little", signed=False)
+        if codepoint >= 0xe000 and codepoint <= 0xf8ff:
+            continue
         codepoints.append(codepoint)
     codepoints.sort()
     codepoints.append(0) # null terminator
 
+    merge_range = 0
+    while True:
+        merge_range = merge_range + 1
         startc = codepoints[0]
         endc = None
         pairs = [startc]
         for codepoint in codepoints:
-        if endc is not None and (endc + 1) != codepoint:
+            if endc is not None and (((endc + merge_range) < codepoint) or (codepoint == 0)):
                 pairs.append(endc)
                 pairs.append(codepoint)
                 startc = codepoint
@@ -102,13 +125,18 @@ def get_pairs(tokens):
                 endc = codepoint
         pairs.append(endc)
 
+        if limit_pairs == -1 or len(pairs) <= (limit_pairs << 1):
+            break
+
+    print("Created {} pairs with a merge range of {}".format(len(pairs) >> 1, merge_range))
     pairs_str = ",".join(list(map("0x{:x}".format, pairs)))
     return pairs_str
 
 with open(dst_file, "r") as f:
     original = f.read()
-    updated = re.sub(out_pattern, "\\1 " + get_pairs(tokens) + " \\2", original)
+    # ImGui asserts if more than 32 ranges are provided for exclusion
-    updated = re.sub(out_pf_pattern, "\\1 " + get_pairs(pf_tokens) + " \\2", updated)
+    # we should also use as few as reasonable for performance reasons
+    updated = re.sub(out_ex_pattern, "\\1 " + get_pairs(pf_tokens, 32) + " \\2", original)
     if original != updated:
         with open(dst_file, "w") as f:
             f.write(updated)