Merge pull request #80650 from bruvzg/comp_char_fix

[TextServer] Fix system font fallback and caret/selection behavior for composite characters.
godotengine · Aug 18, 2023 · b51ee8b · b51ee8b
2 parents a2a1ed1 + 5d3fcc5
commit b51ee8b
Show file tree

Hide file tree

Showing 19 changed files with 398 additions and 54 deletions.
diff --git a/doc/classes/LineEdit.xml b/doc/classes/LineEdit.xml
@@ -197,7 +197,7 @@
 		<member name="caret_force_displayed" type="bool" setter="set_caret_force_displayed" getter="is_caret_force_displayed" default="false">
 			If [code]true[/code], the [LineEdit] will always show the caret, even if focus is lost.
 		</member>
-		<member name="caret_mid_grapheme" type="bool" setter="set_caret_mid_grapheme_enabled" getter="is_caret_mid_grapheme_enabled" default="true">
+		<member name="caret_mid_grapheme" type="bool" setter="set_caret_mid_grapheme_enabled" getter="is_caret_mid_grapheme_enabled" default="false">
 			Allow moving caret, selecting and removing the individual composite character components.
 			[b]Note:[/b] [kbd]Backspace[/kbd] is always removing individual composite character components.
 		</member>

diff --git a/doc/classes/TextEdit.xml b/doc/classes/TextEdit.xml
@@ -1100,7 +1100,7 @@
 		<member name="caret_draw_when_editable_disabled" type="bool" setter="set_draw_caret_when_editable_disabled" getter="is_drawing_caret_when_editable_disabled" default="false">
 			If [code]true[/code], caret will be visible when [member editable] is disabled.
 		</member>
-		<member name="caret_mid_grapheme" type="bool" setter="set_caret_mid_grapheme_enabled" getter="is_caret_mid_grapheme_enabled" default="true">
+		<member name="caret_mid_grapheme" type="bool" setter="set_caret_mid_grapheme_enabled" getter="is_caret_mid_grapheme_enabled" default="false">
 			Allow moving caret, selecting and removing the individual composite character components.
 			[b]Note:[/b] [kbd]Backspace[/kbd] is always removing individual composite character components.
 		</member>

diff --git a/doc/classes/TextServer.xml b/doc/classes/TextServer.xml
@@ -1139,6 +1139,14 @@
 				Clears text buffer (removes text and inline objects).
 			</description>
 		</method>
+		<method name="shaped_text_closest_character_pos" qualifiers="const">
+			<return type="int" />
+			<param index="0" name="shaped" type="RID" />
+			<param index="1" name="pos" type="int" />
+			<description>
+				Returns composite character position closest to the [param pos].
+			</description>
+		</method>
 		<method name="shaped_text_draw" qualifiers="const">
 			<return type="void" />
 			<param index="0" name="shaped" type="RID" />
@@ -1189,6 +1197,13 @@
 				Returns shapes of the carets corresponding to the character offset [param position] in the text. Returned caret shape is 1 pixel wide rectangle.
 			</description>
 		</method>
+		<method name="shaped_text_get_character_breaks" qualifiers="const">
+			<return type="PackedInt32Array" />
+			<param index="0" name="shaped" type="RID" />
+			<description>
+				Returns array of the composite character boundaries.
+			</description>
+		</method>
 		<method name="shaped_text_get_custom_punctuation" qualifiers="const">
 			<return type="String" />
 			<param index="0" name="shaped" type="RID" />
@@ -1432,14 +1447,22 @@
 				Returns [code]true[/code] if buffer is successfully shaped.
 			</description>
 		</method>
-		<method name="shaped_text_next_grapheme_pos" qualifiers="const">
+		<method name="shaped_text_next_character_pos" qualifiers="const">
 			<return type="int" />
 			<param index="0" name="shaped" type="RID" />
 			<param index="1" name="pos" type="int" />
 			<description>
 				Returns composite character end position closest to the [param pos].
 			</description>
 		</method>
+		<method name="shaped_text_next_grapheme_pos" qualifiers="const">
+			<return type="int" />
+			<param index="0" name="shaped" type="RID" />
+			<param index="1" name="pos" type="int" />
+			<description>
+				Returns grapheme end position closest to the [param pos].
+			</description>
+		</method>
 		<method name="shaped_text_overrun_trim_to_width">
 			<return type="void" />
 			<param index="0" name="shaped" type="RID" />
@@ -1449,14 +1472,22 @@
 				Trims text if it exceeds the given width.
 			</description>
 		</method>
-		<method name="shaped_text_prev_grapheme_pos" qualifiers="const">
+		<method name="shaped_text_prev_character_pos" qualifiers="const">
 			<return type="int" />
 			<param index="0" name="shaped" type="RID" />
 			<param index="1" name="pos" type="int" />
 			<description>
 				Returns composite character start position closest to the [param pos].
 			</description>
 		</method>
+		<method name="shaped_text_prev_grapheme_pos" qualifiers="const">
+			<return type="int" />
+			<param index="0" name="shaped" type="RID" />
+			<param index="1" name="pos" type="int" />
+			<description>
+				Returns grapheme start position closest to the [param pos].
+			</description>
+		</method>
 		<method name="shaped_text_resize_object">
 			<return type="bool" />
 			<param index="0" name="shaped" type="RID" />
@@ -1568,6 +1599,18 @@
 				[b]Note:[/b] Always returns [code]false[/code] if the server does not support the [constant FEATURE_UNICODE_SECURITY] feature.
 			</description>
 		</method>
+		<method name="string_get_character_breaks" qualifiers="const">
+			<return type="PackedInt32Array" />
+			<param index="0" name="string" type="String" />
+			<param index="1" name="language" type="String" default="&quot;&quot;" />
+			<description>
+				Returns array of the composite character boundaries.
+				[codeblock]
+				var ts = TextServerManager.get_primary_interface()
+				print(ts.string_get_word_breaks("Test ❤️‍🔥 Test")) # Prints [1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 14]
+				[/codeblock]
+			</description>
+		</method>
 		<method name="string_get_word_breaks" qualifiers="const">
 			<return type="PackedInt32Array" />
 			<param index="0" name="string" type="String" />

diff --git a/doc/classes/TextServerExtension.xml b/doc/classes/TextServerExtension.xml
@@ -981,6 +981,13 @@
 			<description>
 			</description>
 		</method>
+		<method name="_shaped_text_closest_character_pos" qualifiers="virtual const">
+			<return type="int" />
+			<param index="0" name="shaped" type="RID" />
+			<param index="1" name="pos" type="int" />
+			<description>
+			</description>
+		</method>
 		<method name="_shaped_text_draw" qualifiers="virtual const">
 			<return type="void" />
 			<param index="0" name="shaped" type="RID" />
@@ -1026,6 +1033,12 @@
 			<description>
 			</description>
 		</method>
+		<method name="_shaped_text_get_character_breaks" qualifiers="virtual const">
+			<return type="PackedInt32Array" />
+			<param index="0" name="shaped" type="RID" />
+			<description>
+			</description>
+		</method>
 		<method name="_shaped_text_get_custom_punctuation" qualifiers="virtual const">
 			<return type="String" />
 			<param index="0" name="shaped" type="RID" />
@@ -1229,6 +1242,13 @@
 			<description>
 			</description>
 		</method>
+		<method name="_shaped_text_next_character_pos" qualifiers="virtual const">
+			<return type="int" />
+			<param index="0" name="shaped" type="RID" />
+			<param index="1" name="pos" type="int" />
+			<description>
+			</description>
+		</method>
 		<method name="_shaped_text_next_grapheme_pos" qualifiers="virtual const">
 			<return type="int" />
 			<param index="0" name="shaped" type="RID" />
@@ -1244,6 +1264,13 @@
 			<description>
 			</description>
 		</method>
+		<method name="_shaped_text_prev_character_pos" qualifiers="virtual const">
+			<return type="int" />
+			<param index="0" name="shaped" type="RID" />
+			<param index="1" name="pos" type="int" />
+			<description>
+			</description>
+		</method>
 		<method name="_shaped_text_prev_grapheme_pos" qualifiers="virtual const">
 			<return type="int" />
 			<param index="0" name="shaped" type="RID" />
@@ -1356,6 +1383,13 @@
 			<description>
 			</description>
 		</method>
+		<method name="_string_get_character_breaks" qualifiers="virtual const">
+			<return type="PackedInt32Array" />
+			<param index="0" name="string" type="String" />
+			<param index="1" name="language" type="String" />
+			<description>
+			</description>
+		</method>
 		<method name="_string_get_word_breaks" qualifiers="virtual const">
 			<return type="PackedInt32Array" />
 			<param index="0" name="string" type="String" />

diff --git a/modules/text_server_adv/text_server_adv.cpp b/modules/text_server_adv/text_server_adv.cpp
@@ -3780,6 +3780,7 @@ void TextServerAdvanced::invalidate(TextServerAdvanced::ShapedTextDataAdvanced *
 			p_shaped->script_iter = nullptr;
 		}
 		p_shaped->break_ops_valid = false;
+		p_shaped->chars_valid = false;
 		p_shaped->js_ops_valid = false;
 	}
 }
@@ -4833,6 +4834,76 @@ int64_t TextServerAdvanced::_shaped_text_get_ellipsis_glyph_count(const RID &p_s
 	return sd->overrun_trim_data.ellipsis_glyph_buf.size();
 }
 
+void TextServerAdvanced::_update_chars(ShapedTextDataAdvanced *p_sd) const {
+	if (!p_sd->chars_valid) {
+		p_sd->chars.clear();
+
+		const UChar *data = p_sd->utf16.get_data();
+		UErrorCode err = U_ZERO_ERROR;
+		int prev = -1;
+		int i = 0;
+
+		Vector<ShapedTextDataAdvanced::Span> &spans = p_sd->spans;
+		if (p_sd->parent != RID()) {
+			ShapedTextDataAdvanced *parent_sd = shaped_owner.get_or_null(p_sd->parent);
+			ERR_FAIL_COND(!parent_sd->valid);
+			spans = parent_sd->spans;
+		}
+
+		while (i < spans.size()) {
+			if (spans[i].start > p_sd->end) {
+				break;
+			}
+			if (spans[i].end < p_sd->start) {
+				i++;
+				continue;
+			}
+
+			int r_start = MAX(0, spans[i].start - p_sd->start);
+			String language = spans[i].language;
+			while (i + 1 < spans.size() && language == spans[i + 1].language) {
+				i++;
+			}
+			int r_end = MIN(spans[i].end - p_sd->start, p_sd->text.size());
+
+			UBreakIterator *bi = ubrk_open(UBRK_CHARACTER, (language.is_empty()) ? TranslationServer::get_singleton()->get_tool_locale().ascii().get_data() : language.ascii().get_data(), data + _convert_pos_inv(p_sd, r_start), _convert_pos_inv(p_sd, r_end - r_start), &err);
+			if (U_SUCCESS(err)) {
+				while (ubrk_next(bi) != UBRK_DONE) {
+					int pos = _convert_pos(p_sd, ubrk_current(bi)) + r_start + p_sd->start;
+					if (prev != pos) {
+						p_sd->chars.push_back(pos);
+					}
+					prev = pos;
+				}
+				ubrk_close(bi);
+			} else {
+				for (int j = r_start; j <= r_end; j++) {
+					if (prev != j) {
+						p_sd->chars.push_back(j + p_sd->start);
+					}
+					prev = j;
+				}
+			}
+			i++;
+		}
+		p_sd->chars_valid = true;
+	}
+}
+
+PackedInt32Array TextServerAdvanced::_shaped_text_get_character_breaks(const RID &p_shaped) const {
+	ShapedTextDataAdvanced *sd = shaped_owner.get_or_null(p_shaped);
+	ERR_FAIL_COND_V(!sd, PackedInt32Array());
+
+	MutexLock lock(sd->mutex);
+	if (!sd->valid) {
+		const_cast<TextServerAdvanced *>(this)->_shaped_text_shape(p_shaped);
+	}
+
+	_update_chars(sd);
+
+	return sd->chars;
+}
+
 bool TextServerAdvanced::_shaped_text_update_breaks(const RID &p_shaped) {
 	ShapedTextDataAdvanced *sd = shaped_owner.get_or_null(p_shaped);
 	ERR_FAIL_COND_V(!sd, false);
@@ -5336,7 +5407,17 @@ void TextServerAdvanced::_shape_run(ShapedTextDataAdvanced *p_sd, int64_t p_star
 		// Try system fallback.
 		RID fdef = p_fonts[0];
 		if (_font_is_allow_system_fallback(fdef)) {
-			String text = p_sd->text.substr(p_start, 1);
+			_update_chars(p_sd);
+
+			int64_t next = p_end;
+			for (const int32_t &E : p_sd->chars) {
+				if (E > p_start) {
+					next = E;
+					break;
+				}
+			}
+			String text = p_sd->text.substr(p_start, next - p_start);
+
 			String font_name = _font_get_name(fdef);
 			BitField<FontStyle> font_style = _font_get_style(fdef);
 			int font_weight = _font_get_weight(fdef);
@@ -6600,6 +6681,30 @@ PackedInt32Array TextServerAdvanced::_string_get_word_breaks(const String &p_str
 	return ret;
 }
 
+PackedInt32Array TextServerAdvanced::_string_get_character_breaks(const String &p_string, const String &p_language) const {
+	const String lang = (p_language.is_empty()) ? TranslationServer::get_singleton()->get_tool_locale() : p_language;
+	// Convert to UTF-16.
+	Char16String utf16 = p_string.utf16();
+
+	PackedInt32Array ret;
+
+	UErrorCode err = U_ZERO_ERROR;
+	UBreakIterator *bi = ubrk_open(UBRK_CHARACTER, lang.ascii().get_data(), (const UChar *)utf16.get_data(), utf16.length(), &err);
+	if (U_SUCCESS(err)) {
+		while (ubrk_next(bi) != UBRK_DONE) {
+			int pos = _convert_pos(p_string, utf16, ubrk_current(bi));
+			ret.push_back(pos);
+		}
+		ubrk_close(bi);
+	} else {
+		for (int i = 0; i <= p_string.size(); i++) {
+			ret.push_back(i);
+		}
+	}
+
+	return ret;
+}
+
 bool TextServerAdvanced::_is_valid_identifier(const String &p_string) const {
 #ifndef ICU_STATIC_DATA
 	if (!icu_data_loaded) {

diff --git a/modules/text_server_adv/text_server_adv.h b/modules/text_server_adv/text_server_adv.h
@@ -509,9 +509,11 @@ class TextServerAdvanced : public TextServerExtension {
 
 		HashMap<int, bool> jstops;
 		HashMap<int, bool> breaks;
+		PackedInt32Array chars;
 		int break_inserts = 0;
 		bool break_ops_valid = false;
 		bool js_ops_valid = false;
+		bool chars_valid = false;
 
 		~ShapedTextDataAdvanced() {
 			for (int i = 0; i < bidi_iter.size(); i++) {
@@ -609,6 +611,7 @@ class TextServerAdvanced : public TextServerExtension {
 	mutable HashMap<SystemFontKey, SystemFontCache, SystemFontKeyHasher> system_fonts;
 	mutable HashMap<String, PackedByteArray> system_font_data;
 
+	void _update_chars(ShapedTextDataAdvanced *p_sd) const;
 	void _realign(ShapedTextDataAdvanced *p_sd) const;
 	int64_t _convert_pos(const String &p_utf32, const Char16String &p_utf16, int64_t p_pos) const;
 	int64_t _convert_pos(const ShapedTextDataAdvanced *p_sd, int64_t p_pos) const;
@@ -920,11 +923,14 @@ class TextServerAdvanced : public TextServerExtension {
 	MODBIND1RC(double, shaped_text_get_underline_position, const RID &);
 	MODBIND1RC(double, shaped_text_get_underline_thickness, const RID &);
 
+	MODBIND1RC(PackedInt32Array, shaped_text_get_character_breaks, const RID &);
+
 	MODBIND2RC(String, format_number, const String &, const String &);
 	MODBIND2RC(String, parse_number, const String &, const String &);
 	MODBIND1RC(String, percent_sign, const String &);
 
 	MODBIND3RC(PackedInt32Array, string_get_word_breaks, const String &, const String &, int64_t);
+	MODBIND2RC(PackedInt32Array, string_get_character_breaks, const String &, const String &);
 
 	MODBIND2RC(int64_t, is_confusable, const String &, const PackedStringArray &);
 	MODBIND1RC(bool, spoof_check, const String &);

diff --git a/modules/text_server_fb/text_server_fb.cpp b/modules/text_server_fb/text_server_fb.cpp
@@ -4079,6 +4079,20 @@ double TextServerFallback::_shaped_text_get_underline_thickness(const RID &p_sha
 	return sd->uthk;
 }
 
+PackedInt32Array TextServerFallback::_shaped_text_get_character_breaks(const RID &p_shaped) const {
+	const ShapedTextDataFallback *sd = shaped_owner.get_or_null(p_shaped);
+	ERR_FAIL_COND_V(!sd, PackedInt32Array());
+
+	MutexLock lock(sd->mutex);
+
+	PackedInt32Array ret;
+	ret.resize(sd->end - sd->start);
+	for (int i = sd->start; i < sd->end; i++) {
+		ret.write[i] = i;
+	}
+	return ret;
+}
+
 String TextServerFallback::_string_to_upper(const String &p_string, const String &p_language) const {
 	return p_string.to_upper();
 }

diff --git a/modules/text_server_fb/text_server_fb.h b/modules/text_server_fb/text_server_fb.h
@@ -788,6 +788,8 @@ class TextServerFallback : public TextServerExtension {
 	MODBIND1RC(double, shaped_text_get_underline_position, const RID &);
 	MODBIND1RC(double, shaped_text_get_underline_thickness, const RID &);
 
+	MODBIND1RC(PackedInt32Array, shaped_text_get_character_breaks, const RID &);
+
 	MODBIND3RC(PackedInt32Array, string_get_word_breaks, const String &, const String &, int64_t);
 
 	MODBIND2RC(String, string_to_upper, const String &, const String &);