CLDR-17349 One-off import of emoji search keywords (#3689)

unicode-org · May 9, 2024 · af77ae6 · af77ae6
1 parent ee35e91
commit af77ae6
Show file tree

Hide file tree

Showing 68 changed files with 69,690 additions and 68,079 deletions.
diff --git a/common/annotations/af.xml b/common/annotations/af.xml
diff --git a/common/annotations/am.xml b/common/annotations/am.xml
diff --git a/common/annotations/ar.xml b/common/annotations/ar.xml
diff --git a/common/annotations/az.xml b/common/annotations/az.xml
diff --git a/common/annotations/bg.xml b/common/annotations/bg.xml
diff --git a/common/annotations/bn.xml b/common/annotations/bn.xml
diff --git a/common/annotations/ca.xml b/common/annotations/ca.xml
diff --git a/common/annotations/cs.xml b/common/annotations/cs.xml
diff --git a/common/annotations/da.xml b/common/annotations/da.xml
diff --git a/common/annotations/de.xml b/common/annotations/de.xml
diff --git a/common/annotations/el.xml b/common/annotations/el.xml
diff --git a/common/annotations/en.xml b/common/annotations/en.xml
@@ -3907,21 +3907,21 @@ annotations.
 		<annotation cp="³" type="tts">superscript three</annotation>
 		<annotation cp="µ">measure | micro sign</annotation>
 		<annotation cp="µ" type="tts">micro sign</annotation>
-		<!--  start hand-generated lines from E16 -->
-		<annotation cp='🪉' type='tts'>harp</annotation>
-		<annotation cp='🪉'>music | instrument | love | cupid | orchestra</annotation>
-		<annotation cp='🪏' type='tts'>shovel</annotation>
-		<annotation cp='🪏'>dig | spade | hole | scoop</annotation>
-		<annotation cp='🪾' type='tts'>leafless tree</annotation>
-		<annotation cp='🪾'>winter | barren | drought</annotation>
-		<annotation cp='🫆' type='tts'>fingerprint</annotation>
-		<annotation cp='🫆'>forensics | identity | safety</annotation>
-		<annotation cp='🫜' type='tts'>root vegetable</annotation>
-		<annotation cp='🫜'>root | vegetable | garden | beet | turnip</annotation>
-		<annotation cp='🫟' type='tts'>splatter</annotation>
-		<annotation cp='🫟'>paint | holi | spill | stain</annotation>
-		<annotation cp='🫩' type='tts'>face with bags under eyes</annotation>
-		<annotation cp='🫩'>tired | sleepy | exhausted</annotation>
-		<!--  end hand-generated lines from E16 -->
+		<!-- start hand-generated lines from E16 -->
+		<annotation cp="🪉">music | instrument | love | cupid | orchestra</annotation>
+		<annotation cp="🪉" type="tts">harp</annotation>
+		<annotation cp="🪏">dig | spade | hole | scoop</annotation>
+		<annotation cp="🪏" type="tts">shovel</annotation>
+		<annotation cp="🪾">winter | barren | drought</annotation>
+		<annotation cp="🪾" type="tts">leafless tree</annotation>
+		<annotation cp="🫆">forensics | identity | safety</annotation>
+		<annotation cp="🫆" type="tts">fingerprint</annotation>
+		<annotation cp="🫜">root | vegetable | garden | beet | turnip</annotation>
+		<annotation cp="🫜" type="tts">root vegetable</annotation>
+		<annotation cp="🫟">paint | holi | spill | stain</annotation>
+		<annotation cp="🫟" type="tts">splatter</annotation>
+		<annotation cp="🫩">tired | sleepy | exhausted</annotation>
+		<annotation cp="🫩" type="tts">face with bags under eyes</annotation>
+		<!-- end hand-generated lines from E16 -->
 	</annotations>
 </ldml>
diff --git a/common/annotations/es_419.xml b/common/annotations/es_419.xml
diff --git a/common/annotations/et.xml b/common/annotations/et.xml
diff --git a/common/annotations/fa.xml b/common/annotations/fa.xml
diff --git a/common/annotations/fi.xml b/common/annotations/fi.xml
diff --git a/common/annotations/fil.xml b/common/annotations/fil.xml
diff --git a/common/annotations/fr.xml b/common/annotations/fr.xml
diff --git a/common/annotations/ga.xml b/common/annotations/ga.xml
diff --git a/common/annotations/gu.xml b/common/annotations/gu.xml
diff --git a/common/annotations/ha.xml b/common/annotations/ha.xml
diff --git a/common/annotations/he.xml b/common/annotations/he.xml
diff --git a/common/annotations/hi.xml b/common/annotations/hi.xml
diff --git a/common/annotations/hr.xml b/common/annotations/hr.xml
diff --git a/common/annotations/hu.xml b/common/annotations/hu.xml
diff --git a/common/annotations/id.xml b/common/annotations/id.xml
diff --git a/common/annotations/it.xml b/common/annotations/it.xml
diff --git a/common/annotations/ja.xml b/common/annotations/ja.xml
diff --git a/common/annotations/kk.xml b/common/annotations/kk.xml
diff --git a/common/annotations/kn.xml b/common/annotations/kn.xml
diff --git a/common/annotations/ko.xml b/common/annotations/ko.xml
diff --git a/common/annotations/lo.xml b/common/annotations/lo.xml
diff --git a/common/annotations/lt.xml b/common/annotations/lt.xml
diff --git a/common/annotations/lv.xml b/common/annotations/lv.xml
diff --git a/common/annotations/mk.xml b/common/annotations/mk.xml
diff --git a/common/annotations/ml.xml b/common/annotations/ml.xml
diff --git a/common/annotations/mr.xml b/common/annotations/mr.xml
diff --git a/common/annotations/ms.xml b/common/annotations/ms.xml
diff --git a/common/annotations/nl.xml b/common/annotations/nl.xml
diff --git a/common/annotations/no.xml b/common/annotations/no.xml
diff --git a/common/annotations/om.xml b/common/annotations/om.xml
diff --git a/common/annotations/pa.xml b/common/annotations/pa.xml
diff --git a/common/annotations/pl.xml b/common/annotations/pl.xml
diff --git a/common/annotations/pt.xml b/common/annotations/pt.xml
diff --git a/common/annotations/pt_PT.xml b/common/annotations/pt_PT.xml
diff --git a/common/annotations/ro.xml b/common/annotations/ro.xml
diff --git a/common/annotations/root.xml b/common/annotations/root.xml
@@ -3908,21 +3908,21 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
 		<annotation cp="³" type="tts">E13-137</annotation>
 		<annotation cp="µ">E13-150</annotation>
 		<annotation cp="µ" type="tts">E13-150</annotation>
-		<!--  start hand-generated lines from E16 -->
-		<annotation cp='🪉'>E16-001</annotation>
-		<annotation cp='🪉' type='tts'>E16-001</annotation>
-		<annotation cp='🪏'>E16-002</annotation>
-		<annotation cp='🪏' type='tts'>E16-002</annotation>
-		<annotation cp='🪾'>E16-003</annotation>
-		<annotation cp='🪾' type='tts'>E16-003</annotation>
-		<annotation cp='🫆'>E16-004</annotation>
-		<annotation cp='🫆' type='tts'>E16-004</annotation>
-		<annotation cp='🫜'>E16-005</annotation>
-		<annotation cp='🫜' type='tts'>E16-005</annotation>
-		<annotation cp='🫟'>E16-006</annotation>
-		<annotation cp='🫟' type='tts'>E16-006</annotation>
-		<annotation cp='🫩'>E16-007</annotation>
-		<annotation cp='🫩' type='tts'>E16-007</annotation>
-		<!--  end hand-generated lines from E16 -->
+		<!-- start hand-generated lines from E16 -->
+		<annotation cp="🪉">E16-001</annotation>
+		<annotation cp="🪉" type="tts">E16-001</annotation>
+		<annotation cp="🪏">E16-002</annotation>
+		<annotation cp="🪏" type="tts">E16-002</annotation>
+		<annotation cp="🪾">E16-003</annotation>
+		<annotation cp="🪾" type="tts">E16-003</annotation>
+		<annotation cp="🫆">E16-004</annotation>
+		<annotation cp="🫆" type="tts">E16-004</annotation>
+		<annotation cp="🫜">E16-005</annotation>
+		<annotation cp="🫜" type="tts">E16-005</annotation>
+		<annotation cp="🫟">E16-006</annotation>
+		<annotation cp="🫟" type="tts">E16-006</annotation>
+		<annotation cp="🫩">E16-007</annotation>
+		<annotation cp="🫩" type="tts">E16-007</annotation>
+		<!-- end hand-generated lines from E16 -->
 	</annotations>
 </ldml>
diff --git a/common/annotations/ru.xml b/common/annotations/ru.xml
diff --git a/common/annotations/sk.xml b/common/annotations/sk.xml
diff --git a/common/annotations/sl.xml b/common/annotations/sl.xml
diff --git a/common/annotations/sq.xml b/common/annotations/sq.xml
diff --git a/common/annotations/sr.xml b/common/annotations/sr.xml
diff --git a/common/annotations/sv.xml b/common/annotations/sv.xml
diff --git a/common/annotations/sw.xml b/common/annotations/sw.xml
diff --git a/common/annotations/ta.xml b/common/annotations/ta.xml
diff --git a/common/annotations/te.xml b/common/annotations/te.xml
diff --git a/common/annotations/th.xml b/common/annotations/th.xml
diff --git a/common/annotations/tr.xml b/common/annotations/tr.xml
diff --git a/common/annotations/uk.xml b/common/annotations/uk.xml
diff --git a/common/annotations/ur.xml b/common/annotations/ur.xml
diff --git a/common/annotations/uz.xml b/common/annotations/uz.xml
diff --git a/common/annotations/vi.xml b/common/annotations/vi.xml
diff --git a/common/annotations/zh.xml b/common/annotations/zh.xml
diff --git a/common/annotations/zh_Hant.xml b/common/annotations/zh_Hant.xml
diff --git a/common/annotations/zh_Hant_HK.xml b/common/annotations/zh_Hant_HK.xml
diff --git a/common/transforms/Arabic-Latin.xml b/common/transforms/Arabic-Latin.xml
@@ -22,7 +22,7 @@ For terms of use, see http://www.unicode.org/copyright.html
 # Does *not* do assimilation of "al", nor hyphenation.
 # While it could be done, we need to determine whether a prefix "al" could
 # occur other than as the definite article (since no space is used).
-:: [[:Arabic:][:block=ARABIC:][‎ⁿ،؛؟ـً-ٕ٠-٬۰-۹﷼ښ]] ;
+:: [[:Arabic:][:block=ARABIC:][‎ⁿ،؛؟ـً-ٕ٠-٬۰-۹﷼ښ][\u0611\u0670]] ;
 :: NFKD (NFC);
 $disambig =  ̱ ;
 $disambig2 =  ̰ ;
@@ -190,6 +190,9 @@ $notAbove = [[:^ccc=0:] & [:^ccc=230:]];
 ‎ہ‎ → ḥ ;
 ‎ە‎ → ĥ ;
 
+# Delete marks without correspondants
+[\u0611\u0670] → ;
+
 # fallbacks
 | s ← c } [eiy];
 | k ← c ;

diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CLDRModify.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CLDRModify.java
@@ -96,6 +96,7 @@
         description =
                 "Tool for applying modifications to the CLDR files. Use -h to see the options.")
 public class CLDRModify {
+    private static final Splitter SPLIT_ON_SEMI = Splitter.onPattern("\\s*;\\s+");
     static final String DEBUG_PATHS = null; // ".*currency.*";
     static final boolean COMMENT_REMOVALS = false; // append removals as comments
     static final UnicodeSet whitespace = new UnicodeSet("[:whitespace:]").freeze();
@@ -650,7 +651,7 @@ private static void checkSuboptions(int i, UnicodeSet allowedOptions) {
                             + "Use -? for help.");
         }
         if (i == FIX && givenOptions.value != null) {
-            final UnicodeSet allowedFilters = new UnicodeSet().add('P').add('Q').add('V');
+            final UnicodeSet allowedFilters = new UnicodeSet().add('P').add('Q').add('V').add('k');
             for (char c : givenOptions.value.toCharArray()) {
                 if (!allowedFilters.contains(c)) {
                     throw new IllegalArgumentException(
@@ -2376,12 +2377,14 @@ private void fillCache() {
                                     @Override
                                     protected boolean handleLine(int lineCount, String line) {
                                         line = line.trim();
-                                        String[] lineParts = line.split("\\s*;\\s*");
+                                        Iterable<String> lineParts = SPLIT_ON_SEMI.split(line);
                                         Map<ConfigKeys, ConfigMatch> keyValue =
                                                 new EnumMap<>(ConfigKeys.class);
                                         for (String linePart : lineParts) {
                                             int pos = linePart.indexOf('=');
                                             if (pos < 0) {
+                                                // WARNING; the code doesn't allow for ; within
+                                                // values; need to restructure for that.
                                                 throw new IllegalArgumentException(
                                                         lineCount
                                                                 + ":\t No = in command: «"