charset: support utf8mb4_unicode_ci collation (tikv#8420) (tikv#9577)

cherry-pick tikv#8420 to release-4.0
You can switch your code base to this Pull Request by using [git-extras](https://github.com/tj/git-extras):
```bash
# In tikv repo:
git pr tikv#9577
```

After apply modifications, you can push your change to this PR via:
```bash
git push git@github.com:ti-srebot/tikv.git pr/9577:release-4.0-f456abae9e5c
```

---

<!--
Thank you for contributing to TiKV!

If you haven't already, please read TiKV's [CONTRIBUTING](https://github.com/tikv/tikv/blob/master/CONTRIBUTING.md) document.

If you're unsure about anything, just ask; somebody should be along to answer within a day or two.

PR Title Format:
1. module [, module2, module3]: what's changed
2. *: what's changed
-->
Signed-off-by: jwxiong <xiongjiwei1996@outlook.com>

### What problem does this PR solve?

Problem Summary:

support utf8mb4_unicode_ci collation pingcap/tidb#17596

### What is changed and how it works?

add utf8mb4_unicode_ci support in tikv
for detail, please see pingcap/tidb#18776

### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

- Unit test
- Integration test
- Manual test (add detailed scripts or steps below)


### Release note <!-- bugfixes or new feature need a release note -->
- add utf8mb4_unicode_ci implement

components/tidb_query/src/aggr_fn/impl_max_min.rs

@@ -386,10 +386,12 @@ mod tests {
             (Collation::Binary, true, vec!["B", "a"], "a"),
             (Collation::Utf8Mb4Bin, true, vec!["B", "a"], "a"),
             (Collation::Utf8Mb4GeneralCi, true, vec!["B", "a"], "B"),
+            (Collation::Utf8Mb4UnicodeCi, true, vec!["ß", "sr"], "ß"),
             (Collation::Utf8Mb4BinNoPadding, true, vec!["B", "a"], "a"),
             (Collation::Binary, false, vec!["B", "a"], "B"),
             (Collation::Utf8Mb4Bin, false, vec!["B", "a"], "B"),
             (Collation::Utf8Mb4GeneralCi, false, vec!["B", "a"], "a"),
+            (Collation::Utf8Mb4UnicodeCi, false, vec!["ß", "st"], "ß"),
             (Collation::Utf8Mb4BinNoPadding, false, vec!["B", "a"], "B"),
         ];
         for (coll, is_max, args, expected) in cases {

components/tidb_query/src/codec/collation/mod.rs

@@ -1,6 +1,7 @@
 // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.
 
 mod latin1_bin;
+mod unicode_ci_data;
 mod utf8mb4;
 
 pub use self::latin1_bin::*;
@@ -23,6 +24,7 @@ pub macro match_template_collator($t:tt, $($tail:tt)*) {
             Utf8Mb4BinNoPadding => CollatorUtf8Mb4BinNoPadding,
             Utf8Mb4GeneralCi => CollatorUtf8Mb4GeneralCi,
             Latin1Bin => CollatorLatin1Bin,
+            Utf8Mb4UnicodeCi => CollatorUtf8Mb4UnicodeCi,
         ],
         $($tail)*
     }

components/tidb_query/src/codec/collation/utf8mb4.rs

@@ -7,6 +7,7 @@ use std::str;
 use codec::prelude::*;
 
 use super::*;
+use crate::codec::collation::unicode_ci_data::*;
 use crate::codec::Result;
 
 pub struct CharsetUtf8mb4;
@@ -395,6 +396,111 @@ impl Collator for CollatorUtf8Mb4GeneralCi {
     }
 }
 
+#[inline]
+fn unicode_ci_convert(c: char) -> u128 {
+    let r = c as usize;
+    if r > 0xFFFF {
+        return 0xFFFD;
+    }
+
+    let u = UNICODE_CI_TABLE[r];
+    if u == LONG_RUNE {
+        return map_long_rune(r);
+    }
+
+    u as u128
+}
+
+/// Collator for `utf8mb4_unicode_ci` collation with padding behavior (trims right spaces).
+#[derive(Debug)]
+pub struct CollatorUtf8Mb4UnicodeCi;
+
+impl Collator for CollatorUtf8Mb4UnicodeCi {
+    type Charset = CharsetUtf8mb4;
+
+    #[inline]
+    fn validate(bstr: &[u8]) -> Result<()> {
+        str::from_utf8(bstr)?;
+        Ok(())
+    }
+
+    fn write_sort_key<W: BufferWriter>(writer: &mut W, bstr: &[u8]) -> Result<usize> {
+        let s = str::from_utf8(bstr)?.trim_end_matches(TRIM_PADDING_SPACE);
+        let mut n = 0;
+        for ch in s.chars() {
+            let mut convert = unicode_ci_convert(ch);
+            while convert != 0 {
+                writer.write_u16_be((convert & 0xFFFF) as u16)?;
+                n += 1;
+                convert >>= 16
+            }
+        }
+        Ok(n * std::mem::size_of::<u16>())
+    }
+
+    fn sort_compare(a: &[u8], b: &[u8]) -> Result<Ordering> {
+        let mut ca = str::from_utf8(a)?
+            .trim_end_matches(TRIM_PADDING_SPACE)
+            .chars();
+        let mut cb = str::from_utf8(b)?
+            .trim_end_matches(TRIM_PADDING_SPACE)
+            .chars();
+        let mut an = 0;
+        let mut bn = 0;
+
+        loop {
+            if an == 0 {
+                for ach in &mut ca {
+                    an = unicode_ci_convert(ach);
+                    if an != 0 {
+                        break;
+                    }
+                }
+            }
+
+            if bn == 0 {
+                for bch in &mut cb {
+                    bn = unicode_ci_convert(bch);
+                    if bn != 0 {
+                        break;
+                    }
+                }
+            }
+
+            if an == 0 || bn == 0 {
+                return Ok(an.cmp(&bn));
+            }
+
+            if an == bn {
+                an = 0;
+                bn = 0;
+                continue;
+            }
+
+            while an != 0 && bn != 0 {
+                if (an ^ bn) & 0xFFFF == 0 {
+                    an >>= 16;
+                    bn >>= 16;
+                } else {
+                    return Ok((an & 0xFFFF).cmp(&(bn & 0xFFFF)));
+                }
+            }
+        }
+    }
+
+    fn sort_hash<H: Hasher>(state: &mut H, bstr: &[u8]) -> Result<()> {
+        let s = str::from_utf8(bstr)?.trim_end_matches(TRIM_PADDING_SPACE);
+        for ch in s.chars() {
+            let mut convert = unicode_ci_convert(ch);
+            while convert != 0 {
+                (convert & 0xFFFF).hash(state);
+                convert >>= 16;
+            }
+        }
+        Ok(())
+    }
+}
+
 /// Collator for utf8mb4_bin collation with padding behavior (trims right spaces).
 #[derive(Debug)]
 pub struct CollatorUtf8Mb4Bin;
@@ -480,43 +586,89 @@ mod tests {
             (Collation::Utf8Mb4Bin, 0),
             (Collation::Utf8Mb4BinNoPadding, 1),
             (Collation::Utf8Mb4GeneralCi, 2),
+            (Collation::Utf8Mb4UnicodeCi, 3),
         ];
         let cases = vec![
-            // (sa, sb, [Utf8Mb4Bin, Utf8Mb4BinNoPadding, Utf8Mb4GeneralCi])
+            // (sa, sb, [Utf8Mb4Bin, Utf8Mb4BinNoPadding, Utf8Mb4GeneralCi, Utf8Mb4UnicodeCi])
             (
                 "a".as_bytes(),
                 "a".as_bytes(),
-                [Ordering::Equal, Ordering::Equal, Ordering::Equal],
+                [
+                    Ordering::Equal,
+                    Ordering::Equal,
+                    Ordering::Equal,
+                    Ordering::Equal,
+                ],
             ),
             (
                 "a".as_bytes(),
                 "a ".as_bytes(),
-                [Ordering::Equal, Ordering::Less, Ordering::Equal],
+                [
+                    Ordering::Equal,
+                    Ordering::Less,
+                    Ordering::Equal,
+                    Ordering::Equal,
+                ],
             ),
             (
                 "a".as_bytes(),
                 "A ".as_bytes(),
-                [Ordering::Greater, Ordering::Greater, Ordering::Equal],
+                [
+                    Ordering::Greater,
+                    Ordering::Greater,
+                    Ordering::Equal,
+                    Ordering::Equal,
+                ],
             ),
             (
                 "aa ".as_bytes(),
                 "a a".as_bytes(),
-                [Ordering::Greater, Ordering::Greater, Ordering::Greater],
+                [
+                    Ordering::Greater,
+                    Ordering::Greater,
+                    Ordering::Greater,
+                    Ordering::Greater,
+                ],
             ),
             (
                 "A".as_bytes(),
                 "a\t".as_bytes(),
-                [Ordering::Less, Ordering::Less, Ordering::Less],
+                [
+                    Ordering::Less,
+                    Ordering::Less,
+                    Ordering::Less,
+                    Ordering::Less,
+                ],
             ),
             (
                 "cAfe".as_bytes(),
                 "café".as_bytes(),
-                [Ordering::Less, Ordering::Less, Ordering::Equal],
+                [
+                    Ordering::Less,
+                    Ordering::Less,
+                    Ordering::Equal,
+                    Ordering::Equal,
+                ],
             ),
             (
                 "cAfe ".as_bytes(),
                 "café".as_bytes(),
-                [Ordering::Less, Ordering::Less, Ordering::Equal],
+                [
+                    Ordering::Less,
+                    Ordering::Less,
+                    Ordering::Equal,
+                    Ordering::Equal,
+                ],
+            ),
+            (
+                "ß".as_bytes(),
+                "ss".as_bytes(),
+                [
+                    Ordering::Greater,
+                    Ordering::Greater,
+                    Ordering::Less,
+                    Ordering::Equal,
+                ],
             ),
         ];
 
@@ -568,18 +720,34 @@ mod tests {
             (Collation::Utf8Mb4Bin, 0),
             (Collation::Utf8Mb4BinNoPadding, 1),
             (Collation::Utf8Mb4GeneralCi, 2),
+            (Collation::Utf8Mb4UnicodeCi, 3),
         ];
         let cases = vec![
-            // (str, [Utf8Mb4Bin, Utf8Mb4BinNoPadding, Utf8Mb4GeneralCi])
-            ("a", [vec![0x61], vec![0x61], vec![0x00, 0x41]]),
-            ("A ", [vec![0x41], vec![0x41, 0x20], vec![0x00, 0x41]]),
-            ("A", [vec![0x41], vec![0x41], vec![0x00, 0x41]]),
+            // (str, [Utf8Mb4Bin, Utf8Mb4BinNoPadding, Utf8Mb4GeneralCi, Utf8Mb4UnicodeCi])
+            (
+                "a",
+                [vec![0x61], vec![0x61], vec![0x00, 0x41], vec![0x0E, 0x33]],
+            ),
+            (
+                "A ",
+                [
+                    vec![0x41],
+                    vec![0x41, 0x20],
+                    vec![0x00, 0x41],
+                    vec![0x0E, 0x33],
+                ],
+            ),
+            (
+                "A",
+                [vec![0x41], vec![0x41], vec![0x00, 0x41], vec![0x0E, 0x33]],
+            ),
             (
                 "😃",
                 [
                     vec![0xF0, 0x9F, 0x98, 0x83],
                     vec![0xF0, 0x9F, 0x98, 0x83],
                     vec![0xff, 0xfd],
+                    vec![0xff, 0xfd],
                 ],
             ),
             (
@@ -601,6 +769,24 @@ mod tests {
                         0x00, 0x42, 0x00, 0x41, 0x00, 0x5a, 0x00, 0x20, 0x26, 0x3, 0x00, 0x20,
                         0x00, 0x51, 0x00, 0x55, 0x00, 0x58,
                     ],
+                    vec![
+                        0x0E, 0xB9, 0x0F, 0x82, 0x0F, 0x82, 0x02, 0x09, 0x02, 0xC5, 0x02, 0x09,
+                        0x0E, 0x4A, 0x0E, 0x33, 0x0F, 0xC0, 0x02, 0x09, 0xFF, 0xFD, 0x02, 0x09,
+                        0x0E, 0x4A, 0x0E, 0x33, 0x10, 0x6A, 0x02, 0x09, 0x06, 0xFF, 0x02, 0x09,
+                        0x0F, 0xB4, 0x10, 0x1F, 0x10, 0x5A,
+                    ],
+                ],
+            ),
+            (
+                "ﷻ",
+                [
+                    vec![0xEF, 0xB7, 0xBB],
+                    vec![0xEF, 0xB7, 0xBB],
+                    vec![0xFD, 0xFB],
+                    vec![
+                        0x13, 0x5E, 0x13, 0xAB, 0x02, 0x09, 0x13, 0x5E, 0x13, 0xAB, 0x13, 0x50,
+                        0x13, 0xAB, 0x13, 0xB7,
+                    ],
                 ],
             ),
         ];