Merge pull request #4 from yeslogic/wezm/unicode-16

Upgrade to Unicode 16.0
yeslogic · Oct 8, 2024 · ecbd423 · ecbd423
2 parents a592f0e + d4456be
commit ecbd423
Show file tree

Hide file tree

Showing 8 changed files with 91 additions and 124 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -26,10 +26,6 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: test
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --features "serde"
       - uses: actions-rs/cargo@v1
         with:
           command: test

diff --git a/Cargo.toml b/Cargo.toml
@@ -33,10 +33,6 @@ name = "unicode_bidi"
 [dependencies]
 flame = { version = "0.2", optional = true }
 flamer = { version = "0.4", optional = true }
-serde = { version = ">=0.8, <2.0", default-features = false, optional = true, features = ["derive"] }
-
-[dev-dependencies]
-serde_test = ">=0.8, <2.0"
 
 [features]
 # Note: We don't actually use the `std` feature for anything other than making
@@ -47,7 +43,6 @@ std = []
 unstable = []  # travis-cargo needs it
 bench_it = []
 flame_it = ["flame", "flamer"]
-with_serde = ["serde"]  # DEPRECATED, please use `serde` feature, instead.
 
 [[test]]
 name = "conformance_tests"

diff --git a/Makefile b/Makefile
@@ -0,0 +1,9 @@
+UCD:=16.0.0
+
+tables:
+	yeslogic-ucd-generate bidi-class --rust-enum --short-names ../ucd-generate/ucd-$(UCD) > src/char_data/tables.rs
+	cargo fmt
+
+
+.PHONY: tables
+
diff --git a/README.md b/README.md
@@ -4,9 +4,4 @@ This crate implements the [Unicode Bidirectional Algorithm][tr9] for display
 of mixed right-to-left and left-to-right text.  It is written in safe Rust,
 compatible with the current stable release.
 
-[Documentation](https://docs.rs/unicode-bidi/)
-
-[![CI](https://github.com/servo/unicode-bidi/actions/workflows/main.yml/badge.svg)](https://github.com/servo/unicode-bidi/actions)
-[![AppVeyor](https://img.shields.io/appveyor/ci/servo/unicode-bidi/master.svg)](https://ci.appveyor.com/project/servo/unicode-bidi)
-
 [tr9]: https://www.unicode.org/reports/tr9/
diff --git a/src/char_data/mod.rs b/src/char_data/mod.rs
@@ -25,7 +25,7 @@ use crate::BidiClass::*;
 use crate::BidiDataSource;
 
 /// The [Unicode version](http://www.unicode.org/versions/) of data
-pub const UNICODE_VERSION: (u64, u64, u64) = (15, 0, 0);
+pub const UNICODE_VERSION: (u64, u64, u64) = (16, 0, 0);
 
 /// Hardcoded Bidi data that ships with the unicode-bidi crate.
 ///

diff --git a/src/char_data/tables.rs b/src/char_data/tables.rs
@@ -1,10 +1,10 @@
 // DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY:
 //
-//   yeslogic-ucd-generate bidi-class --rust-enum --short-names ../ucd-15.0.0
+//   yeslogic-ucd-generate bidi-class --rust-enum --short-names ../ucd-generate/ucd-16.0.0
 //
-// Unicode version: 15.0.0.
+// Unicode version: 16.0.0.
 //
-// yeslogic-ucd-generate 0.6.0 is available on crates.io.
+// yeslogic-ucd-generate 0.7.0 is available on crates.io.
 
 #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
 pub enum BidiClass {
@@ -175,8 +175,8 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (2144, 2190, BidiClass::AL),
     (2191, 2191, BidiClass::R),
     (2192, 2193, BidiClass::AN),
-    (2194, 2199, BidiClass::R),
-    (2200, 2207, BidiClass::NSM),
+    (2194, 2198, BidiClass::R),
+    (2199, 2207, BidiClass::NSM),
     (2208, 2249, BidiClass::AL),
     (2250, 2273, BidiClass::NSM),
     (2274, 2274, BidiClass::AN),
@@ -574,8 +574,8 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (9014, 9082, BidiClass::L),
     (9083, 9108, BidiClass::ON),
     (9109, 9109, BidiClass::L),
-    (9110, 9254, BidiClass::ON),
-    (9255, 9279, BidiClass::L),
+    (9110, 9257, BidiClass::ON),
+    (9258, 9279, BidiClass::L),
     (9280, 9290, BidiClass::ON),
     (9291, 9311, BidiClass::L),
     (9312, 9351, BidiClass::ON),
@@ -608,8 +608,7 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (12020, 12031, BidiClass::L),
     (12032, 12245, BidiClass::ON),
     (12246, 12271, BidiClass::L),
-    (12272, 12283, BidiClass::ON),
-    (12284, 12287, BidiClass::L),
+    (12272, 12287, BidiClass::ON),
     (12288, 12288, BidiClass::WS),
     (12289, 12292, BidiClass::ON),
     (12293, 12295, BidiClass::L),
@@ -630,8 +629,10 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (12449, 12538, BidiClass::L),
     (12539, 12539, BidiClass::ON),
     (12540, 12735, BidiClass::L),
-    (12736, 12771, BidiClass::ON),
-    (12772, 12828, BidiClass::L),
+    (12736, 12773, BidiClass::ON),
+    (12774, 12782, BidiClass::L),
+    (12783, 12783, BidiClass::ON),
+    (12784, 12828, BidiClass::L),
     (12829, 12830, BidiClass::ON),
     (12831, 12879, BidiClass::L),
     (12880, 12895, BidiClass::ON),
@@ -833,12 +834,18 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (68904, 68911, BidiClass::AL),
     (68912, 68921, BidiClass::AN),
     (68922, 68927, BidiClass::AL),
-    (68928, 69215, BidiClass::R),
+    (68928, 68937, BidiClass::AN),
+    (68938, 68968, BidiClass::R),
+    (68969, 68973, BidiClass::NSM),
+    (68974, 68974, BidiClass::ON),
+    (68975, 69215, BidiClass::R),
     (69216, 69246, BidiClass::AN),
     (69247, 69290, BidiClass::R),
     (69291, 69292, BidiClass::NSM),
-    (69293, 69372, BidiClass::R),
-    (69373, 69375, BidiClass::NSM),
+    (69293, 69313, BidiClass::R),
+    (69314, 69316, BidiClass::AL),
+    (69317, 69371, BidiClass::R),
+    (69372, 69375, BidiClass::NSM),
     (69376, 69423, BidiClass::R),
     (69424, 69445, BidiClass::AL),
     (69446, 69456, BidiClass::NSM),
@@ -904,7 +911,17 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (70502, 70508, BidiClass::NSM),
     (70509, 70511, BidiClass::L),
     (70512, 70516, BidiClass::NSM),
-    (70517, 70711, BidiClass::L),
+    (70517, 70586, BidiClass::L),
+    (70587, 70592, BidiClass::NSM),
+    (70593, 70605, BidiClass::L),
+    (70606, 70606, BidiClass::NSM),
+    (70607, 70607, BidiClass::L),
+    (70608, 70608, BidiClass::NSM),
+    (70609, 70609, BidiClass::L),
+    (70610, 70610, BidiClass::NSM),
+    (70611, 70624, BidiClass::L),
+    (70625, 70626, BidiClass::NSM),
+    (70627, 70711, BidiClass::L),
     (70712, 70719, BidiClass::NSM),
     (70720, 70721, BidiClass::L),
     (70722, 70724, BidiClass::NSM),
@@ -945,7 +962,9 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (71350, 71350, BidiClass::L),
     (71351, 71351, BidiClass::NSM),
     (71352, 71452, BidiClass::L),
-    (71453, 71455, BidiClass::NSM),
+    (71453, 71453, BidiClass::NSM),
+    (71454, 71454, BidiClass::L),
+    (71455, 71455, BidiClass::NSM),
     (71456, 71457, BidiClass::L),
     (71458, 71461, BidiClass::NSM),
     (71462, 71462, BidiClass::L),
@@ -1022,15 +1041,21 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (73536, 73536, BidiClass::NSM),
     (73537, 73537, BidiClass::L),
     (73538, 73538, BidiClass::NSM),
-    (73539, 73684, BidiClass::L),
+    (73539, 73561, BidiClass::L),
+    (73562, 73562, BidiClass::NSM),
+    (73563, 73684, BidiClass::L),
     (73685, 73692, BidiClass::ON),
     (73693, 73696, BidiClass::ET),
     (73697, 73713, BidiClass::ON),
     (73714, 78911, BidiClass::L),
     (78912, 78912, BidiClass::NSM),
     (78913, 78918, BidiClass::L),
     (78919, 78933, BidiClass::NSM),
-    (78934, 92911, BidiClass::L),
+    (78934, 90397, BidiClass::L),
+    (90398, 90409, BidiClass::NSM),
+    (90410, 90412, BidiClass::L),
+    (90413, 90415, BidiClass::NSM),
+    (90416, 92911, BidiClass::L),
     (92912, 92916, BidiClass::NSM),
     (92917, 92975, BidiClass::L),
     (92976, 92982, BidiClass::NSM),
@@ -1046,7 +1071,13 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (113821, 113822, BidiClass::NSM),
     (113823, 113823, BidiClass::L),
     (113824, 113827, BidiClass::BN),
-    (113828, 118527, BidiClass::L),
+    (113828, 117759, BidiClass::L),
+    (117760, 117973, BidiClass::ON),
+    (117974, 117999, BidiClass::L),
+    (118000, 118009, BidiClass::EN),
+    (118010, 118015, BidiClass::L),
+    (118016, 118451, BidiClass::ON),
+    (118452, 118527, BidiClass::L),
     (118528, 118573, BidiClass::NSM),
     (118574, 118575, BidiClass::L),
     (118576, 118598, BidiClass::NSM),
@@ -1067,15 +1098,25 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (119365, 119365, BidiClass::ON),
     (119366, 119551, BidiClass::L),
     (119552, 119638, BidiClass::ON),
-    (119639, 120538, BidiClass::L),
+    (119639, 120512, BidiClass::L),
+    (120513, 120513, BidiClass::ON),
+    (120514, 120538, BidiClass::L),
     (120539, 120539, BidiClass::ON),
-    (120540, 120596, BidiClass::L),
+    (120540, 120570, BidiClass::L),
+    (120571, 120571, BidiClass::ON),
+    (120572, 120596, BidiClass::L),
     (120597, 120597, BidiClass::ON),
-    (120598, 120654, BidiClass::L),
+    (120598, 120628, BidiClass::L),
+    (120629, 120629, BidiClass::ON),
+    (120630, 120654, BidiClass::L),
     (120655, 120655, BidiClass::ON),
-    (120656, 120712, BidiClass::L),
+    (120656, 120686, BidiClass::L),
+    (120687, 120687, BidiClass::ON),
+    (120688, 120712, BidiClass::L),
     (120713, 120713, BidiClass::ON),
-    (120714, 120770, BidiClass::L),
+    (120714, 120744, BidiClass::L),
+    (120745, 120745, BidiClass::ON),
+    (120746, 120770, BidiClass::L),
     (120771, 120771, BidiClass::ON),
     (120772, 120781, BidiClass::L),
     (120782, 120831, BidiClass::EN),
@@ -1113,7 +1154,9 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (123647, 123647, BidiClass::ET),
     (123648, 124139, BidiClass::L),
     (124140, 124143, BidiClass::NSM),
-    (124144, 124927, BidiClass::L),
+    (124144, 124397, BidiClass::L),
+    (124398, 124399, BidiClass::NSM),
+    (124400, 124927, BidiClass::L),
     (124928, 125135, BidiClass::R),
     (125136, 125142, BidiClass::NSM),
     (125143, 125251, BidiClass::R),
@@ -1174,30 +1217,29 @@ pub const BIDI_CLASS: &'static [(u32, u32, BidiClass)] = &[
     (129160, 129167, BidiClass::L),
     (129168, 129197, BidiClass::ON),
     (129198, 129199, BidiClass::L),
-    (129200, 129201, BidiClass::ON),
-    (129202, 129279, BidiClass::L),
+    (129200, 129211, BidiClass::ON),
+    (129212, 129215, BidiClass::L),
+    (129216, 129217, BidiClass::ON),
+    (129218, 129279, BidiClass::L),
     (129280, 129619, BidiClass::ON),
     (129620, 129631, BidiClass::L),
     (129632, 129645, BidiClass::ON),
     (129646, 129647, BidiClass::L),
     (129648, 129660, BidiClass::ON),
     (129661, 129663, BidiClass::L),
-    (129664, 129672, BidiClass::ON),
-    (129673, 129679, BidiClass::L),
-    (129680, 129725, BidiClass::ON),
-    (129726, 129726, BidiClass::L),
-    (129727, 129733, BidiClass::ON),
-    (129734, 129741, BidiClass::L),
-    (129742, 129755, BidiClass::ON),
-    (129756, 129759, BidiClass::L),
-    (129760, 129768, BidiClass::ON),
-    (129769, 129775, BidiClass::L),
+    (129664, 129673, BidiClass::ON),
+    (129674, 129678, BidiClass::L),
+    (129679, 129734, BidiClass::ON),
+    (129735, 129741, BidiClass::L),
+    (129742, 129756, BidiClass::ON),
+    (129757, 129758, BidiClass::L),
+    (129759, 129769, BidiClass::ON),
+    (129770, 129775, BidiClass::L),
     (129776, 129784, BidiClass::ON),
     (129785, 129791, BidiClass::L),
     (129792, 129938, BidiClass::ON),
     (129939, 129939, BidiClass::L),
-    (129940, 129994, BidiClass::ON),
-    (129995, 130031, BidiClass::L),
+    (129940, 130031, BidiClass::ON),
     (130032, 130041, BidiClass::EN),
     (130042, 917503, BidiClass::L),
     (917504, 917759, BidiClass::BN),

diff --git a/src/level.rs b/src/level.rs
@@ -30,7 +30,6 @@ use super::char_data::BidiClass;
 ///
 /// <http://www.unicode.org/reports/tr9/#BD2>
 #[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct Level(u8);
 
 pub const LTR_LEVEL: Level = Level(0);
@@ -356,30 +355,3 @@ mod tests {
         );
     }
 }
-
-#[cfg(all(feature = "serde", test))]
-mod serde_tests {
-    use super::*;
-    use serde_test::{assert_tokens, Token};
-
-    #[test]
-    fn test_statics() {
-        assert_tokens(
-            &Level::ltr(),
-            &[Token::NewtypeStruct { name: "Level" }, Token::U8(0)],
-        );
-        assert_tokens(
-            &Level::rtl(),
-            &[Token::NewtypeStruct { name: "Level" }, Token::U8(1)],
-        );
-    }
-
-    #[test]
-    fn test_new() {
-        let level = Level::new(42).unwrap();
-        assert_tokens(
-            &level,
-            &[Token::NewtypeStruct { name: "Level" }, Token::U8(42)],
-        );
-    }
-}
diff --git a/src/lib.rs b/src/lib.rs
@@ -1087,45 +1087,3 @@ mod tests {
         assert_eq!(p_mixed.level_at(ltr_text.len()), RTL_LEVEL);
     }
 }
-
-#[cfg(all(feature = "serde", test))]
-mod serde_tests {
-    use super::*;
-    use serde_test::{assert_tokens, Token};
-
-    #[test]
-    fn test_levels() {
-        let text = "abc אבג";
-        let bidi_info = BidiInfo::new(text, None);
-        let levels = bidi_info.levels;
-        assert_eq!(text.as_bytes().len(), 10);
-        assert_eq!(levels.len(), 10);
-        assert_tokens(
-            &levels,
-            &[
-                Token::Seq { len: Some(10) },
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(0),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(0),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(0),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(0),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(1),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(1),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(1),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(1),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(1),
-                Token::NewtypeStruct { name: "Level" },
-                Token::U8(1),
-                Token::SeqEnd,
-            ],
-        );
-    }
-}