From 4265117d841bd4696ef67fcaf30f96cbb3b85d69 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Fri, 26 Feb 2021 10:51:04 -0600 Subject: [PATCH 01/22] Part 1: Directly seralize patterns and skeletons in the CLDR and provider --- Cargo.lock | 2 + components/datetime/Cargo.toml | 1 + components/datetime/src/fields/length.rs | 2 +- components/datetime/src/fields/mod.rs | 6 +- components/datetime/src/fields/symbols.rs | 81 +++- components/datetime/src/pattern/mod.rs | 171 +++++++- components/datetime/src/provider/helpers.rs | 8 +- components/datetime/src/provider/mod.rs | 391 +++++++++++++++++- components/datetime/tests/datetime.rs | 13 +- components/provider_cldr/Cargo.toml | 1 + .../provider_cldr/src/transform/dates.rs | 99 ++++- 11 files changed, 760 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c6f34ba712..ef00cda03df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -853,6 +853,7 @@ dependencies = [ "icu_testdata", "serde", "serde_json", + "smallvec", "tinystr", "writeable", ] @@ -940,6 +941,7 @@ dependencies = [ "serde", "serde-tuple-vec-map", "serde_json", + "smallvec", "tinystr", "unzip", "urlencoding", diff --git a/components/datetime/Cargo.toml b/components/datetime/Cargo.toml index 012165016c8..34f5093707e 100644 --- a/components/datetime/Cargo.toml +++ b/components/datetime/Cargo.toml @@ -28,6 +28,7 @@ icu_provider = { version = "0.1", path = "../provider" } writeable = { version = "0.2", path = "../../utils/writeable" } tinystr = { version = "0.4.1" } serde = { version = "1.0", features = ["derive"], optional = true } +smallvec = "1.4" [dev-dependencies] criterion = "0.3" diff --git a/components/datetime/src/fields/length.rs b/components/datetime/src/fields/length.rs index 684014d051c..b59f5471a21 100644 --- a/components/datetime/src/fields/length.rs +++ b/components/datetime/src/fields/length.rs @@ -3,7 +3,7 @@ // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). use std::convert::TryFrom; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum LengthError { TooLong, } diff --git a/components/datetime/src/fields/mod.rs b/components/datetime/src/fields/mod.rs index 192f6894e1d..d24d4dbb713 100644 --- a/components/datetime/src/fields/mod.rs +++ b/components/datetime/src/fields/mod.rs @@ -2,9 +2,9 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). mod length; -mod symbols; +pub(crate) mod symbols; -pub use length::FieldLength; +pub use length::{FieldLength, LengthError}; pub use symbols::*; use std::convert::{TryFrom, TryInto}; @@ -20,8 +20,6 @@ pub struct Field { pub length: FieldLength, } -impl Field {} - impl From<(FieldSymbol, FieldLength)> for Field { fn from(input: (FieldSymbol, FieldLength)) -> Self { Self { diff --git a/components/datetime/src/fields/symbols.rs b/components/datetime/src/fields/symbols.rs index 9a4b729ed87..c34d018cdbe 100644 --- a/components/datetime/src/fields/symbols.rs +++ b/components/datetime/src/fields/symbols.rs @@ -3,7 +3,7 @@ // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). use std::convert::TryFrom; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum SymbolError { /// Unknown field symbol Unknown(u8), @@ -23,6 +23,43 @@ pub enum FieldSymbol { Second(Second), } +impl FieldSymbol { + /// Skeletons are a Vec, and represent the Fields that can be used to match to a + /// specific pattern. The order of the Vec does not affect the Pattern that is output. + /// However, it's more performant when matching these fields, and it's more deterministic + /// when serializing them to present them in a consistent order. + /// + /// This ordering is taken by the order of the fields listed in the [UTS 35 Date Field Symbol Table] + /// (https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table), and are generally + /// ordered most significant to least significant. + /// + pub fn get_canonical_order(&self) -> u8 { + match self { + FieldSymbol::Year(Year::Calendar) => 0, + FieldSymbol::Year(Year::WeekOf) => 1, + FieldSymbol::Month(Month::Format) => 2, + FieldSymbol::Month(Month::StandAlone) => 3, + FieldSymbol::Day(Day::DayOfMonth) => 4, + FieldSymbol::Day(Day::DayOfYear) => 5, + FieldSymbol::Day(Day::DayOfWeekInMonth) => 6, + FieldSymbol::Day(Day::ModifiedJulianDay) => 7, + FieldSymbol::Weekday(Weekday::Format) => 8, + FieldSymbol::Weekday(Weekday::Local) => 9, + FieldSymbol::Weekday(Weekday::StandAlone) => 10, + FieldSymbol::DayPeriod(DayPeriod::AmPm) => 11, + FieldSymbol::DayPeriod(DayPeriod::NoonMidnight) => 12, + FieldSymbol::Hour(Hour::H11) => 13, + FieldSymbol::Hour(Hour::H12) => 14, + FieldSymbol::Hour(Hour::H23) => 15, + FieldSymbol::Hour(Hour::H24) => 16, + FieldSymbol::Minute => 17, + FieldSymbol::Second(Second::Second) => 18, + FieldSymbol::Second(Second::FractionalSecond) => 19, + FieldSymbol::Second(Second::Millisecond) => 20, + } + } +} + impl TryFrom for FieldSymbol { type Error = SymbolError; fn try_from(b: u8) -> Result { @@ -52,6 +89,48 @@ impl TryFrom for FieldSymbol { } } +impl From for char { + fn from(symbol: FieldSymbol) -> Self { + match symbol { + FieldSymbol::Year(year) => match year { + Year::Calendar => 'y', + Year::WeekOf => 'Y', + }, + FieldSymbol::Month(month) => match month { + Month::Format => 'M', + Month::StandAlone => 'L', + }, + FieldSymbol::Day(day) => match day { + Day::DayOfMonth => 'd', + Day::DayOfYear => 'D', + Day::DayOfWeekInMonth => 'F', + Day::ModifiedJulianDay => 'g', + }, + FieldSymbol::Weekday(weekday) => match weekday { + Weekday::Format => 'E', + Weekday::Local => 'e', + Weekday::StandAlone => 'c', + }, + FieldSymbol::DayPeriod(dayperiod) => match dayperiod { + DayPeriod::AmPm => 'a', + DayPeriod::NoonMidnight => 'b', + }, + FieldSymbol::Hour(hour) => match hour { + Hour::H11 => 'K', + Hour::H12 => 'h', + Hour::H23 => 'H', + Hour::H24 => 'k', + }, + FieldSymbol::Minute => 'm', + FieldSymbol::Second(second) => match second { + Second::Second => 's', + Second::FractionalSecond => 'S', + Second::Millisecond => 'A', + }, + } + } +} + #[derive(Debug, PartialEq, Clone, Copy)] pub enum Year { Calendar, diff --git a/components/datetime/src/pattern/mod.rs b/components/datetime/src/pattern/mod.rs index 028e155ca10..ae27fec456a 100644 --- a/components/datetime/src/pattern/mod.rs +++ b/components/datetime/src/pattern/mod.rs @@ -7,8 +7,11 @@ mod parser; use crate::fields::{self, Field, FieldLength, FieldSymbol}; pub use error::Error; use parser::Parser; -use std::convert::TryFrom; use std::iter::FromIterator; +use std::{convert::TryFrom, fmt}; + +#[cfg(feature = "provider_serde")] +use serde::{de, ser, Deserialize, Deserializer, Serialize}; #[derive(Debug, PartialEq, Clone)] pub enum PatternItem { @@ -107,8 +110,174 @@ impl From> for Pattern { } } +impl From<&Pattern> for String { + fn from(pattern: &Pattern) -> Self { + let mut string = String::new(); + + for pattern_item in pattern.items().iter() { + match pattern_item { + PatternItem::Field(field) => { + let ch: char = field.symbol.into(); + for _ in 0..field.length as usize { + string.push(ch); + } + } + PatternItem::Literal(literal) => { + // Determine if the literal contains any characters that would need to be escaped. + let mut needs_escaping = false; + for ch in literal.chars() { + if ch.is_ascii_alphabetic() || ch == '\'' { + needs_escaping = true; + break; + } + } + + if needs_escaping { + let mut ch_iter = literal.trim_end().chars().peekable(); + + // Do not escape the leading whitespace. + while let Some(ch) = ch_iter.peek() { + if ch.is_whitespace() { + string.push(*ch); + ch_iter.next(); + } else { + break; + } + } + + // Wrap in "'" and escape "'". + string.push('\''); + for ch in ch_iter { + if ch == '\'' { + // Escape a single quote. + string.push('\\'); + } + string.push(ch); + } + string.push('\''); + + // Add the trailing whitespace + for ch in literal.chars().rev() { + if ch.is_whitespace() { + string.push(ch); + } else { + break; + } + } + } else { + string.push_str(literal); + } + } + } + } + + string + } +} + impl FromIterator for Pattern { fn from_iter>(iter: I) -> Self { Self::from(iter.into_iter().collect::>()) } } + +#[cfg(feature = "provider_serde")] +struct DeserializePatternVisitor; + +#[cfg(feature = "provider_serde")] +impl<'de> de::Visitor<'de> for DeserializePatternVisitor { + type Value = Pattern; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "Expected to find a valid pattern.") + } + + fn visit_str(self, pattern_string: &str) -> Result + where + E: de::Error, + { + // Parse a string into a list of fields. + let pattern = match Pattern::from_bytes(pattern_string) { + Ok(p) => p, + Err(err) => { + return Err(E::custom(format!( + "The pattern \"{}\" could not be parsed: {:?}", + pattern_string, err + ))); + } + }; + Ok(pattern) + } +} + +#[cfg(feature = "provider_serde")] +impl<'de> Deserialize<'de> for Pattern { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(DeserializePatternVisitor) + } +} + +#[cfg(feature = "provider_serde")] +impl Serialize for Pattern { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + let string: String = String::from(self); + serializer.serialize_str(&string) + } +} + +#[cfg(all(test, feature = "provider_serde"))] +mod test { + use super::*; + + // Provide a few patterns to sanity check serialization. + const PATTERNS: [&str; 6] = [ + "d", + "E, M/d/y", + "h:mm:ss a", + // TODO(#502) - The W field isn't supported yet, so swap it out for a field we do know. + "'week' d 'of' MMMM", + // Arabic "week of" in the availableFormats. + "الأسبوع d من MMMM", + // Cherokee "week of" in the availableFormats. + "ᏒᎾᏙᏓᏆᏍᏗ’ d ’ᎾᎿ’ MMMM", + ]; + + #[test] + fn test_pattern_serialization_roundtrip() { + for pattern_string in &PATTERNS { + // Wrap the string in quotes so it's a JSON string. + let json_in: String = serde_json::to_string(pattern_string).unwrap(); + + let pattern: Pattern = match serde_json::from_str(&json_in) { + Ok(p) => p, + Err(err) => { + panic!( + "Unable to parse the pattern {:?}. {:?}", + pattern_string, err + ); + } + }; + + let json_out = match serde_json::to_string(&pattern) { + Ok(s) => s, + Err(err) => { + panic!( + "Unable to re-serialize the pattern {:?}. {:?}", + pattern_string, err + ); + } + }; + + assert_eq!( + json_in, json_out, + "The roundtrip serialization for the pattern matched." + ); + } + } +} diff --git a/components/datetime/src/provider/helpers.rs b/components/datetime/src/provider/helpers.rs index b1086028b6d..bdd016e0ed2 100644 --- a/components/datetime/src/provider/helpers.rs +++ b/components/datetime/src/provider/helpers.rs @@ -89,10 +89,10 @@ impl DateTimePatterns for provider::gregory::PatternsV1 { ) -> Result { let date_time = &self.date_time; let s = match style { - style::Date::Full => &date_time.full, - style::Date::Long => &date_time.long, - style::Date::Medium => &date_time.medium, - style::Date::Short => &date_time.short, + style::Date::Full => &date_time.style_patterns.full, + style::Date::Long => &date_time.style_patterns.long, + style::Date::Medium => &date_time.style_patterns.medium, + style::Date::Short => &date_time.style_patterns.short, }; Ok(Pattern::from_bytes_combination(s, date, time)?) } diff --git a/components/datetime/src/provider/mod.rs b/components/datetime/src/provider/mod.rs index 2cff862aca8..8ee481b41b5 100644 --- a/components/datetime/src/provider/mod.rs +++ b/components/datetime/src/provider/mod.rs @@ -14,6 +14,7 @@ pub mod key { } pub mod gregory { + use smallvec::SmallVec; use std::borrow::Cow; #[derive(Debug, PartialEq, Clone, Default)] @@ -50,7 +51,7 @@ pub mod gregory { pub time: patterns::StylePatternsV1, - pub date_time: patterns::StylePatternsV1, + pub date_time: patterns::DateTimeFormatsV1, } macro_rules! symbols { @@ -182,6 +183,19 @@ pub mod gregory { pub mod patterns { use super::*; + use crate::{ + fields::{self, Field, FieldLength, FieldSymbol, LengthError, SymbolError}, + pattern::{self, Pattern}, + }; + use std::fmt; + use std::{cmp::Ordering, convert::TryFrom}; + + #[cfg(feature = "provider_serde")] + use serde::{ + de::{self, Visitor}, + ser, Deserialize, Deserializer, Serialize, + }; + #[derive(Debug, PartialEq, Clone, Default)] #[cfg_attr( feature = "provider_serde", @@ -193,5 +207,380 @@ pub mod gregory { pub medium: Cow<'static, str>, pub short: Cow<'static, str>, } + + #[derive(Debug, PartialEq, Clone, Default)] + pub struct SkeletonFieldsV1(pub SmallVec<[fields::Field; 5]>); + + impl SkeletonFieldsV1 { + /// [`SkeletonTupleV1`] structs should be ordered by the `SkeletonFieldsV1` canonical + /// order. This helps ensure that the skeleton serialization is sorted deterministically. + pub fn compare_canonical_order(&self, other: &SkeletonFieldsV1) -> Ordering { + let fields_a = &self.0; + let fields_b = &other.0; + let max_len = fields_a.len().max(fields_b.len()); + + for i in 0..max_len { + let maybe_field_a = fields_a.get(i); + let maybe_field_b = fields_b.get(i); + + match maybe_field_a { + Some(field_a) => match maybe_field_b { + Some(field_b) => { + let order_a = field_a.symbol.get_canonical_order(); + let order_b = field_b.symbol.get_canonical_order(); + if order_a < order_b { + return Ordering::Less; + } + if order_a > order_b { + return Ordering::Greater; + } + + // If the fields are equivalent, try to sort by field length. + let length_a = field_a.length as u8; + let length_b = field_b.length as u8; + + if length_a < length_b { + return Ordering::Less; + } + if length_a > length_b { + return Ordering::Greater; + } + } + None => { + // Field A has more fields. + return Ordering::Greater; + } + }, + None => { + // Field B has more fields. + return Ordering::Less; + } + }; + } + + Ordering::Equal + } + } + + /// This struct is a public wrapper around the internal Pattern struct. This allows + /// access to the serialization and deserialization capabilities, without exposing the + /// internals of the pattern machinery. + #[derive(Debug, PartialEq, Clone, Default)] + #[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) + )] + pub struct PatternV1(pub Pattern); + + impl From for PatternV1 { + fn from(pattern: Pattern) -> Self { + Self(pattern) + } + } + + impl TryFrom<&str> for PatternV1 { + type Error = pattern::Error; + + fn try_from(pattern_string: &str) -> Result { + let pattern = Pattern::from_bytes(pattern_string); + match pattern { + Ok(pattern) => Ok(PatternV1::from(pattern)), + Err(err) => Err(err), + } + } + } + + /// The serialization and deserialization errors need to be mapped to a public-facing + /// error enum. + #[derive(Debug, PartialEq)] + pub enum SkeletonFieldsV1Error { + SymbolUnimplemented(u8), + SymbolUnknown(u8), + SymbolInvalid(char), + FieldTooLong, + } + + impl From for SkeletonFieldsV1Error { + fn from(err: SymbolError) -> Self { + match err { + SymbolError::Invalid(ch) => SkeletonFieldsV1Error::SymbolInvalid(ch), + SymbolError::Unknown(byte) => { + match byte { + // TODO(#487) - Flexible day periods + b'B' + // TODO(#486) - Era + | b'G' + // TODO(#418) - Timezones + | b'Z' | b'v' + // TODO(#502) - Week of month + | b'W' + // TODO(#501) - Quarters + | b'Q' + // TODO (#488) - Week of year + | b'w' + => SkeletonFieldsV1Error::SymbolUnimplemented(byte), + _ => SkeletonFieldsV1Error::SymbolUnknown(byte), + } + } + } + } + } + + impl From for SkeletonFieldsV1Error { + fn from(err: LengthError) -> Self { + match err { + LengthError::TooLong => SkeletonFieldsV1Error::FieldTooLong, + } + } + } + + impl TryFrom<&str> for SkeletonFieldsV1 { + type Error = SkeletonFieldsV1Error; + fn try_from(skeleton_string: &str) -> Result { + // Parse a string into a list of fields. + let mut fields = SmallVec::new(); + + let mut iter = skeleton_string.bytes().peekable(); + while let Some(byte) = iter.next() { + // Convert the byte to a valid field symbol. + let field_symbol = FieldSymbol::try_from(byte)?; + + // Go through the bytes to count how often it's repeated. + let mut field_length: u8 = 1; + while let Some(next_byte) = iter.peek() { + if *next_byte != byte { + break; + } + field_length += 1; + iter.next(); + } + + // Add the field. + fields.push(Field::from(( + field_symbol, + FieldLength::try_from(field_length)?, + ))); + } + + Ok(SkeletonFieldsV1(fields)) + } + } + + /// This is an implementation of the serde deserialization visitor pattern. + #[cfg(feature = "provider_serde")] + struct SkeletonFieldsDeserializeVisitor; + + #[cfg(feature = "provider_serde")] + impl<'de> Visitor<'de> for SkeletonFieldsDeserializeVisitor { + type Value = SkeletonFieldsV1; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "Expected to find a valid skeleton.") + } + + /// A skeleton serialized into a string follows UTS 35. + /// https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table + /// This string consists of a symbol that is repeated N times. This string is + /// deserialized here into the SkeletonFieldsV1 format which is used in memory + /// when working with formatting date times. + fn visit_str(self, skeleton_string: &str) -> Result + where + E: de::Error, + { + SkeletonFieldsV1::try_from(skeleton_string).map_err(|err| match err { + SkeletonFieldsV1Error::SymbolUnknown(byte) => E::custom(format!( + "Skeleton {:?} contained an unknown symbol, {:?}", + skeleton_string, byte as char + )), + SkeletonFieldsV1Error::SymbolUnimplemented(byte) => E::custom(format!( + "Skeleton {:?} contained an unimplemented symbol, {:?}", + skeleton_string, byte as char + )), + SkeletonFieldsV1Error::FieldTooLong => E::custom(format!( + "Skeleton \"{}\" contained a field that was too long.", + skeleton_string + )), + SkeletonFieldsV1Error::SymbolInvalid(ch) => E::custom(format!( + "Skeleton {:?} contained an invalid symbol, {:?}", + skeleton_string, ch + )), + }) + } + } + + #[cfg(feature = "provider_serde")] + impl<'de> Deserialize<'de> for SkeletonFieldsV1 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(SkeletonFieldsDeserializeVisitor) + } + } + + #[cfg(feature = "provider_serde")] + impl Serialize for SkeletonFieldsV1 { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + let mut string = String::new(); + + for field in self.0.iter() { + let ch: char = field.symbol.into(); + for _ in 0..field.length as usize { + string.push(ch); + } + } + + serializer.serialize_str(&string) + } + } + + #[derive(Debug, PartialEq, Clone, Default)] + #[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) + )] + pub struct SkeletonTupleV1(pub SkeletonFieldsV1, pub PatternV1); + + #[derive(Debug, PartialEq, Clone, Default)] + #[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) + )] + pub struct DateTimeFormatsV1 { + pub style_patterns: StylePatternsV1, + pub skeletons: Vec, + } + } +} + +#[cfg(test)] +mod test { + use super::gregory::patterns::*; + use crate::fields::{Day, Field, FieldLength, Month, Weekday}; + use std::{cmp::Ordering, convert::TryFrom}; + + // These were all of the skeletons from the "available formats" in the CLDR as of 2021-01 + // Generated with: + // https://gist.github.com/gregtatum/1d76bbdb87132f71a969a10f0c1d2d9c + + #[rustfmt::skip] + const SUPPORTED_STRING_SKELETONS: [&str; 51] = [ + "E", "EEEEd", "EHm", "EHms", "Ed", "Ehm", "Ehms", "H", "HHmm", "HHmmss", "Hm", "Hms", "M", + "MEEEEd", "MEd", "MMM", "MMMEEEEd", "MMMEd", "MMMM", "MMMMEEEEd", "MMMMEd", "MMMMd", + "MMMMdd", "MMMd", "MMMdd", "MMd", "MMdd", "Md", "Mdd", "d", "h", "hm", "hms", "mmss", "ms", + "y", "yM", "yMEEEEd", "yMEd", "yMM", "yMMM", "yMMMEEEEd", "yMMMEd", "yMMMM", "yMMMMEEEEd", + "yMMMMEd", "yMMMMccccd", "yMMMMd", "yMMMd", "yMMdd", "yMd", + ]; + + #[rustfmt::skip] + const UNSUPPORTED_STRING_SKELETONS: [&str; 28] = [ + // TODO(#487) - Flexible day periods + "Bh", "Bhm", "Bhms", "EBhm", "EBhms", + // TODO(#486) - Era + "Gy", "GyM", "GyMMM", "GyMMMEEEEd", "GyMMMEd", "GyMMMM", "GyMMMMEd", "GyMMMMd", "GyMMMd", + // TODO(#418) - Timezones + "HHmmZ", "Hmsv", "Hmsvvvv", "Hmv", "Hmvvvv", "hmsv", "hmsvvvv", "hmv", "hmvvvv", + // TODO(#502) - Week of month + "MMMMW", + // TODO(#501) - Quarters + "yQ", "yQQQ", "yQQQQ", + // TODO (#488) - Week of year + "yw" + ]; + + #[test] + fn test_known_skeletons_ok() { + for string_skeleton in &SUPPORTED_STRING_SKELETONS { + match SkeletonFieldsV1::try_from(*string_skeleton) { + Ok(_) => {} + Err(err) => { + panic!( + "Unable to parse string_skeleton {:?} with error, {:?}", + string_skeleton, err + ) + } + } + } + } + + #[test] + fn test_unsupported_skeletons_skeletons_err() { + for string_skeleton in &UNSUPPORTED_STRING_SKELETONS { + match SkeletonFieldsV1::try_from(*string_skeleton) { + Ok(_) => { + panic!( + "An unsupported field is now supported, consider moving {:?} to the \ + supported skeletons, and ensure the skeleton is properly implemented.", + string_skeleton + ) + } + Err(err) => match err { + SkeletonFieldsV1Error::SymbolUnimplemented(_) => { + // Every skeleton should return this error. + } + SkeletonFieldsV1Error::SymbolUnknown(byte) => { + panic!( + "An unknown symbol {:?} was found in the skeleton {:?}", + byte as char, string_skeleton + ) + } + SkeletonFieldsV1Error::FieldTooLong => { + panic!("An field was too longin the skeleton {:?}", string_skeleton) + } + SkeletonFieldsV1Error::SymbolInvalid(ch) => { + panic!( + "An invalid symbol {:?} was found in the skeleton {:?}", + ch, string_skeleton + ) + } + }, + } + } + } + + #[test] + fn test_skeleton_deserialization() { + assert_eq!( + SkeletonFieldsV1::try_from("MMMMEEEEd"), + Ok(SkeletonFieldsV1( + vec![ + Field { + symbol: Month::Format.into(), + length: FieldLength::Wide + }, + Field { + symbol: Weekday::Format.into(), + length: FieldLength::Wide + }, + Field { + symbol: Day::DayOfMonth.into(), + length: FieldLength::One + } + ] + .into() + )) + ); + } + + #[test] + fn test_skeleton_tuple_ordering() { + let skeletons_strings = Vec::from([ + "y", "yM", "yMEd", "yMEEEEd", "yMMM", "M", "Md", "Mdd", "MMd", "MMdd", "d", "h", "hm", + "hms", "Hm", "Hms", "ms", "mmss", + ]); + + let skeleton_fields: Vec = skeletons_strings + .iter() + .map(|skeleton_string| SkeletonFieldsV1::try_from(*skeleton_string).unwrap()) + .collect(); + + for (strings, fields) in skeletons_strings.windows(2).zip(skeleton_fields.windows(2)) { + if fields[0].compare_canonical_order(&fields[1]) != Ordering::Less { + panic!("Expected {:?} < {:?}", strings[0], strings[1]); + } + } } } diff --git a/components/datetime/tests/datetime.rs b/components/datetime/tests/datetime.rs index 7a9bef2a2b2..395f303a389 100644 --- a/components/datetime/tests/datetime.rs +++ b/components/datetime/tests/datetime.rs @@ -20,7 +20,10 @@ use std::{borrow::Cow, fmt::Write}; fn test_fixture(fixture_name: &str) { let provider = icu_testdata::get_provider(); - for fx in fixtures::get_fixture(fixture_name).unwrap().0 { + for fx in fixtures::get_fixture(fixture_name) + .expect("Unable to get fixture.") + .0 + { let locale: Locale = fx.input.locale.parse().unwrap(); let options = fixtures::get_options(&fx.input.options); let dtf = DateTimeFormat::try_new(locale, &provider, &options).unwrap(); @@ -63,7 +66,13 @@ fn test_dayperiod_patterns() { .unwrap() .take_payload() .unwrap(); - *data.to_mut().patterns.date_time.long.to_mut() = String::from("{0}"); + *data + .to_mut() + .patterns + .date_time + .style_patterns + .long + .to_mut() = String::from("{0}"); for test_case in &test.test_cases { for dt_input in &test_case.date_times { let date_time: MockDateTime = dt_input.parse().unwrap(); diff --git a/components/provider_cldr/Cargo.toml b/components/provider_cldr/Cargo.toml index e3c7c6ccf0d..983e285a149 100644 --- a/components/provider_cldr/Cargo.toml +++ b/components/provider_cldr/Cargo.toml @@ -37,6 +37,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde-tuple-vec-map = "1.0" tinystr = "0.4" +smallvec = "1.4" # Dependencies for the download feature urlencoding = { version = "1.1", optional = true } diff --git a/components/provider_cldr/src/transform/dates.rs b/components/provider_cldr/src/transform/dates.rs index b88a59a45cc..5ba51000477 100644 --- a/components/provider_cldr/src/transform/dates.rs +++ b/components/provider_cldr/src/transform/dates.rs @@ -115,6 +115,83 @@ impl From<&cldr_json::StylePatterns> for gregory::patterns::StylePatternsV1 { } } +impl From<&cldr_json::DateTimeFormats> for gregory::patterns::DateTimeFormatsV1 { + fn from(other: &cldr_json::DateTimeFormats) -> Self { + use gregory::patterns::{ + PatternV1, SkeletonFieldsV1, SkeletonFieldsV1Error, SkeletonTupleV1, + }; + + // TODO(#308): Support numbering system variations. We currently throw them away. + Self { + style_patterns: gregory::patterns::StylePatternsV1 { + full: other.full.get_pattern().clone(), + long: other.long.get_pattern().clone(), + medium: other.medium.get_pattern().clone(), + short: other.short.get_pattern().clone(), + }, + skeletons: { + let mut skeleton_tuples: Vec = Vec::new(); + + // The CLDR keys for available_formats can have duplicate skeletons with either + // an additional variant, or with multiple variants for different plurals. + for (skeleton_str, pattern_str) in other.available_formats.0.iter() { + let mut unique_skeleton = String::new(); + let mut variant_text = String::new(); + + // Split out the unique skeleton component, and the variant text. + // "MMMMW-count-two" + // unique_skeleton: "MMMMW" + // variant_text: "-count-two" + let chars = skeleton_str.chars(); + let mut target = &mut unique_skeleton; + for ch in chars { + if ch == '-' { + target = &mut variant_text; + } + target.push(ch); + } + + let skeleton_fields_v1 = + match SkeletonFieldsV1::try_from(unique_skeleton.as_str()) { + Ok(s) => s, + Err(err) => match err { + // Ignore unimplemented fields for now. + SkeletonFieldsV1Error::SymbolUnimplemented(_) => continue, + SkeletonFieldsV1Error::SymbolUnknown(symbol) => panic!( + "Unknown symbol {:?} in skeleton {:?}", + symbol, unique_skeleton + ), + SkeletonFieldsV1Error::FieldTooLong => { + panic!("Field too long in skeleton {:?}", unique_skeleton) + } + SkeletonFieldsV1Error::SymbolInvalid(symbol) => panic!( + "Symbol invalid {:?} in skeleton {:?}", + symbol, unique_skeleton + ), + }, + }; + + if !variant_text.is_empty() && variant_text != "-alt-variant" { + unimplemented!( + "This skeleton string is not yet supported: {:?}", + skeleton_str + ); + } + + let pattern_v1 = PatternV1::try_from(pattern_str as &str) + .expect("Unable to parse a pattern"); + + skeleton_tuples.push(SkeletonTupleV1(skeleton_fields_v1, pattern_v1)); + } + + // Sort the skeletons deterministically by their canonical sort order. + skeleton_tuples.sort_by(|a, b| a.0.compare_canonical_order(&b.0)); + skeleton_tuples + }, + } + } +} + impl From<&cldr_json::Dates> for gregory::DatesV1 { fn from(other: &cldr_json::Dates) -> Self { Self { @@ -374,6 +451,26 @@ pub(self) mod cldr_json { pub short: StylePattern, } + #[derive(PartialEq, Debug, Deserialize)] + pub struct DateTimeFormats { + pub full: StylePattern, + pub long: StylePattern, + pub medium: StylePattern, + pub short: StylePattern, + #[serde(rename = "availableFormats")] + pub available_formats: AvailableFormats, + } + + #[derive(PartialEq, Clone, Debug, Deserialize)] + pub struct AvailableFormats( + #[serde(with = "tuple_vec_map")] pub(crate) Vec<(Cow<'static, str>, Cow<'static, str>)>, + ); + + /// This struct represents a 1:1 mapping of the CLDR ca-gregorian.json data at the key + /// "main.LANGID.dates.calendars.gregorian" where "LANGID" is the identifier. + /// + /// e.g. + /// https://github.com/unicode-org/cldr-json/blob/master/cldr-json/cldr-dates-full/main/en/ca-gregorian.json #[derive(PartialEq, Debug, Deserialize)] pub struct GregoryDates { pub months: months::Contexts, @@ -385,7 +482,7 @@ pub(self) mod cldr_json { #[serde(rename = "timeFormats")] pub time_formats: StylePatterns, #[serde(rename = "dateTimeFormats")] - pub date_time_formats: StylePatterns, + pub date_time_formats: DateTimeFormats, } #[derive(PartialEq, Debug, Deserialize)] From 2884ccdea972372c27d41e5a4af8d4722a34b51c Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Fri, 26 Feb 2021 10:46:34 -0600 Subject: [PATCH 02/22] Part 2: Re-compute the testdata for skeletons --- .../data/json/dates/gregory@1/ar-EG.json | 140 ++++++++++++++- .../data/json/dates/gregory@1/ar.json | 140 ++++++++++++++- .../data/json/dates/gregory@1/bn.json | 140 ++++++++++++++- .../data/json/dates/gregory@1/ccp.json | 140 ++++++++++++++- .../json/dates/gregory@1/en-US-posix.json | 128 +++++++++++++- .../data/json/dates/gregory@1/en-ZA.json | 132 ++++++++++++++- .../data/json/dates/gregory@1/en.json | 128 +++++++++++++- .../data/json/dates/gregory@1/es-AR.json | 156 ++++++++++++++++- .../data/json/dates/gregory@1/es.json | 152 ++++++++++++++++- .../data/json/dates/gregory@1/fr.json | 128 +++++++++++++- .../data/json/dates/gregory@1/ja.json | 152 ++++++++++++++++- .../data/json/dates/gregory@1/ru.json | 136 ++++++++++++++- .../data/json/dates/gregory@1/sr-Cyrl.json | 148 +++++++++++++++- .../data/json/dates/gregory@1/sr-Latn.json | 148 +++++++++++++++- .../data/json/dates/gregory@1/sr.json | 148 +++++++++++++++- .../data/json/dates/gregory@1/th.json | 160 +++++++++++++++++- .../data/json/dates/gregory@1/tr.json | 140 ++++++++++++++- .../data/json/dates/gregory@1/und.json | 128 +++++++++++++- 18 files changed, 2472 insertions(+), 72 deletions(-) diff --git a/resources/testdata/data/json/dates/gregory@1/ar-EG.json b/resources/testdata/data/json/dates/gregory@1/ar-EG.json index ff240b1cf51..72a8119daa1 100644 --- a/resources/testdata/data/json/dates/gregory@1/ar-EG.json +++ b/resources/testdata/data/json/dates/gregory@1/ar-EG.json @@ -123,10 +123,142 @@ "short": "h:mm a" }, "date_time": { - "full": "{1} في {0}", - "long": "{1} في {0}", - "medium": "{1}, {0}", - "short": "{1}, {0}" + "style_patterns": { + "full": "{1} في {0}", + "long": "{1} في {0}", + "medium": "{1}, {0}", + "short": "{1}, {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M‏/y" + ], + [ + "yMd", + "d‏/M‏/y" + ], + [ + "yMEd", + "E، d/‏M/‏y" + ], + [ + "yMM", + "MM‏/y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d MMM y" + ], + [ + "yMMMEd", + "E، d MMM y" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "d/‏M" + ], + [ + "MEd", + "E، d/‏M" + ], + [ + "MMdd", + "dd‏/MM" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "E، d MMM" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "MMMMEd", + "E، d MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "E، d" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/ar.json b/resources/testdata/data/json/dates/gregory@1/ar.json index ff240b1cf51..72a8119daa1 100644 --- a/resources/testdata/data/json/dates/gregory@1/ar.json +++ b/resources/testdata/data/json/dates/gregory@1/ar.json @@ -123,10 +123,142 @@ "short": "h:mm a" }, "date_time": { - "full": "{1} في {0}", - "long": "{1} في {0}", - "medium": "{1}, {0}", - "short": "{1}, {0}" + "style_patterns": { + "full": "{1} في {0}", + "long": "{1} في {0}", + "medium": "{1}, {0}", + "short": "{1}, {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M‏/y" + ], + [ + "yMd", + "d‏/M‏/y" + ], + [ + "yMEd", + "E، d/‏M/‏y" + ], + [ + "yMM", + "MM‏/y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d MMM y" + ], + [ + "yMMMEd", + "E، d MMM y" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "d/‏M" + ], + [ + "MEd", + "E، d/‏M" + ], + [ + "MMdd", + "dd‏/MM" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "E، d MMM" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "MMMMEd", + "E، d MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "E، d" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/bn.json b/resources/testdata/data/json/dates/gregory@1/bn.json index 326d3da0eec..6302713cc6b 100644 --- a/resources/testdata/data/json/dates/gregory@1/bn.json +++ b/resources/testdata/data/json/dates/gregory@1/bn.json @@ -133,10 +133,142 @@ "short": "h:mm a" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M/y" + ], + [ + "yMd", + "d/M/y" + ], + [ + "yMEd", + "E, d/M/y" + ], + [ + "yMM", + "MM-y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d MMM, y" + ], + [ + "yMMMEd", + "E, d MMM, y" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "d/M" + ], + [ + "MEd", + "E, d-M" + ], + [ + "MMdd", + "dd-MM" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "E d MMM" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "MMMMEd", + "E d MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "d E" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/ccp.json b/resources/testdata/data/json/dates/gregory@1/ccp.json index dbc2db536da..c8281da3340 100644 --- a/resources/testdata/data/json/dates/gregory@1/ccp.json +++ b/resources/testdata/data/json/dates/gregory@1/ccp.json @@ -147,10 +147,142 @@ "short": "h:mm a" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M/y" + ], + [ + "yMd", + "d/M/y" + ], + [ + "yMEd", + "E, d/M/y" + ], + [ + "yMM", + "MM-y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d MMM, y" + ], + [ + "yMMMEd", + "E, d MMM, y" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "d/M" + ], + [ + "MEd", + "E, d-M" + ], + [ + "MMdd", + "dd-MM" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "E d MMM" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "MMMMEd", + "E d MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "d E" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/en-US-posix.json b/resources/testdata/data/json/dates/gregory@1/en-US-posix.json index 9f2d6455431..91bc4ed8d35 100644 --- a/resources/testdata/data/json/dates/gregory@1/en-US-posix.json +++ b/resources/testdata/data/json/dates/gregory@1/en-US-posix.json @@ -131,10 +131,130 @@ "short": "h:mm a" }, "date_time": { - "full": "{1} 'at' {0}", - "long": "{1} 'at' {0}", - "medium": "{1}, {0}", - "short": "{1}, {0}" + "style_patterns": { + "full": "{1} 'at' {0}", + "long": "{1} 'at' {0}", + "medium": "{1}, {0}", + "short": "{1}, {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M/y" + ], + [ + "yMd", + "M/d/y" + ], + [ + "yMEd", + "E, M/d/y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "MMM d, y" + ], + [ + "yMMMEd", + "E, MMM d, y" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "M/d" + ], + [ + "MEd", + "E, M/d" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "MMM d" + ], + [ + "MMMEd", + "E, MMM d" + ], + [ + "MMMMd", + "MMMM d" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "d E" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/en-ZA.json b/resources/testdata/data/json/dates/gregory@1/en-ZA.json index 00092e2f8dd..97636b9fddd 100644 --- a/resources/testdata/data/json/dates/gregory@1/en-ZA.json +++ b/resources/testdata/data/json/dates/gregory@1/en-ZA.json @@ -131,10 +131,134 @@ "short": "HH:mm" }, "date_time": { - "full": "{1} 'at' {0}", - "long": "{1} 'at' {0}", - "medium": "{1}, {0}", - "short": "{1}, {0}" + "style_patterns": { + "full": "{1} 'at' {0}", + "long": "{1} 'at' {0}", + "medium": "{1}, {0}", + "short": "{1}, {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "MM/y" + ], + [ + "yMd", + "y/MM/dd" + ], + [ + "yMEd", + "E, y/MM/dd" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "dd MMM y" + ], + [ + "yMMMEd", + "E, dd MMM y" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "MM/dd" + ], + [ + "MEd", + "E, MM/dd" + ], + [ + "MMdd", + "dd/MM" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "dd MMM" + ], + [ + "MMMEd", + "E, dd MMM" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "E d" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/en.json b/resources/testdata/data/json/dates/gregory@1/en.json index 9f2d6455431..91bc4ed8d35 100644 --- a/resources/testdata/data/json/dates/gregory@1/en.json +++ b/resources/testdata/data/json/dates/gregory@1/en.json @@ -131,10 +131,130 @@ "short": "h:mm a" }, "date_time": { - "full": "{1} 'at' {0}", - "long": "{1} 'at' {0}", - "medium": "{1}, {0}", - "short": "{1}, {0}" + "style_patterns": { + "full": "{1} 'at' {0}", + "long": "{1} 'at' {0}", + "medium": "{1}, {0}", + "short": "{1}, {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M/y" + ], + [ + "yMd", + "M/d/y" + ], + [ + "yMEd", + "E, M/d/y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "MMM d, y" + ], + [ + "yMMMEd", + "E, MMM d, y" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "M/d" + ], + [ + "MEd", + "E, M/d" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "MMM d" + ], + [ + "MMMEd", + "E, MMM d" + ], + [ + "MMMMd", + "MMMM d" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "d E" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/es-AR.json b/resources/testdata/data/json/dates/gregory@1/es-AR.json index 4bd729e48d0..696f23f9b88 100644 --- a/resources/testdata/data/json/dates/gregory@1/es-AR.json +++ b/resources/testdata/data/json/dates/gregory@1/es-AR.json @@ -138,10 +138,158 @@ "short": "HH:mm" }, "date_time": { - "full": "{1}, {0}", - "long": "{1}, {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1}, {0}", + "long": "{1}, {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M-y" + ], + [ + "yMd", + "d/M/y" + ], + [ + "yMEd", + "E, d/M/y" + ], + [ + "yMM", + "M/y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d 'de' MMM 'de' y" + ], + [ + "yMMMEd", + "E, d MMM y" + ], + [ + "yMMMM", + "MMMM 'de' y" + ], + [ + "yMMMMd", + "d 'de' MMMM 'de' y" + ], + [ + "yMMMMEd", + "EEE, d 'de' MMMM 'de' y" + ], + [ + "M", + "L" + ], + [ + "Md", + "d/M" + ], + [ + "MEd", + "E d-M" + ], + [ + "MMd", + "d/M" + ], + [ + "MMdd", + "d/M" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMdd", + "dd-MMM" + ], + [ + "MMMEd", + "E, d MMM" + ], + [ + "MMMMd", + "d 'de' MMMM" + ], + [ + "MMMMEd", + "E, d 'de' MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "E d" + ], + [ + "Ehm", + "E, h:mm a" + ], + [ + "Ehms", + "E, h:mm:ss a" + ], + [ + "EHm", + "E, HH:mm" + ], + [ + "EHms", + "E, HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "hh:mm:ss" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/es.json b/resources/testdata/data/json/dates/gregory@1/es.json index 4f65f34304a..a8a29cb01af 100644 --- a/resources/testdata/data/json/dates/gregory@1/es.json +++ b/resources/testdata/data/json/dates/gregory@1/es.json @@ -137,10 +137,154 @@ "short": "H:mm" }, "date_time": { - "full": "{1}, {0}", - "long": "{1}, {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1}, {0}", + "long": "{1}, {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M/y" + ], + [ + "yMd", + "d/M/y" + ], + [ + "yMEd", + "EEE, d/M/y" + ], + [ + "yMM", + "M/y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d MMM y" + ], + [ + "yMMMEd", + "EEE, d MMM y" + ], + [ + "yMMMM", + "MMMM 'de' y" + ], + [ + "yMMMMd", + "d 'de' MMMM 'de' y" + ], + [ + "yMMMMEd", + "EEE, d 'de' MMMM 'de' y" + ], + [ + "M", + "L" + ], + [ + "Md", + "d/M" + ], + [ + "MEd", + "E, d/M" + ], + [ + "MMd", + "d/M" + ], + [ + "MMdd", + "d/M" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "E, d MMM" + ], + [ + "MMMMd", + "d 'de' MMMM" + ], + [ + "MMMMEd", + "E, d 'de' MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "E d" + ], + [ + "Ehm", + "E, h:mm a" + ], + [ + "Ehms", + "E, h:mm:ss a" + ], + [ + "EHm", + "E, H:mm" + ], + [ + "EHms", + "E, H:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "H" + ], + [ + "Hm", + "H:mm" + ], + [ + "Hms", + "H:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/fr.json b/resources/testdata/data/json/dates/gregory@1/fr.json index 7460fc17138..10ac353a439 100644 --- a/resources/testdata/data/json/dates/gregory@1/fr.json +++ b/resources/testdata/data/json/dates/gregory@1/fr.json @@ -123,10 +123,130 @@ "short": "HH:mm" }, "date_time": { - "full": "{1} 'à' {0}", - "long": "{1} 'à' {0}", - "medium": "{1}, {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} 'à' {0}", + "long": "{1} 'à' {0}", + "medium": "{1}, {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "MM/y" + ], + [ + "yMd", + "dd/MM/y" + ], + [ + "yMEd", + "E dd/MM/y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d MMM y" + ], + [ + "yMMMEd", + "E d MMM y" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "dd/MM" + ], + [ + "MEd", + "E dd/MM" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "E d MMM" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "E" + ], + [ + "Ed", + "E d" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH 'h'" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/ja.json b/resources/testdata/data/json/dates/gregory@1/ja.json index e601cccb4a6..e8c08dccaa7 100644 --- a/resources/testdata/data/json/dates/gregory@1/ja.json +++ b/resources/testdata/data/json/dates/gregory@1/ja.json @@ -123,10 +123,154 @@ "short": "H:mm" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y年" + ], + [ + "yM", + "y/M" + ], + [ + "yMd", + "y/M/d" + ], + [ + "yMEd", + "y/M/d(E)" + ], + [ + "yMEEEEd", + "y/M/dEEEE" + ], + [ + "yMM", + "y/MM" + ], + [ + "yMMM", + "y年M月" + ], + [ + "yMMMd", + "y年M月d日" + ], + [ + "yMMMEd", + "y年M月d日(E)" + ], + [ + "yMMMEEEEd", + "y年M月d日EEEE" + ], + [ + "yMMMM", + "y年M月" + ], + [ + "M", + "M月" + ], + [ + "Md", + "M/d" + ], + [ + "MEd", + "M/d(E)" + ], + [ + "MEEEEd", + "M/dEEEE" + ], + [ + "MMM", + "M月" + ], + [ + "MMMd", + "M月d日" + ], + [ + "MMMEd", + "M月d日(E)" + ], + [ + "MMMEEEEd", + "M月d日EEEE" + ], + [ + "MMMMd", + "M月d日" + ], + [ + "d", + "d日" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "d日(E)" + ], + [ + "Ehm", + "aK:mm (E)" + ], + [ + "Ehms", + "aK:mm:ss (E)" + ], + [ + "EHm", + "H:mm (E)" + ], + [ + "EHms", + "H:mm:ss (E)" + ], + [ + "EEEEd", + "d日EEEE" + ], + [ + "h", + "aK時" + ], + [ + "hm", + "aK:mm" + ], + [ + "hms", + "aK:mm:ss" + ], + [ + "H", + "H時" + ], + [ + "Hm", + "H:mm" + ], + [ + "Hms", + "H:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/ru.json b/resources/testdata/data/json/dates/gregory@1/ru.json index 6e68c510889..b71e189382d 100644 --- a/resources/testdata/data/json/dates/gregory@1/ru.json +++ b/resources/testdata/data/json/dates/gregory@1/ru.json @@ -164,10 +164,138 @@ "short": "HH:mm" }, "date_time": { - "full": "{1}, {0}", - "long": "{1}, {0}", - "medium": "{1}, {0}", - "short": "{1}, {0}" + "style_patterns": { + "full": "{1}, {0}", + "long": "{1}, {0}", + "medium": "{1}, {0}", + "short": "{1}, {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "MM.y" + ], + [ + "yMd", + "dd.MM.y" + ], + [ + "yMEd", + "ccc, dd.MM.y г." + ], + [ + "yMM", + "MM.y" + ], + [ + "yMMM", + "LLL y г." + ], + [ + "yMMMd", + "d MMM y г." + ], + [ + "yMMMEd", + "E, d MMM y г." + ], + [ + "yMMMM", + "LLLL y г." + ], + [ + "M", + "L" + ], + [ + "Md", + "dd.MM" + ], + [ + "MEd", + "E, dd.MM" + ], + [ + "MMdd", + "dd.MM" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "ccc, d MMM" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "ccc, d" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json b/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json index 739d33f3a25..e641123993d 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json +++ b/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json @@ -131,10 +131,150 @@ "short": "HH:mm" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y." + ], + [ + "yM", + "M.y." + ], + [ + "yMd", + "d.M.y." + ], + [ + "yMEd", + "E, d.M.y." + ], + [ + "yMM", + "MM.y." + ], + [ + "yMMdd", + "dd.MM.y." + ], + [ + "yMMM", + "MMM y." + ], + [ + "yMMMd", + "d. MMM y." + ], + [ + "yMMMEd", + "E, d. MMM y." + ], + [ + "yMMMM", + "MMMM y." + ], + [ + "M", + "L" + ], + [ + "Md", + "d.M." + ], + [ + "MEd", + "E, d.M." + ], + [ + "MMdd", + "dd.MM." + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d. MMM" + ], + [ + "MMMdd", + "dd.MMM" + ], + [ + "MMMEd", + "E d. MMM" + ], + [ + "MMMMd", + "d. MMMM" + ], + [ + "MMMMEd", + "E, d. MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "E" + ], + [ + "Ed", + "E d." + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/sr-Latn.json b/resources/testdata/data/json/dates/gregory@1/sr-Latn.json index 7d24081d047..e3975bf3035 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr-Latn.json +++ b/resources/testdata/data/json/dates/gregory@1/sr-Latn.json @@ -131,10 +131,150 @@ "short": "HH:mm" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y." + ], + [ + "yM", + "M.y." + ], + [ + "yMd", + "d.M.y." + ], + [ + "yMEd", + "E, d.M.y." + ], + [ + "yMM", + "MM.y." + ], + [ + "yMMdd", + "dd.MM.y." + ], + [ + "yMMM", + "MMM y." + ], + [ + "yMMMd", + "d. MMM y." + ], + [ + "yMMMEd", + "E, d. MMM y." + ], + [ + "yMMMM", + "MMMM y." + ], + [ + "M", + "L" + ], + [ + "Md", + "d.M." + ], + [ + "MEd", + "E, d.M." + ], + [ + "MMdd", + "dd.MM." + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d. MMM" + ], + [ + "MMMdd", + "dd.MMM" + ], + [ + "MMMEd", + "E d. MMM" + ], + [ + "MMMMd", + "d. MMMM" + ], + [ + "MMMMEd", + "E, d. MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "E" + ], + [ + "Ed", + "E d." + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/sr.json b/resources/testdata/data/json/dates/gregory@1/sr.json index 739d33f3a25..e641123993d 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr.json +++ b/resources/testdata/data/json/dates/gregory@1/sr.json @@ -131,10 +131,150 @@ "short": "HH:mm" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y." + ], + [ + "yM", + "M.y." + ], + [ + "yMd", + "d.M.y." + ], + [ + "yMEd", + "E, d.M.y." + ], + [ + "yMM", + "MM.y." + ], + [ + "yMMdd", + "dd.MM.y." + ], + [ + "yMMM", + "MMM y." + ], + [ + "yMMMd", + "d. MMM y." + ], + [ + "yMMMEd", + "E, d. MMM y." + ], + [ + "yMMMM", + "MMMM y." + ], + [ + "M", + "L" + ], + [ + "Md", + "d.M." + ], + [ + "MEd", + "E, d.M." + ], + [ + "MMdd", + "dd.MM." + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d. MMM" + ], + [ + "MMMdd", + "dd.MMM" + ], + [ + "MMMEd", + "E d. MMM" + ], + [ + "MMMMd", + "d. MMMM" + ], + [ + "MMMMEd", + "E, d. MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "E" + ], + [ + "Ed", + "E d." + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/th.json b/resources/testdata/data/json/dates/gregory@1/th.json index 1e9c5c2267f..1beebdaa548 100644 --- a/resources/testdata/data/json/dates/gregory@1/th.json +++ b/resources/testdata/data/json/dates/gregory@1/th.json @@ -131,10 +131,162 @@ "short": "HH:mm" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "M/y" + ], + [ + "yMd", + "d/M/y" + ], + [ + "yMEd", + "E d/M/y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d MMM y" + ], + [ + "yMMMEd", + "E d MMM y" + ], + [ + "yMMMEEEEd", + "EEEEที่ d MMM y" + ], + [ + "yMMMM", + "MMMM 'G' y" + ], + [ + "yMMMMd", + "d MMMM 'G' y" + ], + [ + "yMMMMEd", + "E d MMMM 'G' y" + ], + [ + "yMMMMEEEEd", + "EEEEที่ d MMMM 'G' y" + ], + [ + "M", + "L" + ], + [ + "Md", + "d/M" + ], + [ + "MEd", + "E d/M" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "E d MMM" + ], + [ + "MMMEEEEd", + "EEEEที่ d MMM" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "MMMMEd", + "E d MMMM" + ], + [ + "MMMMEEEEd", + "EEEEที่ d MMMM" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "E d" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm น." + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm น." + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ], + [ + "mmss", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/tr.json b/resources/testdata/data/json/dates/gregory@1/tr.json index 7751f9f6ee2..9a702d87739 100644 --- a/resources/testdata/data/json/dates/gregory@1/tr.json +++ b/resources/testdata/data/json/dates/gregory@1/tr.json @@ -131,10 +131,142 @@ "short": "HH:mm" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "MM/y" + ], + [ + "yMd", + "dd.MM.y" + ], + [ + "yMEd", + "d.M.y E" + ], + [ + "yMM", + "MM.y" + ], + [ + "yMMM", + "MMM y" + ], + [ + "yMMMd", + "d MMM y" + ], + [ + "yMMMEd", + "d MMM y E" + ], + [ + "yMMMM", + "MMMM y" + ], + [ + "M", + "L" + ], + [ + "Md", + "d/M" + ], + [ + "MEd", + "d/MM E" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "d MMM" + ], + [ + "MMMEd", + "d MMMM E" + ], + [ + "MMMMd", + "d MMMM" + ], + [ + "MMMMEd", + "d MMMM E" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "d E" + ], + [ + "Ehm", + "E a h:mm" + ], + [ + "Ehms", + "E a h:mm:ss" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "a h" + ], + [ + "hm", + "a h:mm" + ], + [ + "hms", + "a h:mm:ss" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ], + [ + "mmss", + "mm:ss" + ] + ] } } } diff --git a/resources/testdata/data/json/dates/gregory@1/und.json b/resources/testdata/data/json/dates/gregory@1/und.json index 00074b72a6b..67e79091093 100644 --- a/resources/testdata/data/json/dates/gregory@1/und.json +++ b/resources/testdata/data/json/dates/gregory@1/und.json @@ -117,10 +117,130 @@ "short": "HH:mm" }, "date_time": { - "full": "{1} {0}", - "long": "{1} {0}", - "medium": "{1} {0}", - "short": "{1} {0}" + "style_patterns": { + "full": "{1} {0}", + "long": "{1} {0}", + "medium": "{1} {0}", + "short": "{1} {0}" + }, + "skeletons": [ + [ + "y", + "y" + ], + [ + "yM", + "y-MM" + ], + [ + "yMd", + "y-MM-dd" + ], + [ + "yMEd", + "y-MM-dd, E" + ], + [ + "yMMM", + "y MMM" + ], + [ + "yMMMd", + "y MMM d" + ], + [ + "yMMMEd", + "y MMM d, E" + ], + [ + "yMMMM", + "y MMMM" + ], + [ + "M", + "L" + ], + [ + "Md", + "MM-dd" + ], + [ + "MEd", + "MM-dd, E" + ], + [ + "MMM", + "LLL" + ], + [ + "MMMd", + "MMM d" + ], + [ + "MMMEd", + "MMM d, E" + ], + [ + "MMMMd", + "MMMM d" + ], + [ + "d", + "d" + ], + [ + "E", + "ccc" + ], + [ + "Ed", + "d, E" + ], + [ + "Ehm", + "E h:mm a" + ], + [ + "Ehms", + "E h:mm:ss a" + ], + [ + "EHm", + "E HH:mm" + ], + [ + "EHms", + "E HH:mm:ss" + ], + [ + "h", + "h a" + ], + [ + "hm", + "h:mm a" + ], + [ + "hms", + "h:mm:ss a" + ], + [ + "H", + "HH" + ], + [ + "Hm", + "HH:mm" + ], + [ + "Hms", + "HH:mm:ss" + ], + [ + "ms", + "mm:ss" + ] + ] } } } From 59789122b0edd723412dc544f8efc229eef95425 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Wed, 10 Mar 2021 14:30:34 -0600 Subject: [PATCH 03/22] Part 3: Add bincode serialization support for Patterns --- Cargo.lock | 1 + components/datetime/Cargo.toml | 1 + components/datetime/src/fields/length.rs | 4 + components/datetime/src/fields/mod.rs | 4 + components/datetime/src/fields/symbols.rs | 32 +++++ components/datetime/src/pattern/mod.rs | 111 ++++++++-------- .../tests/fixtures/tests/patterns.bin | Bin 0 -> 2601 bytes .../tests/fixtures/tests/patterns.json | 24 ++++ .../datetime/tests/pattern_serialization.rs | 120 ++++++++++++++++++ 9 files changed, 241 insertions(+), 56 deletions(-) create mode 100644 components/datetime/tests/fixtures/tests/patterns.bin create mode 100644 components/datetime/tests/fixtures/tests/patterns.json create mode 100644 components/datetime/tests/pattern_serialization.rs diff --git a/Cargo.lock b/Cargo.lock index ef00cda03df..6dd9d094658 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -845,6 +845,7 @@ dependencies = [ name = "icu_datetime" version = "0.1.0" dependencies = [ + "bincode", "criterion", "icu_benchmark_macros", "icu_locid", diff --git a/components/datetime/Cargo.toml b/components/datetime/Cargo.toml index 34f5093707e..a36d9a0973d 100644 --- a/components/datetime/Cargo.toml +++ b/components/datetime/Cargo.toml @@ -38,6 +38,7 @@ icu_testdata = { version = "0.1", path = "../../resources/testdata" } icu_locid_macros = { version = "0.1", path = "../locid/macros" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +bincode = "1.3" [lib] bench = false # This option is required for Benchmark CI diff --git a/components/datetime/src/fields/length.rs b/components/datetime/src/fields/length.rs index b59f5471a21..295038fdc42 100644 --- a/components/datetime/src/fields/length.rs +++ b/components/datetime/src/fields/length.rs @@ -9,6 +9,10 @@ pub enum LengthError { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum FieldLength { One = 1, TwoDigit = 2, diff --git a/components/datetime/src/fields/mod.rs b/components/datetime/src/fields/mod.rs index d24d4dbb713..5a28ecae58c 100644 --- a/components/datetime/src/fields/mod.rs +++ b/components/datetime/src/fields/mod.rs @@ -15,6 +15,10 @@ pub enum Error { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub struct Field { pub symbol: FieldSymbol, pub length: FieldLength, diff --git a/components/datetime/src/fields/symbols.rs b/components/datetime/src/fields/symbols.rs index c34d018cdbe..b1a4513eea1 100644 --- a/components/datetime/src/fields/symbols.rs +++ b/components/datetime/src/fields/symbols.rs @@ -12,6 +12,10 @@ pub enum SymbolError { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum FieldSymbol { Year(Year), Month(Month), @@ -132,6 +136,10 @@ impl From for char { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum Year { Calendar, WeekOf, @@ -155,6 +163,10 @@ impl From for FieldSymbol { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum Month { Format, StandAlone, @@ -178,6 +190,10 @@ impl From for FieldSymbol { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum Day { DayOfMonth, DayOfYear, @@ -205,6 +221,10 @@ impl From for FieldSymbol { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum Hour { H11, H12, @@ -232,6 +252,10 @@ impl From for FieldSymbol { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum Second { Second, FractionalSecond, @@ -257,6 +281,10 @@ impl From for FieldSymbol { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum Weekday { Format, Local, @@ -282,6 +310,10 @@ impl From for FieldSymbol { } #[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum DayPeriod { AmPm, NoonMidnight, diff --git a/components/datetime/src/pattern/mod.rs b/components/datetime/src/pattern/mod.rs index ae27fec456a..c01b49b6c4b 100644 --- a/components/datetime/src/pattern/mod.rs +++ b/components/datetime/src/pattern/mod.rs @@ -11,9 +11,17 @@ use std::iter::FromIterator; use std::{convert::TryFrom, fmt}; #[cfg(feature = "provider_serde")] -use serde::{de, ser, Deserialize, Deserializer, Serialize}; +use serde::{ + de, + ser::{self, SerializeSeq}, + Deserialize, Deserializer, Serialize, +}; #[derive(Debug, PartialEq, Clone)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub enum PatternItem { Field(fields::Field), Literal(String), @@ -54,6 +62,10 @@ impl<'p> From for PatternItem { /// The granularity of time represented in a pattern item. /// Ordered from least granular to most granular for comparsion. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) +)] pub(super) enum TimeGranularity { Hours, Minutes, @@ -182,10 +194,10 @@ impl FromIterator for Pattern { } #[cfg(feature = "provider_serde")] -struct DeserializePatternVisitor; +struct DeserializePatternUTS35String; #[cfg(feature = "provider_serde")] -impl<'de> de::Visitor<'de> for DeserializePatternVisitor { +impl<'de> de::Visitor<'de> for DeserializePatternUTS35String { type Value = Pattern; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { @@ -210,13 +222,40 @@ impl<'de> de::Visitor<'de> for DeserializePatternVisitor { } } +#[cfg(feature = "provider_serde")] +struct DeserializePatternBincode; + +#[cfg(feature = "provider_serde")] +impl<'de> de::Visitor<'de> for DeserializePatternBincode { + type Value = Pattern; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "Unable to deserialize a bincode Pattern.") + } + + fn visit_seq(self, mut seq: V) -> Result + where + V: de::SeqAccess<'de>, + { + let mut items = Vec::new(); + while let Some(item) = seq.next_element()? { + items.push(item) + } + Ok(Pattern::from(items)) + } +} + #[cfg(feature = "provider_serde")] impl<'de> Deserialize<'de> for Pattern { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { - deserializer.deserialize_str(DeserializePatternVisitor) + if deserializer.is_human_readable() { + deserializer.deserialize_str(DeserializePatternUTS35String) + } else { + deserializer.deserialize_seq(DeserializePatternBincode) + } } } @@ -226,58 +265,18 @@ impl Serialize for Pattern { where S: ser::Serializer, { - let string: String = String::from(self); - serializer.serialize_str(&string) - } -} - -#[cfg(all(test, feature = "provider_serde"))] -mod test { - use super::*; - - // Provide a few patterns to sanity check serialization. - const PATTERNS: [&str; 6] = [ - "d", - "E, M/d/y", - "h:mm:ss a", - // TODO(#502) - The W field isn't supported yet, so swap it out for a field we do know. - "'week' d 'of' MMMM", - // Arabic "week of" in the availableFormats. - "الأسبوع d من MMMM", - // Cherokee "week of" in the availableFormats. - "ᏒᎾᏙᏓᏆᏍᏗ’ d ’ᎾᎿ’ MMMM", - ]; - - #[test] - fn test_pattern_serialization_roundtrip() { - for pattern_string in &PATTERNS { - // Wrap the string in quotes so it's a JSON string. - let json_in: String = serde_json::to_string(pattern_string).unwrap(); - - let pattern: Pattern = match serde_json::from_str(&json_in) { - Ok(p) => p, - Err(err) => { - panic!( - "Unable to parse the pattern {:?}. {:?}", - pattern_string, err - ); - } - }; - - let json_out = match serde_json::to_string(&pattern) { - Ok(s) => s, - Err(err) => { - panic!( - "Unable to re-serialize the pattern {:?}. {:?}", - pattern_string, err - ); - } - }; - - assert_eq!( - json_in, json_out, - "The roundtrip serialization for the pattern matched." - ); + if serializer.is_human_readable() { + // Serialize into the UTS 35 string representation. + let string: String = String::from(self); + serializer.serialize_str(&string) + } else { + // Serialize into a bincode-friendly representation. This means that pattern parsing + // will not be needed when deserializing. + let mut seq = serializer.serialize_seq(Some(self.items.len()))?; + for item in self.items.iter() { + seq.serialize_element(item)?; + } + seq.end() } } } diff --git a/components/datetime/tests/fixtures/tests/patterns.bin b/components/datetime/tests/fixtures/tests/patterns.bin new file mode 100644 index 0000000000000000000000000000000000000000..f808adaa6c642cd1540c52db5f31ce681f2a6c24 GIT binary patch literal 2601 zcmaKtJ#G|15QPUYSP03^5M*@#BO(&tfE0-%*aQwBBP9Tf06Y=|mSX}SCL+Ptm<@-M zFyC%}R_}H-C2du8y?Rw|djB2{8+%mZtr7{htNcyL_p92SDu-wPQ2^AF$UQ#=w3WO8 zDQagWsrTMpYEG-k$MoJSU;{WozzG3s4_EAYiPmUjCHD5@Bs8h^gF2Tj^>H5SbbQX2U$J*6@|s>;g2_^VIIHC7k}pQ zY97DN?6uX)LyP5!dn!q%2&q2|HfafhR)nsR?N$`J~>>--pthH!ddWJ2q zs|iL|6Cj$NEia5ljfq;bRuJGt>GIpjQ90x(*`krgz{w=`eh%H9edNG=jC6Q?88l#l zlS%9{Yc{07$>b`3QdsQ4Cak?A4$Wji0U{PS!Bmr=F=e93(M$}S4BoPyU{+!&xz^LH z1*{vG5;)cZxISfe1;a#e_O3(H8o`1BTWbYV!Gft^L1Isnt6Yu7z(KNTq#iiIt|k~= zO@K^zvb-=FHUGt$mHuI8zXbkgGo&F30%RX0a4;+-TQt%bIGO1!2&T6n=shwT3tSEM b1hW!LHPw2W$$)hOQv%0YfUaP`u3*7`7=b;? literal 0 HcmV?d00001 diff --git a/components/datetime/tests/fixtures/tests/patterns.json b/components/datetime/tests/fixtures/tests/patterns.json new file mode 100644 index 00000000000..1846c0160b2 --- /dev/null +++ b/components/datetime/tests/fixtures/tests/patterns.json @@ -0,0 +1,24 @@ +[ + "d", + "E, M/d/y", + "h:mm:ss a", + "'week' d 'of' MMMM", + "الأسبوع d من MMMM", + "ᏒᎾᏙᏓᏆᏍᏗ’ d ’ᎾᎿ’ MMMM", + "y yy yyy yyyy yyyyy", + "Y YY YYY YYYY YYYYY", + "M MM MMM MMMM MMMMM", + "L LL LLL LLLL LLLLL", + "d dd", + "D DD DDD", + "F", + "g gg ggg gggg ggggg", + "e ee eee eeee eeeee eeeeee", + "c cc ccc cccc ccccc cccccc", + "a aa aaa aaaa aaaaa", + "b bb bbb bbbb bbbbb", + "m mm", + "s ss", + "S SS SSS SSSS SSSSS SSSSS", + "A AA AAA AAAA AAAAA AAAAA" +] diff --git a/components/datetime/tests/pattern_serialization.rs b/components/datetime/tests/pattern_serialization.rs new file mode 100644 index 00000000000..61537dd481b --- /dev/null +++ b/components/datetime/tests/pattern_serialization.rs @@ -0,0 +1,120 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +#![cfg(all(test, feature = "provider_serde"))] + +use icu_datetime::pattern::Pattern; +use std::{fs::File, io::BufReader}; + +pub fn get_pattern_strings() -> Vec { + let file = File::open(format!("./tests/fixtures/tests/patterns.json")) + .expect("Unable to open ./tests/fixtures/tests/patterns.json"); + let reader = BufReader::new(file); + serde_json::from_reader(reader).expect("Unable to deserialize pattern strings.") +} + +pub fn get_pattern_bincode_write_handle() -> File { + File::create("./tests/fixtures/tests/patterns.bin") + .expect("Unable to create ./tests/fixtures/tests/patterns.bin") +} + +pub fn get_pattern_bincode_from_file() -> Vec> { + bincode::deserialize_from( + File::open("./tests/fixtures/tests/patterns.bin") + .expect("Unable to ./tests/fixtures/tests/patterns.bin"), + ) + .expect("Unable to deserialize bytes.") +} + +#[test] +fn test_pattern_json_serialization_roundtrip() { + for pattern_string in &get_pattern_strings() { + // Wrap the string in quotes so it's a JSON string. + let json_in: String = serde_json::to_string(pattern_string).unwrap(); + + let pattern: Pattern = match serde_json::from_str(&json_in) { + Ok(p) => p, + Err(err) => { + panic!( + "Unable to parse the pattern {:?}. {:?}", + pattern_string, err + ); + } + }; + + let json_out = match serde_json::to_string(&pattern) { + Ok(s) => s, + Err(err) => { + panic!( + "Unable to re-serialize the pattern {:?}. {:?}", + pattern_string, err + ); + } + }; + + assert_eq!( + json_in, json_out, + "The roundtrip serialization for the pattern matched." + ); + } +} + +/// Bincode representation of patterns need to be stable across time. This test checks the +/// current serialization against historic serialization to ensure that this remains stable. +#[test] +fn test_pattern_bincode_serialization_roundtrip() { + let patterns = get_pattern_strings(); + let update_bincode = std::env::var_os("ICU4X_REGEN_FIXTURE").is_some(); + let mut result_vec = Vec::new(); + let expect_vec = if update_bincode { + None + } else { + Some(get_pattern_bincode_from_file()) + }; + + if let Some(ref expect_vec) = expect_vec { + if expect_vec.len() != patterns.len() { + panic!( + "Expected the bincode to have the same number of entries as the string patterns. \ + The bincode can be re-generated by re-running the test with the environment + variable ICU4X_REGEN_FIXTURE set." + ); + } + } + + for (i, pattern_string) in patterns.iter().enumerate() { + // Wrap the string in quotes so it's a JSON string. + let json_in: String = serde_json::to_string(pattern_string).unwrap(); + + let pattern: Pattern = match serde_json::from_str(&json_in) { + Ok(p) => p, + Err(err) => { + panic!( + "Unable to parse the pattern {:?}. {:?}", + pattern_string, err + ); + } + }; + + let bincode: Vec = bincode::serialize(&pattern).unwrap(); + + if let Some(ref expect_vec) = expect_vec { + if bincode != *expect_vec.get(i).unwrap() { + panic!( + "The bincode representations of the pattern {:?} did not match the stored \ + representation. Patterns are supposed to have stable bincode representations. \ + Something changed to make it different than what it was in the past. If this is \ + expected, then the bincode can be updated by re-running the test with the \ + environment variable ICU4X_REGEN_FIXTURE set.", + json_in + ) + } + } + result_vec.push(bincode); + } + if update_bincode { + eprintln!("Writing the bincode into a file"); + bincode::serialize_into(&mut get_pattern_bincode_write_handle(), &result_vec).unwrap(); + } +} From 981f17d4a0c58ecc088dbea684a598731a269195 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 11 Mar 2021 15:20:28 -0600 Subject: [PATCH 04/22] Part 4: Add impl Display for Pattern --- components/datetime/src/pattern/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/datetime/src/pattern/mod.rs b/components/datetime/src/pattern/mod.rs index c01b49b6c4b..658a8c2c086 100644 --- a/components/datetime/src/pattern/mod.rs +++ b/components/datetime/src/pattern/mod.rs @@ -122,11 +122,11 @@ impl From> for Pattern { } } -impl From<&Pattern> for String { - fn from(pattern: &Pattern) -> Self { +impl fmt::Display for Pattern { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { let mut string = String::new(); - for pattern_item in pattern.items().iter() { + for pattern_item in self.items().iter() { match pattern_item { PatternItem::Field(field) => { let ch: char = field.symbol.into(); @@ -183,7 +183,7 @@ impl From<&Pattern> for String { } } - string + formatter.write_str(&string) } } @@ -267,7 +267,7 @@ impl Serialize for Pattern { { if serializer.is_human_readable() { // Serialize into the UTS 35 string representation. - let string: String = String::from(self); + let string: String = self.to_string(); serializer.serialize_str(&string) } else { // Serialize into a bincode-friendly representation. This means that pattern parsing From dd48759b39921c659e05c80953cd14dec78304c1 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Tue, 16 Mar 2021 16:39:19 -0500 Subject: [PATCH 05/22] Review 1: Add comment about suitability of the Writeable trait for Pattern --- components/datetime/src/pattern/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/datetime/src/pattern/mod.rs b/components/datetime/src/pattern/mod.rs index 658a8c2c086..3c0cc301acb 100644 --- a/components/datetime/src/pattern/mod.rs +++ b/components/datetime/src/pattern/mod.rs @@ -122,6 +122,11 @@ impl From> for Pattern { } } +/// This trait is implemented in order to provide the machinery to convert a `Pattern` to a UTS 35 +/// pattern string. It could also be implemented as the Writeable trait, but at the time of writing +/// this was not done, as this code would need to implement the `write_len` method, which would +/// need to duplicate the branching logic of the `fmt` method here. This code is used in generating +/// the data providers and is not as performance sensitive. impl fmt::Display for Pattern { fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { let mut string = String::new(); From 0f94f33a5131cf6d931d5c8792ef74f8f83de71f Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Tue, 16 Mar 2021 16:55:53 -0500 Subject: [PATCH 06/22] Review 2: Adjust the serde Error for Pattern to de::Error::invalid_value --- components/datetime/src/pattern/error.rs | 17 ++++++ components/datetime/src/pattern/mod.rs | 16 ++--- .../tests/fixtures/tests/patterns.json | 60 +++++++++++-------- .../datetime/tests/pattern_serialization.rs | 46 ++++++++++++-- 4 files changed, 101 insertions(+), 38 deletions(-) diff --git a/components/datetime/src/pattern/error.rs b/components/datetime/src/pattern/error.rs index 073b4092825..5ef80e25d20 100644 --- a/components/datetime/src/pattern/error.rs +++ b/components/datetime/src/pattern/error.rs @@ -2,6 +2,7 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). use crate::fields; +use std::fmt; #[derive(Debug, PartialEq)] pub enum Error { @@ -11,6 +12,22 @@ pub enum Error { UnclosedPlaceholder, } +/// These strings follow the recommendations for the serde::de::Unexpected::Other type. +/// https://docs.serde.rs/serde/de/enum.Unexpected.html#variant.Other +/// +/// Serde will generate an error such as: +/// "invalid value: unclosed literal in pattern, expected a valid UTS 35 pattern string at line 1 column 12" +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::FieldTooLong(symbol) => write!(f, "{:?} field too long in pattern", symbol), + Error::UnknownSubstitution(ch) => write!(f, "unknown substitution {} in pattern", ch), + Error::UnclosedLiteral => write!(f, "unclosed literal in pattern"), + Error::UnclosedPlaceholder => write!(f, "unclosed placeholder in pattern"), + } + } +} + impl From for Error { fn from(input: fields::Error) -> Self { match input { diff --git a/components/datetime/src/pattern/mod.rs b/components/datetime/src/pattern/mod.rs index 3c0cc301acb..7f19d2ec037 100644 --- a/components/datetime/src/pattern/mod.rs +++ b/components/datetime/src/pattern/mod.rs @@ -214,16 +214,12 @@ impl<'de> de::Visitor<'de> for DeserializePatternUTS35String { E: de::Error, { // Parse a string into a list of fields. - let pattern = match Pattern::from_bytes(pattern_string) { - Ok(p) => p, - Err(err) => { - return Err(E::custom(format!( - "The pattern \"{}\" could not be parsed: {:?}", - pattern_string, err - ))); - } - }; - Ok(pattern) + Pattern::from_bytes(pattern_string).map_err(|err| { + de::Error::invalid_value( + de::Unexpected::Other(&format!("{}", err)), + &"a valid UTS 35 pattern string", + ) + }) } } diff --git a/components/datetime/tests/fixtures/tests/patterns.json b/components/datetime/tests/fixtures/tests/patterns.json index 1846c0160b2..08d65a0b255 100644 --- a/components/datetime/tests/fixtures/tests/patterns.json +++ b/components/datetime/tests/fixtures/tests/patterns.json @@ -1,24 +1,36 @@ -[ - "d", - "E, M/d/y", - "h:mm:ss a", - "'week' d 'of' MMMM", - "الأسبوع d من MMMM", - "ᏒᎾᏙᏓᏆᏍᏗ’ d ’ᎾᎿ’ MMMM", - "y yy yyy yyyy yyyyy", - "Y YY YYY YYYY YYYYY", - "M MM MMM MMMM MMMMM", - "L LL LLL LLLL LLLLL", - "d dd", - "D DD DDD", - "F", - "g gg ggg gggg ggggg", - "e ee eee eeee eeeee eeeeee", - "c cc ccc cccc ccccc cccccc", - "a aa aaa aaaa aaaaa", - "b bb bbb bbbb bbbbb", - "m mm", - "s ss", - "S SS SSS SSSS SSSSS SSSSS", - "A AA AAA AAAA AAAAA AAAAA" -] +{ + "valid_patterns": [ + "d", + "E, M/d/y", + "h:mm:ss a", + "'week' d 'of' MMMM", + "الأسبوع d من MMMM", + "ᏒᎾᏙᏓᏆᏍᏗ’ d ’ᎾᎿ’ MMMM", + "y yy yyy yyyy yyyyy", + "Y YY YYY YYYY YYYYY", + "M MM MMM MMMM MMMMM", + "L LL LLL LLLL LLLLL", + "d dd", + "D DD DDD", + "F", + "g gg ggg gggg ggggg", + "e ee eee eeee eeeee eeeeee", + "c cc ccc cccc ccccc cccccc", + "a aa aaa aaaa aaaaa", + "b bb bbb bbbb bbbbb", + "m mm", + "s ss", + "S SS SSS SSSS SSSSS SSSSS", + "A AA AAA AAAA AAAAA AAAAA" + ], + "invalid_patterns": [ + { + "pattern": "yyyyyyy", + "error": "invalid value: Year(Calendar) field too long in pattern, expected a valid UTS 35 pattern string at line 1 column 9" + }, + { + "pattern": " 'unclosed", + "error": "invalid value: unclosed literal in pattern, expected a valid UTS 35 pattern string at line 1 column 12" + } + ] +} diff --git a/components/datetime/tests/pattern_serialization.rs b/components/datetime/tests/pattern_serialization.rs index 61537dd481b..9efbaa81754 100644 --- a/components/datetime/tests/pattern_serialization.rs +++ b/components/datetime/tests/pattern_serialization.rs @@ -7,19 +7,39 @@ use icu_datetime::pattern::Pattern; use std::{fs::File, io::BufReader}; -pub fn get_pattern_strings() -> Vec { +#[derive(serde::Serialize, serde::Deserialize)] +struct InvalidPatternFixture { + pub pattern: String, + pub error: String, +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct PatternFixtures { + pub valid_patterns: Vec, + pub invalid_patterns: Vec, +} + +fn get_pattern_fixtures() -> PatternFixtures { let file = File::open(format!("./tests/fixtures/tests/patterns.json")) .expect("Unable to open ./tests/fixtures/tests/patterns.json"); let reader = BufReader::new(file); - serde_json::from_reader(reader).expect("Unable to deserialize pattern strings.") + serde_json::from_reader(reader).expect("Unable to deserialize pattern fixtures.") } -pub fn get_pattern_bincode_write_handle() -> File { +fn get_pattern_strings() -> Vec { + get_pattern_fixtures().valid_patterns +} + +fn get_invalid_pattern_strings() -> Vec { + get_pattern_fixtures().invalid_patterns +} + +fn get_pattern_bincode_write_handle() -> File { File::create("./tests/fixtures/tests/patterns.bin") .expect("Unable to create ./tests/fixtures/tests/patterns.bin") } -pub fn get_pattern_bincode_from_file() -> Vec> { +fn get_pattern_bincode_from_file() -> Vec> { bincode::deserialize_from( File::open("./tests/fixtures/tests/patterns.bin") .expect("Unable to ./tests/fixtures/tests/patterns.bin"), @@ -118,3 +138,21 @@ fn test_pattern_bincode_serialization_roundtrip() { bincode::serialize_into(&mut get_pattern_bincode_write_handle(), &result_vec).unwrap(); } } + +/// Test that pattern serialization produces sensible error messages given the Serde +/// serde::de::Unexpected type and the use of fmt::Display traits on the Error objects. +#[test] +fn test_pattern_json_errors() { + for InvalidPatternFixture { pattern, error } in &get_invalid_pattern_strings() { + // Wrap the string in quotes so it's a JSON string. + let json_in: String = serde_json::to_string(pattern).unwrap(); + + // Wrap the string in quotes so it's a JSON string. + match serde_json::from_str::(&json_in) { + Ok(_) => panic!("Expected an invalid pattern. {}", json_in), + Err(serde_err) => { + assert_eq!(format!("{}", serde_err), *error); + } + }; + } +} From d5b4a0f85da389cb0d287d604aa51e1636e72cba Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Tue, 16 Mar 2021 17:04:01 -0500 Subject: [PATCH 07/22] Review 3: Adjust pattern serialization write directly into the formatter --- components/datetime/src/pattern/mod.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/components/datetime/src/pattern/mod.rs b/components/datetime/src/pattern/mod.rs index 7f19d2ec037..899739d3b5c 100644 --- a/components/datetime/src/pattern/mod.rs +++ b/components/datetime/src/pattern/mod.rs @@ -7,8 +7,8 @@ mod parser; use crate::fields::{self, Field, FieldLength, FieldSymbol}; pub use error::Error; use parser::Parser; -use std::iter::FromIterator; use std::{convert::TryFrom, fmt}; +use std::{fmt::Write, iter::FromIterator}; #[cfg(feature = "provider_serde")] use serde::{ @@ -129,14 +129,12 @@ impl From> for Pattern { /// the data providers and is not as performance sensitive. impl fmt::Display for Pattern { fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - let mut string = String::new(); - for pattern_item in self.items().iter() { match pattern_item { PatternItem::Field(field) => { let ch: char = field.symbol.into(); for _ in 0..field.length as usize { - string.push(ch); + formatter.write_char(ch)?; } } PatternItem::Literal(literal) => { @@ -155,7 +153,7 @@ impl fmt::Display for Pattern { // Do not escape the leading whitespace. while let Some(ch) = ch_iter.peek() { if ch.is_whitespace() { - string.push(*ch); + formatter.write_char(*ch)?; ch_iter.next(); } else { break; @@ -163,32 +161,31 @@ impl fmt::Display for Pattern { } // Wrap in "'" and escape "'". - string.push('\''); + formatter.write_char('\'')?; for ch in ch_iter { if ch == '\'' { // Escape a single quote. - string.push('\\'); + formatter.write_char('\\')?; } - string.push(ch); + formatter.write_char(ch)?; } - string.push('\''); + formatter.write_char('\'')?; // Add the trailing whitespace for ch in literal.chars().rev() { if ch.is_whitespace() { - string.push(ch); + formatter.write_char(ch)?; } else { break; } } } else { - string.push_str(literal); + formatter.write_str(literal)?; } } } } - - formatter.write_str(&string) + Ok(()) } } From dd688d8efee45053a50789db64d90eb753fe22a2 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Wed, 17 Mar 2021 10:26:55 -0500 Subject: [PATCH 08/22] Review 4: Use a split iterator when deserializing skeletons --- .../provider_cldr/src/transform/dates.rs | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/components/provider_cldr/src/transform/dates.rs b/components/provider_cldr/src/transform/dates.rs index 5ba51000477..98e5a70ed17 100644 --- a/components/provider_cldr/src/transform/dates.rs +++ b/components/provider_cldr/src/transform/dates.rs @@ -135,43 +135,40 @@ impl From<&cldr_json::DateTimeFormats> for gregory::patterns::DateTimeFormatsV1 // The CLDR keys for available_formats can have duplicate skeletons with either // an additional variant, or with multiple variants for different plurals. for (skeleton_str, pattern_str) in other.available_formats.0.iter() { - let mut unique_skeleton = String::new(); - let mut variant_text = String::new(); - - // Split out the unique skeleton component, and the variant text. - // "MMMMW-count-two" - // unique_skeleton: "MMMMW" - // variant_text: "-count-two" - let chars = skeleton_str.chars(); - let mut target = &mut unique_skeleton; - for ch in chars { - if ch == '-' { - target = &mut variant_text; + let mut unique_skeleton = None; + let mut variant_parts = Vec::new(); + + for part in skeleton_str.split('-') { + match unique_skeleton { + None => { + unique_skeleton = Some(part); + } + Some(_) => variant_parts.push(part), } - target.push(ch); } - let skeleton_fields_v1 = - match SkeletonFieldsV1::try_from(unique_skeleton.as_str()) { - Ok(s) => s, - Err(err) => match err { - // Ignore unimplemented fields for now. - SkeletonFieldsV1Error::SymbolUnimplemented(_) => continue, - SkeletonFieldsV1Error::SymbolUnknown(symbol) => panic!( - "Unknown symbol {:?} in skeleton {:?}", - symbol, unique_skeleton - ), - SkeletonFieldsV1Error::FieldTooLong => { - panic!("Field too long in skeleton {:?}", unique_skeleton) - } - SkeletonFieldsV1Error::SymbolInvalid(symbol) => panic!( - "Symbol invalid {:?} in skeleton {:?}", - symbol, unique_skeleton - ), - }, - }; - - if !variant_text.is_empty() && variant_text != "-alt-variant" { + let unique_skeleton = unique_skeleton.expect("Expected to find a skeleton."); + + let skeleton_fields_v1 = match SkeletonFieldsV1::try_from(unique_skeleton) { + Ok(s) => s, + Err(err) => match err { + // Ignore unimplemented fields for now. + SkeletonFieldsV1Error::SymbolUnimplemented(_) => continue, + SkeletonFieldsV1Error::SymbolUnknown(symbol) => panic!( + "Unknown symbol {:?} in skeleton {:?}", + symbol, unique_skeleton + ), + SkeletonFieldsV1Error::FieldTooLong => { + panic!("Field too long in skeleton {:?}", unique_skeleton) + } + SkeletonFieldsV1Error::SymbolInvalid(symbol) => panic!( + "Symbol invalid {:?} in skeleton {:?}", + symbol, unique_skeleton + ), + }, + }; + + if !variant_parts.is_empty() { unimplemented!( "This skeleton string is not yet supported: {:?}", skeleton_str From 6a34025f35b167c5d1a581a61964046c87501167 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Wed, 17 Mar 2021 12:19:38 -0500 Subject: [PATCH 09/22] Review 5: Derive the Eq trait for Fields, which is needed for Litemap support --- components/datetime/src/fields/length.rs | 2 +- components/datetime/src/fields/mod.rs | 2 +- components/datetime/src/fields/symbols.rs | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/datetime/src/fields/length.rs b/components/datetime/src/fields/length.rs index 295038fdc42..cbe54036cc4 100644 --- a/components/datetime/src/fields/length.rs +++ b/components/datetime/src/fields/length.rs @@ -8,7 +8,7 @@ pub enum LengthError { TooLong, } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) diff --git a/components/datetime/src/fields/mod.rs b/components/datetime/src/fields/mod.rs index 5a28ecae58c..d0644aff504 100644 --- a/components/datetime/src/fields/mod.rs +++ b/components/datetime/src/fields/mod.rs @@ -14,7 +14,7 @@ pub enum Error { TooLong(FieldSymbol), } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) diff --git a/components/datetime/src/fields/symbols.rs b/components/datetime/src/fields/symbols.rs index b1a4513eea1..278d86375cd 100644 --- a/components/datetime/src/fields/symbols.rs +++ b/components/datetime/src/fields/symbols.rs @@ -11,7 +11,7 @@ pub enum SymbolError { Invalid(char), } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) @@ -135,7 +135,7 @@ impl From for char { } } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) @@ -162,7 +162,7 @@ impl From for FieldSymbol { } } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) @@ -189,7 +189,7 @@ impl From for FieldSymbol { } } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) @@ -220,7 +220,7 @@ impl From for FieldSymbol { } } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) @@ -251,7 +251,7 @@ impl From for FieldSymbol { } } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) @@ -280,7 +280,7 @@ impl From for FieldSymbol { } } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) @@ -309,7 +309,7 @@ impl From for FieldSymbol { } } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) From e12a4bb6761a4f7355791f292bc0711f89df3eea Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Wed, 17 Mar 2021 12:43:14 -0500 Subject: [PATCH 10/22] Review 6: Convert skeleton tuples to LiteMap --- Cargo.lock | 2 ++ components/datetime/Cargo.toml | 3 +- components/datetime/src/provider/mod.rs | 32 ++++++++++++++++--- components/provider_cldr/Cargo.toml | 1 + .../provider_cldr/src/transform/dates.rs | 13 +++----- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6dd9d094658..ced4c469f35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -852,6 +852,7 @@ dependencies = [ "icu_locid_macros", "icu_provider", "icu_testdata", + "litemap", "serde", "serde_json", "smallvec", @@ -936,6 +937,7 @@ dependencies = [ "icu_provider", "icu_testdata", "json", + "litemap", "log", "mktemp", "reqwest", diff --git a/components/datetime/Cargo.toml b/components/datetime/Cargo.toml index a36d9a0973d..e4d3851c427 100644 --- a/components/datetime/Cargo.toml +++ b/components/datetime/Cargo.toml @@ -26,6 +26,7 @@ skip_optional_dependencies = true icu_locid = { version = "0.1", path = "../locid" } icu_provider = { version = "0.1", path = "../provider" } writeable = { version = "0.2", path = "../../utils/writeable" } +litemap = { version = "0.1.1", path = "../../utils/litemap" } tinystr = { version = "0.4.1" } serde = { version = "1.0", features = ["derive"], optional = true } smallvec = "1.4" @@ -46,7 +47,7 @@ bench = false # This option is required for Benchmark CI [features] default = ["provider_serde"] bench = [] -provider_serde = ["serde"] +provider_serde = ["serde", "litemap/serde"] serialize_none = [] [[bench]] diff --git a/components/datetime/src/provider/mod.rs b/components/datetime/src/provider/mod.rs index 8ee481b41b5..4f1a11aba9e 100644 --- a/components/datetime/src/provider/mod.rs +++ b/components/datetime/src/provider/mod.rs @@ -187,6 +187,7 @@ pub mod gregory { fields::{self, Field, FieldLength, FieldSymbol, LengthError, SymbolError}, pattern::{self, Pattern}, }; + use litemap::LiteMap; use std::fmt; use std::{cmp::Ordering, convert::TryFrom}; @@ -208,12 +209,21 @@ pub mod gregory { pub short: Cow<'static, str>, } - #[derive(Debug, PartialEq, Clone, Default)] + /// This struct represents the serialization form of the skeleton fields. It contains + /// a single "exotic type", the fields::Field, which must remain stable between versions. + /// The skeletons are stored in a LiteMap, which is sorted according to the canonical + /// sort order. + #[derive(Debug, Eq, PartialEq, Clone, Default)] pub struct SkeletonFieldsV1(pub SmallVec<[fields::Field; 5]>); impl SkeletonFieldsV1 { - /// [`SkeletonTupleV1`] structs should be ordered by the `SkeletonFieldsV1` canonical - /// order. This helps ensure that the skeleton serialization is sorted deterministically. + /// The LiteMap structs should be ordered by the + /// `SkeletonFieldsV1` canonical order. Thishelps ensure that the skeleton + /// serialization is sorted deterministically. This order is determined by the order + /// of the fields listed in UTS 35, which is ordered from most significant, to least + /// significant. + /// + /// https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table pub fn compare_canonical_order(&self, other: &SkeletonFieldsV1) -> Ordering { let fields_a = &self.0; let fields_b = &other.0; @@ -262,6 +272,18 @@ pub mod gregory { } } + impl PartialOrd for SkeletonFieldsV1 { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.compare_canonical_order(other)) + } + } + + impl Ord for SkeletonFieldsV1 { + fn cmp(&self, other: &Self) -> Ordering { + self.compare_canonical_order(other) + } + } + /// This struct is a public wrapper around the internal Pattern struct. This allows /// access to the serialization and deserialization capabilities, without exposing the /// internals of the pattern machinery. @@ -442,7 +464,7 @@ pub mod gregory { feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) )] - pub struct SkeletonTupleV1(pub SkeletonFieldsV1, pub PatternV1); + pub struct SkeletonsV1(pub LiteMap); #[derive(Debug, PartialEq, Clone, Default)] #[cfg_attr( @@ -451,7 +473,7 @@ pub mod gregory { )] pub struct DateTimeFormatsV1 { pub style_patterns: StylePatternsV1, - pub skeletons: Vec, + pub skeletons: SkeletonsV1, } } } diff --git a/components/provider_cldr/Cargo.toml b/components/provider_cldr/Cargo.toml index 983e285a149..06253c8510e 100644 --- a/components/provider_cldr/Cargo.toml +++ b/components/provider_cldr/Cargo.toml @@ -38,6 +38,7 @@ serde_json = "1.0" serde-tuple-vec-map = "1.0" tinystr = "0.4" smallvec = "1.4" +litemap = { version = "0.1.1", path = "../../utils/litemap" } # Dependencies for the download feature urlencoding = { version = "1.1", optional = true } diff --git a/components/provider_cldr/src/transform/dates.rs b/components/provider_cldr/src/transform/dates.rs index 98e5a70ed17..e8c549b7a27 100644 --- a/components/provider_cldr/src/transform/dates.rs +++ b/components/provider_cldr/src/transform/dates.rs @@ -117,9 +117,8 @@ impl From<&cldr_json::StylePatterns> for gregory::patterns::StylePatternsV1 { impl From<&cldr_json::DateTimeFormats> for gregory::patterns::DateTimeFormatsV1 { fn from(other: &cldr_json::DateTimeFormats) -> Self { - use gregory::patterns::{ - PatternV1, SkeletonFieldsV1, SkeletonFieldsV1Error, SkeletonTupleV1, - }; + use gregory::patterns::{PatternV1, SkeletonFieldsV1, SkeletonFieldsV1Error, SkeletonsV1}; + use litemap::LiteMap; // TODO(#308): Support numbering system variations. We currently throw them away. Self { @@ -130,7 +129,7 @@ impl From<&cldr_json::DateTimeFormats> for gregory::patterns::DateTimeFormatsV1 short: other.short.get_pattern().clone(), }, skeletons: { - let mut skeleton_tuples: Vec = Vec::new(); + let mut skeletons = SkeletonsV1(LiteMap::new()); // The CLDR keys for available_formats can have duplicate skeletons with either // an additional variant, or with multiple variants for different plurals. @@ -178,12 +177,10 @@ impl From<&cldr_json::DateTimeFormats> for gregory::patterns::DateTimeFormatsV1 let pattern_v1 = PatternV1::try_from(pattern_str as &str) .expect("Unable to parse a pattern"); - skeleton_tuples.push(SkeletonTupleV1(skeleton_fields_v1, pattern_v1)); + skeletons.0.insert(skeleton_fields_v1, pattern_v1); } - // Sort the skeletons deterministically by their canonical sort order. - skeleton_tuples.sort_by(|a, b| a.0.compare_canonical_order(&b.0)); - skeleton_tuples + skeletons }, } } From 858c6b7e13c606cb39566b2ed82a1fa97d2fa420 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Wed, 17 Mar 2021 12:16:37 -0500 Subject: [PATCH 11/22] Review 7: Re-generate the testdata for skeleton LiteMaps --- .../data/json/dates/gregory@1/ar-EG.json | 164 ++++----------- .../data/json/dates/gregory@1/ar.json | 164 ++++----------- .../data/json/dates/gregory@1/bn.json | 164 ++++----------- .../data/json/dates/gregory@1/ccp.json | 164 ++++----------- .../json/dates/gregory@1/en-US-posix.json | 149 +++----------- .../data/json/dates/gregory@1/en-ZA.json | 154 +++----------- .../data/json/dates/gregory@1/en.json | 149 +++----------- .../data/json/dates/gregory@1/es-AR.json | 184 ++++------------- .../data/json/dates/gregory@1/es.json | 179 ++++------------- .../data/json/dates/gregory@1/fr.json | 149 +++----------- .../data/json/dates/gregory@1/ja.json | 179 ++++------------- .../data/json/dates/gregory@1/ru.json | 159 +++------------ .../data/json/dates/gregory@1/sr-Cyrl.json | 174 ++++------------ .../data/json/dates/gregory@1/sr-Latn.json | 174 ++++------------ .../data/json/dates/gregory@1/sr.json | 174 ++++------------ .../data/json/dates/gregory@1/th.json | 189 ++++-------------- .../data/json/dates/gregory@1/tr.json | 164 ++++----------- .../data/json/dates/gregory@1/und.json | 149 +++----------- 18 files changed, 618 insertions(+), 2364 deletions(-) diff --git a/resources/testdata/data/json/dates/gregory@1/ar-EG.json b/resources/testdata/data/json/dates/gregory@1/ar-EG.json index 72a8119daa1..1344653ab83 100644 --- a/resources/testdata/data/json/dates/gregory@1/ar-EG.json +++ b/resources/testdata/data/json/dates/gregory@1/ar-EG.json @@ -129,136 +129,40 @@ "medium": "{1}, {0}", "short": "{1}, {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M‏/y" - ], - [ - "yMd", - "d‏/M‏/y" - ], - [ - "yMEd", - "E، d/‏M/‏y" - ], - [ - "yMM", - "MM‏/y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d MMM y" - ], - [ - "yMMMEd", - "E، d MMM y" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "d/‏M" - ], - [ - "MEd", - "E، d/‏M" - ], - [ - "MMdd", - "dd‏/MM" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "E، d MMM" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "MMMMEd", - "E، d MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "E، d" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M‏/y", + "yMd": "d‏/M‏/y", + "yMEd": "E، d/‏M/‏y", + "yMM": "MM‏/y", + "yMMM": "MMM y", + "yMMMd": "d MMM y", + "yMMMEd": "E، d MMM y", + "yMMMM": "MMMM y", + "M": "L", + "Md": "d/‏M", + "MEd": "E، d/‏M", + "MMdd": "dd‏/MM", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "E، d MMM", + "MMMMd": "d MMMM", + "MMMMEd": "E، d MMMM", + "d": "d", + "E": "ccc", + "Ed": "E، d", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/ar.json b/resources/testdata/data/json/dates/gregory@1/ar.json index 72a8119daa1..1344653ab83 100644 --- a/resources/testdata/data/json/dates/gregory@1/ar.json +++ b/resources/testdata/data/json/dates/gregory@1/ar.json @@ -129,136 +129,40 @@ "medium": "{1}, {0}", "short": "{1}, {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M‏/y" - ], - [ - "yMd", - "d‏/M‏/y" - ], - [ - "yMEd", - "E، d/‏M/‏y" - ], - [ - "yMM", - "MM‏/y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d MMM y" - ], - [ - "yMMMEd", - "E، d MMM y" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "d/‏M" - ], - [ - "MEd", - "E، d/‏M" - ], - [ - "MMdd", - "dd‏/MM" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "E، d MMM" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "MMMMEd", - "E، d MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "E، d" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M‏/y", + "yMd": "d‏/M‏/y", + "yMEd": "E، d/‏M/‏y", + "yMM": "MM‏/y", + "yMMM": "MMM y", + "yMMMd": "d MMM y", + "yMMMEd": "E، d MMM y", + "yMMMM": "MMMM y", + "M": "L", + "Md": "d/‏M", + "MEd": "E، d/‏M", + "MMdd": "dd‏/MM", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "E، d MMM", + "MMMMd": "d MMMM", + "MMMMEd": "E، d MMMM", + "d": "d", + "E": "ccc", + "Ed": "E، d", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/bn.json b/resources/testdata/data/json/dates/gregory@1/bn.json index 6302713cc6b..2eaefd66fa2 100644 --- a/resources/testdata/data/json/dates/gregory@1/bn.json +++ b/resources/testdata/data/json/dates/gregory@1/bn.json @@ -139,136 +139,40 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M/y" - ], - [ - "yMd", - "d/M/y" - ], - [ - "yMEd", - "E, d/M/y" - ], - [ - "yMM", - "MM-y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d MMM, y" - ], - [ - "yMMMEd", - "E, d MMM, y" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "d/M" - ], - [ - "MEd", - "E, d-M" - ], - [ - "MMdd", - "dd-MM" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "E d MMM" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "MMMMEd", - "E d MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "d E" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M/y", + "yMd": "d/M/y", + "yMEd": "E, d/M/y", + "yMM": "MM-y", + "yMMM": "MMM y", + "yMMMd": "d MMM, y", + "yMMMEd": "E, d MMM, y", + "yMMMM": "MMMM y", + "M": "L", + "Md": "d/M", + "MEd": "E, d-M", + "MMdd": "dd-MM", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "E d MMM", + "MMMMd": "d MMMM", + "MMMMEd": "E d MMMM", + "d": "d", + "E": "ccc", + "Ed": "d E", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/ccp.json b/resources/testdata/data/json/dates/gregory@1/ccp.json index c8281da3340..a2db41ac287 100644 --- a/resources/testdata/data/json/dates/gregory@1/ccp.json +++ b/resources/testdata/data/json/dates/gregory@1/ccp.json @@ -153,136 +153,40 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M/y" - ], - [ - "yMd", - "d/M/y" - ], - [ - "yMEd", - "E, d/M/y" - ], - [ - "yMM", - "MM-y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d MMM, y" - ], - [ - "yMMMEd", - "E, d MMM, y" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "d/M" - ], - [ - "MEd", - "E, d-M" - ], - [ - "MMdd", - "dd-MM" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "E d MMM" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "MMMMEd", - "E d MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "d E" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M/y", + "yMd": "d/M/y", + "yMEd": "E, d/M/y", + "yMM": "MM-y", + "yMMM": "MMM y", + "yMMMd": "d MMM, y", + "yMMMEd": "E, d MMM, y", + "yMMMM": "MMMM y", + "M": "L", + "Md": "d/M", + "MEd": "E, d-M", + "MMdd": "dd-MM", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "E d MMM", + "MMMMd": "d MMMM", + "MMMMEd": "E d MMMM", + "d": "d", + "E": "ccc", + "Ed": "d E", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/en-US-posix.json b/resources/testdata/data/json/dates/gregory@1/en-US-posix.json index 91bc4ed8d35..ae911caaafe 100644 --- a/resources/testdata/data/json/dates/gregory@1/en-US-posix.json +++ b/resources/testdata/data/json/dates/gregory@1/en-US-posix.json @@ -137,124 +137,37 @@ "medium": "{1}, {0}", "short": "{1}, {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M/y" - ], - [ - "yMd", - "M/d/y" - ], - [ - "yMEd", - "E, M/d/y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "MMM d, y" - ], - [ - "yMMMEd", - "E, MMM d, y" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "M/d" - ], - [ - "MEd", - "E, M/d" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "MMM d" - ], - [ - "MMMEd", - "E, MMM d" - ], - [ - "MMMMd", - "MMMM d" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "d E" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M/y", + "yMd": "M/d/y", + "yMEd": "E, M/d/y", + "yMMM": "MMM y", + "yMMMd": "MMM d, y", + "yMMMEd": "E, MMM d, y", + "yMMMM": "MMMM y", + "M": "L", + "Md": "M/d", + "MEd": "E, M/d", + "MMM": "LLL", + "MMMd": "MMM d", + "MMMEd": "E, MMM d", + "MMMMd": "MMMM d", + "d": "d", + "E": "ccc", + "Ed": "d E", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/en-ZA.json b/resources/testdata/data/json/dates/gregory@1/en-ZA.json index 97636b9fddd..effcd9d76ba 100644 --- a/resources/testdata/data/json/dates/gregory@1/en-ZA.json +++ b/resources/testdata/data/json/dates/gregory@1/en-ZA.json @@ -137,128 +137,38 @@ "medium": "{1}, {0}", "short": "{1}, {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "MM/y" - ], - [ - "yMd", - "y/MM/dd" - ], - [ - "yMEd", - "E, y/MM/dd" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "dd MMM y" - ], - [ - "yMMMEd", - "E, dd MMM y" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "MM/dd" - ], - [ - "MEd", - "E, MM/dd" - ], - [ - "MMdd", - "dd/MM" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "dd MMM" - ], - [ - "MMMEd", - "E, dd MMM" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "E d" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "MM/y", + "yMd": "y/MM/dd", + "yMEd": "E, y/MM/dd", + "yMMM": "MMM y", + "yMMMd": "dd MMM y", + "yMMMEd": "E, dd MMM y", + "yMMMM": "MMMM y", + "M": "L", + "Md": "MM/dd", + "MEd": "E, MM/dd", + "MMdd": "dd/MM", + "MMM": "LLL", + "MMMd": "dd MMM", + "MMMEd": "E, dd MMM", + "MMMMd": "d MMMM", + "d": "d", + "E": "ccc", + "Ed": "E d", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/en.json b/resources/testdata/data/json/dates/gregory@1/en.json index 91bc4ed8d35..ae911caaafe 100644 --- a/resources/testdata/data/json/dates/gregory@1/en.json +++ b/resources/testdata/data/json/dates/gregory@1/en.json @@ -137,124 +137,37 @@ "medium": "{1}, {0}", "short": "{1}, {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M/y" - ], - [ - "yMd", - "M/d/y" - ], - [ - "yMEd", - "E, M/d/y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "MMM d, y" - ], - [ - "yMMMEd", - "E, MMM d, y" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "M/d" - ], - [ - "MEd", - "E, M/d" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "MMM d" - ], - [ - "MMMEd", - "E, MMM d" - ], - [ - "MMMMd", - "MMMM d" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "d E" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M/y", + "yMd": "M/d/y", + "yMEd": "E, M/d/y", + "yMMM": "MMM y", + "yMMMd": "MMM d, y", + "yMMMEd": "E, MMM d, y", + "yMMMM": "MMMM y", + "M": "L", + "Md": "M/d", + "MEd": "E, M/d", + "MMM": "LLL", + "MMMd": "MMM d", + "MMMEd": "E, MMM d", + "MMMMd": "MMMM d", + "d": "d", + "E": "ccc", + "Ed": "d E", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/es-AR.json b/resources/testdata/data/json/dates/gregory@1/es-AR.json index 696f23f9b88..3f661dfea59 100644 --- a/resources/testdata/data/json/dates/gregory@1/es-AR.json +++ b/resources/testdata/data/json/dates/gregory@1/es-AR.json @@ -144,152 +144,44 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M-y" - ], - [ - "yMd", - "d/M/y" - ], - [ - "yMEd", - "E, d/M/y" - ], - [ - "yMM", - "M/y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d 'de' MMM 'de' y" - ], - [ - "yMMMEd", - "E, d MMM y" - ], - [ - "yMMMM", - "MMMM 'de' y" - ], - [ - "yMMMMd", - "d 'de' MMMM 'de' y" - ], - [ - "yMMMMEd", - "EEE, d 'de' MMMM 'de' y" - ], - [ - "M", - "L" - ], - [ - "Md", - "d/M" - ], - [ - "MEd", - "E d-M" - ], - [ - "MMd", - "d/M" - ], - [ - "MMdd", - "d/M" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMdd", - "dd-MMM" - ], - [ - "MMMEd", - "E, d MMM" - ], - [ - "MMMMd", - "d 'de' MMMM" - ], - [ - "MMMMEd", - "E, d 'de' MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "E d" - ], - [ - "Ehm", - "E, h:mm a" - ], - [ - "Ehms", - "E, h:mm:ss a" - ], - [ - "EHm", - "E, HH:mm" - ], - [ - "EHms", - "E, HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "hh:mm:ss" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M-y", + "yMd": "d/M/y", + "yMEd": "E, d/M/y", + "yMM": "M/y", + "yMMM": "MMM y", + "yMMMd": "d 'de' MMM 'de' y", + "yMMMEd": "E, d MMM y", + "yMMMM": "MMMM 'de' y", + "yMMMMd": "d 'de' MMMM 'de' y", + "yMMMMEd": "EEE, d 'de' MMMM 'de' y", + "M": "L", + "Md": "d/M", + "MEd": "E d-M", + "MMd": "d/M", + "MMdd": "d/M", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMdd": "dd-MMM", + "MMMEd": "E, d MMM", + "MMMMd": "d 'de' MMMM", + "MMMMEd": "E, d 'de' MMMM", + "d": "d", + "E": "ccc", + "Ed": "E d", + "Ehm": "E, h:mm a", + "Ehms": "E, h:mm:ss a", + "EHm": "E, HH:mm", + "EHms": "E, HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "hh:mm:ss", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/es.json b/resources/testdata/data/json/dates/gregory@1/es.json index a8a29cb01af..d9bedf185fe 100644 --- a/resources/testdata/data/json/dates/gregory@1/es.json +++ b/resources/testdata/data/json/dates/gregory@1/es.json @@ -143,148 +143,43 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M/y" - ], - [ - "yMd", - "d/M/y" - ], - [ - "yMEd", - "EEE, d/M/y" - ], - [ - "yMM", - "M/y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d MMM y" - ], - [ - "yMMMEd", - "EEE, d MMM y" - ], - [ - "yMMMM", - "MMMM 'de' y" - ], - [ - "yMMMMd", - "d 'de' MMMM 'de' y" - ], - [ - "yMMMMEd", - "EEE, d 'de' MMMM 'de' y" - ], - [ - "M", - "L" - ], - [ - "Md", - "d/M" - ], - [ - "MEd", - "E, d/M" - ], - [ - "MMd", - "d/M" - ], - [ - "MMdd", - "d/M" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "E, d MMM" - ], - [ - "MMMMd", - "d 'de' MMMM" - ], - [ - "MMMMEd", - "E, d 'de' MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "E d" - ], - [ - "Ehm", - "E, h:mm a" - ], - [ - "Ehms", - "E, h:mm:ss a" - ], - [ - "EHm", - "E, H:mm" - ], - [ - "EHms", - "E, H:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "H" - ], - [ - "Hm", - "H:mm" - ], - [ - "Hms", - "H:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M/y", + "yMd": "d/M/y", + "yMEd": "EEE, d/M/y", + "yMM": "M/y", + "yMMM": "MMM y", + "yMMMd": "d MMM y", + "yMMMEd": "EEE, d MMM y", + "yMMMM": "MMMM 'de' y", + "yMMMMd": "d 'de' MMMM 'de' y", + "yMMMMEd": "EEE, d 'de' MMMM 'de' y", + "M": "L", + "Md": "d/M", + "MEd": "E, d/M", + "MMd": "d/M", + "MMdd": "d/M", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "E, d MMM", + "MMMMd": "d 'de' MMMM", + "MMMMEd": "E, d 'de' MMMM", + "d": "d", + "E": "ccc", + "Ed": "E d", + "Ehm": "E, h:mm a", + "Ehms": "E, h:mm:ss a", + "EHm": "E, H:mm", + "EHms": "E, H:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "H", + "Hm": "H:mm", + "Hms": "H:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/fr.json b/resources/testdata/data/json/dates/gregory@1/fr.json index 10ac353a439..05ac7658560 100644 --- a/resources/testdata/data/json/dates/gregory@1/fr.json +++ b/resources/testdata/data/json/dates/gregory@1/fr.json @@ -129,124 +129,37 @@ "medium": "{1}, {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "MM/y" - ], - [ - "yMd", - "dd/MM/y" - ], - [ - "yMEd", - "E dd/MM/y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d MMM y" - ], - [ - "yMMMEd", - "E d MMM y" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "dd/MM" - ], - [ - "MEd", - "E dd/MM" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "E d MMM" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "E" - ], - [ - "Ed", - "E d" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH 'h'" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "MM/y", + "yMd": "dd/MM/y", + "yMEd": "E dd/MM/y", + "yMMM": "MMM y", + "yMMMd": "d MMM y", + "yMMMEd": "E d MMM y", + "yMMMM": "MMMM y", + "M": "L", + "Md": "dd/MM", + "MEd": "E dd/MM", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "E d MMM", + "MMMMd": "d MMMM", + "d": "d", + "E": "E", + "Ed": "E d", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH 'h'", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/ja.json b/resources/testdata/data/json/dates/gregory@1/ja.json index e8c08dccaa7..678bb66dd56 100644 --- a/resources/testdata/data/json/dates/gregory@1/ja.json +++ b/resources/testdata/data/json/dates/gregory@1/ja.json @@ -129,148 +129,43 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y年" - ], - [ - "yM", - "y/M" - ], - [ - "yMd", - "y/M/d" - ], - [ - "yMEd", - "y/M/d(E)" - ], - [ - "yMEEEEd", - "y/M/dEEEE" - ], - [ - "yMM", - "y/MM" - ], - [ - "yMMM", - "y年M月" - ], - [ - "yMMMd", - "y年M月d日" - ], - [ - "yMMMEd", - "y年M月d日(E)" - ], - [ - "yMMMEEEEd", - "y年M月d日EEEE" - ], - [ - "yMMMM", - "y年M月" - ], - [ - "M", - "M月" - ], - [ - "Md", - "M/d" - ], - [ - "MEd", - "M/d(E)" - ], - [ - "MEEEEd", - "M/dEEEE" - ], - [ - "MMM", - "M月" - ], - [ - "MMMd", - "M月d日" - ], - [ - "MMMEd", - "M月d日(E)" - ], - [ - "MMMEEEEd", - "M月d日EEEE" - ], - [ - "MMMMd", - "M月d日" - ], - [ - "d", - "d日" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "d日(E)" - ], - [ - "Ehm", - "aK:mm (E)" - ], - [ - "Ehms", - "aK:mm:ss (E)" - ], - [ - "EHm", - "H:mm (E)" - ], - [ - "EHms", - "H:mm:ss (E)" - ], - [ - "EEEEd", - "d日EEEE" - ], - [ - "h", - "aK時" - ], - [ - "hm", - "aK:mm" - ], - [ - "hms", - "aK:mm:ss" - ], - [ - "H", - "H時" - ], - [ - "Hm", - "H:mm" - ], - [ - "Hms", - "H:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y年", + "yM": "y/M", + "yMd": "y/M/d", + "yMEd": "y/M/d(E)", + "yMEEEEd": "y/M/dEEEE", + "yMM": "y/MM", + "yMMM": "y年M月", + "yMMMd": "y年M月d日", + "yMMMEd": "y年M月d日(E)", + "yMMMEEEEd": "y年M月d日EEEE", + "yMMMM": "y年M月", + "M": "M月", + "Md": "M/d", + "MEd": "M/d(E)", + "MEEEEd": "M/dEEEE", + "MMM": "M月", + "MMMd": "M月d日", + "MMMEd": "M月d日(E)", + "MMMEEEEd": "M月d日EEEE", + "MMMMd": "M月d日", + "d": "d日", + "E": "ccc", + "Ed": "d日(E)", + "Ehm": "aK:mm (E)", + "Ehms": "aK:mm:ss (E)", + "EHm": "H:mm (E)", + "EHms": "H:mm:ss (E)", + "EEEEd": "d日EEEE", + "h": "aK時", + "hm": "aK:mm", + "hms": "aK:mm:ss", + "H": "H時", + "Hm": "H:mm", + "Hms": "H:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/ru.json b/resources/testdata/data/json/dates/gregory@1/ru.json index b71e189382d..0efd9a9822a 100644 --- a/resources/testdata/data/json/dates/gregory@1/ru.json +++ b/resources/testdata/data/json/dates/gregory@1/ru.json @@ -170,132 +170,39 @@ "medium": "{1}, {0}", "short": "{1}, {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "MM.y" - ], - [ - "yMd", - "dd.MM.y" - ], - [ - "yMEd", - "ccc, dd.MM.y г." - ], - [ - "yMM", - "MM.y" - ], - [ - "yMMM", - "LLL y г." - ], - [ - "yMMMd", - "d MMM y г." - ], - [ - "yMMMEd", - "E, d MMM y г." - ], - [ - "yMMMM", - "LLLL y г." - ], - [ - "M", - "L" - ], - [ - "Md", - "dd.MM" - ], - [ - "MEd", - "E, dd.MM" - ], - [ - "MMdd", - "dd.MM" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "ccc, d MMM" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "ccc, d" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "MM.y", + "yMd": "dd.MM.y", + "yMEd": "ccc, dd.MM.y г.", + "yMM": "MM.y", + "yMMM": "LLL y г.", + "yMMMd": "d MMM y г.", + "yMMMEd": "E, d MMM y г.", + "yMMMM": "LLLL y г.", + "M": "L", + "Md": "dd.MM", + "MEd": "E, dd.MM", + "MMdd": "dd.MM", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "ccc, d MMM", + "MMMMd": "d MMMM", + "d": "d", + "E": "ccc", + "Ed": "ccc, d", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json b/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json index e641123993d..26784a9e7ad 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json +++ b/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json @@ -137,144 +137,42 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y." - ], - [ - "yM", - "M.y." - ], - [ - "yMd", - "d.M.y." - ], - [ - "yMEd", - "E, d.M.y." - ], - [ - "yMM", - "MM.y." - ], - [ - "yMMdd", - "dd.MM.y." - ], - [ - "yMMM", - "MMM y." - ], - [ - "yMMMd", - "d. MMM y." - ], - [ - "yMMMEd", - "E, d. MMM y." - ], - [ - "yMMMM", - "MMMM y." - ], - [ - "M", - "L" - ], - [ - "Md", - "d.M." - ], - [ - "MEd", - "E, d.M." - ], - [ - "MMdd", - "dd.MM." - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d. MMM" - ], - [ - "MMMdd", - "dd.MMM" - ], - [ - "MMMEd", - "E d. MMM" - ], - [ - "MMMMd", - "d. MMMM" - ], - [ - "MMMMEd", - "E, d. MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "E" - ], - [ - "Ed", - "E d." - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y.", + "yM": "M.y.", + "yMd": "d.M.y.", + "yMEd": "E, d.M.y.", + "yMM": "MM.y.", + "yMMdd": "dd.MM.y.", + "yMMM": "MMM y.", + "yMMMd": "d. MMM y.", + "yMMMEd": "E, d. MMM y.", + "yMMMM": "MMMM y.", + "M": "L", + "Md": "d.M.", + "MEd": "E, d.M.", + "MMdd": "dd.MM.", + "MMM": "LLL", + "MMMd": "d. MMM", + "MMMdd": "dd.MMM", + "MMMEd": "E d. MMM", + "MMMMd": "d. MMMM", + "MMMMEd": "E, d. MMMM", + "d": "d", + "E": "E", + "Ed": "E d.", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/sr-Latn.json b/resources/testdata/data/json/dates/gregory@1/sr-Latn.json index e3975bf3035..322fd96a2dd 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr-Latn.json +++ b/resources/testdata/data/json/dates/gregory@1/sr-Latn.json @@ -137,144 +137,42 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y." - ], - [ - "yM", - "M.y." - ], - [ - "yMd", - "d.M.y." - ], - [ - "yMEd", - "E, d.M.y." - ], - [ - "yMM", - "MM.y." - ], - [ - "yMMdd", - "dd.MM.y." - ], - [ - "yMMM", - "MMM y." - ], - [ - "yMMMd", - "d. MMM y." - ], - [ - "yMMMEd", - "E, d. MMM y." - ], - [ - "yMMMM", - "MMMM y." - ], - [ - "M", - "L" - ], - [ - "Md", - "d.M." - ], - [ - "MEd", - "E, d.M." - ], - [ - "MMdd", - "dd.MM." - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d. MMM" - ], - [ - "MMMdd", - "dd.MMM" - ], - [ - "MMMEd", - "E d. MMM" - ], - [ - "MMMMd", - "d. MMMM" - ], - [ - "MMMMEd", - "E, d. MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "E" - ], - [ - "Ed", - "E d." - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y.", + "yM": "M.y.", + "yMd": "d.M.y.", + "yMEd": "E, d.M.y.", + "yMM": "MM.y.", + "yMMdd": "dd.MM.y.", + "yMMM": "MMM y.", + "yMMMd": "d. MMM y.", + "yMMMEd": "E, d. MMM y.", + "yMMMM": "MMMM y.", + "M": "L", + "Md": "d.M.", + "MEd": "E, d.M.", + "MMdd": "dd.MM.", + "MMM": "LLL", + "MMMd": "d. MMM", + "MMMdd": "dd.MMM", + "MMMEd": "E d. MMM", + "MMMMd": "d. MMMM", + "MMMMEd": "E, d. MMMM", + "d": "d", + "E": "E", + "Ed": "E d.", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/sr.json b/resources/testdata/data/json/dates/gregory@1/sr.json index e641123993d..26784a9e7ad 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr.json +++ b/resources/testdata/data/json/dates/gregory@1/sr.json @@ -137,144 +137,42 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y." - ], - [ - "yM", - "M.y." - ], - [ - "yMd", - "d.M.y." - ], - [ - "yMEd", - "E, d.M.y." - ], - [ - "yMM", - "MM.y." - ], - [ - "yMMdd", - "dd.MM.y." - ], - [ - "yMMM", - "MMM y." - ], - [ - "yMMMd", - "d. MMM y." - ], - [ - "yMMMEd", - "E, d. MMM y." - ], - [ - "yMMMM", - "MMMM y." - ], - [ - "M", - "L" - ], - [ - "Md", - "d.M." - ], - [ - "MEd", - "E, d.M." - ], - [ - "MMdd", - "dd.MM." - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d. MMM" - ], - [ - "MMMdd", - "dd.MMM" - ], - [ - "MMMEd", - "E d. MMM" - ], - [ - "MMMMd", - "d. MMMM" - ], - [ - "MMMMEd", - "E, d. MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "E" - ], - [ - "Ed", - "E d." - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y.", + "yM": "M.y.", + "yMd": "d.M.y.", + "yMEd": "E, d.M.y.", + "yMM": "MM.y.", + "yMMdd": "dd.MM.y.", + "yMMM": "MMM y.", + "yMMMd": "d. MMM y.", + "yMMMEd": "E, d. MMM y.", + "yMMMM": "MMMM y.", + "M": "L", + "Md": "d.M.", + "MEd": "E, d.M.", + "MMdd": "dd.MM.", + "MMM": "LLL", + "MMMd": "d. MMM", + "MMMdd": "dd.MMM", + "MMMEd": "E d. MMM", + "MMMMd": "d. MMMM", + "MMMMEd": "E, d. MMMM", + "d": "d", + "E": "E", + "Ed": "E d.", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/th.json b/resources/testdata/data/json/dates/gregory@1/th.json index 1beebdaa548..51849f3ba59 100644 --- a/resources/testdata/data/json/dates/gregory@1/th.json +++ b/resources/testdata/data/json/dates/gregory@1/th.json @@ -137,156 +137,45 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "M/y" - ], - [ - "yMd", - "d/M/y" - ], - [ - "yMEd", - "E d/M/y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d MMM y" - ], - [ - "yMMMEd", - "E d MMM y" - ], - [ - "yMMMEEEEd", - "EEEEที่ d MMM y" - ], - [ - "yMMMM", - "MMMM 'G' y" - ], - [ - "yMMMMd", - "d MMMM 'G' y" - ], - [ - "yMMMMEd", - "E d MMMM 'G' y" - ], - [ - "yMMMMEEEEd", - "EEEEที่ d MMMM 'G' y" - ], - [ - "M", - "L" - ], - [ - "Md", - "d/M" - ], - [ - "MEd", - "E d/M" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "E d MMM" - ], - [ - "MMMEEEEd", - "EEEEที่ d MMM" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "MMMMEd", - "E d MMMM" - ], - [ - "MMMMEEEEd", - "EEEEที่ d MMMM" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "E d" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm น." - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm น." - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ], - [ - "mmss", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "M/y", + "yMd": "d/M/y", + "yMEd": "E d/M/y", + "yMMM": "MMM y", + "yMMMd": "d MMM y", + "yMMMEd": "E d MMM y", + "yMMMEEEEd": "EEEEที่ d MMM y", + "yMMMM": "MMMM 'G' y", + "yMMMMd": "d MMMM 'G' y", + "yMMMMEd": "E d MMMM 'G' y", + "yMMMMEEEEd": "EEEEที่ d MMMM 'G' y", + "M": "L", + "Md": "d/M", + "MEd": "E d/M", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "E d MMM", + "MMMEEEEd": "EEEEที่ d MMM", + "MMMMd": "d MMMM", + "MMMMEd": "E d MMMM", + "MMMMEEEEd": "EEEEที่ d MMMM", + "d": "d", + "E": "ccc", + "Ed": "E d", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm น.", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm น.", + "Hms": "HH:mm:ss", + "ms": "mm:ss", + "mmss": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/tr.json b/resources/testdata/data/json/dates/gregory@1/tr.json index 9a702d87739..9cc6c818943 100644 --- a/resources/testdata/data/json/dates/gregory@1/tr.json +++ b/resources/testdata/data/json/dates/gregory@1/tr.json @@ -137,136 +137,40 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "MM/y" - ], - [ - "yMd", - "dd.MM.y" - ], - [ - "yMEd", - "d.M.y E" - ], - [ - "yMM", - "MM.y" - ], - [ - "yMMM", - "MMM y" - ], - [ - "yMMMd", - "d MMM y" - ], - [ - "yMMMEd", - "d MMM y E" - ], - [ - "yMMMM", - "MMMM y" - ], - [ - "M", - "L" - ], - [ - "Md", - "d/M" - ], - [ - "MEd", - "d/MM E" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "d MMM" - ], - [ - "MMMEd", - "d MMMM E" - ], - [ - "MMMMd", - "d MMMM" - ], - [ - "MMMMEd", - "d MMMM E" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "d E" - ], - [ - "Ehm", - "E a h:mm" - ], - [ - "Ehms", - "E a h:mm:ss" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "a h" - ], - [ - "hm", - "a h:mm" - ], - [ - "hms", - "a h:mm:ss" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ], - [ - "mmss", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "MM/y", + "yMd": "dd.MM.y", + "yMEd": "d.M.y E", + "yMM": "MM.y", + "yMMM": "MMM y", + "yMMMd": "d MMM y", + "yMMMEd": "d MMM y E", + "yMMMM": "MMMM y", + "M": "L", + "Md": "d/M", + "MEd": "d/MM E", + "MMM": "LLL", + "MMMd": "d MMM", + "MMMEd": "d MMMM E", + "MMMMd": "d MMMM", + "MMMMEd": "d MMMM E", + "d": "d", + "E": "ccc", + "Ed": "d E", + "Ehm": "E a h:mm", + "Ehms": "E a h:mm:ss", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "a h", + "hm": "a h:mm", + "hms": "a h:mm:ss", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss", + "mmss": "mm:ss" + } } } } diff --git a/resources/testdata/data/json/dates/gregory@1/und.json b/resources/testdata/data/json/dates/gregory@1/und.json index 67e79091093..3dd8892e5fd 100644 --- a/resources/testdata/data/json/dates/gregory@1/und.json +++ b/resources/testdata/data/json/dates/gregory@1/und.json @@ -123,124 +123,37 @@ "medium": "{1} {0}", "short": "{1} {0}" }, - "skeletons": [ - [ - "y", - "y" - ], - [ - "yM", - "y-MM" - ], - [ - "yMd", - "y-MM-dd" - ], - [ - "yMEd", - "y-MM-dd, E" - ], - [ - "yMMM", - "y MMM" - ], - [ - "yMMMd", - "y MMM d" - ], - [ - "yMMMEd", - "y MMM d, E" - ], - [ - "yMMMM", - "y MMMM" - ], - [ - "M", - "L" - ], - [ - "Md", - "MM-dd" - ], - [ - "MEd", - "MM-dd, E" - ], - [ - "MMM", - "LLL" - ], - [ - "MMMd", - "MMM d" - ], - [ - "MMMEd", - "MMM d, E" - ], - [ - "MMMMd", - "MMMM d" - ], - [ - "d", - "d" - ], - [ - "E", - "ccc" - ], - [ - "Ed", - "d, E" - ], - [ - "Ehm", - "E h:mm a" - ], - [ - "Ehms", - "E h:mm:ss a" - ], - [ - "EHm", - "E HH:mm" - ], - [ - "EHms", - "E HH:mm:ss" - ], - [ - "h", - "h a" - ], - [ - "hm", - "h:mm a" - ], - [ - "hms", - "h:mm:ss a" - ], - [ - "H", - "HH" - ], - [ - "Hm", - "HH:mm" - ], - [ - "Hms", - "HH:mm:ss" - ], - [ - "ms", - "mm:ss" - ] - ] + "skeletons": { + "y": "y", + "yM": "y-MM", + "yMd": "y-MM-dd", + "yMEd": "y-MM-dd, E", + "yMMM": "y MMM", + "yMMMd": "y MMM d", + "yMMMEd": "y MMM d, E", + "yMMMM": "y MMMM", + "M": "L", + "Md": "MM-dd", + "MEd": "MM-dd, E", + "MMM": "LLL", + "MMMd": "MMM d", + "MMMEd": "MMM d, E", + "MMMMd": "MMMM d", + "d": "d", + "E": "ccc", + "Ed": "d, E", + "Ehm": "E h:mm a", + "Ehms": "E h:mm:ss a", + "EHm": "E HH:mm", + "EHms": "E HH:mm:ss", + "h": "h a", + "hm": "h:mm a", + "hms": "h:mm:ss a", + "H": "HH", + "Hm": "HH:mm", + "Hms": "HH:mm:ss", + "ms": "mm:ss" + } } } } From e4ecc1acaf2be9ff2ca6ae4a156cc11c555b41a5 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Wed, 17 Mar 2021 15:18:48 -0500 Subject: [PATCH 12/22] Review 8: Consolidate the skeleton serialization into skeleton.rs --- components/datetime/src/lib.rs | 1 + components/datetime/src/provider/mod.rs | 378 ++---------------- components/datetime/src/skeleton.rs | 375 +++++++++++++++++ .../provider_cldr/src/transform/dates.rs | 20 +- 4 files changed, 404 insertions(+), 370 deletions(-) create mode 100644 components/datetime/src/skeleton.rs diff --git a/components/datetime/src/lib.rs b/components/datetime/src/lib.rs index 7b4a2a70c19..bc997068a1a 100644 --- a/components/datetime/src/lib.rs +++ b/components/datetime/src/lib.rs @@ -83,6 +83,7 @@ pub mod options; #[doc(hidden)] pub mod pattern; pub mod provider; +pub mod skeleton; use crate::provider::helpers::DateTimePatterns; use date::DateTimeInput; diff --git a/components/datetime/src/provider/mod.rs b/components/datetime/src/provider/mod.rs index 4f1a11aba9e..49c1dafee63 100644 --- a/components/datetime/src/provider/mod.rs +++ b/components/datetime/src/provider/mod.rs @@ -14,7 +14,6 @@ pub mod key { } pub mod gregory { - use smallvec::SmallVec; use std::borrow::Cow; #[derive(Debug, PartialEq, Clone, Default)] @@ -184,18 +183,11 @@ pub mod gregory { pub mod patterns { use super::*; use crate::{ - fields::{self, Field, FieldLength, FieldSymbol, LengthError, SymbolError}, pattern::{self, Pattern}, + skeleton::{Skeleton, SkeletonError}, }; use litemap::LiteMap; - use std::fmt; - use std::{cmp::Ordering, convert::TryFrom}; - - #[cfg(feature = "provider_serde")] - use serde::{ - de::{self, Visitor}, - ser, Deserialize, Deserializer, Serialize, - }; + use std::convert::TryFrom; #[derive(Debug, PartialEq, Clone, Default)] #[cfg_attr( @@ -209,84 +201,12 @@ pub mod gregory { pub short: Cow<'static, str>, } - /// This struct represents the serialization form of the skeleton fields. It contains - /// a single "exotic type", the fields::Field, which must remain stable between versions. - /// The skeletons are stored in a LiteMap, which is sorted according to the canonical - /// sort order. - #[derive(Debug, Eq, PartialEq, Clone, Default)] - pub struct SkeletonFieldsV1(pub SmallVec<[fields::Field; 5]>); - - impl SkeletonFieldsV1 { - /// The LiteMap structs should be ordered by the - /// `SkeletonFieldsV1` canonical order. Thishelps ensure that the skeleton - /// serialization is sorted deterministically. This order is determined by the order - /// of the fields listed in UTS 35, which is ordered from most significant, to least - /// significant. - /// - /// https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table - pub fn compare_canonical_order(&self, other: &SkeletonFieldsV1) -> Ordering { - let fields_a = &self.0; - let fields_b = &other.0; - let max_len = fields_a.len().max(fields_b.len()); - - for i in 0..max_len { - let maybe_field_a = fields_a.get(i); - let maybe_field_b = fields_b.get(i); - - match maybe_field_a { - Some(field_a) => match maybe_field_b { - Some(field_b) => { - let order_a = field_a.symbol.get_canonical_order(); - let order_b = field_b.symbol.get_canonical_order(); - if order_a < order_b { - return Ordering::Less; - } - if order_a > order_b { - return Ordering::Greater; - } - - // If the fields are equivalent, try to sort by field length. - let length_a = field_a.length as u8; - let length_b = field_b.length as u8; - - if length_a < length_b { - return Ordering::Less; - } - if length_a > length_b { - return Ordering::Greater; - } - } - None => { - // Field A has more fields. - return Ordering::Greater; - } - }, - None => { - // Field B has more fields. - return Ordering::Less; - } - }; - } - - Ordering::Equal - } - } - - impl PartialOrd for SkeletonFieldsV1 { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.compare_canonical_order(other)) - } - } - - impl Ord for SkeletonFieldsV1 { - fn cmp(&self, other: &Self) -> Ordering { - self.compare_canonical_order(other) - } - } - /// This struct is a public wrapper around the internal Pattern struct. This allows /// access to the serialization and deserialization capabilities, without exposing the /// internals of the pattern machinery. + /// + /// The Pattern is an "exotic type" in the serialization process, and handles its own + /// custom serialization practices. #[derive(Debug, PartialEq, Clone, Default)] #[cfg_attr( feature = "provider_serde", @@ -312,150 +232,27 @@ pub mod gregory { } } - /// The serialization and deserialization errors need to be mapped to a public-facing - /// error enum. - #[derive(Debug, PartialEq)] - pub enum SkeletonFieldsV1Error { - SymbolUnimplemented(u8), - SymbolUnknown(u8), - SymbolInvalid(char), - FieldTooLong, - } - - impl From for SkeletonFieldsV1Error { - fn from(err: SymbolError) -> Self { - match err { - SymbolError::Invalid(ch) => SkeletonFieldsV1Error::SymbolInvalid(ch), - SymbolError::Unknown(byte) => { - match byte { - // TODO(#487) - Flexible day periods - b'B' - // TODO(#486) - Era - | b'G' - // TODO(#418) - Timezones - | b'Z' | b'v' - // TODO(#502) - Week of month - | b'W' - // TODO(#501) - Quarters - | b'Q' - // TODO (#488) - Week of year - | b'w' - => SkeletonFieldsV1Error::SymbolUnimplemented(byte), - _ => SkeletonFieldsV1Error::SymbolUnknown(byte), - } - } - } - } - } + /// This struct is a public wrapper around the internal Skeleton struct. This allows + /// access to the serialization and deserialization capabilities, without exposing the + /// internals of the skeleton machinery. + /// + /// The Skeleton is an "exotic type" in the serialization process, and handles its own + /// custom serialization practices. + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] + #[cfg_attr( + feature = "provider_serde", + derive(serde::Serialize, serde::Deserialize) + )] + pub struct SkeletonV1(pub Skeleton); - impl From for SkeletonFieldsV1Error { - fn from(err: LengthError) -> Self { - match err { - LengthError::TooLong => SkeletonFieldsV1Error::FieldTooLong, - } - } - } + impl TryFrom<&str> for SkeletonV1 { + type Error = SkeletonError; - impl TryFrom<&str> for SkeletonFieldsV1 { - type Error = SkeletonFieldsV1Error; fn try_from(skeleton_string: &str) -> Result { - // Parse a string into a list of fields. - let mut fields = SmallVec::new(); - - let mut iter = skeleton_string.bytes().peekable(); - while let Some(byte) = iter.next() { - // Convert the byte to a valid field symbol. - let field_symbol = FieldSymbol::try_from(byte)?; - - // Go through the bytes to count how often it's repeated. - let mut field_length: u8 = 1; - while let Some(next_byte) = iter.peek() { - if *next_byte != byte { - break; - } - field_length += 1; - iter.next(); - } - - // Add the field. - fields.push(Field::from(( - field_symbol, - FieldLength::try_from(field_length)?, - ))); - } - - Ok(SkeletonFieldsV1(fields)) - } - } - - /// This is an implementation of the serde deserialization visitor pattern. - #[cfg(feature = "provider_serde")] - struct SkeletonFieldsDeserializeVisitor; - - #[cfg(feature = "provider_serde")] - impl<'de> Visitor<'de> for SkeletonFieldsDeserializeVisitor { - type Value = SkeletonFieldsV1; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - write!(formatter, "Expected to find a valid skeleton.") - } - - /// A skeleton serialized into a string follows UTS 35. - /// https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table - /// This string consists of a symbol that is repeated N times. This string is - /// deserialized here into the SkeletonFieldsV1 format which is used in memory - /// when working with formatting date times. - fn visit_str(self, skeleton_string: &str) -> Result - where - E: de::Error, - { - SkeletonFieldsV1::try_from(skeleton_string).map_err(|err| match err { - SkeletonFieldsV1Error::SymbolUnknown(byte) => E::custom(format!( - "Skeleton {:?} contained an unknown symbol, {:?}", - skeleton_string, byte as char - )), - SkeletonFieldsV1Error::SymbolUnimplemented(byte) => E::custom(format!( - "Skeleton {:?} contained an unimplemented symbol, {:?}", - skeleton_string, byte as char - )), - SkeletonFieldsV1Error::FieldTooLong => E::custom(format!( - "Skeleton \"{}\" contained a field that was too long.", - skeleton_string - )), - SkeletonFieldsV1Error::SymbolInvalid(ch) => E::custom(format!( - "Skeleton {:?} contained an invalid symbol, {:?}", - skeleton_string, ch - )), - }) - } - } - - #[cfg(feature = "provider_serde")] - impl<'de> Deserialize<'de> for SkeletonFieldsV1 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_str(SkeletonFieldsDeserializeVisitor) - } - } - - #[cfg(feature = "provider_serde")] - impl Serialize for SkeletonFieldsV1 { - fn serialize(&self, serializer: S) -> Result - where - S: ser::Serializer, - { - let mut string = String::new(); - - for field in self.0.iter() { - let ch: char = field.symbol.into(); - for _ in 0..field.length as usize { - string.push(ch); - } + match Skeleton::try_from(skeleton_string) { + Ok(skeleton) => Ok(SkeletonV1(skeleton)), + Err(err) => Err(err), } - - serializer.serialize_str(&string) } } @@ -464,7 +261,7 @@ pub mod gregory { feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) )] - pub struct SkeletonsV1(pub LiteMap); + pub struct SkeletonsV1(pub LiteMap); #[derive(Debug, PartialEq, Clone, Default)] #[cfg_attr( @@ -477,132 +274,3 @@ pub mod gregory { } } } - -#[cfg(test)] -mod test { - use super::gregory::patterns::*; - use crate::fields::{Day, Field, FieldLength, Month, Weekday}; - use std::{cmp::Ordering, convert::TryFrom}; - - // These were all of the skeletons from the "available formats" in the CLDR as of 2021-01 - // Generated with: - // https://gist.github.com/gregtatum/1d76bbdb87132f71a969a10f0c1d2d9c - - #[rustfmt::skip] - const SUPPORTED_STRING_SKELETONS: [&str; 51] = [ - "E", "EEEEd", "EHm", "EHms", "Ed", "Ehm", "Ehms", "H", "HHmm", "HHmmss", "Hm", "Hms", "M", - "MEEEEd", "MEd", "MMM", "MMMEEEEd", "MMMEd", "MMMM", "MMMMEEEEd", "MMMMEd", "MMMMd", - "MMMMdd", "MMMd", "MMMdd", "MMd", "MMdd", "Md", "Mdd", "d", "h", "hm", "hms", "mmss", "ms", - "y", "yM", "yMEEEEd", "yMEd", "yMM", "yMMM", "yMMMEEEEd", "yMMMEd", "yMMMM", "yMMMMEEEEd", - "yMMMMEd", "yMMMMccccd", "yMMMMd", "yMMMd", "yMMdd", "yMd", - ]; - - #[rustfmt::skip] - const UNSUPPORTED_STRING_SKELETONS: [&str; 28] = [ - // TODO(#487) - Flexible day periods - "Bh", "Bhm", "Bhms", "EBhm", "EBhms", - // TODO(#486) - Era - "Gy", "GyM", "GyMMM", "GyMMMEEEEd", "GyMMMEd", "GyMMMM", "GyMMMMEd", "GyMMMMd", "GyMMMd", - // TODO(#418) - Timezones - "HHmmZ", "Hmsv", "Hmsvvvv", "Hmv", "Hmvvvv", "hmsv", "hmsvvvv", "hmv", "hmvvvv", - // TODO(#502) - Week of month - "MMMMW", - // TODO(#501) - Quarters - "yQ", "yQQQ", "yQQQQ", - // TODO (#488) - Week of year - "yw" - ]; - - #[test] - fn test_known_skeletons_ok() { - for string_skeleton in &SUPPORTED_STRING_SKELETONS { - match SkeletonFieldsV1::try_from(*string_skeleton) { - Ok(_) => {} - Err(err) => { - panic!( - "Unable to parse string_skeleton {:?} with error, {:?}", - string_skeleton, err - ) - } - } - } - } - - #[test] - fn test_unsupported_skeletons_skeletons_err() { - for string_skeleton in &UNSUPPORTED_STRING_SKELETONS { - match SkeletonFieldsV1::try_from(*string_skeleton) { - Ok(_) => { - panic!( - "An unsupported field is now supported, consider moving {:?} to the \ - supported skeletons, and ensure the skeleton is properly implemented.", - string_skeleton - ) - } - Err(err) => match err { - SkeletonFieldsV1Error::SymbolUnimplemented(_) => { - // Every skeleton should return this error. - } - SkeletonFieldsV1Error::SymbolUnknown(byte) => { - panic!( - "An unknown symbol {:?} was found in the skeleton {:?}", - byte as char, string_skeleton - ) - } - SkeletonFieldsV1Error::FieldTooLong => { - panic!("An field was too longin the skeleton {:?}", string_skeleton) - } - SkeletonFieldsV1Error::SymbolInvalid(ch) => { - panic!( - "An invalid symbol {:?} was found in the skeleton {:?}", - ch, string_skeleton - ) - } - }, - } - } - } - - #[test] - fn test_skeleton_deserialization() { - assert_eq!( - SkeletonFieldsV1::try_from("MMMMEEEEd"), - Ok(SkeletonFieldsV1( - vec![ - Field { - symbol: Month::Format.into(), - length: FieldLength::Wide - }, - Field { - symbol: Weekday::Format.into(), - length: FieldLength::Wide - }, - Field { - symbol: Day::DayOfMonth.into(), - length: FieldLength::One - } - ] - .into() - )) - ); - } - - #[test] - fn test_skeleton_tuple_ordering() { - let skeletons_strings = Vec::from([ - "y", "yM", "yMEd", "yMEEEEd", "yMMM", "M", "Md", "Mdd", "MMd", "MMdd", "d", "h", "hm", - "hms", "Hm", "Hms", "ms", "mmss", - ]); - - let skeleton_fields: Vec = skeletons_strings - .iter() - .map(|skeleton_string| SkeletonFieldsV1::try_from(*skeleton_string).unwrap()) - .collect(); - - for (strings, fields) in skeletons_strings.windows(2).zip(skeleton_fields.windows(2)) { - if fields[0].compare_canonical_order(&fields[1]) != Ordering::Less { - panic!("Expected {:?} < {:?}", strings[0], strings[1]); - } - } - } -} diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs new file mode 100644 index 00000000000..d45ba11342c --- /dev/null +++ b/components/datetime/src/skeleton.rs @@ -0,0 +1,375 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/master/LICENSE ). + +//! Skeletons are used for pattern matching. See the [`Skeleton`] struct for more information. + +use smallvec::SmallVec; +use std::{cmp::Ordering, convert::TryFrom, fmt}; + +use crate::fields::{self, Field, FieldLength, FieldSymbol}; + +#[cfg(feature = "provider_serde")] +use serde::{de, ser, Deserialize, Deserializer, Serialize}; + +#[derive(Debug, PartialEq)] +struct FieldIndex(usize); + +/// A `Skeleton` is used to represent what types of `Field`s are present in a `Pattern`. The +/// ordering of the `Skeleton`'s `Field`s have no bearing on the ordering of the `Field`s and +/// `Literal`s in the `Pattern`. +/// +/// A `Skeleton` is a `Vec`, but with the invariant that it is sorted according to the canonical +/// sort order. This order is sorted according to the most significant `Field` to the least significant. +/// For example, a field with a `Minute` symbol would preceed a field with a `Second` symbol. +/// This order is documented as the order of fields as presented in the +/// [UTS 35 Date Field Symbol Table](https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table) +/// +/// The `Field`s are only sorted in the `Skeleton` in order to provide a deterministic +/// serialization strategy, and to provide a faster `Skeleton` matching operation. +#[derive(Debug, Eq, PartialEq, Clone)] +pub struct Skeleton(SmallVec<[fields::Field; 5]>); + +impl Skeleton { + /// The LiteMap structs should be ordered by the `Skeleton` canonical + /// order. This helps ensure that the skeleton serialization is sorted deterministically. + /// This order is determined by the order of the fields listed in UTS 35, which is ordered + /// from most significant, to least significant. + /// + /// https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table + pub fn compare_canonical_order(&self, other: &Skeleton) -> Ordering { + let fields_a = &self.0; + let fields_b = &other.0; + let max_len = fields_a.len().max(fields_b.len()); + + for i in 0..max_len { + let maybe_field_a = fields_a.get(i); + let maybe_field_b = fields_b.get(i); + + match maybe_field_a { + Some(field_a) => match maybe_field_b { + Some(field_b) => { + let order_a = field_a.symbol.get_canonical_order(); + let order_b = field_b.symbol.get_canonical_order(); + if order_a < order_b { + return Ordering::Less; + } + if order_a > order_b { + return Ordering::Greater; + } + + // If the fields are equivalent, try to sort by field length. + let length_a = field_a.length as u8; + let length_b = field_b.length as u8; + + if length_a < length_b { + return Ordering::Less; + } + if length_a > length_b { + return Ordering::Greater; + } + } + None => { + // Field A has more fields. + return Ordering::Greater; + } + }, + None => { + // Field B has more fields. + return Ordering::Less; + } + }; + } + + Ordering::Equal + } +} + +impl PartialOrd for Skeleton { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.compare_canonical_order(other)) + } +} + +impl Ord for Skeleton { + fn cmp(&self, other: &Self) -> Ordering { + self.compare_canonical_order(other) + } +} + +/// This is an implementation of the serde deserialization visitor pattern. +#[cfg(feature = "provider_serde")] +struct SkeletonFieldsDeserializeVisitor; + +#[cfg(feature = "provider_serde")] +impl<'de> de::Visitor<'de> for SkeletonFieldsDeserializeVisitor { + type Value = Skeleton; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "Expected to find a valid skeleton.") + } + + /// A skeleton serialized into a string follows UTS 35. + /// https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table + /// This string consists of a symbol that is repeated N times. This string is + /// deserialized here into the Skeleton format which is used in memory + /// when working with formatting date times. + fn visit_str(self, skeleton_string: &str) -> Result + where + E: de::Error, + { + Skeleton::try_from(skeleton_string).map_err(|err| match err { + SkeletonError::SymbolUnknown(byte) => E::custom(format!( + "Skeleton {:?} contained an unknown symbol, {:?}", + skeleton_string, byte as char + )), + SkeletonError::SymbolUnimplemented(byte) => E::custom(format!( + "Skeleton {:?} contained an unimplemented symbol, {:?}", + skeleton_string, byte as char + )), + SkeletonError::FieldLengthTooLong => E::custom(format!( + "Skeleton \"{}\" contained a field that was too long.", + skeleton_string + )), + SkeletonError::SymbolInvalid(ch) => E::custom(format!( + "Skeleton {:?} contained an invalid symbol, {:?}", + skeleton_string, ch + )), + SkeletonError::FieldOutOfOrder + | SkeletonError::DuplicateField + | SkeletonError::UnimplementedField(_) + | SkeletonError::SkeletonFieldOutOfOrder + | SkeletonError::Fields(_) => E::custom("TODO - Following commit"), + }) + } +} + +#[cfg(feature = "provider_serde")] +impl<'de> Deserialize<'de> for Skeleton { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(SkeletonFieldsDeserializeVisitor) + } +} + +#[cfg(feature = "provider_serde")] +impl Serialize for Skeleton { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + let mut string = String::new(); + + for field in self.0.iter() { + let ch: char = field.symbol.into(); + for _ in 0..field.length as usize { + string.push(ch); + } + } + + serializer.serialize_str(&string) + } +} + +/// Parse a string into a list of fields. This trait implementation validates the input string to +/// verify that fields are correct. If the fields are out of order, this returns an error that +/// contains the fields, which gives the callee a chance to sort the fields with the +/// `From> for Skeleton` trait. +impl TryFrom<&str> for Skeleton { + type Error = SkeletonError; + fn try_from(skeleton_string: &str) -> Result { + // Parse a string into a list of fields. + let mut fields = SmallVec::new(); + + let mut iter = skeleton_string.bytes().peekable(); + while let Some(byte) = iter.next() { + // Convert the byte to a valid field symbol. + let field_symbol = FieldSymbol::try_from(byte)?; + + // Go through the bytes to count how often it's repeated. + let mut field_length: u8 = 1; + while let Some(next_byte) = iter.peek() { + if *next_byte != byte { + break; + } + field_length += 1; + iter.next(); + } + + // Add the field. + fields.push(Field::from(( + field_symbol, + FieldLength::try_from(field_length)?, + ))); + } + + Ok(Skeleton(fields)) + } +} + +#[derive(Debug)] +pub enum SkeletonError { + FieldLengthTooLong, + FieldOutOfOrder, + DuplicateField, + SymbolUnknown(char), + SymbolInvalid(char), + SymbolUnimplemented(char), + UnimplementedField(char), + SkeletonFieldOutOfOrder, + Fields(fields::Error), +} + +impl From for SkeletonError { + fn from(fields_error: fields::Error) -> Self { + SkeletonError::Fields(fields_error) + } +} + +impl From for SkeletonError { + fn from(_: fields::LengthError) -> Self { + SkeletonError::FieldLengthTooLong + } +} + +impl From for SkeletonError { + fn from(symbol_error: fields::SymbolError) -> Self { + match symbol_error { + fields::SymbolError::Invalid(ch) => SkeletonError::SymbolInvalid(ch), + fields::SymbolError::Unknown(byte) => { + match byte { + // TODO(#487) - Flexible day periods + b'B' + // TODO(#486) - Era + | b'G' + // TODO(#418) - Timezones + | b'Z' | b'v' + // TODO(#502) - Week of month + | b'W' + // TODO(#501) - Quarters + | b'Q' + // TODO (#488) - Week of year + | b'w' + => SkeletonError::SymbolUnimplemented(byte.into()), + _ => SkeletonError::SymbolUnknown(byte.into()), + } + } + } + } +} + +#[cfg(all(test, feature = "provider_serde"))] +mod test { + use super::*; + use crate::fields::{Day, Field, FieldLength, Month, Weekday}; + + // These were all of the skeletons from the "available formats" in the CLDR as of 2021-01 + // Generated with: + // https://gist.github.com/gregtatum/1d76bbdb87132f71a969a10f0c1d2d9c + + #[rustfmt::skip] + const SUPPORTED_STRING_SKELETONS: [&str; 51] = [ + "E", "EEEEd", "EHm", "EHms", "Ed", "Ehm", "Ehms", "H", "HHmm", "HHmmss", "Hm", "Hms", "M", + "MEEEEd", "MEd", "MMM", "MMMEEEEd", "MMMEd", "MMMM", "MMMMEEEEd", "MMMMEd", "MMMMd", + "MMMMdd", "MMMd", "MMMdd", "MMd", "MMdd", "Md", "Mdd", "d", "h", "hm", "hms", "mmss", "ms", + "y", "yM", "yMEEEEd", "yMEd", "yMM", "yMMM", "yMMMEEEEd", "yMMMEd", "yMMMM", "yMMMMEEEEd", + "yMMMMEd", "yMMMMccccd", "yMMMMd", "yMMMd", "yMMdd", "yMd", + ]; + + #[rustfmt::skip] + const UNSUPPORTED_STRING_SKELETONS: [&str; 28] = [ + // TODO(#487) - Flexible day periods + "Bh", "Bhm", "Bhms", "EBhm", "EBhms", + // TODO(#486) - Era + "Gy", "GyM", "GyMMM", "GyMMMEEEEd", "GyMMMEd", "GyMMMM", "GyMMMMEd", "GyMMMMd", "GyMMMd", + // TODO(#418) - Timezones + "HHmmZ", "Hmsv", "Hmsvvvv", "Hmv", "Hmvvvv", "hmsv", "hmsvvvv", "hmv", "hmvvvv", + // TODO(#502) - Week of month + "MMMMW", + // TODO(#501) - Quarters + "yQ", "yQQQ", "yQQQQ", + // TODO (#488) - Week of year + "yw" + ]; + + #[test] + fn test_known_skeletons_ok() { + for string_skeleton in &SUPPORTED_STRING_SKELETONS { + match Skeleton::try_from(*string_skeleton) { + Ok(_) => {} + Err(err) => { + panic!( + "Unable to parse string_skeleton {:?} with error, {:?}", + string_skeleton, err + ) + } + } + } + } + + #[test] + fn test_unsupported_skeletons_skeletons_err() { + for string_skeleton in &UNSUPPORTED_STRING_SKELETONS { + match Skeleton::try_from(*string_skeleton) { + Ok(_) => { + panic!( + "An unsupported field is now supported, consider moving {:?} to the \ + supported skeletons, and ensure the skeleton is properly implemented.", + string_skeleton + ) + } + Err(err) => match err { + SkeletonError::SymbolUnimplemented(_) => { + // Every skeleton should return this error. + } + _ => panic!("TODO - {:?}", err), + }, + } + } + } + + #[test] + fn test_skeleton_deserialization() { + assert_eq!( + Skeleton::try_from("MMMMEEEEd").unwrap(), + Skeleton( + vec![ + Field { + symbol: Month::Format.into(), + length: FieldLength::Wide + }, + Field { + symbol: Weekday::Format.into(), + length: FieldLength::Wide + }, + Field { + symbol: Day::DayOfMonth.into(), + length: FieldLength::One + } + ] + .into() + ) + ); + } + + #[test] + fn test_skeleton_tuple_ordering() { + let skeletons_strings = Vec::from([ + "y", "yM", "yMEd", "yMEEEEd", "yMMM", "M", "Md", "Mdd", "MMd", "MMdd", "d", "h", "hm", + "hms", "Hm", "Hms", "ms", "mmss", + ]); + + let skeleton_fields: Vec = skeletons_strings + .iter() + .map(|skeleton_string| Skeleton::try_from(*skeleton_string).unwrap()) + .collect(); + + for (strings, fields) in skeletons_strings.windows(2).zip(skeleton_fields.windows(2)) { + if fields[0].compare_canonical_order(&fields[1]) != Ordering::Less { + panic!("Expected {:?} < {:?}", strings[0], strings[1]); + } + } + } +} diff --git a/components/provider_cldr/src/transform/dates.rs b/components/provider_cldr/src/transform/dates.rs index e8c549b7a27..94e7513599f 100644 --- a/components/provider_cldr/src/transform/dates.rs +++ b/components/provider_cldr/src/transform/dates.rs @@ -6,7 +6,7 @@ use crate::cldr_langid::CldrLangID; use crate::error::Error; use crate::reader::{get_subdirectories, open_reader}; use crate::CldrPaths; -use icu_datetime::provider::*; +use icu_datetime::{provider::*, skeleton::SkeletonError}; use icu_provider::prelude::*; use std::borrow::Cow; use std::convert::TryFrom; @@ -117,7 +117,7 @@ impl From<&cldr_json::StylePatterns> for gregory::patterns::StylePatternsV1 { impl From<&cldr_json::DateTimeFormats> for gregory::patterns::DateTimeFormatsV1 { fn from(other: &cldr_json::DateTimeFormats) -> Self { - use gregory::patterns::{PatternV1, SkeletonFieldsV1, SkeletonFieldsV1Error, SkeletonsV1}; + use gregory::patterns::{PatternV1, SkeletonV1, SkeletonsV1}; use litemap::LiteMap; // TODO(#308): Support numbering system variations. We currently throw them away. @@ -148,22 +148,12 @@ impl From<&cldr_json::DateTimeFormats> for gregory::patterns::DateTimeFormatsV1 let unique_skeleton = unique_skeleton.expect("Expected to find a skeleton."); - let skeleton_fields_v1 = match SkeletonFieldsV1::try_from(unique_skeleton) { + let skeleton_fields_v1 = match SkeletonV1::try_from(unique_skeleton) { Ok(s) => s, Err(err) => match err { // Ignore unimplemented fields for now. - SkeletonFieldsV1Error::SymbolUnimplemented(_) => continue, - SkeletonFieldsV1Error::SymbolUnknown(symbol) => panic!( - "Unknown symbol {:?} in skeleton {:?}", - symbol, unique_skeleton - ), - SkeletonFieldsV1Error::FieldTooLong => { - panic!("Field too long in skeleton {:?}", unique_skeleton) - } - SkeletonFieldsV1Error::SymbolInvalid(symbol) => panic!( - "Symbol invalid {:?} in skeleton {:?}", - symbol, unique_skeleton - ), + SkeletonError::SymbolUnimplemented(_) => continue, + _ => panic!("TODO - {:?}", err), }, }; From bcd0197f8f6c38e9f7d2021a5e706c6d83b9c1bf Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 18 Mar 2021 09:30:39 -0500 Subject: [PATCH 13/22] Review 9: Regenerate testdata to fix canonical field order in skeletons --- .../data/json/dates/gregory@1/ar-EG.json | 12 +++++----- .../data/json/dates/gregory@1/ar.json | 12 +++++----- .../data/json/dates/gregory@1/bn.json | 12 +++++----- .../data/json/dates/gregory@1/ccp.json | 12 +++++----- .../json/dates/gregory@1/en-US-posix.json | 10 ++++----- .../data/json/dates/gregory@1/en-ZA.json | 10 ++++----- .../data/json/dates/gregory@1/en.json | 10 ++++----- .../data/json/dates/gregory@1/es-AR.json | 14 ++++++------ .../data/json/dates/gregory@1/es.json | 14 ++++++------ .../data/json/dates/gregory@1/fr.json | 10 ++++----- .../data/json/dates/gregory@1/ja.json | 20 ++++++++--------- .../data/json/dates/gregory@1/ru.json | 10 ++++----- .../data/json/dates/gregory@1/sr-Cyrl.json | 12 +++++----- .../data/json/dates/gregory@1/sr-Latn.json | 12 +++++----- .../data/json/dates/gregory@1/sr.json | 12 +++++----- .../data/json/dates/gregory@1/th.json | 22 +++++++++---------- .../data/json/dates/gregory@1/tr.json | 12 +++++----- .../data/json/dates/gregory@1/und.json | 10 ++++----- 18 files changed, 113 insertions(+), 113 deletions(-) diff --git a/resources/testdata/data/json/dates/gregory@1/ar-EG.json b/resources/testdata/data/json/dates/gregory@1/ar-EG.json index 1344653ab83..f6e6c2460cc 100644 --- a/resources/testdata/data/json/dates/gregory@1/ar-EG.json +++ b/resources/testdata/data/json/dates/gregory@1/ar-EG.json @@ -133,24 +133,24 @@ "y": "y", "yM": "M‏/y", "yMd": "d‏/M‏/y", - "yMEd": "E، d/‏M/‏y", + "yMdE": "E، d/‏M/‏y", "yMM": "MM‏/y", "yMMM": "MMM y", "yMMMd": "d MMM y", - "yMMMEd": "E، d MMM y", + "yMMMdE": "E، d MMM y", "yMMMM": "MMMM y", "M": "L", "Md": "d/‏M", - "MEd": "E، d/‏M", + "MdE": "E، d/‏M", "MMdd": "dd‏/MM", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "E، d MMM", + "MMMdE": "E، d MMM", "MMMMd": "d MMMM", - "MMMMEd": "E، d MMMM", + "MMMMdE": "E، d MMMM", "d": "d", + "dE": "E، d", "E": "ccc", - "Ed": "E، d", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/ar.json b/resources/testdata/data/json/dates/gregory@1/ar.json index 1344653ab83..f6e6c2460cc 100644 --- a/resources/testdata/data/json/dates/gregory@1/ar.json +++ b/resources/testdata/data/json/dates/gregory@1/ar.json @@ -133,24 +133,24 @@ "y": "y", "yM": "M‏/y", "yMd": "d‏/M‏/y", - "yMEd": "E، d/‏M/‏y", + "yMdE": "E، d/‏M/‏y", "yMM": "MM‏/y", "yMMM": "MMM y", "yMMMd": "d MMM y", - "yMMMEd": "E، d MMM y", + "yMMMdE": "E، d MMM y", "yMMMM": "MMMM y", "M": "L", "Md": "d/‏M", - "MEd": "E، d/‏M", + "MdE": "E، d/‏M", "MMdd": "dd‏/MM", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "E، d MMM", + "MMMdE": "E، d MMM", "MMMMd": "d MMMM", - "MMMMEd": "E، d MMMM", + "MMMMdE": "E، d MMMM", "d": "d", + "dE": "E، d", "E": "ccc", - "Ed": "E، d", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/bn.json b/resources/testdata/data/json/dates/gregory@1/bn.json index 2eaefd66fa2..8cd6e12d90f 100644 --- a/resources/testdata/data/json/dates/gregory@1/bn.json +++ b/resources/testdata/data/json/dates/gregory@1/bn.json @@ -143,24 +143,24 @@ "y": "y", "yM": "M/y", "yMd": "d/M/y", - "yMEd": "E, d/M/y", + "yMdE": "E, d/M/y", "yMM": "MM-y", "yMMM": "MMM y", "yMMMd": "d MMM, y", - "yMMMEd": "E, d MMM, y", + "yMMMdE": "E, d MMM, y", "yMMMM": "MMMM y", "M": "L", "Md": "d/M", - "MEd": "E, d-M", + "MdE": "E, d-M", "MMdd": "dd-MM", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "E d MMM", + "MMMdE": "E d MMM", "MMMMd": "d MMMM", - "MMMMEd": "E d MMMM", + "MMMMdE": "E d MMMM", "d": "d", + "dE": "d E", "E": "ccc", - "Ed": "d E", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/ccp.json b/resources/testdata/data/json/dates/gregory@1/ccp.json index a2db41ac287..3b63ab16a67 100644 --- a/resources/testdata/data/json/dates/gregory@1/ccp.json +++ b/resources/testdata/data/json/dates/gregory@1/ccp.json @@ -157,24 +157,24 @@ "y": "y", "yM": "M/y", "yMd": "d/M/y", - "yMEd": "E, d/M/y", + "yMdE": "E, d/M/y", "yMM": "MM-y", "yMMM": "MMM y", "yMMMd": "d MMM, y", - "yMMMEd": "E, d MMM, y", + "yMMMdE": "E, d MMM, y", "yMMMM": "MMMM y", "M": "L", "Md": "d/M", - "MEd": "E, d-M", + "MdE": "E, d-M", "MMdd": "dd-MM", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "E d MMM", + "MMMdE": "E d MMM", "MMMMd": "d MMMM", - "MMMMEd": "E d MMMM", + "MMMMdE": "E d MMMM", "d": "d", + "dE": "d E", "E": "ccc", - "Ed": "d E", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/en-US-posix.json b/resources/testdata/data/json/dates/gregory@1/en-US-posix.json index ae911caaafe..c623dc2e869 100644 --- a/resources/testdata/data/json/dates/gregory@1/en-US-posix.json +++ b/resources/testdata/data/json/dates/gregory@1/en-US-posix.json @@ -141,21 +141,21 @@ "y": "y", "yM": "M/y", "yMd": "M/d/y", - "yMEd": "E, M/d/y", + "yMdE": "E, M/d/y", "yMMM": "MMM y", "yMMMd": "MMM d, y", - "yMMMEd": "E, MMM d, y", + "yMMMdE": "E, MMM d, y", "yMMMM": "MMMM y", "M": "L", "Md": "M/d", - "MEd": "E, M/d", + "MdE": "E, M/d", "MMM": "LLL", "MMMd": "MMM d", - "MMMEd": "E, MMM d", + "MMMdE": "E, MMM d", "MMMMd": "MMMM d", "d": "d", + "dE": "d E", "E": "ccc", - "Ed": "d E", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/en-ZA.json b/resources/testdata/data/json/dates/gregory@1/en-ZA.json index effcd9d76ba..e252c0509b4 100644 --- a/resources/testdata/data/json/dates/gregory@1/en-ZA.json +++ b/resources/testdata/data/json/dates/gregory@1/en-ZA.json @@ -141,22 +141,22 @@ "y": "y", "yM": "MM/y", "yMd": "y/MM/dd", - "yMEd": "E, y/MM/dd", + "yMdE": "E, y/MM/dd", "yMMM": "MMM y", "yMMMd": "dd MMM y", - "yMMMEd": "E, dd MMM y", + "yMMMdE": "E, dd MMM y", "yMMMM": "MMMM y", "M": "L", "Md": "MM/dd", - "MEd": "E, MM/dd", + "MdE": "E, MM/dd", "MMdd": "dd/MM", "MMM": "LLL", "MMMd": "dd MMM", - "MMMEd": "E, dd MMM", + "MMMdE": "E, dd MMM", "MMMMd": "d MMMM", "d": "d", + "dE": "E d", "E": "ccc", - "Ed": "E d", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/en.json b/resources/testdata/data/json/dates/gregory@1/en.json index ae911caaafe..c623dc2e869 100644 --- a/resources/testdata/data/json/dates/gregory@1/en.json +++ b/resources/testdata/data/json/dates/gregory@1/en.json @@ -141,21 +141,21 @@ "y": "y", "yM": "M/y", "yMd": "M/d/y", - "yMEd": "E, M/d/y", + "yMdE": "E, M/d/y", "yMMM": "MMM y", "yMMMd": "MMM d, y", - "yMMMEd": "E, MMM d, y", + "yMMMdE": "E, MMM d, y", "yMMMM": "MMMM y", "M": "L", "Md": "M/d", - "MEd": "E, M/d", + "MdE": "E, M/d", "MMM": "LLL", "MMMd": "MMM d", - "MMMEd": "E, MMM d", + "MMMdE": "E, MMM d", "MMMMd": "MMMM d", "d": "d", + "dE": "d E", "E": "ccc", - "Ed": "d E", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/es-AR.json b/resources/testdata/data/json/dates/gregory@1/es-AR.json index 3f661dfea59..6c8db988304 100644 --- a/resources/testdata/data/json/dates/gregory@1/es-AR.json +++ b/resources/testdata/data/json/dates/gregory@1/es-AR.json @@ -148,28 +148,28 @@ "y": "y", "yM": "M-y", "yMd": "d/M/y", - "yMEd": "E, d/M/y", + "yMdE": "E, d/M/y", "yMM": "M/y", "yMMM": "MMM y", "yMMMd": "d 'de' MMM 'de' y", - "yMMMEd": "E, d MMM y", + "yMMMdE": "E, d MMM y", "yMMMM": "MMMM 'de' y", "yMMMMd": "d 'de' MMMM 'de' y", - "yMMMMEd": "EEE, d 'de' MMMM 'de' y", + "yMMMMdE": "EEE, d 'de' MMMM 'de' y", "M": "L", "Md": "d/M", - "MEd": "E d-M", + "MdE": "E d-M", "MMd": "d/M", "MMdd": "d/M", "MMM": "LLL", "MMMd": "d MMM", + "MMMdE": "E, d MMM", "MMMdd": "dd-MMM", - "MMMEd": "E, d MMM", "MMMMd": "d 'de' MMMM", - "MMMMEd": "E, d 'de' MMMM", + "MMMMdE": "E, d 'de' MMMM", "d": "d", + "dE": "E d", "E": "ccc", - "Ed": "E d", "Ehm": "E, h:mm a", "Ehms": "E, h:mm:ss a", "EHm": "E, HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/es.json b/resources/testdata/data/json/dates/gregory@1/es.json index d9bedf185fe..0d65e5b83b2 100644 --- a/resources/testdata/data/json/dates/gregory@1/es.json +++ b/resources/testdata/data/json/dates/gregory@1/es.json @@ -147,27 +147,27 @@ "y": "y", "yM": "M/y", "yMd": "d/M/y", - "yMEd": "EEE, d/M/y", + "yMdE": "EEE, d/M/y", "yMM": "M/y", "yMMM": "MMM y", "yMMMd": "d MMM y", - "yMMMEd": "EEE, d MMM y", + "yMMMdE": "EEE, d MMM y", "yMMMM": "MMMM 'de' y", "yMMMMd": "d 'de' MMMM 'de' y", - "yMMMMEd": "EEE, d 'de' MMMM 'de' y", + "yMMMMdE": "EEE, d 'de' MMMM 'de' y", "M": "L", "Md": "d/M", - "MEd": "E, d/M", + "MdE": "E, d/M", "MMd": "d/M", "MMdd": "d/M", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "E, d MMM", + "MMMdE": "E, d MMM", "MMMMd": "d 'de' MMMM", - "MMMMEd": "E, d 'de' MMMM", + "MMMMdE": "E, d 'de' MMMM", "d": "d", + "dE": "E d", "E": "ccc", - "Ed": "E d", "Ehm": "E, h:mm a", "Ehms": "E, h:mm:ss a", "EHm": "E, H:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/fr.json b/resources/testdata/data/json/dates/gregory@1/fr.json index 05ac7658560..ad8a6939d66 100644 --- a/resources/testdata/data/json/dates/gregory@1/fr.json +++ b/resources/testdata/data/json/dates/gregory@1/fr.json @@ -133,21 +133,21 @@ "y": "y", "yM": "MM/y", "yMd": "dd/MM/y", - "yMEd": "E dd/MM/y", + "yMdE": "E dd/MM/y", "yMMM": "MMM y", "yMMMd": "d MMM y", - "yMMMEd": "E d MMM y", + "yMMMdE": "E d MMM y", "yMMMM": "MMMM y", "M": "L", "Md": "dd/MM", - "MEd": "E dd/MM", + "MdE": "E dd/MM", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "E d MMM", + "MMMdE": "E d MMM", "MMMMd": "d MMMM", "d": "d", + "dE": "E d", "E": "E", - "Ed": "E d", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/ja.json b/resources/testdata/data/json/dates/gregory@1/ja.json index 678bb66dd56..a9ab5921761 100644 --- a/resources/testdata/data/json/dates/gregory@1/ja.json +++ b/resources/testdata/data/json/dates/gregory@1/ja.json @@ -133,31 +133,31 @@ "y": "y年", "yM": "y/M", "yMd": "y/M/d", - "yMEd": "y/M/d(E)", - "yMEEEEd": "y/M/dEEEE", + "yMdE": "y/M/d(E)", + "yMdEEEE": "y/M/dEEEE", "yMM": "y/MM", "yMMM": "y年M月", "yMMMd": "y年M月d日", - "yMMMEd": "y年M月d日(E)", - "yMMMEEEEd": "y年M月d日EEEE", + "yMMMdE": "y年M月d日(E)", + "yMMMdEEEE": "y年M月d日EEEE", "yMMMM": "y年M月", "M": "M月", "Md": "M/d", - "MEd": "M/d(E)", - "MEEEEd": "M/dEEEE", + "MdE": "M/d(E)", + "MdEEEE": "M/dEEEE", "MMM": "M月", "MMMd": "M月d日", - "MMMEd": "M月d日(E)", - "MMMEEEEd": "M月d日EEEE", + "MMMdE": "M月d日(E)", + "MMMdEEEE": "M月d日EEEE", "MMMMd": "M月d日", "d": "d日", + "dE": "d日(E)", + "dEEEE": "d日EEEE", "E": "ccc", - "Ed": "d日(E)", "Ehm": "aK:mm (E)", "Ehms": "aK:mm:ss (E)", "EHm": "H:mm (E)", "EHms": "H:mm:ss (E)", - "EEEEd": "d日EEEE", "h": "aK時", "hm": "aK:mm", "hms": "aK:mm:ss", diff --git a/resources/testdata/data/json/dates/gregory@1/ru.json b/resources/testdata/data/json/dates/gregory@1/ru.json index 0efd9a9822a..4cbfca94c63 100644 --- a/resources/testdata/data/json/dates/gregory@1/ru.json +++ b/resources/testdata/data/json/dates/gregory@1/ru.json @@ -174,23 +174,23 @@ "y": "y", "yM": "MM.y", "yMd": "dd.MM.y", - "yMEd": "ccc, dd.MM.y г.", + "yMdE": "ccc, dd.MM.y г.", "yMM": "MM.y", "yMMM": "LLL y г.", "yMMMd": "d MMM y г.", - "yMMMEd": "E, d MMM y г.", + "yMMMdE": "E, d MMM y г.", "yMMMM": "LLLL y г.", "M": "L", "Md": "dd.MM", - "MEd": "E, dd.MM", + "MdE": "E, dd.MM", "MMdd": "dd.MM", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "ccc, d MMM", + "MMMdE": "ccc, d MMM", "MMMMd": "d MMMM", "d": "d", + "dE": "ccc, d", "E": "ccc", - "Ed": "ccc, d", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json b/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json index 26784a9e7ad..6bcc921548b 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json +++ b/resources/testdata/data/json/dates/gregory@1/sr-Cyrl.json @@ -141,26 +141,26 @@ "y": "y.", "yM": "M.y.", "yMd": "d.M.y.", - "yMEd": "E, d.M.y.", + "yMdE": "E, d.M.y.", "yMM": "MM.y.", "yMMdd": "dd.MM.y.", "yMMM": "MMM y.", "yMMMd": "d. MMM y.", - "yMMMEd": "E, d. MMM y.", + "yMMMdE": "E, d. MMM y.", "yMMMM": "MMMM y.", "M": "L", "Md": "d.M.", - "MEd": "E, d.M.", + "MdE": "E, d.M.", "MMdd": "dd.MM.", "MMM": "LLL", "MMMd": "d. MMM", + "MMMdE": "E d. MMM", "MMMdd": "dd.MMM", - "MMMEd": "E d. MMM", "MMMMd": "d. MMMM", - "MMMMEd": "E, d. MMMM", + "MMMMdE": "E, d. MMMM", "d": "d", + "dE": "E d.", "E": "E", - "Ed": "E d.", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/sr-Latn.json b/resources/testdata/data/json/dates/gregory@1/sr-Latn.json index 322fd96a2dd..9762de52d80 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr-Latn.json +++ b/resources/testdata/data/json/dates/gregory@1/sr-Latn.json @@ -141,26 +141,26 @@ "y": "y.", "yM": "M.y.", "yMd": "d.M.y.", - "yMEd": "E, d.M.y.", + "yMdE": "E, d.M.y.", "yMM": "MM.y.", "yMMdd": "dd.MM.y.", "yMMM": "MMM y.", "yMMMd": "d. MMM y.", - "yMMMEd": "E, d. MMM y.", + "yMMMdE": "E, d. MMM y.", "yMMMM": "MMMM y.", "M": "L", "Md": "d.M.", - "MEd": "E, d.M.", + "MdE": "E, d.M.", "MMdd": "dd.MM.", "MMM": "LLL", "MMMd": "d. MMM", + "MMMdE": "E d. MMM", "MMMdd": "dd.MMM", - "MMMEd": "E d. MMM", "MMMMd": "d. MMMM", - "MMMMEd": "E, d. MMMM", + "MMMMdE": "E, d. MMMM", "d": "d", + "dE": "E d.", "E": "E", - "Ed": "E d.", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/sr.json b/resources/testdata/data/json/dates/gregory@1/sr.json index 26784a9e7ad..6bcc921548b 100644 --- a/resources/testdata/data/json/dates/gregory@1/sr.json +++ b/resources/testdata/data/json/dates/gregory@1/sr.json @@ -141,26 +141,26 @@ "y": "y.", "yM": "M.y.", "yMd": "d.M.y.", - "yMEd": "E, d.M.y.", + "yMdE": "E, d.M.y.", "yMM": "MM.y.", "yMMdd": "dd.MM.y.", "yMMM": "MMM y.", "yMMMd": "d. MMM y.", - "yMMMEd": "E, d. MMM y.", + "yMMMdE": "E, d. MMM y.", "yMMMM": "MMMM y.", "M": "L", "Md": "d.M.", - "MEd": "E, d.M.", + "MdE": "E, d.M.", "MMdd": "dd.MM.", "MMM": "LLL", "MMMd": "d. MMM", + "MMMdE": "E d. MMM", "MMMdd": "dd.MMM", - "MMMEd": "E d. MMM", "MMMMd": "d. MMMM", - "MMMMEd": "E, d. MMMM", + "MMMMdE": "E, d. MMMM", "d": "d", + "dE": "E d.", "E": "E", - "Ed": "E d.", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/th.json b/resources/testdata/data/json/dates/gregory@1/th.json index 51849f3ba59..7213ca7e582 100644 --- a/resources/testdata/data/json/dates/gregory@1/th.json +++ b/resources/testdata/data/json/dates/gregory@1/th.json @@ -141,28 +141,28 @@ "y": "y", "yM": "M/y", "yMd": "d/M/y", - "yMEd": "E d/M/y", + "yMdE": "E d/M/y", "yMMM": "MMM y", "yMMMd": "d MMM y", - "yMMMEd": "E d MMM y", - "yMMMEEEEd": "EEEEที่ d MMM y", + "yMMMdE": "E d MMM y", + "yMMMdEEEE": "EEEEที่ d MMM y", "yMMMM": "MMMM 'G' y", "yMMMMd": "d MMMM 'G' y", - "yMMMMEd": "E d MMMM 'G' y", - "yMMMMEEEEd": "EEEEที่ d MMMM 'G' y", + "yMMMMdE": "E d MMMM 'G' y", + "yMMMMdEEEE": "EEEEที่ d MMMM 'G' y", "M": "L", "Md": "d/M", - "MEd": "E d/M", + "MdE": "E d/M", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "E d MMM", - "MMMEEEEd": "EEEEที่ d MMM", + "MMMdE": "E d MMM", + "MMMdEEEE": "EEEEที่ d MMM", "MMMMd": "d MMMM", - "MMMMEd": "E d MMMM", - "MMMMEEEEd": "EEEEที่ d MMMM", + "MMMMdE": "E d MMMM", + "MMMMdEEEE": "EEEEที่ d MMMM", "d": "d", + "dE": "E d", "E": "ccc", - "Ed": "E d", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm น.", diff --git a/resources/testdata/data/json/dates/gregory@1/tr.json b/resources/testdata/data/json/dates/gregory@1/tr.json index 9cc6c818943..029e897a0a5 100644 --- a/resources/testdata/data/json/dates/gregory@1/tr.json +++ b/resources/testdata/data/json/dates/gregory@1/tr.json @@ -141,23 +141,23 @@ "y": "y", "yM": "MM/y", "yMd": "dd.MM.y", - "yMEd": "d.M.y E", + "yMdE": "d.M.y E", "yMM": "MM.y", "yMMM": "MMM y", "yMMMd": "d MMM y", - "yMMMEd": "d MMM y E", + "yMMMdE": "d MMM y E", "yMMMM": "MMMM y", "M": "L", "Md": "d/M", - "MEd": "d/MM E", + "MdE": "d/MM E", "MMM": "LLL", "MMMd": "d MMM", - "MMMEd": "d MMMM E", + "MMMdE": "d MMMM E", "MMMMd": "d MMMM", - "MMMMEd": "d MMMM E", + "MMMMdE": "d MMMM E", "d": "d", + "dE": "d E", "E": "ccc", - "Ed": "d E", "Ehm": "E a h:mm", "Ehms": "E a h:mm:ss", "EHm": "E HH:mm", diff --git a/resources/testdata/data/json/dates/gregory@1/und.json b/resources/testdata/data/json/dates/gregory@1/und.json index 3dd8892e5fd..4cfcddd7475 100644 --- a/resources/testdata/data/json/dates/gregory@1/und.json +++ b/resources/testdata/data/json/dates/gregory@1/und.json @@ -127,21 +127,21 @@ "y": "y", "yM": "y-MM", "yMd": "y-MM-dd", - "yMEd": "y-MM-dd, E", + "yMdE": "y-MM-dd, E", "yMMM": "y MMM", "yMMMd": "y MMM d", - "yMMMEd": "y MMM d, E", + "yMMMdE": "y MMM d, E", "yMMMM": "y MMMM", "M": "L", "Md": "MM-dd", - "MEd": "MM-dd, E", + "MdE": "MM-dd, E", "MMM": "LLL", "MMMd": "MMM d", - "MMMEd": "MMM d, E", + "MMMdE": "MMM d, E", "MMMMd": "MMMM d", "d": "d", + "dE": "d, E", "E": "ccc", - "Ed": "d, E", "Ehm": "E h:mm a", "Ehms": "E h:mm:ss a", "EHm": "E HH:mm", From 9565e673d39d4fcacf4d14d63bfc4308d7d0423f Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 18 Mar 2021 09:34:51 -0500 Subject: [PATCH 14/22] Review 10: Update tests to fix canonical field order in skeletons --- components/datetime/src/skeleton.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs index d45ba11342c..52107ea36e0 100644 --- a/components/datetime/src/skeleton.rs +++ b/components/datetime/src/skeleton.rs @@ -271,11 +271,11 @@ mod test { #[rustfmt::skip] const SUPPORTED_STRING_SKELETONS: [&str; 51] = [ - "E", "EEEEd", "EHm", "EHms", "Ed", "Ehm", "Ehms", "H", "HHmm", "HHmmss", "Hm", "Hms", "M", - "MEEEEd", "MEd", "MMM", "MMMEEEEd", "MMMEd", "MMMM", "MMMMEEEEd", "MMMMEd", "MMMMd", + "E", "dEEEE", "EHm", "EHms", "dE", "Ehm", "Ehms", "H", "HHmm", "HHmmss", "Hm", "Hms", "M", + "MdEEEE", "MdE", "MMM", "MMMdEEEE", "MMMdE", "MMMM", "MMMMdEEEE", "MMMMdE", "MMMMd", "MMMMdd", "MMMd", "MMMdd", "MMd", "MMdd", "Md", "Mdd", "d", "h", "hm", "hms", "mmss", "ms", - "y", "yM", "yMEEEEd", "yMEd", "yMM", "yMMM", "yMMMEEEEd", "yMMMEd", "yMMMM", "yMMMMEEEEd", - "yMMMMEd", "yMMMMccccd", "yMMMMd", "yMMMd", "yMMdd", "yMd", + "y", "yM", "yMdEEEE", "yMdE", "yMM", "yMMM", "yMMMdEEEE", "yMMMdE", "yMMMM", "yMMMMdEEEE", + "yMMMMdE", "yMMMMdcccc", "yMMMMd", "yMMMd", "yMMdd", "yMd", ]; #[rustfmt::skip] @@ -283,7 +283,7 @@ mod test { // TODO(#487) - Flexible day periods "Bh", "Bhm", "Bhms", "EBhm", "EBhms", // TODO(#486) - Era - "Gy", "GyM", "GyMMM", "GyMMMEEEEd", "GyMMMEd", "GyMMMM", "GyMMMMEd", "GyMMMMd", "GyMMMd", + "Gy", "GyM", "GyMMM", "GyMMMdEEEE", "GyMMMdE", "GyMMMM", "GyMMMMdE", "GyMMMMd", "GyMMMd", // TODO(#418) - Timezones "HHmmZ", "Hmsv", "Hmsvvvv", "Hmv", "Hmvvvv", "hmsv", "hmsvvvv", "hmv", "hmvvvv", // TODO(#502) - Week of month @@ -333,21 +333,21 @@ mod test { #[test] fn test_skeleton_deserialization() { assert_eq!( - Skeleton::try_from("MMMMEEEEd").unwrap(), + Skeleton::try_from("MMMMdEEEE").unwrap(), Skeleton( vec![ Field { symbol: Month::Format.into(), length: FieldLength::Wide }, + Field { + symbol: Day::DayOfMonth.into(), + length: FieldLength::One + }, Field { symbol: Weekday::Format.into(), length: FieldLength::Wide }, - Field { - symbol: Day::DayOfMonth.into(), - length: FieldLength::One - } ] .into() ) @@ -357,7 +357,7 @@ mod test { #[test] fn test_skeleton_tuple_ordering() { let skeletons_strings = Vec::from([ - "y", "yM", "yMEd", "yMEEEEd", "yMMM", "M", "Md", "Mdd", "MMd", "MMdd", "d", "h", "hm", + "y", "yM", "yMdE", "yMdEEEE", "yMMM", "M", "Md", "Mdd", "MMd", "MMdd", "d", "h", "hm", "hms", "Hm", "Hms", "ms", "mmss", ]); From ae2338fa51b7ba4641a9d5083fcfdfeb28785afb Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 18 Mar 2021 09:40:25 -0500 Subject: [PATCH 15/22] Review 11: Add fmt::Display for serde errors with skeletons --- components/datetime/src/fields/mod.rs | 13 +++++++- components/datetime/src/skeleton.rs | 31 +++++++++++++++++-- .../provider_cldr/src/transform/dates.rs | 2 +- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/components/datetime/src/fields/mod.rs b/components/datetime/src/fields/mod.rs index d0644aff504..1c030c5fc21 100644 --- a/components/datetime/src/fields/mod.rs +++ b/components/datetime/src/fields/mod.rs @@ -7,13 +7,24 @@ pub(crate) mod symbols; pub use length::{FieldLength, LengthError}; pub use symbols::*; -use std::convert::{TryFrom, TryInto}; +use std::{ + convert::{TryFrom, TryInto}, + fmt, +}; #[derive(Debug)] pub enum Error { TooLong(FieldSymbol), } +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::TooLong(symbol) => write!(f, "field {:?} is too long", symbol), + } + } +} + #[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs index 52107ea36e0..29a4328439a 100644 --- a/components/datetime/src/skeleton.rs +++ b/components/datetime/src/skeleton.rs @@ -212,16 +212,41 @@ impl TryFrom<&str> for Skeleton { #[derive(Debug)] pub enum SkeletonError { FieldLengthTooLong, - FieldOutOfOrder, + FieldsOutOfOrder(SmallVec<[fields::Field; 5]>), DuplicateField, SymbolUnknown(char), SymbolInvalid(char), SymbolUnimplemented(char), UnimplementedField(char), - SkeletonFieldOutOfOrder, Fields(fields::Error), } +/// These strings follow the recommendations for the serde::de::Unexpected::Other type. +/// https://docs.serde.rs/serde/de/enum.Unexpected.html#variant.Other +/// +/// Serde will generate an error such as: +/// "invalid value: unclosed literal in pattern, expected a valid UTS 35 pattern string at line 1 column 12" +impl fmt::Display for SkeletonError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + SkeletonError::FieldLengthTooLong => write!(f, "field too long in skeleton"), + SkeletonError::DuplicateField => write!(f, "duplicate field in skeleton"), + SkeletonError::SymbolUnknown(ch) => write!(f, "symbol unknown {} in skeleton", ch), + SkeletonError::SymbolInvalid(ch) => write!(f, "symbol invalid {} in skeleton", ch), + SkeletonError::SymbolUnimplemented(ch) => { + write!(f, "symbol unimplemented {} in skeleton", ch) + } + SkeletonError::UnimplementedField(ch) => { + write!(f, "unimplemented field {} in skeleton", ch) + } + SkeletonError::FieldsOutOfOrder(fields) => { + write!(f, "skeleton fields {:?} out of order", fields) + } + SkeletonError::Fields(err) => write!(f, "{} in skeleton", err), + } + } +} + impl From for SkeletonError { fn from(fields_error: fields::Error) -> Self { SkeletonError::Fields(fields_error) @@ -324,7 +349,7 @@ mod test { SkeletonError::SymbolUnimplemented(_) => { // Every skeleton should return this error. } - _ => panic!("TODO - {:?}", err), + _ => panic!("{}", err), }, } } diff --git a/components/provider_cldr/src/transform/dates.rs b/components/provider_cldr/src/transform/dates.rs index 94e7513599f..1512c407aa0 100644 --- a/components/provider_cldr/src/transform/dates.rs +++ b/components/provider_cldr/src/transform/dates.rs @@ -153,7 +153,7 @@ impl From<&cldr_json::DateTimeFormats> for gregory::patterns::DateTimeFormatsV1 Err(err) => match err { // Ignore unimplemented fields for now. SkeletonError::SymbolUnimplemented(_) => continue, - _ => panic!("TODO - {:?}", err), + _ => panic!("{:?} {}", unique_skeleton, err), }, }; From 1584020710681b22348138453040a0618a0c4fdd Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 18 Mar 2021 09:41:25 -0500 Subject: [PATCH 16/22] Review 12: Enforce sorting of skeleton fields coming from CLDR --- components/datetime/src/provider/mod.rs | 5 ++ components/datetime/src/skeleton.rs | 94 ++++++++++++++----------- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/components/datetime/src/provider/mod.rs b/components/datetime/src/provider/mod.rs index 49c1dafee63..22054469035 100644 --- a/components/datetime/src/provider/mod.rs +++ b/components/datetime/src/provider/mod.rs @@ -251,6 +251,11 @@ pub mod gregory { fn try_from(skeleton_string: &str) -> Result { match Skeleton::try_from(skeleton_string) { Ok(skeleton) => Ok(SkeletonV1(skeleton)), + Err(SkeletonError::FieldsOutOfOrder(fields)) => { + // The fields were out of order, converting fields to a skeleton will ensure + // that these fields are sorted. This sorting is only done in the provider. + Ok(SkeletonV1(Skeleton::from(fields))) + } Err(err) => Err(err), } } diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs index 29a4328439a..920dc63d44a 100644 --- a/components/datetime/src/skeleton.rs +++ b/components/datetime/src/skeleton.rs @@ -49,24 +49,9 @@ impl Skeleton { match maybe_field_a { Some(field_a) => match maybe_field_b { Some(field_b) => { - let order_a = field_a.symbol.get_canonical_order(); - let order_b = field_b.symbol.get_canonical_order(); - if order_a < order_b { - return Ordering::Less; - } - if order_a > order_b { - return Ordering::Greater; - } - - // If the fields are equivalent, try to sort by field length. - let length_a = field_a.length as u8; - let length_b = field_b.length as u8; - - if length_a < length_b { - return Ordering::Less; - } - if length_a > length_b { - return Ordering::Greater; + let ordering = compare_fields(field_a, field_b); + if ordering != Ordering::Equal { + return ordering; } } None => { @@ -85,6 +70,29 @@ impl Skeleton { } } +fn compare_fields(field_a: &Field, field_b: &Field) -> Ordering { + let order_a = field_a.symbol.get_canonical_order(); + let order_b = field_b.symbol.get_canonical_order(); + if order_a < order_b { + return Ordering::Less; + } + if order_a > order_b { + return Ordering::Greater; + } + + // If the fields are equivalent, try to sort by field length. + let length_a = field_a.length as u8; + let length_b = field_b.length as u8; + + if length_a < length_b { + return Ordering::Less; + } + if length_a > length_b { + return Ordering::Greater; + } + return Ordering::Equal; +} + impl PartialOrd for Skeleton { fn partial_cmp(&self, other: &Self) -> Option { Some(self.compare_canonical_order(other)) @@ -118,28 +126,11 @@ impl<'de> de::Visitor<'de> for SkeletonFieldsDeserializeVisitor { where E: de::Error, { - Skeleton::try_from(skeleton_string).map_err(|err| match err { - SkeletonError::SymbolUnknown(byte) => E::custom(format!( - "Skeleton {:?} contained an unknown symbol, {:?}", - skeleton_string, byte as char - )), - SkeletonError::SymbolUnimplemented(byte) => E::custom(format!( - "Skeleton {:?} contained an unimplemented symbol, {:?}", - skeleton_string, byte as char - )), - SkeletonError::FieldLengthTooLong => E::custom(format!( - "Skeleton \"{}\" contained a field that was too long.", - skeleton_string - )), - SkeletonError::SymbolInvalid(ch) => E::custom(format!( - "Skeleton {:?} contained an invalid symbol, {:?}", - skeleton_string, ch - )), - SkeletonError::FieldOutOfOrder - | SkeletonError::DuplicateField - | SkeletonError::UnimplementedField(_) - | SkeletonError::SkeletonFieldOutOfOrder - | SkeletonError::Fields(_) => E::custom("TODO - Following commit"), + Skeleton::try_from(skeleton_string).map_err(|err| { + de::Error::invalid_value( + de::Unexpected::Other(&format!("{:?} {}", skeleton_string, err)), + &"a UTS 35 sorted field symbols representing a skeleton", + ) }) } } @@ -180,8 +171,8 @@ impl Serialize for Skeleton { impl TryFrom<&str> for Skeleton { type Error = SkeletonError; fn try_from(skeleton_string: &str) -> Result { - // Parse a string into a list of fields. - let mut fields = SmallVec::new(); + let mut fields: SmallVec<[fields::Field; 5]> = SmallVec::new(); + let mut in_order = true; let mut iter = skeleton_string.bytes().peekable(); while let Some(byte) = iter.next() { @@ -198,6 +189,11 @@ impl TryFrom<&str> for Skeleton { iter.next(); } + if let Some(prev_field) = fields.last() { + in_order = in_order + && prev_field.symbol.get_canonical_order() < field_symbol.get_canonical_order(); + } + // Add the field. fields.push(Field::from(( field_symbol, @@ -205,7 +201,19 @@ impl TryFrom<&str> for Skeleton { ))); } - Ok(Skeleton(fields)) + if in_order { + Ok(Skeleton(fields)) + } else { + Err(SkeletonError::FieldsOutOfOrder(fields)) + } + } +} + +/// Apply the sorting invarants for an unknown list of fields. +impl From> for Skeleton { + fn from(mut fields: SmallVec<[fields::Field; 5]>) -> Skeleton { + fields.sort_by(compare_fields); + Skeleton(fields) } } From 01eaba9e015a49531a0295db0c4a412defbc6242 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 25 Mar 2021 10:25:03 -0500 Subject: [PATCH 17/22] ReviewB 1: Add support for skeleton bincode serialization --- components/datetime/src/skeleton.rs | 72 ++++++++-- .../tests/fixtures/tests/skeletons.bin | Bin 0 -> 728 bytes .../tests/fixtures/tests/skeletons.json | 22 +++ .../datetime/tests/skeleton_serialization.rs | 130 ++++++++++++++++++ 4 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 components/datetime/tests/fixtures/tests/skeletons.bin create mode 100644 components/datetime/tests/fixtures/tests/skeletons.json create mode 100644 components/datetime/tests/skeleton_serialization.rs diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs index 920dc63d44a..eff11524704 100644 --- a/components/datetime/src/skeleton.rs +++ b/components/datetime/src/skeleton.rs @@ -10,7 +10,11 @@ use std::{cmp::Ordering, convert::TryFrom, fmt}; use crate::fields::{self, Field, FieldLength, FieldSymbol}; #[cfg(feature = "provider_serde")] -use serde::{de, ser, Deserialize, Deserializer, Serialize}; +use serde::{ + de, + ser::{self, SerializeSeq}, + Deserialize, Deserializer, Serialize, +}; #[derive(Debug, PartialEq)] struct FieldIndex(usize); @@ -31,6 +35,14 @@ struct FieldIndex(usize); pub struct Skeleton(SmallVec<[fields::Field; 5]>); impl Skeleton { + fn fields_iter<'a>(&'a self) -> impl Iterator + 'a { + self.0.iter() + } + + fn fields_len(&self) -> usize { + self.0.len() + } + /// The LiteMap structs should be ordered by the `Skeleton` canonical /// order. This helps ensure that the skeleton serialization is sorted deterministically. /// This order is determined by the order of the fields listed in UTS 35, which is ordered @@ -107,10 +119,10 @@ impl Ord for Skeleton { /// This is an implementation of the serde deserialization visitor pattern. #[cfg(feature = "provider_serde")] -struct SkeletonFieldsDeserializeVisitor; +struct DeserializeSkeletonFieldsUTS35String; #[cfg(feature = "provider_serde")] -impl<'de> de::Visitor<'de> for SkeletonFieldsDeserializeVisitor { +impl<'de> de::Visitor<'de> for DeserializeSkeletonFieldsUTS35String { type Value = Skeleton; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { @@ -135,13 +147,40 @@ impl<'de> de::Visitor<'de> for SkeletonFieldsDeserializeVisitor { } } +#[cfg(feature = "provider_serde")] +struct DeserializeSkeletonBincode; + +#[cfg(feature = "provider_serde")] +impl<'de> de::Visitor<'de> for DeserializeSkeletonBincode { + type Value = Skeleton; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "Unable to deserialize a bincode Pattern.") + } + + fn visit_seq(self, mut seq: V) -> Result + where + V: de::SeqAccess<'de>, + { + let mut items: SmallVec<[fields::Field; 5]> = SmallVec::new(); + while let Some(item) = seq.next_element()? { + items.push(item) + } + Ok(Skeleton(items)) + } +} + #[cfg(feature = "provider_serde")] impl<'de> Deserialize<'de> for Skeleton { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { - deserializer.deserialize_str(SkeletonFieldsDeserializeVisitor) + if deserializer.is_human_readable() { + deserializer.deserialize_str(DeserializeSkeletonFieldsUTS35String) + } else { + deserializer.deserialize_seq(DeserializeSkeletonBincode) + } } } @@ -151,16 +190,27 @@ impl Serialize for Skeleton { where S: ser::Serializer, { - let mut string = String::new(); + if serializer.is_human_readable() { + // Serialize into the UTS 35 string representation. + let mut string = String::new(); + + for field in self.0.iter() { + let ch: char = field.symbol.into(); + for _ in 0..field.length as usize { + string.push(ch); + } + } - for field in self.0.iter() { - let ch: char = field.symbol.into(); - for _ in 0..field.length as usize { - string.push(ch); + serializer.serialize_str(&string) + } else { + // Serialize into a bincode-friendly representation. This means that pattern parsing + // will not be needed when deserializing. + let mut seq = serializer.serialize_seq(Some(self.fields_len()))?; + for item in self.fields_iter() { + seq.serialize_element(item)?; } + seq.end() } - - serializer.serialize_str(&string) } } diff --git a/components/datetime/tests/fixtures/tests/skeletons.bin b/components/datetime/tests/fixtures/tests/skeletons.bin new file mode 100644 index 0000000000000000000000000000000000000000..a13d16d822a252fb1150a7c52fc878a71384d374 GIT binary patch literal 728 zcmbu6F%E-33V+|I6@*UCFXeP-F|+c;{9JY}2Wa`a`|8(8IGXHCO(p z-F(BlJX88RgSse s-8^jf-a3!BQ&Q%ux184WRrfy`F{>EbDAV~@nVq@Hbj}^aw=u(f06`c70RR91 literal 0 HcmV?d00001 diff --git a/components/datetime/tests/fixtures/tests/skeletons.json b/components/datetime/tests/fixtures/tests/skeletons.json new file mode 100644 index 00000000000..4b61ec27452 --- /dev/null +++ b/components/datetime/tests/fixtures/tests/skeletons.json @@ -0,0 +1,22 @@ +{ + "skeletons": [ + "y", + "yM", + "yMdE", + "yMdEEEE", + "yMMM", + "M", + "Md", + "Mdd", + "MMd", + "MMdd", + "d", + "h", + "hm", + "hms", + "Hm", + "Hms", + "ms", + "mmss" + ] +} diff --git a/components/datetime/tests/skeleton_serialization.rs b/components/datetime/tests/skeleton_serialization.rs new file mode 100644 index 00000000000..ef8e8282fb5 --- /dev/null +++ b/components/datetime/tests/skeleton_serialization.rs @@ -0,0 +1,130 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +#![cfg(all(test, feature = "provider_serde"))] + +use icu_datetime::skeleton::Skeleton; +use std::{fs::File, io::BufReader}; + +/// Note that this file tests only valid skeleton cases for the stability of the serialization +/// pipeline. For tests failure cases see the file where skeletons are defined. + +#[derive(serde::Serialize, serde::Deserialize)] +struct SkeletonFixtures { + pub skeletons: Vec, +} + +fn get_skeleton_fixtures() -> Vec { + let file = File::open("./tests/fixtures/tests/skeletons.json".to_string()) + .expect("Unable to open ./tests/fixtures/tests/skeletons.json"); + let reader = BufReader::new(file); + let fixtures: SkeletonFixtures = + serde_json::from_reader(reader).expect("Unable to deserialize skeleton fixtures."); + fixtures.skeletons +} + +fn get_skeleton_bincode_write_handle() -> File { + File::create("./tests/fixtures/tests/skeletons.bin") + .expect("Unable to create ./tests/fixtures/tests/skeletons.bin") +} + +fn get_skeleton_bincode_from_file() -> Vec> { + bincode::deserialize_from( + File::open("./tests/fixtures/tests/skeletons.bin") + .expect("Unable to ./tests/fixtures/tests/skeletons.bin"), + ) + .expect("Unable to deserialize bytes.") +} + +#[test] +fn test_skeleton_json_serialization_roundtrip() { + for skeleton_string in &get_skeleton_fixtures() { + // Wrap the string in quotes so it's a JSON string. + let json_in: String = serde_json::to_string(skeleton_string).unwrap(); + + let skeleton: Skeleton = match serde_json::from_str(&json_in) { + Ok(p) => p, + Err(err) => { + panic!( + "Unable to parse the skeleton {:?}. {:?}", + skeleton_string, err + ); + } + }; + + let json_out = match serde_json::to_string(&skeleton) { + Ok(s) => s, + Err(err) => { + panic!( + "Unable to re-serialize the skeleton {:?}. {:?}", + skeleton_string, err + ); + } + }; + + assert_eq!( + json_in, json_out, + "The roundtrip serialization for the skeleton matched." + ); + } +} + +/// Bincode representation of skeletons need to be stable across time. This test checks the +/// current serialization against historic serialization to ensure that this remains stable. +#[test] +fn test_skeleton_bincode_serialization_roundtrip() { + let skeletons = get_skeleton_fixtures(); + let update_bincode = std::env::var_os("ICU4X_REGEN_FIXTURE").is_some(); + let mut result_vec = Vec::new(); + let expect_vec = if update_bincode { + None + } else { + Some(get_skeleton_bincode_from_file()) + }; + + if let Some(ref expect_vec) = expect_vec { + if expect_vec.len() != skeletons.len() { + panic!( + "Expected the bincode to have the same number of entries as the string skeletons. \ + The bincode can be re-generated by re-running the test with the environment + variable ICU4X_REGEN_FIXTURE set." + ); + } + } + + for (i, skeleton_string) in skeletons.iter().enumerate() { + // Wrap the string in quotes so it's a JSON string. + let json_in: String = serde_json::to_string(skeleton_string).unwrap(); + + let skeleton: Skeleton = match serde_json::from_str(&json_in) { + Ok(p) => p, + Err(err) => { + panic!( + "Unable to parse the skeleton {:?}. {:?}", + skeleton_string, err + ); + } + }; + + let bincode: Vec = bincode::serialize(&skeleton).unwrap(); + + if let Some(ref expect_vec) = expect_vec { + if bincode != *expect_vec.get(i).unwrap() { + panic!( + "The bincode representations of the skeleton {:?} did not match the stored \ + representation. Skeletons are supposed to have stable bincode representations. \ + Something changed to make it different than what it was in the past. If this is \ + expected, then the bincode can be updated by re-running the test with the \ + environment variable ICU4X_REGEN_FIXTURE set.", + json_in + ) + } + } + result_vec.push(bincode); + } + if update_bincode { + eprintln!("Writing the bincode into a file"); + bincode::serialize_into(&mut get_skeleton_bincode_write_handle(), &result_vec).unwrap(); + } +} From 439aa4e8719eedad471cd31a7c4af794a605b668 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 25 Mar 2021 10:44:37 -0500 Subject: [PATCH 18/22] ReviewB 2: Add PartialOrd and Ord for Fields and derive Ord for Skeleton --- components/datetime/src/fields/length.rs | 7 +- components/datetime/src/fields/mod.rs | 3 +- components/datetime/src/fields/symbols.rs | 14 ++- components/datetime/src/provider/mod.rs | 5 - components/datetime/src/skeleton.rs | 141 +++++++--------------- 5 files changed, 64 insertions(+), 106 deletions(-) diff --git a/components/datetime/src/fields/length.rs b/components/datetime/src/fields/length.rs index cbe54036cc4..c30050d0435 100644 --- a/components/datetime/src/fields/length.rs +++ b/components/datetime/src/fields/length.rs @@ -1,14 +1,17 @@ // This file is part of ICU4X. For terms of use, please see the file // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use std::convert::TryFrom; +use std::{ + cmp::{Ord, PartialOrd}, + convert::TryFrom, +}; #[derive(Debug, PartialEq)] pub enum LengthError { TooLong, } -#[derive(Debug, Eq, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy, Ord, PartialOrd)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) diff --git a/components/datetime/src/fields/mod.rs b/components/datetime/src/fields/mod.rs index 1c030c5fc21..c7402bb1e16 100644 --- a/components/datetime/src/fields/mod.rs +++ b/components/datetime/src/fields/mod.rs @@ -8,6 +8,7 @@ pub use length::{FieldLength, LengthError}; pub use symbols::*; use std::{ + cmp::{Ord, PartialOrd}, convert::{TryFrom, TryInto}, fmt, }; @@ -25,7 +26,7 @@ impl fmt::Display for Error { } } -#[derive(Debug, Eq, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy, Ord, PartialOrd)] #[cfg_attr( feature = "provider_serde", derive(serde::Serialize, serde::Deserialize) diff --git a/components/datetime/src/fields/symbols.rs b/components/datetime/src/fields/symbols.rs index 278d86375cd..6183b098850 100644 --- a/components/datetime/src/fields/symbols.rs +++ b/components/datetime/src/fields/symbols.rs @@ -1,7 +1,7 @@ // This file is part of ICU4X. For terms of use, please see the file // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use std::convert::TryFrom; +use std::{cmp::Ordering, convert::TryFrom}; #[derive(Debug, PartialEq)] pub enum SymbolError { @@ -135,6 +135,18 @@ impl From for char { } } +impl PartialOrd for FieldSymbol { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for FieldSymbol { + fn cmp(&self, other: &Self) -> Ordering { + self.get_canonical_order().cmp(&other.get_canonical_order()) + } +} + #[derive(Debug, Eq, PartialEq, Clone, Copy)] #[cfg_attr( feature = "provider_serde", diff --git a/components/datetime/src/provider/mod.rs b/components/datetime/src/provider/mod.rs index 22054469035..49c1dafee63 100644 --- a/components/datetime/src/provider/mod.rs +++ b/components/datetime/src/provider/mod.rs @@ -251,11 +251,6 @@ pub mod gregory { fn try_from(skeleton_string: &str) -> Result { match Skeleton::try_from(skeleton_string) { Ok(skeleton) => Ok(SkeletonV1(skeleton)), - Err(SkeletonError::FieldsOutOfOrder(fields)) => { - // The fields were out of order, converting fields to a skeleton will ensure - // that these fields are sorted. This sorting is only done in the provider. - Ok(SkeletonV1(Skeleton::from(fields))) - } Err(err) => Err(err), } } diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs index eff11524704..235d6373406 100644 --- a/components/datetime/src/skeleton.rs +++ b/components/datetime/src/skeleton.rs @@ -5,7 +5,7 @@ //! Skeletons are used for pattern matching. See the [`Skeleton`] struct for more information. use smallvec::SmallVec; -use std::{cmp::Ordering, convert::TryFrom, fmt}; +use std::{convert::TryFrom, fmt}; use crate::fields::{self, Field, FieldLength, FieldSymbol}; @@ -31,7 +31,7 @@ struct FieldIndex(usize); /// /// The `Field`s are only sorted in the `Skeleton` in order to provide a deterministic /// serialization strategy, and to provide a faster `Skeleton` matching operation. -#[derive(Debug, Eq, PartialEq, Clone)] +#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd)] pub struct Skeleton(SmallVec<[fields::Field; 5]>); impl Skeleton { @@ -42,79 +42,6 @@ impl Skeleton { fn fields_len(&self) -> usize { self.0.len() } - - /// The LiteMap structs should be ordered by the `Skeleton` canonical - /// order. This helps ensure that the skeleton serialization is sorted deterministically. - /// This order is determined by the order of the fields listed in UTS 35, which is ordered - /// from most significant, to least significant. - /// - /// https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table - pub fn compare_canonical_order(&self, other: &Skeleton) -> Ordering { - let fields_a = &self.0; - let fields_b = &other.0; - let max_len = fields_a.len().max(fields_b.len()); - - for i in 0..max_len { - let maybe_field_a = fields_a.get(i); - let maybe_field_b = fields_b.get(i); - - match maybe_field_a { - Some(field_a) => match maybe_field_b { - Some(field_b) => { - let ordering = compare_fields(field_a, field_b); - if ordering != Ordering::Equal { - return ordering; - } - } - None => { - // Field A has more fields. - return Ordering::Greater; - } - }, - None => { - // Field B has more fields. - return Ordering::Less; - } - }; - } - - Ordering::Equal - } -} - -fn compare_fields(field_a: &Field, field_b: &Field) -> Ordering { - let order_a = field_a.symbol.get_canonical_order(); - let order_b = field_b.symbol.get_canonical_order(); - if order_a < order_b { - return Ordering::Less; - } - if order_a > order_b { - return Ordering::Greater; - } - - // If the fields are equivalent, try to sort by field length. - let length_a = field_a.length as u8; - let length_b = field_b.length as u8; - - if length_a < length_b { - return Ordering::Less; - } - if length_a > length_b { - return Ordering::Greater; - } - return Ordering::Equal; -} - -impl PartialOrd for Skeleton { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.compare_canonical_order(other)) - } -} - -impl Ord for Skeleton { - fn cmp(&self, other: &Self) -> Ordering { - self.compare_canonical_order(other) - } } /// This is an implementation of the serde deserialization visitor pattern. @@ -141,7 +68,7 @@ impl<'de> de::Visitor<'de> for DeserializeSkeletonFieldsUTS35String { Skeleton::try_from(skeleton_string).map_err(|err| { de::Error::invalid_value( de::Unexpected::Other(&format!("{:?} {}", skeleton_string, err)), - &"a UTS 35 sorted field symbols representing a skeleton", + &"field symbols representing a skeleton", ) }) } @@ -222,7 +149,6 @@ impl TryFrom<&str> for Skeleton { type Error = SkeletonError; fn try_from(skeleton_string: &str) -> Result { let mut fields: SmallVec<[fields::Field; 5]> = SmallVec::new(); - let mut in_order = true; let mut iter = skeleton_string.bytes().peekable(); while let Some(byte) = iter.next() { @@ -239,30 +165,22 @@ impl TryFrom<&str> for Skeleton { iter.next(); } - if let Some(prev_field) = fields.last() { - in_order = in_order - && prev_field.symbol.get_canonical_order() < field_symbol.get_canonical_order(); - } + let field = Field::from((field_symbol, FieldLength::try_from(field_length)?)); - // Add the field. - fields.push(Field::from(( - field_symbol, - FieldLength::try_from(field_length)?, - ))); + match fields.binary_search(&field) { + Ok(_) => return Err(SkeletonError::DuplicateField), + Err(pos) => fields.insert(pos, field), + } } - if in_order { - Ok(Skeleton(fields)) - } else { - Err(SkeletonError::FieldsOutOfOrder(fields)) - } + Ok(Skeleton(fields)) } } /// Apply the sorting invarants for an unknown list of fields. impl From> for Skeleton { fn from(mut fields: SmallVec<[fields::Field; 5]>) -> Skeleton { - fields.sort_by(compare_fields); + fields.sort(); Skeleton(fields) } } @@ -270,7 +188,6 @@ impl From> for Skeleton { #[derive(Debug)] pub enum SkeletonError { FieldLengthTooLong, - FieldsOutOfOrder(SmallVec<[fields::Field; 5]>), DuplicateField, SymbolUnknown(char), SymbolInvalid(char), @@ -297,9 +214,6 @@ impl fmt::Display for SkeletonError { SkeletonError::UnimplementedField(ch) => { write!(f, "unimplemented field {} in skeleton", ch) } - SkeletonError::FieldsOutOfOrder(fields) => { - write!(f, "skeleton fields {:?} out of order", fields) - } SkeletonError::Fields(err) => write!(f, "{} in skeleton", err), } } @@ -450,9 +364,42 @@ mod test { .collect(); for (strings, fields) in skeletons_strings.windows(2).zip(skeleton_fields.windows(2)) { - if fields[0].compare_canonical_order(&fields[1]) != Ordering::Less { + if fields[0].cmp(&fields[1]) != std::cmp::Ordering::Less { panic!("Expected {:?} < {:?}", strings[0], strings[1]); } } } + + #[test] + fn test_skeleton_json_reordering() { + let unordered_skeleton = "EEEEyMd"; + let ordered_skeleton = "yMdEEEE"; + + // Wrap the string in quotes so it's a JSON string. + let json: String = serde_json::to_string(unordered_skeleton).unwrap(); + + // Wrap the string in quotes so it's a JSON string. + let skeleton = serde_json::from_str::(&json) + .expect("Unable to parse an unordered skeletons."); + + assert_eq!( + serde_json::to_string(&skeleton).unwrap(), + serde_json::to_string(ordered_skeleton).unwrap() + ); + } + + /// This test handles a branch in the skeleton serialization code that takes into account + /// duplicate field errors when deserializing from string. + #[test] + fn test_skeleton_json_duplicate_fields() { + // Wrap the string in quotes so it's a JSON string. + let json: String = serde_json::to_string("EEEEyMdEEEE").unwrap(); + let err = + serde_json::from_str::(&json).expect_err("Expected a duplicate field error."); + + assert_eq!( + format!("{}", err), + "invalid value: \"EEEEyMdEEEE\" duplicate field in skeleton, expected field symbols representing a skeleton at line 1 column 13" + ); + } } From 8d0d73b19905e8e2c8e20ba802f2ef4fcd0765d6 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 25 Mar 2021 16:02:06 -0500 Subject: [PATCH 19/22] fixup! Review 2: Adjust the serde Error for Pattern to de::Error::invalid_value --- components/datetime/tests/pattern_serialization.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/datetime/tests/pattern_serialization.rs b/components/datetime/tests/pattern_serialization.rs index 9efbaa81754..aefa70cd644 100644 --- a/components/datetime/tests/pattern_serialization.rs +++ b/components/datetime/tests/pattern_serialization.rs @@ -20,7 +20,7 @@ struct PatternFixtures { } fn get_pattern_fixtures() -> PatternFixtures { - let file = File::open(format!("./tests/fixtures/tests/patterns.json")) + let file = File::open("./tests/fixtures/tests/patterns.json".to_string()) .expect("Unable to open ./tests/fixtures/tests/patterns.json"); let reader = BufReader::new(file); serde_json::from_reader(reader).expect("Unable to deserialize pattern fixtures.") From 067e657c2356936de1923d12dd0b7368da485a7a Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 25 Mar 2021 16:09:37 -0500 Subject: [PATCH 20/22] Fix license check errors --- components/datetime/src/fields/length.rs | 1 + components/datetime/src/fields/symbols.rs | 1 + components/datetime/src/skeleton.rs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/components/datetime/src/fields/length.rs b/components/datetime/src/fields/length.rs index c30050d0435..469f1c604d5 100644 --- a/components/datetime/src/fields/length.rs +++ b/components/datetime/src/fields/length.rs @@ -1,6 +1,7 @@ // This file is part of ICU4X. For terms of use, please see the file // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + use std::{ cmp::{Ord, PartialOrd}, convert::TryFrom, diff --git a/components/datetime/src/fields/symbols.rs b/components/datetime/src/fields/symbols.rs index 6183b098850..879f1e79171 100644 --- a/components/datetime/src/fields/symbols.rs +++ b/components/datetime/src/fields/symbols.rs @@ -1,6 +1,7 @@ // This file is part of ICU4X. For terms of use, please see the file // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + use std::{cmp::Ordering, convert::TryFrom}; #[derive(Debug, PartialEq)] diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs index 235d6373406..dd40981dac7 100644 --- a/components/datetime/src/skeleton.rs +++ b/components/datetime/src/skeleton.rs @@ -1,6 +1,6 @@ // This file is part of ICU4X. For terms of use, please see the file // called LICENSE at the top level of the ICU4X source tree -// (online at: https://github.com/unicode-org/icu4x/blob/master/LICENSE ). +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). //! Skeletons are used for pattern matching. See the [`Skeleton`] struct for more information. From 8295df19988373e269e95b8f24f3bed9e3f9172b Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Fri, 26 Mar 2021 09:21:59 -0500 Subject: [PATCH 21/22] Apply review feedback on field ordering --- components/datetime/src/fields/symbols.rs | 2 +- components/datetime/src/skeleton.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/components/datetime/src/fields/symbols.rs b/components/datetime/src/fields/symbols.rs index 879f1e79171..dae218a29c0 100644 --- a/components/datetime/src/fields/symbols.rs +++ b/components/datetime/src/fields/symbols.rs @@ -38,7 +38,7 @@ impl FieldSymbol { /// (https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table), and are generally /// ordered most significant to least significant. /// - pub fn get_canonical_order(&self) -> u8 { + fn get_canonical_order(&self) -> u8 { match self { FieldSymbol::Year(Year::Calendar) => 0, FieldSymbol::Year(Year::WeekOf) => 1, diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs index dd40981dac7..305675b1694 100644 --- a/components/datetime/src/skeleton.rs +++ b/components/datetime/src/skeleton.rs @@ -91,6 +91,14 @@ impl<'de> de::Visitor<'de> for DeserializeSkeletonBincode { { let mut items: SmallVec<[fields::Field; 5]> = SmallVec::new(); while let Some(item) = seq.next_element()? { + if let Some(prev_item) = items.last() { + if prev_item > &item { + return Err(de::Error::invalid_value( + de::Unexpected::Other(&format!("field item out of order: {:?}", item)), + &"ordered field symbols representing a skeleton", + )); + } + } items.push(item) } Ok(Skeleton(items)) @@ -177,14 +185,6 @@ impl TryFrom<&str> for Skeleton { } } -/// Apply the sorting invarants for an unknown list of fields. -impl From> for Skeleton { - fn from(mut fields: SmallVec<[fields::Field; 5]>) -> Skeleton { - fields.sort(); - Skeleton(fields) - } -} - #[derive(Debug)] pub enum SkeletonError { FieldLengthTooLong, From 9ce602175b0c13b796329d7409dd390b045d727d Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Fri, 26 Mar 2021 10:00:04 -0500 Subject: [PATCH 22/22] Add tests for invalid bincode and duplicate field check --- components/datetime/src/skeleton.rs | 66 +++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/components/datetime/src/skeleton.rs b/components/datetime/src/skeleton.rs index 305675b1694..4a1072119df 100644 --- a/components/datetime/src/skeleton.rs +++ b/components/datetime/src/skeleton.rs @@ -98,6 +98,12 @@ impl<'de> de::Visitor<'de> for DeserializeSkeletonBincode { &"ordered field symbols representing a skeleton", )); } + if prev_item == &item { + return Err(de::Error::invalid_value( + de::Unexpected::Other(&format!("duplicate field: {:?}", item)), + &"ordered field symbols representing a skeleton", + )); + } } items.push(item) } @@ -402,4 +408,64 @@ mod test { "invalid value: \"EEEEyMdEEEE\" duplicate field in skeleton, expected field symbols representing a skeleton at line 1 column 13" ); } + + /// Skeletons are represented in bincode as a vec of field, but bincode shouldn't be completely + /// trusted, test that the bincode gets validated correctly. + struct TestInvalidSkeleton(Vec); + + #[cfg(feature = "provider_serde")] + impl Serialize for TestInvalidSkeleton { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + let fields = &self.0; + let mut seq = serializer.serialize_seq(Some(fields.len()))?; + for item in fields.iter() { + seq.serialize_element(item)?; + } + seq.end() + } + } + + #[test] + fn test_skeleton_bincode_reordering() { + let unordered_skeleton = TestInvalidSkeleton(vec![ + Field::from((FieldSymbol::Day(Day::DayOfMonth), FieldLength::One)), + Field::from((FieldSymbol::Month(Month::Format), FieldLength::One)), + ]); + + let mut buffer: Vec = Vec::new(); + + bincode::serialize_into(&mut buffer, &unordered_skeleton).unwrap(); + + let err = + bincode::deserialize::(&buffer).expect_err("Expected an unordered error"); + + assert_eq!( + format!("{}", err), + "invalid value: field item out of order: Field { symbol: Month(Format), length: One }, expected ordered field symbols representing a skeleton" + ); + } + + #[test] + fn test_skeleton_bincode_duplicate_field() { + let unordered_skeleton = TestInvalidSkeleton(vec![ + Field::from((FieldSymbol::Month(Month::Format), FieldLength::One)), + Field::from((FieldSymbol::Day(Day::DayOfMonth), FieldLength::One)), + Field::from((FieldSymbol::Day(Day::DayOfMonth), FieldLength::One)), + ]); + + let mut buffer: Vec = Vec::new(); + + bincode::serialize_into(&mut buffer, &unordered_skeleton).unwrap(); + + let err = bincode::deserialize::(&buffer) + .expect_err("Expected a duplicate field error"); + + assert_eq!( + format!("{}", err), + "invalid value: duplicate field: Field { symbol: Day(DayOfMonth), length: One }, expected ordered field symbols representing a skeleton" + ); + } }