Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New properties API #5548

Merged
merged 14 commits into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions components/casemap/src/titlecase.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ use crate::provider::CaseMapV1Marker;
use crate::CaseMapper;
use alloc::string::String;
use icu_locale_core::LanguageIdentifier;
use icu_properties::maps::CodePointMapData;
use icu_properties::props::{GeneralCategory, GeneralCategoryGroup};
use icu_properties::provider::GeneralCategoryV1Marker;
use icu_properties::{GeneralCategory, GeneralCategoryGroup};
use icu_properties::CodePointMapData;
use icu_provider::prelude::*;
use writeable::Writeable;

Expand Down Expand Up @@ -221,7 +221,8 @@ impl TitlecaseMapper<CaseMapper> {
pub const fn new() -> Self {
Self {
cm: CaseMapper::new(),
gc: icu_properties::maps::general_category().static_to_owned(),
gc: icu_properties::CodePointMapData::<icu_properties::props::GeneralCategory>::new()
.static_to_owned(),
}
}

Expand All @@ -240,7 +241,7 @@ impl TitlecaseMapper<CaseMapper> {
P: DataProvider<CaseMapV1Marker> + DataProvider<GeneralCategoryV1Marker> + ?Sized,
{
let cm = CaseMapper::try_new_unstable(provider)?;
let gc = icu_properties::maps::load_general_category(provider)?;
let gc = icu_properties::CodePointMapData::<icu_properties::props::GeneralCategory>::try_new_unstable(provider)?;
Ok(Self { cm, gc })
}
}
Expand All @@ -266,7 +267,8 @@ impl<CM: AsRef<CaseMapper>> TitlecaseMapper<CM> {
pub const fn new_with_mapper(casemapper: CM) -> Self {
Self {
cm: casemapper,
gc: icu_properties::maps::general_category().static_to_owned(),
gc: icu_properties::CodePointMapData::<icu_properties::props::GeneralCategory>::new()
.static_to_owned(),
}
}

Expand All @@ -276,7 +278,7 @@ impl<CM: AsRef<CaseMapper>> TitlecaseMapper<CM> {
where
P: DataProvider<CaseMapV1Marker> + DataProvider<GeneralCategoryV1Marker> + ?Sized,
{
let gc = icu_properties::maps::load_general_category(provider)?;
let gc = icu_properties::CodePointMapData::<icu_properties::props::GeneralCategory>::try_new_unstable(provider)?;
Ok(Self { cm: casemapper, gc })
}

Expand Down
9 changes: 6 additions & 3 deletions components/casemap/tests/gen_greek_to_me.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,17 @@ use icu_casemap::greek_to_me::{
};
use icu_casemap::CaseMapper;
use icu_normalizer::DecomposingNormalizerBorrowed;
use icu_properties::{maps, GeneralCategoryGroup, Script};
use icu_properties::{
props::{GeneralCategory, GeneralCategoryGroup, Script},
CodePointMapData,
};
use std::collections::BTreeMap;
use std::fmt::Write;

fn main() {
let decomposer = DecomposingNormalizerBorrowed::new_nfd();
let script = maps::script();
let gc = maps::general_category();
let script = CodePointMapData::<Script>::new();
let gc = CodePointMapData::<GeneralCategory>::new();
let cm = CaseMapper::new();

let mut vec_370 = vec![0; 0x400 - 0x370];
Expand Down
2 changes: 1 addition & 1 deletion components/collator/src/elements.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use icu_collections::char16trie::TrieResult;
use icu_collections::codepointtrie::CodePointTrie;
use icu_normalizer::provider::DecompositionDataV1;
use icu_normalizer::provider::DecompositionTablesV1;
use icu_properties::CanonicalCombiningClass;
use icu_properties::props::CanonicalCombiningClass;
use smallvec::SmallVec;
use zerovec::ule::AsULE;
use zerovec::ule::RawBytesULE;
Expand Down
178 changes: 80 additions & 98 deletions components/collections/src/iterator_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,20 +62,18 @@ where
mod tests {
use core::fmt::Debug;
use icu::collections::codepointinvlist::CodePointInversionListBuilder;
use icu::collections::codepointtrie::TrieValue;
use icu::properties::maps::{self, CodePointMapDataBorrowed};
use icu::properties::sets::{self, CodePointSetDataBorrowed};
use icu::properties::{GeneralCategory, Script};
use icu::properties::props::{BinaryProperty, EnumeratedProperty};
use icu::properties::{CodePointMapData, CodePointSetData};

fn test_set(data: CodePointSetDataBorrowed<'static>, name: &str) {
fn test_set<P: BinaryProperty>(name: &str) {
let mut builder = CodePointInversionListBuilder::new();
let mut builder_complement = CodePointInversionListBuilder::new();

for range in data.iter_ranges() {
for range in CodePointSetData::new::<P>().iter_ranges() {
builder.add_range32(range)
}

for range in data.iter_ranges_complemented() {
for range in CodePointSetData::new::<P>().iter_ranges_complemented() {
builder_complement.add_range32(range)
}

Expand All @@ -85,19 +83,15 @@ mod tests {
assert_eq!(set1, set2, "Set {name} failed to complement correctly");
}

fn test_map<T: TrieValue + Debug>(
data: &CodePointMapDataBorrowed<'static, T>,
value: T,
name: &str,
) {
fn test_map<T: EnumeratedProperty + Debug>(value: T, name: &str) {
let mut builder = CodePointInversionListBuilder::new();
let mut builder_complement = CodePointInversionListBuilder::new();

for range in data.iter_ranges_for_value(value) {
for range in CodePointMapData::<T>::new().iter_ranges_for_value(value) {
builder.add_range32(range)
}

for range in data.iter_ranges_for_value_complemented(value) {
for range in CodePointMapData::<T>::new().iter_ranges_for_value_complemented(value) {
builder_complement.add_range32(range)
}

Expand All @@ -112,95 +106,83 @@ mod tests {

#[test]
fn test_complement_sets() {
use icu::properties::props::*;
// Stress test the RangeListIteratorComplementer logic by ensuring it works for
// a whole bunch of binary properties
test_set(sets::ascii_hex_digit(), "ASCII_Hex_Digit");
test_set(sets::alnum(), "Alnum");
test_set(sets::alphabetic(), "Alphabetic");
test_set(sets::bidi_control(), "Bidi_Control");
test_set(sets::bidi_mirrored(), "Bidi_Mirrored");
test_set(sets::blank(), "Blank");
test_set(sets::cased(), "Cased");
test_set(sets::case_ignorable(), "Case_Ignorable");
test_set(
sets::full_composition_exclusion(),
"Full_Composition_Exclusion",
);
test_set(sets::changes_when_casefolded(), "Changes_When_Casefolded");
test_set(sets::changes_when_casemapped(), "Changes_When_Casemapped");
test_set(
sets::changes_when_nfkc_casefolded(),
"Changes_When_NFKC_Casefolded",
);
test_set(sets::changes_when_lowercased(), "Changes_When_Lowercased");
test_set(sets::changes_when_titlecased(), "Changes_When_Titlecased");
test_set(sets::changes_when_uppercased(), "Changes_When_Uppercased");
test_set(sets::dash(), "Dash");
test_set(sets::deprecated(), "Deprecated");
test_set(
sets::default_ignorable_code_point(),
"Default_Ignorable_Code_Point",
);
test_set(sets::diacritic(), "Diacritic");
test_set(sets::emoji_modifier_base(), "Emoji_Modifier_Base");
test_set(sets::emoji_component(), "Emoji_Component");
test_set(sets::emoji_modifier(), "Emoji_Modifier");
test_set(sets::emoji(), "Emoji");
test_set(sets::emoji_presentation(), "Emoji_Presentation");
test_set(sets::extender(), "Extender");
test_set(sets::extended_pictographic(), "Extended_Pictographic");
test_set(sets::graph(), "Graph");
test_set(sets::grapheme_base(), "Grapheme_Base");
test_set(sets::grapheme_extend(), "Grapheme_Extend");
test_set(sets::grapheme_link(), "Grapheme_Link");
test_set(sets::hex_digit(), "Hex_Digit");
test_set(sets::hyphen(), "Hyphen");
test_set(sets::id_continue(), "Id_Continue");
test_set(sets::ideographic(), "Ideographic");
test_set(sets::id_start(), "Id_Start");
test_set(sets::ids_binary_operator(), "Ids_Binary_Operator");
test_set(sets::ids_trinary_operator(), "Ids_Trinary_Operator");
test_set(sets::join_control(), "Join_Control");
test_set(sets::logical_order_exception(), "Logical_Order_Exception");
test_set(sets::lowercase(), "Lowercase");
test_set(sets::math(), "Math");
test_set(sets::noncharacter_code_point(), "Noncharacter_Code_Point");
test_set(sets::nfc_inert(), "NFC_Inert");
test_set(sets::nfd_inert(), "NFD_Inert");
test_set(sets::nfkc_inert(), "NFKC_Inert");
test_set(sets::nfkd_inert(), "NFKD_Inert");
test_set(sets::pattern_syntax(), "Pattern_Syntax");
test_set(sets::pattern_white_space(), "Pattern_White_Space");
test_set(
sets::prepended_concatenation_mark(),
"Prepended_Concatenation_Mark",
);
test_set(sets::print(), "Print");
test_set(sets::quotation_mark(), "Quotation_Mark");
test_set(sets::radical(), "Radical");
test_set(sets::regional_indicator(), "Regional_Indicator");
test_set(sets::soft_dotted(), "Soft_Dotted");
test_set(sets::segment_starter(), "Segment_Starter");
test_set(sets::case_sensitive(), "Case_Sensitive");
test_set(sets::sentence_terminal(), "Sentence_Terminal");
test_set(sets::terminal_punctuation(), "Terminal_Punctuation");
test_set(sets::unified_ideograph(), "Unified_Ideograph");
test_set(sets::uppercase(), "Uppercase");
test_set(sets::variation_selector(), "Variation_Selector");
test_set(sets::white_space(), "White_Space");
test_set(sets::xdigit(), "Xdigit");
test_set(sets::xid_continue(), "XID_Continue");
test_set(sets::xid_start(), "XID_Start");
test_set::<AsciiHexDigit>("ASCII_Hex_Digit");
test_set::<Alnum>("Alnum");
test_set::<Alphabetic>("Alphabetic");
test_set::<BidiControl>("Bidi_Control");
test_set::<BidiMirrored>("Bidi_Mirrored");
test_set::<Blank>("Blank");
test_set::<Cased>("Cased");
test_set::<CaseIgnorable>("Case_Ignorable");
test_set::<FullCompositionExclusion>("Full_Composition_Exclusion");
test_set::<ChangesWhenCasefolded>("Changes_When_Casefolded");
test_set::<ChangesWhenCasemapped>("Changes_When_Casemapped");
test_set::<ChangesWhenNfkcCasefolded>("Changes_When_NFKC_Casefolded");
test_set::<ChangesWhenLowercased>("Changes_When_Lowercased");
test_set::<ChangesWhenTitlecased>("Changes_When_Titlecased");
test_set::<ChangesWhenUppercased>("Changes_When_Uppercased");
test_set::<Dash>("Dash");
test_set::<Deprecated>("Deprecated");
test_set::<DefaultIgnorableCodePoint>("Default_Ignorable_Code_Point");
test_set::<Diacritic>("Diacritic");
test_set::<EmojiModifierBase>("Emoji_Modifier_Base");
test_set::<EmojiComponent>("Emoji_Component");
test_set::<EmojiModifier>("Emoji_Modifier");
test_set::<Emoji>("Emoji");
test_set::<EmojiPresentation>("Emoji_Presentation");
test_set::<Extender>("Extender");
test_set::<ExtendedPictographic>("Extended_Pictographic");
test_set::<Graph>("Graph");
test_set::<GraphemeBase>("Grapheme_Base");
test_set::<GraphemeExtend>("Grapheme_Extend");
test_set::<GraphemeLink>("Grapheme_Link");
test_set::<HexDigit>("Hex_Digit");
test_set::<Hyphen>("Hyphen");
test_set::<IdContinue>("Id_Continue");
test_set::<Ideographic>("Ideographic");
test_set::<IdStart>("Id_Start");
test_set::<IdsBinaryOperator>("Ids_Binary_Operator");
test_set::<IdsTrinaryOperator>("Ids_Trinary_Operator");
test_set::<JoinControl>("Join_Control");
test_set::<LogicalOrderException>("Logical_Order_Exception");
test_set::<Lowercase>("Lowercase");
test_set::<Math>("Math");
test_set::<NoncharacterCodePoint>("Noncharacter_Code_Point");
test_set::<NfcInert>("NFC_Inert");
test_set::<NfdInert>("NFD_Inert");
test_set::<NfkcInert>("NFKC_Inert");
test_set::<NfkdInert>("NFKD_Inert");
test_set::<PatternSyntax>("Pattern_Syntax");
test_set::<PatternWhiteSpace>("Pattern_White_Space");
test_set::<PrependedConcatenationMark>("Prepended_Concatenation_Mark");
test_set::<Print>("Print");
test_set::<QuotationMark>("Quotation_Mark");
test_set::<Radical>("Radical");
test_set::<RegionalIndicator>("Regional_Indicator");
test_set::<SoftDotted>("Soft_Dotted");
test_set::<SegmentStarter>("Segment_Starter");
test_set::<CaseSensitive>("Case_Sensitive");
test_set::<SentenceTerminal>("Sentence_Terminal");
test_set::<TerminalPunctuation>("Terminal_Punctuation");
test_set::<UnifiedIdeograph>("Unified_Ideograph");
test_set::<Uppercase>("Uppercase");
test_set::<VariationSelector>("Variation_Selector");
test_set::<WhiteSpace>("White_Space");
test_set::<Xdigit>("Xdigit");
test_set::<XidContinue>("XID_Continue");
test_set::<XidStart>("XID_Start");
}

#[test]
fn test_complement_maps() {
let gc = maps::general_category();
let script = maps::script();
test_map(&gc, GeneralCategory::UppercaseLetter, "gc");
test_map(&gc, GeneralCategory::OtherPunctuation, "gc");
test_map(&script, Script::Devanagari, "script");
test_map(&script, Script::Latin, "script");
test_map(&script, Script::Common, "script");
use icu::properties::props::{GeneralCategory, Script};
test_map(GeneralCategory::UppercaseLetter, "gc");
test_map(GeneralCategory::OtherPunctuation, "gc");
test_map(Script::Devanagari, "script");
test_map(Script::Latin, "script");
test_map(Script::Common, "script");
}
}
9 changes: 4 additions & 5 deletions components/experimental/src/personnames/formatter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@
use alloc::string::String;
use alloc::vec::Vec;
use icu_locale::LocaleFallbacker;
use icu_properties::names::PropertyScriptToIcuScriptMapper;
use icu_properties::script::ScriptWithExtensions;
use icu_properties::script::{ScriptMapper, ScriptWithExtensions};

use super::api::{
FormattingOrder, NameField, NameFieldKind, PersonName, PersonNamesFormatterError,
Expand All @@ -24,7 +23,7 @@ use zerofrom::ZeroFrom;
pub struct PersonNamesFormatter {
pub(crate) default_options: PersonNamesFormatterOptions,
swe: ScriptWithExtensions,
scripts: PropertyScriptToIcuScriptMapper<icu_properties::Script>,
scripts: ScriptMapper,
fallbacker: LocaleFallbacker,
}

Expand All @@ -49,8 +48,8 @@ impl PersonNamesFormatter {
+ DataProvider<icu_locale::provider::LikelySubtagsForLanguageV1Marker>
+ DataProvider<icu_locale::provider::ParentsV1Marker>,
{
let swe = icu_properties::script::load_script_with_extensions_unstable(provider)?;
let scripts = icu_properties::Script::get_enum_to_icu_script_mapper(provider)?;
let swe = icu_properties::script::ScriptWithExtensions::try_new_unstable(provider)?;
let scripts = ScriptMapper::try_new_unstable(provider)?;
let fallbacker = LocaleFallbacker::try_new_unstable(provider)?;
Ok(PersonNamesFormatter {
default_options: options,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@

use icu_locale_core::subtags::script;
use icu_locale_core::{subtags, Locale};
use icu_properties::names::PropertyScriptToIcuScriptMapperBorrowed;
use icu_properties::script::ScriptWithExtensionsBorrowed;
use icu_properties::script::{ScriptMapperBorrowed, ScriptWithExtensionsBorrowed};

use crate::personnames::api::NameFieldKind::{Given, Surname};
use crate::personnames::api::{NameFieldKind, PersonName, PersonNamesFormatterError};
Expand Down Expand Up @@ -45,7 +44,7 @@ fn compatible_scripts(sc1: subtags::Script, sc2: subtags::Script) -> bool {
pub fn likely_person_name_locale<N>(
person_name: &N,
swe: ScriptWithExtensionsBorrowed,
scripts: PropertyScriptToIcuScriptMapperBorrowed<icu_properties::Script>,
scripts: ScriptMapperBorrowed,
) -> Result<Locale, PersonNamesFormatterError>
where
N: PersonName,
Expand All @@ -54,14 +53,9 @@ where
if found_name_script.is_none() {
found_name_script = find_script(person_name, swe, Given);
}
let name_script = found_name_script.unwrap_or(icu_properties::Script::Unknown);

let locid_script = scripts
.get(name_script)
.unwrap()
.as_str()
.parse::<subtags::Script>()
.map_err(|_err| PersonNamesFormatterError::InvalidPersonName)?;
let name_script = found_name_script.unwrap_or(icu_properties::props::Script::Unknown);

let locid_script = scripts.get(name_script).unwrap();
person_name.name_locale().map_or_else(
|| {
let mut effective_locale = Locale::default();
Expand All @@ -80,11 +74,11 @@ fn find_script<N>(
person_name: &N,
swe: ScriptWithExtensionsBorrowed,
kind: NameFieldKind,
) -> Option<icu_properties::Script>
) -> Option<icu_properties::props::Script>
where
N: PersonName,
{
use icu_properties::Script;
use icu_properties::props::Script;

person_name
.available_name_fields()
Expand Down Expand Up @@ -176,8 +170,8 @@ mod tests {

#[test]
fn test_likely_person_names_locale() {
let swe = icu_properties::script::script_with_extensions();
let scripts = icu_properties::Script::enum_to_icu_script_mapper();
let swe = icu_properties::script::ScriptWithExtensions::new();
let scripts = icu_properties::script::ScriptMapper::new();
assert_eq!(
likely_person_name_locale(&person_name("Miyazaki", "Hayao").unwrap(), swe, scripts),
Ok(locale!("und_Latn"))
Expand Down
Loading
Loading