Skip to content

Commit

Permalink
Compiled data cleanups (#3597)
Browse files Browse the repository at this point in the history
  • Loading branch information
@robertbastian
robertbastian authored Jun 30, 2023
1 parent 7b8ad79 commit 5458916
Show file tree
Hide file tree
Showing 41 changed files with 332 additions and 846 deletions.
5 changes: 1 addition & 4 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Makefile.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ description = "Run full data generation on latest CLDR and ICU"
category = "CI"
dependencies = [
"bakeddata-check",
"full-data",
]

[tasks.ci-job-ffi]
Expand Down
2 changes: 1 addition & 1 deletion components/collections/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ criterion = "0.4"
icu_benchmark_macros = { path = "../../tools/benchmark/macros" }
iai = "0.1.1"
icu = { path = "../icu", default-features = false }
icu_testdata = { path = "../../provider/testdata", default-features = false, features = ["icu_properties", "icu_locid_transform"] }
icu_properties = { path = "../properties", features = ["data"] }

[features]
std = []
Expand Down
208 changes: 77 additions & 131 deletions components/collections/src/iterator_utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,11 @@ mod tests {
use core::fmt::Debug;
use icu::collections::codepointinvlist::CodePointInversionListBuilder;
use icu::collections::codepointtrie::TrieValue;
use icu::properties::maps::{self, CodePointMapData};
use icu::properties::sets::{self, CodePointSetData};
use icu::properties::maps::{self, CodePointMapDataBorrowed};
use icu::properties::sets::{self, CodePointSetDataBorrowed};
use icu::properties::{GeneralCategory, Script};

fn test_set(d: CodePointSetData, name: &str) {
let data = d.as_borrowed();
fn test_set(data: CodePointSetDataBorrowed<'static>, name: &str) {
let mut builder = CodePointInversionListBuilder::new();
let mut builder_complement = CodePointInversionListBuilder::new();

Expand All @@ -86,8 +85,11 @@ mod tests {
assert_eq!(set1, set2, "Set {name} failed to complement correctly");
}

fn test_map<T: TrieValue + Debug>(d: &CodePointMapData<T>, value: T, name: &str) {
let data = d.as_borrowed();
fn test_map<T: TrieValue + Debug>(
data: &CodePointMapDataBorrowed<'static, T>,
value: T,
name: &str,
) {
let mut builder = CodePointInversionListBuilder::new();
let mut builder_complement = CodePointInversionListBuilder::new();

Expand All @@ -112,145 +114,89 @@ mod tests {
fn test_complement_sets() {
// Stress test the RangeListIteratorComplementer logic by ensuring it works for
// a whole bunch of binary properties
let dp = icu_testdata::unstable();
test_set(sets::load_ascii_hex_digit(&dp).unwrap(), "ASCII_Hex_Digit");
test_set(sets::load_alnum(&dp).unwrap(), "Alnum");
test_set(sets::load_alphabetic(&dp).unwrap(), "Alphabetic");
test_set(sets::load_bidi_control(&dp).unwrap(), "Bidi_Control");
test_set(sets::load_bidi_mirrored(&dp).unwrap(), "Bidi_Mirrored");
test_set(sets::load_blank(&dp).unwrap(), "Blank");
test_set(sets::load_cased(&dp).unwrap(), "Cased");
test_set(sets::load_case_ignorable(&dp).unwrap(), "Case_Ignorable");
test_set(
sets::load_full_composition_exclusion(&dp).unwrap(),
test_set(sets::ascii_hex_digit(), "ASCII_Hex_Digit");
test_set(sets::alnum(), "Alnum");
test_set(sets::alphabetic(), "Alphabetic");
test_set(sets::bidi_control(), "Bidi_Control");
test_set(sets::bidi_mirrored(), "Bidi_Mirrored");
test_set(sets::blank(), "Blank");
test_set(sets::cased(), "Cased");
test_set(sets::case_ignorable(), "Case_Ignorable");
test_set(
sets::full_composition_exclusion(),
"Full_Composition_Exclusion",
);
test_set(sets::changes_when_casefolded(), "Changes_When_Casefolded");
test_set(sets::changes_when_casemapped(), "Changes_When_Casemapped");
test_set(
sets::load_changes_when_casefolded(&dp).unwrap(),
"Changes_When_Casefolded",
);
test_set(
sets::load_changes_when_casemapped(&dp).unwrap(),
"Changes_When_Casemapped",
);
test_set(
sets::load_changes_when_nfkc_casefolded(&dp).unwrap(),
sets::changes_when_nfkc_casefolded(),
"Changes_When_NFKC_Casefolded",
);
test_set(sets::changes_when_lowercased(), "Changes_When_Lowercased");
test_set(sets::changes_when_titlecased(), "Changes_When_Titlecased");
test_set(sets::changes_when_uppercased(), "Changes_When_Uppercased");
test_set(sets::dash(), "Dash");
test_set(sets::deprecated(), "Deprecated");
test_set(
sets::load_changes_when_lowercased(&dp).unwrap(),
"Changes_When_Lowercased",
);
test_set(
sets::load_changes_when_titlecased(&dp).unwrap(),
"Changes_When_Titlecased",
);
test_set(
sets::load_changes_when_uppercased(&dp).unwrap(),
"Changes_When_Uppercased",
);
test_set(sets::load_dash(&dp).unwrap(), "Dash");
test_set(sets::load_deprecated(&dp).unwrap(), "Deprecated");
test_set(
sets::load_default_ignorable_code_point(&dp).unwrap(),
sets::default_ignorable_code_point(),
"Default_Ignorable_Code_Point",
);
test_set(sets::load_diacritic(&dp).unwrap(), "Diacritic");
test_set(
sets::load_emoji_modifier_base(&dp).unwrap(),
"Emoji_Modifier_Base",
);
test_set(sets::load_emoji_component(&dp).unwrap(), "Emoji_Component");
test_set(sets::load_emoji_modifier(&dp).unwrap(), "Emoji_Modifier");
test_set(sets::load_emoji(&dp).unwrap(), "Emoji");
test_set(
sets::load_emoji_presentation(&dp).unwrap(),
"Emoji_Presentation",
);
test_set(sets::load_extender(&dp).unwrap(), "Extender");
test_set(
sets::load_extended_pictographic(&dp).unwrap(),
"Extended_Pictographic",
);
test_set(sets::load_graph(&dp).unwrap(), "Graph");
test_set(sets::load_grapheme_base(&dp).unwrap(), "Grapheme_Base");
test_set(sets::load_grapheme_extend(&dp).unwrap(), "Grapheme_Extend");
test_set(sets::load_grapheme_link(&dp).unwrap(), "Grapheme_Link");
test_set(sets::load_hex_digit(&dp).unwrap(), "Hex_Digit");
test_set(sets::load_hyphen(&dp).unwrap(), "Hyphen");
test_set(sets::load_id_continue(&dp).unwrap(), "Id_Continue");
test_set(sets::load_ideographic(&dp).unwrap(), "Ideographic");
test_set(sets::load_id_start(&dp).unwrap(), "Id_Start");
test_set(
sets::load_ids_binary_operator(&dp).unwrap(),
"Ids_Binary_Operator",
);
test_set(
sets::load_ids_trinary_operator(&dp).unwrap(),
"Ids_Trinary_Operator",
);
test_set(sets::load_join_control(&dp).unwrap(), "Join_Control");
test_set(
sets::load_logical_order_exception(&dp).unwrap(),
"Logical_Order_Exception",
);
test_set(sets::load_lowercase(&dp).unwrap(), "Lowercase");
test_set(sets::load_math(&dp).unwrap(), "Math");
test_set(
sets::load_noncharacter_code_point(&dp).unwrap(),
"Noncharacter_Code_Point",
);
test_set(sets::load_nfc_inert(&dp).unwrap(), "NFC_Inert");
test_set(sets::load_nfd_inert(&dp).unwrap(), "NFD_Inert");
test_set(sets::load_nfkc_inert(&dp).unwrap(), "NFKC_Inert");
test_set(sets::load_nfkd_inert(&dp).unwrap(), "NFKD_Inert");
test_set(sets::load_pattern_syntax(&dp).unwrap(), "Pattern_Syntax");
test_set(
sets::load_pattern_white_space(&dp).unwrap(),
"Pattern_White_Space",
);
test_set(
sets::load_prepended_concatenation_mark(&dp).unwrap(),
test_set(sets::diacritic(), "Diacritic");
test_set(sets::emoji_modifier_base(), "Emoji_Modifier_Base");
test_set(sets::emoji_component(), "Emoji_Component");
test_set(sets::emoji_modifier(), "Emoji_Modifier");
test_set(sets::emoji(), "Emoji");
test_set(sets::emoji_presentation(), "Emoji_Presentation");
test_set(sets::extender(), "Extender");
test_set(sets::extended_pictographic(), "Extended_Pictographic");
test_set(sets::graph(), "Graph");
test_set(sets::grapheme_base(), "Grapheme_Base");
test_set(sets::grapheme_extend(), "Grapheme_Extend");
test_set(sets::grapheme_link(), "Grapheme_Link");
test_set(sets::hex_digit(), "Hex_Digit");
test_set(sets::hyphen(), "Hyphen");
test_set(sets::id_continue(), "Id_Continue");
test_set(sets::ideographic(), "Ideographic");
test_set(sets::id_start(), "Id_Start");
test_set(sets::ids_binary_operator(), "Ids_Binary_Operator");
test_set(sets::ids_trinary_operator(), "Ids_Trinary_Operator");
test_set(sets::join_control(), "Join_Control");
test_set(sets::logical_order_exception(), "Logical_Order_Exception");
test_set(sets::lowercase(), "Lowercase");
test_set(sets::math(), "Math");
test_set(sets::noncharacter_code_point(), "Noncharacter_Code_Point");
test_set(sets::nfc_inert(), "NFC_Inert");
test_set(sets::nfd_inert(), "NFD_Inert");
test_set(sets::nfkc_inert(), "NFKC_Inert");
test_set(sets::nfkd_inert(), "NFKD_Inert");
test_set(sets::pattern_syntax(), "Pattern_Syntax");
test_set(sets::pattern_white_space(), "Pattern_White_Space");
test_set(
sets::prepended_concatenation_mark(),
"Prepended_Concatenation_Mark",
);
test_set(sets::load_print(&dp).unwrap(), "Print");
test_set(sets::load_quotation_mark(&dp).unwrap(), "Quotation_Mark");
test_set(sets::load_radical(&dp).unwrap(), "Radical");
test_set(
sets::load_regional_indicator(&dp).unwrap(),
"Regional_Indicator",
);
test_set(sets::load_soft_dotted(&dp).unwrap(), "Soft_Dotted");
test_set(sets::load_segment_starter(&dp).unwrap(), "Segment_Starter");
test_set(sets::load_case_sensitive(&dp).unwrap(), "Case_Sensitive");
test_set(
sets::load_sentence_terminal(&dp).unwrap(),
"Sentence_Terminal",
);
test_set(
sets::load_terminal_punctuation(&dp).unwrap(),
"Terminal_Punctuation",
);
test_set(
sets::load_unified_ideograph(&dp).unwrap(),
"Unified_Ideograph",
);
test_set(sets::load_uppercase(&dp).unwrap(), "Uppercase");
test_set(
sets::load_variation_selector(&dp).unwrap(),
"Variation_Selector",
);
test_set(sets::load_white_space(&dp).unwrap(), "White_Space");
test_set(sets::load_xdigit(&dp).unwrap(), "Xdigit");
test_set(sets::load_xid_continue(&dp).unwrap(), "XID_Continue");
test_set(sets::load_xid_start(&dp).unwrap(), "XID_Start");
test_set(sets::print(), "Print");
test_set(sets::quotation_mark(), "Quotation_Mark");
test_set(sets::radical(), "Radical");
test_set(sets::regional_indicator(), "Regional_Indicator");
test_set(sets::soft_dotted(), "Soft_Dotted");
test_set(sets::segment_starter(), "Segment_Starter");
test_set(sets::case_sensitive(), "Case_Sensitive");
test_set(sets::sentence_terminal(), "Sentence_Terminal");
test_set(sets::terminal_punctuation(), "Terminal_Punctuation");
test_set(sets::unified_ideograph(), "Unified_Ideograph");
test_set(sets::uppercase(), "Uppercase");
test_set(sets::variation_selector(), "Variation_Selector");
test_set(sets::white_space(), "White_Space");
test_set(sets::xdigit(), "Xdigit");
test_set(sets::xid_continue(), "XID_Continue");
test_set(sets::xid_start(), "XID_Start");
}

#[test]
fn test_complement_maps() {
let dp = icu_testdata::unstable();
let gc = maps::load_general_category(&dp).unwrap();
let script = maps::load_script(&dp).unwrap();
let gc = maps::general_category();
let script = maps::script();
test_map(&gc, GeneralCategory::UppercaseLetter, "gc");
test_map(&gc, GeneralCategory::OtherPunctuation, "gc");
test_map(&script, Script::Devanagari, "script");
Expand Down
20 changes: 16 additions & 4 deletions components/icu/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,14 @@ icu_timezone = { version = "1.2.0", path = "../timezone", default-features = fal

# Experimental components
icu_casemapping = { version = "0.7.1", path = "../../experimental/casemapping", default-features = false, optional = true }
icu_compactdecimal = { version = "0.2.0", path = "../../experimental/compactdecimal", default-features = false, optional = true }
icu_displaynames = { version = "0.10.0", path = "../../experimental/displaynames", default-features = false, optional = true }
icu_relativetime = { version = "0.1.0", path = "../../experimental/relativetime", default-features = false, optional = true }
icu_compactdecimal = { version = "0.2.0", path = "../../experimental/compactdecimal", default-features = false, optional = true }

# For docs links
icu_provider = { version = "1.2.0", path = "../../provider/core", default-features = false }

[dev-dependencies]
icu_testdata = { path = "../../provider/testdata" }
writeable = { path = "../../utils/writeable" }

[features]
Expand All @@ -73,8 +72,8 @@ std = [
"icu_locid/std",
"icu_normalizer/std",
"icu_plurals/std",
"icu_relativetime?/std",
"icu_properties/std",
"icu_relativetime?/std",
"icu_segmenter/std",
"icu_timezone/std",
]
Expand All @@ -91,14 +90,27 @@ serde = [
"icu_locid/serde",
"icu_normalizer/serde",
"icu_plurals/serde",
"icu_relativetime?/serde",
"icu_properties/serde",
"icu_relativetime?/serde",
"icu_segmenter/serde",
"icu_timezone/serde",
]
data = [
"icu_calendar/data",
"icu_casemapping?/data",
"icu_collator/data",
"icu_compactdecimal?/data",
"icu_datetime/data",
"icu_decimal/data",
"icu_displaynames?/data",
"icu_list/data",
"icu_locid_transform/data",
"icu_normalizer/data",
"icu_plurals/data",
"icu_properties/data",
"icu_relativetime?/data",
"icu_segmenter/data",
"icu_timezone/data",
]
serde_human = [
"icu_list/serde_human"
Expand Down
7 changes: 1 addition & 6 deletions components/icu/README.md
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5458916

Please sign in to comment.