Change icuexportdata trie format to improve normalizer performance (u…

…nicode-org#5813) Co-authored-by: Elango Cheran <[email protected]> Co-authored-by: Shane F. Carr <[email protected]>
hsivonen · Dec 18, 2024 · 5f103cb · 5f103cb
1 parent c047d8a
commit 5f103cb
Show file tree

Hide file tree

Showing 165 changed files with 38,837 additions and 28,490 deletions.
diff --git a/components/collator/src/comparison.rs b/components/collator/src/comparison.rs
@@ -29,9 +29,9 @@ use crate::provider::CollationTailoringV1Marker;
 use crate::{AlternateHandling, CollatorOptions, MaxVariable, ResolvedCollatorOptions, Strength};
 use core::cmp::Ordering;
 use core::convert::TryFrom;
-use icu_normalizer::provider::CanonicalDecompositionDataV1Marker;
+use icu_normalizer::provider::CanonicalDecompositionDataV2Marker;
 use icu_normalizer::provider::CanonicalDecompositionTablesV1Marker;
-use icu_normalizer::provider::DecompositionDataV1;
+use icu_normalizer::provider::DecompositionDataV2;
 use icu_normalizer::provider::DecompositionTablesV1;
 use icu_normalizer::Decomposition;
 use icu_provider::prelude::*;
@@ -220,7 +220,7 @@ pub struct Collator {
     diacritics: DataPayload<CollationDiacriticsV1Marker>,
     options: CollatorOptionsBitField,
     reordering: Option<DataPayload<CollationReorderingV1Marker>>,
-    decompositions: DataPayload<CanonicalDecompositionDataV1Marker>,
+    decompositions: DataPayload<CanonicalDecompositionDataV2Marker>,
     tables: DataPayload<CanonicalDecompositionTablesV1Marker>,
     lithuanian_dot_above: bool,
 }
@@ -276,7 +276,7 @@ impl Collator {
             + DataProvider<CollationJamoV1Marker>
             + DataProvider<CollationMetadataV1Marker>
             + DataProvider<CollationReorderingV1Marker>
-            + DataProvider<CanonicalDecompositionDataV1Marker>
+            + DataProvider<CanonicalDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + ?Sized,
     {
@@ -296,7 +296,7 @@ impl Collator {
     fn try_new_unstable_internal<D>(
         provider: &D,
         root: DataPayload<CollationRootV1Marker>,
-        decompositions: DataPayload<CanonicalDecompositionDataV1Marker>,
+        decompositions: DataPayload<CanonicalDecompositionDataV2Marker>,
         tables: DataPayload<CanonicalDecompositionTablesV1Marker>,
         jamo: DataPayload<CollationJamoV1Marker>,
         special_primaries: impl FnOnce() -> Result<
@@ -364,7 +364,7 @@ pub struct CollatorBorrowed<'a> {
     diacritics: &'a CollationDiacriticsV1<'a>,
     options: CollatorOptionsBitField,
     reordering: Option<&'a CollationReorderingV1<'a>>,
-    decompositions: &'a DecompositionDataV1<'a>,
+    decompositions: &'a DecompositionDataV2<'a>,
     tables: &'a DecompositionTablesV1<'a>,
     lithuanian_dot_above: bool,
 }
@@ -381,7 +381,7 @@ impl CollatorBorrowed<'static> {
 
         let provider = &crate::provider::Baked;
         let decompositions =
-            icu_normalizer::provider::Baked::SINGLETON_CANONICAL_DECOMPOSITION_DATA_V1_MARKER;
+            icu_normalizer::provider::Baked::SINGLETON_CANONICAL_DECOMPOSITION_DATA_V2_MARKER;
         let tables =
             icu_normalizer::provider::Baked::SINGLETON_CANONICAL_DECOMPOSITION_TABLES_V1_MARKER;
         let root = crate::provider::Baked::SINGLETON_COLLATION_ROOT_V1_MARKER;

diff --git a/components/collator/src/elements.rs b/components/collator/src/elements.rs
diff --git a/components/collator/tests/tests.rs b/components/collator/tests/tests.rs
@@ -32,11 +32,11 @@ const _: () = {
 
     icu_normalizer_data::impl_canonical_compositions_v1_marker!(TestingProvider);
     icu_normalizer_data::impl_non_recursive_decomposition_supplement_v1_marker!(TestingProvider);
-    icu_normalizer_data::impl_canonical_decomposition_data_v1_marker!(TestingProvider);
+    icu_normalizer_data::impl_canonical_decomposition_data_v2_marker!(TestingProvider);
     icu_normalizer_data::impl_canonical_decomposition_tables_v1_marker!(TestingProvider);
-    icu_normalizer_data::impl_compatibility_decomposition_supplement_v1_marker!(TestingProvider);
+    icu_normalizer_data::impl_compatibility_decomposition_data_v2_marker!(TestingProvider);
     icu_normalizer_data::impl_compatibility_decomposition_tables_v1_marker!(TestingProvider);
-    icu_normalizer_data::impl_uts46_decomposition_supplement_v1_marker!(TestingProvider);
+    icu_normalizer_data::impl_uts46_decomposition_data_v2_marker!(TestingProvider);
 };
 
 type StackString = arraystring::ArrayString<arraystring::typenum::U32>;

diff --git a/components/experimental/src/transliterate/compile/mod.rs b/components/experimental/src/transliterate/compile/mod.rs
@@ -69,7 +69,7 @@ impl Direction {
 ///       {$AE} [:Lowercase:] → Ae;
 ///       {$OE} [:Lowercase:] → Oe;
 ///       {$UE} [:Lowercase:] → Ue;
-///  
+///
 ///       $AE → AE;
 ///       $OE → OE;
 ///       $UE → UE;
@@ -232,8 +232,8 @@ impl RuleCollection {
             + DataProvider<ScriptWithExtensionsPropertyV1Marker>
             + DataProvider<XidStartV1Marker>,
         NP: ?Sized
-            + DataProvider<CanonicalDecompositionDataV1Marker>
-            + DataProvider<CompatibilityDecompositionSupplementV1Marker>
+            + DataProvider<CanonicalDecompositionDataV2Marker>
+            + DataProvider<CompatibilityDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CompatibilityDecompositionTablesV1Marker>
             + DataProvider<CanonicalCompositionsV1Marker>,
@@ -414,8 +414,8 @@ macro_rules! redirect {
 }
 
 redirect!(
-    CanonicalDecompositionDataV1Marker,
-    CompatibilityDecompositionSupplementV1Marker,
+    CanonicalDecompositionDataV2Marker,
+    CompatibilityDecompositionDataV2Marker,
     CanonicalDecompositionTablesV1Marker,
     CompatibilityDecompositionTablesV1Marker,
     CanonicalCompositionsV1Marker

diff --git a/components/experimental/src/transliterate/transliterator/mod.rs b/components/experimental/src/transliterate/transliterator/mod.rs
@@ -51,7 +51,7 @@ struct ComposingTransliterator(ComposingNormalizer);
 impl ComposingTransliterator {
     fn try_nfc<P>(provider: &P) -> Result<Self, DataError>
     where
-        P: DataProvider<CanonicalDecompositionDataV1Marker>
+        P: DataProvider<CanonicalDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CanonicalCompositionsV1Marker>
             + ?Sized,
@@ -63,8 +63,7 @@ impl ComposingTransliterator {
 
     fn try_nfkc<P>(provider: &P) -> Result<Self, DataError>
     where
-        P: DataProvider<CanonicalDecompositionDataV1Marker>
-            + DataProvider<CompatibilityDecompositionSupplementV1Marker>
+        P: DataProvider<CompatibilityDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CompatibilityDecompositionTablesV1Marker>
             + DataProvider<CanonicalCompositionsV1Marker>
@@ -90,7 +89,7 @@ struct DecomposingTransliterator(DecomposingNormalizer);
 impl DecomposingTransliterator {
     fn try_nfd<P>(provider: &P) -> Result<Self, DataError>
     where
-        P: DataProvider<CanonicalDecompositionDataV1Marker>
+        P: DataProvider<CanonicalDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + ?Sized,
     {
@@ -101,8 +100,7 @@ impl DecomposingTransliterator {
 
     fn try_nfkd<P>(provider: &P) -> Result<Self, DataError>
     where
-        P: DataProvider<CanonicalDecompositionDataV1Marker>
-            + DataProvider<CompatibilityDecompositionSupplementV1Marker>
+        P: DataProvider<CompatibilityDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CompatibilityDecompositionTablesV1Marker>
             + ?Sized,
@@ -279,8 +277,8 @@ impl Transliterator {
     ) -> Result<Self, DataError>
     where
         PT: DataProvider<TransliteratorRulesV1Marker> + ?Sized,
-        PN: DataProvider<CanonicalDecompositionDataV1Marker>
-            + DataProvider<CompatibilityDecompositionSupplementV1Marker>
+        PN: DataProvider<CanonicalDecompositionDataV2Marker>
+            + DataProvider<CompatibilityDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CompatibilityDecompositionTablesV1Marker>
             + DataProvider<CanonicalCompositionsV1Marker>
@@ -391,8 +389,8 @@ impl Transliterator {
     ) -> Result<Transliterator, DataError>
     where
         PT: DataProvider<TransliteratorRulesV1Marker> + ?Sized,
-        PN: DataProvider<CanonicalDecompositionDataV1Marker>
-            + DataProvider<CompatibilityDecompositionSupplementV1Marker>
+        PN: DataProvider<CanonicalDecompositionDataV2Marker>
+            + DataProvider<CompatibilityDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CompatibilityDecompositionTablesV1Marker>
             + DataProvider<CanonicalCompositionsV1Marker>
@@ -415,8 +413,8 @@ impl Transliterator {
     ) -> Result<Transliterator, DataError>
     where
         PT: DataProvider<TransliteratorRulesV1Marker> + ?Sized,
-        PN: DataProvider<CanonicalDecompositionDataV1Marker>
-            + DataProvider<CompatibilityDecompositionSupplementV1Marker>
+        PN: DataProvider<CanonicalDecompositionDataV2Marker>
+            + DataProvider<CompatibilityDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CompatibilityDecompositionTablesV1Marker>
             + DataProvider<CanonicalCompositionsV1Marker>
@@ -451,8 +449,8 @@ impl Transliterator {
     ) -> Result<DataPayload<TransliteratorRulesV1Marker>, DataError>
     where
         PT: DataProvider<TransliteratorRulesV1Marker> + ?Sized,
-        PN: DataProvider<CanonicalDecompositionDataV1Marker>
-            + DataProvider<CompatibilityDecompositionSupplementV1Marker>
+        PN: DataProvider<CanonicalDecompositionDataV2Marker>
+            + DataProvider<CompatibilityDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CompatibilityDecompositionTablesV1Marker>
             + DataProvider<CanonicalCompositionsV1Marker>
@@ -500,8 +498,8 @@ impl Transliterator {
         normalizer_provider: &P,
     ) -> Option<Result<InternalTransliterator, DataError>>
     where
-        P: DataProvider<CanonicalDecompositionDataV1Marker>
-            + DataProvider<CompatibilityDecompositionSupplementV1Marker>
+        P: DataProvider<CanonicalDecompositionDataV2Marker>
+            + DataProvider<CompatibilityDecompositionDataV2Marker>
             + DataProvider<CanonicalDecompositionTablesV1Marker>
             + DataProvider<CompatibilityDecompositionTablesV1Marker>
             + DataProvider<CanonicalCompositionsV1Marker>