From 141701d5d081d91f580a7b9a930d44fb4924efd3 Mon Sep 17 00:00:00 2001 From: Robert Bastian Date: Tue, 25 Jun 2024 21:47:46 +0200 Subject: [PATCH] change --- components/calendar/src/week_of.rs | 6 +- components/collator/src/comparison.rs | 2 +- components/collator/tests/tests.rs | 6 +- components/datetime/src/calendar.rs | 8 +- components/datetime/src/external_loaders.rs | 29 ++ components/datetime/src/format/datetime.rs | 2 +- components/datetime/src/format/neo.rs | 18 +- components/datetime/src/neo.rs | 2 +- components/datetime/src/neo_skeleton.rs | 218 ++++++---- components/datetime/src/provider/date_time.rs | 6 +- components/datetime/src/provider/neo.rs | 149 ++++--- .../datetime/src/provider/neo/adapter.rs | 163 ++++---- components/datetime/src/raw/datetime.rs | 12 +- components/datetime/src/raw/neo.rs | 16 +- components/datetime/src/raw/zoned_datetime.rs | 6 +- components/datetime/src/skeleton/mod.rs | 2 +- components/datetime/src/time_zone.rs | 6 +- components/datetime/tests/datetime.rs | 6 +- components/decimal/src/lib.rs | 2 +- .../src/compactdecimal/compactdecimal.rs | 8 +- .../src/dimension/currency/formatter.rs | 4 +- .../src/displaynames/displaynames.rs | 10 +- .../experimental/src/personnames/formatter.rs | 2 +- .../src/relativetime/relativetime.rs | 4 +- .../src/transliterate/compile/mod.rs | 23 +- .../src/transliterate/compile/pass1.rs | 6 +- .../src/transliterate/transliterator/mod.rs | 6 +- .../src/unicodeset_parse/parse.rs | 6 +- components/experimental/src/units/ratio.rs | 4 - components/list/src/list_formatter.rs | 2 +- components/plurals/benches/parser.rs | 2 +- components/plurals/src/lib.rs | 6 +- components/plurals/tests/plurals.rs | 2 +- components/properties/src/exemplar_chars.rs | 4 +- .../segmenter/src/complex/dictionary.rs | 4 +- components/segmenter/src/complex/lstm/mod.rs | 6 +- components/segmenter/src/complex/mod.rs | 224 +++++----- provider/adapters/src/either.rs | 16 +- provider/adapters/src/empty.rs | 8 +- provider/adapters/src/fallback/mod.rs | 37 +- provider/adapters/src/filter/impls.rs | 87 ++-- provider/adapters/src/filter/mod.rs | 24 +- provider/adapters/src/fork/by_error.rs | 14 +- provider/adapters/src/fork/mod.rs | 10 +- provider/adapters/src/fork/predicates.rs | 8 +- provider/baked/src/binary_search.rs | 66 +-- provider/baked/src/export.rs | 42 +- provider/baked/src/lib.rs | 6 +- .../tests/data/hello_world_v1_marker.rs.data | 4 +- provider/baked/tests/test-baked-source.rs | 7 +- .../bikeshed/src/calendar/chinese_based.rs | 8 +- provider/bikeshed/src/calendar/islamic.rs | 8 +- provider/bikeshed/src/calendar/japanese.rs | 8 +- provider/bikeshed/src/characters/mod.rs | 8 +- .../src/cldr_serde/currencies/data.rs | 3 +- provider/bikeshed/src/collator/mod.rs | 10 +- provider/bikeshed/src/currency/essentials.rs | 27 +- provider/bikeshed/src/currency/extended.rs | 44 +- provider/bikeshed/src/datetime/mod.rs | 29 +- provider/bikeshed/src/datetime/neo.rs | 89 ++-- .../bikeshed/src/datetime/neo_skeleton.rs | 41 +- provider/bikeshed/src/datetime/week_data.rs | 28 +- provider/bikeshed/src/decimal/compact.rs | 24 +- provider/bikeshed/src/decimal/mod.rs | 9 +- provider/bikeshed/src/decimal/symbols.rs | 12 +- .../bikeshed/src/displaynames/language.rs | 26 +- provider/bikeshed/src/displaynames/region.rs | 12 +- provider/bikeshed/src/displaynames/script.rs | 12 +- provider/bikeshed/src/displaynames/variant.rs | 10 +- provider/bikeshed/src/fallback/mod.rs | 8 +- provider/bikeshed/src/hello_world/mod.rs | 6 +- provider/bikeshed/src/lib.rs | 44 +- provider/bikeshed/src/list/mod.rs | 8 +- .../src/locale_canonicalizer/aliases.rs | 8 +- .../locale_canonicalizer/directionality.rs | 4 +- .../locale_canonicalizer/likely_subtags.rs | 16 +- provider/bikeshed/src/normalizer/mod.rs | 4 +- provider/bikeshed/src/percent/mod.rs | 16 +- .../person_names_format_data_providers.rs | 14 +- provider/bikeshed/src/plurals/mod.rs | 22 +- provider/bikeshed/src/properties/bidi_data.rs | 4 +- .../bikeshed/src/properties/bin_cp_set.rs | 2 +- .../bikeshed/src/properties/bin_uniset.rs | 2 +- .../src/properties/enum_codepointtrie.rs | 20 +- provider/bikeshed/src/properties/script.rs | 4 +- provider/bikeshed/src/relativetime/mod.rs | 10 +- provider/bikeshed/src/segmenter/dictionary.rs | 16 +- provider/bikeshed/src/segmenter/lstm.rs | 25 +- provider/bikeshed/src/segmenter/mod.rs | 2 +- provider/bikeshed/src/tests/make_testdata.rs | 13 +- provider/bikeshed/src/time_zones/mod.rs | 18 +- provider/bikeshed/src/time_zones/names.rs | 12 +- provider/bikeshed/src/transforms/mod.rs | 22 +- provider/bikeshed/src/ucase/mod.rs | 8 +- provider/bikeshed/src/units/data.rs | 35 +- provider/bikeshed/src/units/info.rs | 6 +- provider/blob/benches/auxkey_bench.rs | 44 +- provider/blob/benches/blob_version_bench.rs | 12 +- provider/blob/src/blob_data_provider.rs | 4 +- provider/blob/src/blob_schema.rs | 33 +- provider/blob/src/export/blob_exporter.rs | 9 +- provider/blob/src/export/iter.rs | 4 +- provider/blob/tests/test_versions.rs | 26 +- provider/core/src/any.rs | 16 +- provider/core/src/buf.rs | 15 +- provider/core/src/data_provider.rs | 26 +- provider/core/src/dynutil.rs | 36 +- provider/core/src/error.rs | 2 +- provider/core/src/export/mod.rs | 15 +- provider/core/src/hello_world.rs | 86 ++-- provider/core/src/lib.rs | 3 +- provider/core/src/marker.rs | 4 +- provider/core/src/request.rs | 387 ++++++++++-------- provider/core/src/response.rs | 3 +- provider/datagen/src/driver.rs | 236 ++++++----- provider/datagen/tests/test-options.rs | 13 +- provider/datagen/tests/testutil.rs | 16 +- provider/fs/benches/provider_fs.rs | 14 +- provider/fs/src/export/fs_exporter.rs | 9 +- provider/fs/src/fs_data_provider.rs | 8 +- provider/fs/tests/test_file_io.rs | 13 +- provider/icu4x-datagen/src/main.rs | 6 +- tools/bakeddata-scripts/src/main.rs | 15 +- tutorials/data_provider.md | 8 +- 124 files changed, 1613 insertions(+), 1518 deletions(-) diff --git a/components/calendar/src/week_of.rs b/components/calendar/src/week_of.rs index a286285784d..af140f1b4ed 100644 --- a/components/calendar/src/week_of.rs +++ b/components/calendar/src/week_of.rs @@ -65,7 +65,7 @@ impl WeekCalculator { DataProvider::::load( &provider.as_downcasting(), DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, ) @@ -84,7 +84,7 @@ impl WeekCalculator { DataProvider::::load( &provider.as_deserializing(), DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, ) @@ -100,7 +100,7 @@ impl WeekCalculator { { provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }) .map(|response| WeekCalculator { diff --git a/components/collator/src/comparison.rs b/components/collator/src/comparison.rs index d122e0d591f..e878c1be511 100644 --- a/components/collator/src/comparison.rs +++ b/components/collator/src/comparison.rs @@ -150,7 +150,7 @@ impl Collator { + ?Sized, { let req = DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }; diff --git a/components/collator/tests/tests.rs b/components/collator/tests/tests.rs index 57884b3dc4d..731f84616b2 100644 --- a/components/collator/tests/tests.rs +++ b/components/collator/tests/tests.rs @@ -1191,7 +1191,7 @@ fn test_nb_nn_no() { DataProvider::::load( &icu_collator::provider::Baked, DataRequest { - locale: &locale, + id: DataIdentifierBorrowed::for_locale(&locale), ..Default::default() } ) @@ -1211,7 +1211,7 @@ fn test_nb_nn_no() { DataProvider::::load( &icu_collator::provider::Baked, DataRequest { - locale: &locale, + id: DataIdentifierBorrowed::for_locale(&locale), ..Default::default() } ) @@ -1231,7 +1231,7 @@ fn test_nb_nn_no() { DataProvider::::load( &icu_collator::provider::Baked, DataRequest { - locale: &locale, + id: DataIdentifierBorrowed::for_locale(&locale), ..Default::default() } ) diff --git a/components/datetime/src/calendar.rs b/components/datetime/src/calendar.rs index 5ddc96ec2d4..47cb1c6511e 100644 --- a/components/datetime/src/calendar.rs +++ b/components/datetime/src/calendar.rs @@ -388,7 +388,7 @@ where { let payload = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; @@ -405,7 +405,7 @@ where { let payload = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; @@ -434,7 +434,7 @@ where + ?Sized, { let req = DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }; let payload = match kind { @@ -555,7 +555,7 @@ where + ?Sized, { let req = DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }; let payload = match kind { diff --git a/components/datetime/src/external_loaders.rs b/components/datetime/src/external_loaders.rs index f1d565e860c..8897a40c1ca 100644 --- a/components/datetime/src/external_loaders.rs +++ b/components/datetime/src/external_loaders.rs @@ -35,6 +35,35 @@ pub(crate) trait AnyCalendarLoader { fn load(&self, locale: &DataLocale) -> Result; } +/// Helper for type resolution with optional loader arguments +pub(crate) struct PhantomLoader { + _not_constructible: core::convert::Infallible, +} + +impl FixedDecimalFormatterLoader for PhantomLoader { + fn load( + &self, + _locale: &DataLocale, + _options: FixedDecimalFormatterOptions, + ) -> Result { + unreachable!() // not constructible + } +} + +impl WeekCalculatorLoader for PhantomLoader { + #[inline] + fn load(&self, _locale: &DataLocale) -> Result { + unreachable!() // not constructible + } +} + +impl AnyCalendarLoader for PhantomLoader { + #[inline] + fn load(&self, _locale: &DataLocale) -> Result { + unreachable!() // not constructible + } +} + /// Loader for types from other crates using compiled data. #[cfg(feature = "compiled_data")] pub(crate) struct ExternalLoaderCompiledData; diff --git a/components/datetime/src/format/datetime.rs b/components/datetime/src/format/datetime.rs index 264d7112f13..451becb4d8f 100644 --- a/components/datetime/src/format/datetime.rs +++ b/components/datetime/src/format/datetime.rs @@ -956,7 +956,7 @@ mod tests { let locale = "en-u-ca-gregory".parse().unwrap(); let req = DataRequest { - locale: &locale, + id: DataIdentifierBorrowed::for_locale(&locale), ..Default::default() }; let date_data = diff --git a/components/datetime/src/format/neo.rs b/components/datetime/src/format/neo.rs index af12369f71a..c16a77b6dcb 100644 --- a/components/datetime/src/format/neo.rs +++ b/components/datetime/src/format/neo.rs @@ -1025,8 +1025,7 @@ impl RawDateTimeNames { }; let payload = provider .load_bound(DataRequest { - locale, - marker_attributes: &DataMarkerAttributes::from_tinystr( + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( marker_attrs::symbol_attr_for( marker_attrs::Context::Format, match field_length { @@ -1036,6 +1035,7 @@ impl RawDateTimeNames { _ => return Err(SingleLoadError::UnsupportedField(field)), }, ), + locale, ), ..Default::default() }) @@ -1075,8 +1075,7 @@ impl RawDateTimeNames { }; let payload = provider .load_bound(DataRequest { - locale, - marker_attributes: &DataMarkerAttributes::from_tinystr( + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( marker_attrs::symbol_attr_for( match field_symbol { fields::Month::Format => marker_attrs::Context::Format, @@ -1089,6 +1088,7 @@ impl RawDateTimeNames { _ => return Err(SingleLoadError::UnsupportedField(field)), }, ), + locale, ), ..Default::default() }) @@ -1127,8 +1127,7 @@ impl RawDateTimeNames { let payload = R::DayPeriodNames::load_from( provider, DataRequest { - locale, - marker_attributes: &DataMarkerAttributes::from_tinystr( + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( marker_attrs::symbol_attr_for( marker_attrs::Context::Format, match field_length { @@ -1138,6 +1137,7 @@ impl RawDateTimeNames { _ => return Err(SingleLoadError::UnsupportedField(field)), }, ), + locale, ), ..Default::default() }, @@ -1179,8 +1179,7 @@ impl RawDateTimeNames { }; let payload = provider .load_bound(DataRequest { - locale, - marker_attributes: &DataMarkerAttributes::from_tinystr( + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( marker_attrs::symbol_attr_for( match field_symbol { // UTS 35 says that "e" and "E" have the same non-numeric names @@ -1197,6 +1196,7 @@ impl RawDateTimeNames { _ => return Err(SingleLoadError::UnsupportedField(field)), }, ), + locale, ), ..Default::default() }) @@ -1234,7 +1234,7 @@ impl RawDateTimeNames { let payload = R::ZoneGenericShortNames::load_from( provider, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, ) diff --git a/components/datetime/src/neo.rs b/components/datetime/src/neo.rs index ef189e71503..81684ac698f 100644 --- a/components/datetime/src/neo.rs +++ b/components/datetime/src/neo.rs @@ -1286,7 +1286,7 @@ impl NeoFormatter { /// ``` pub fn convert_and_format<'a, I>(&'a self, datetime: &I) -> FormattedNeoDateTime where - I: ConvertCalendar, + I: ?Sized + ConvertCalendar, I::Converted<'a>: NeoGetField<::YearInput> + NeoGetField<::MonthInput> + NeoGetField<::DayOfMonthInput> diff --git a/components/datetime/src/neo_skeleton.rs b/components/datetime/src/neo_skeleton.rs index fb1731a2a6b..eb812957411 100644 --- a/components/datetime/src/neo_skeleton.rs +++ b/components/datetime/src/neo_skeleton.rs @@ -7,8 +7,7 @@ use crate::options::components; use crate::options::length; use crate::DateTimeFormatterOptions; -use tinystr::tinystr; -use tinystr::TinyAsciiStr; +use icu_provider::DataMarkerAttributes; /// A specification for the length of a date or component of a date. /// @@ -141,17 +140,37 @@ impl NeoDayComponents { Self::AutoWeekday, ]; - const DAY_STR: TinyAsciiStr<8> = tinystr!(8, "d"); - const MONTH_DAY_STR: TinyAsciiStr<8> = tinystr!(8, "m0d"); - const YEAR_MONTH_DAY_STR: TinyAsciiStr<8> = tinystr!(8, "ym0d"); - const ERA_YEAR_MONTH_DAY_STR: TinyAsciiStr<8> = tinystr!(8, "gym0d"); - const DAY_WEEKDAY_STR: TinyAsciiStr<8> = tinystr!(8, "de"); - const MONTH_DAY_WEEKDAY_STR: TinyAsciiStr<8> = tinystr!(8, "m0de"); - const YEAR_MONTH_DAY_WEEKDAY_STR: TinyAsciiStr<8> = tinystr!(8, "ym0de"); - const ERA_YEAR_MONTH_DAY_WEEKDAY_STR: TinyAsciiStr<8> = tinystr!(8, "gym0de"); - const WEEKDAY_STR: TinyAsciiStr<8> = tinystr!(8, "e"); - const AUTO_STR: TinyAsciiStr<8> = tinystr!(8, "a1"); - const AUTO_WEEKDAY_STR: TinyAsciiStr<8> = tinystr!(8, "a1e"); + const DAY: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("d"); + const MONTH_DAY: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("m0d"); + const YEAR_MONTH_DAY: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("ym0d"); + const ERA_YEAR_MONTH_DAY: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("gym0d"); + const DAY_WEEKDAY: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("de"); + const MONTH_DAY_WEEKDAY: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("m0de"); + const YEAR_MONTH_DAY_WEEKDAY: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("ym0de"); + const ERA_YEAR_MONTH_DAY_WEEKDAY: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("gym0de"); + const WEEKDAY: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("e"); + const AUTO: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("a1"); + const AUTO_WEEKDAY: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("a1e"); + + // For matching + const DAY_STR: &'static str = Self::DAY.as_str(); + const MONTH_DAY_STR: &'static str = Self::MONTH_DAY.as_str(); + const YEAR_MONTH_DAY_STR: &'static str = Self::YEAR_MONTH_DAY.as_str(); + const ERA_YEAR_MONTH_DAY_STR: &'static str = Self::ERA_YEAR_MONTH_DAY.as_str(); + const DAY_WEEKDAY_STR: &'static str = Self::DAY_WEEKDAY.as_str(); + const MONTH_DAY_WEEKDAY_STR: &'static str = Self::MONTH_DAY_WEEKDAY.as_str(); + const YEAR_MONTH_DAY_WEEKDAY_STR: &'static str = Self::YEAR_MONTH_DAY_WEEKDAY.as_str(); + const ERA_YEAR_MONTH_DAY_WEEKDAY_STR: &'static str = Self::ERA_YEAR_MONTH_DAY_WEEKDAY.as_str(); + const WEEKDAY_STR: &'static str = Self::WEEKDAY.as_str(); + const AUTO_STR: &'static str = Self::AUTO.as_str(); + const AUTO_WEEKDAY_STR: &'static str = Self::AUTO_WEEKDAY.as_str(); /// Returns a stable string identifying this set of components. /// @@ -173,22 +192,22 @@ impl NeoDayComponents { /// /// assert_eq!( /// "gym0de", - /// &*NeoDayComponents::EraYearMonthDayWeekday.id_str() + /// NeoDayComponents::EraYearMonthDayWeekday.id_str().as_str() /// ); /// ``` - pub const fn id_str(self) -> TinyAsciiStr<8> { + pub const fn id_str(self) -> &'static DataMarkerAttributes { match self { - Self::Day => Self::DAY_STR, - Self::MonthDay => Self::MONTH_DAY_STR, - Self::YearMonthDay => Self::YEAR_MONTH_DAY_STR, - Self::EraYearMonthDay => Self::ERA_YEAR_MONTH_DAY_STR, - Self::DayWeekday => Self::DAY_WEEKDAY_STR, - Self::MonthDayWeekday => Self::MONTH_DAY_WEEKDAY_STR, - Self::YearMonthDayWeekday => Self::YEAR_MONTH_DAY_WEEKDAY_STR, - Self::EraYearMonthDayWeekday => Self::ERA_YEAR_MONTH_DAY_WEEKDAY_STR, - Self::Weekday => Self::WEEKDAY_STR, - Self::Auto => Self::AUTO_STR, - Self::AutoWeekday => Self::AUTO_WEEKDAY_STR, + Self::Day => Self::DAY, + Self::MonthDay => Self::MONTH_DAY, + Self::YearMonthDay => Self::YEAR_MONTH_DAY, + Self::EraYearMonthDay => Self::ERA_YEAR_MONTH_DAY, + Self::DayWeekday => Self::DAY_WEEKDAY, + Self::MonthDayWeekday => Self::MONTH_DAY_WEEKDAY, + Self::YearMonthDayWeekday => Self::YEAR_MONTH_DAY_WEEKDAY, + Self::EraYearMonthDayWeekday => Self::ERA_YEAR_MONTH_DAY_WEEKDAY, + Self::Weekday => Self::WEEKDAY, + Self::Auto => Self::AUTO, + Self::AutoWeekday => Self::AUTO_WEEKDAY, } } @@ -198,15 +217,15 @@ impl NeoDayComponents { /// /// ``` /// use icu::datetime::neo_skeleton::NeoDayComponents; - /// use tinystr::tinystr; + /// use icu_provider::prelude::*; /// /// assert_eq!( - /// NeoDayComponents::from_id_str(tinystr!(8, "gym0de")), + /// NeoDayComponents::from_id_str(DataMarkerAttributes::from_str_or_panic("gym0de")), /// Some(NeoDayComponents::EraYearMonthDayWeekday) /// ); /// ``` - pub const fn from_id_str(id_str: TinyAsciiStr<8>) -> Option { - match id_str { + pub fn from_id_str(id_str: &DataMarkerAttributes) -> Option { + match &**id_str { Self::DAY_STR => Some(Self::Day), Self::MONTH_DAY_STR => Some(Self::MonthDay), Self::YEAR_MONTH_DAY_STR => Some(Self::YearMonthDay), @@ -367,37 +386,50 @@ impl NeoDateComponents { Self::YearQuarter, ]; - const MONTH_STR: TinyAsciiStr<8> = tinystr!(8, "m0"); - const YEAR_MONTH_STR: TinyAsciiStr<8> = tinystr!(8, "ym0"); - const ERA_YEAR_MONTH_STR: TinyAsciiStr<8> = tinystr!(8, "gym0"); - const YEAR_STR: TinyAsciiStr<8> = tinystr!(8, "y"); - const ERA_YEAR_STR: TinyAsciiStr<8> = tinystr!(8, "gy"); - const YEAR_WEEK_STR: TinyAsciiStr<8> = tinystr!(8, "y0w"); - const QUARTER_STR: TinyAsciiStr<8> = tinystr!(8, "q"); - const YEAR_QUARTER_STR: TinyAsciiStr<8> = tinystr!(8, "yq"); + const MONTH: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("m0"); + const YEAR_MONTH: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("ym0"); + const ERA_YEAR_MONTH: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("gym0"); + const YEAR: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("y"); + const ERA_YEAR: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("gy"); + const YEAR_WEEK: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("y0w"); + const QUARTER: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("q"); + const YEAR_QUARTER: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("yq"); + + // For matching + const MONTH_STR: &'static str = Self::MONTH.as_str(); + const YEAR_MONTH_STR: &'static str = Self::YEAR_MONTH.as_str(); + const ERA_YEAR_MONTH_STR: &'static str = Self::ERA_YEAR_MONTH.as_str(); + const YEAR_STR: &'static str = Self::YEAR.as_str(); + const ERA_YEAR_STR: &'static str = Self::ERA_YEAR.as_str(); + const YEAR_WEEK_STR: &'static str = Self::YEAR_WEEK.as_str(); + const QUARTER_STR: &'static str = Self::QUARTER.as_str(); + const YEAR_QUARTER_STR: &'static str = Self::YEAR_QUARTER.as_str(); /// Returns a stable string identifying this set of components. /// /// For details, see [`NeoDayComponents::id_str()`]. - pub const fn id_str(self) -> TinyAsciiStr<8> { + pub const fn id_str(self) -> &'static DataMarkerAttributes { match self { Self::Day(day_components) => day_components.id_str(), - Self::Month => Self::MONTH_STR, - Self::YearMonth => Self::YEAR_MONTH_STR, - Self::EraYearMonth => Self::ERA_YEAR_MONTH_STR, - Self::Year => Self::YEAR_STR, - Self::EraYear => Self::ERA_YEAR_STR, - Self::YearWeek => Self::YEAR_WEEK_STR, - Self::Quarter => Self::QUARTER_STR, - Self::YearQuarter => Self::YEAR_QUARTER_STR, + Self::Month => Self::MONTH, + Self::YearMonth => Self::YEAR_MONTH, + Self::EraYearMonth => Self::ERA_YEAR_MONTH, + Self::Year => Self::YEAR, + Self::EraYear => Self::ERA_YEAR, + Self::YearWeek => Self::YEAR_WEEK, + Self::Quarter => Self::QUARTER, + Self::YearQuarter => Self::YEAR_QUARTER, } } /// Returns the set of components for the given stable string. /// /// For details, see [`NeoDayComponents::from_id_str()`]. - pub const fn from_id_str(id_str: TinyAsciiStr<8>) -> Option { - match id_str { + pub fn from_id_str(id_str: &DataMarkerAttributes) -> Option { + match &**id_str { Self::MONTH_STR => Some(Self::Month), Self::YEAR_MONTH_STR => Some(Self::YearMonth), Self::ERA_YEAR_MONTH_STR => Some(Self::EraYearMonth), @@ -406,10 +438,7 @@ impl NeoDateComponents { Self::YEAR_WEEK_STR => Some(Self::YearWeek), Self::QUARTER_STR => Some(Self::Quarter), Self::YEAR_QUARTER_STR => Some(Self::YearQuarter), - other => match NeoDayComponents::from_id_str(other) { - Some(day_components) => Some(Self::Day(day_components)), - None => None, - }, + _ => NeoDayComponents::from_id_str(id_str).map(Self::Day), } } @@ -521,46 +550,71 @@ impl NeoTimeComponents { Self::Auto, ]; - const HOUR_STR: TinyAsciiStr<8> = tinystr!(8, "j"); - const HOUR_MINUTE_STR: TinyAsciiStr<8> = tinystr!(8, "jm"); - const HOUR_MINUTE_SECOND_STR: TinyAsciiStr<8> = tinystr!(8, "jms"); - const DAY_PERIOD_HOUR12_STR: TinyAsciiStr<8> = tinystr!(8, "bh"); - const HOUR12_STR: TinyAsciiStr<8> = tinystr!(8, "h"); - const DAY_PERIOD_HOUR12_MINUTE_STR: TinyAsciiStr<8> = tinystr!(8, "bhm"); - const HOUR12_MINUTE_STR: TinyAsciiStr<8> = tinystr!(8, "hm"); - const DAY_PERIOD_HOUR12_MINUTE_SECOND_STR: TinyAsciiStr<8> = tinystr!(8, "bhms"); - const HOUR12_MINUTE_SECOND_STR: TinyAsciiStr<8> = tinystr!(8, "hms"); - const HOUR24_STR: TinyAsciiStr<8> = tinystr!(8, "h0"); - const HOUR24_MINUTE_STR: TinyAsciiStr<8> = tinystr!(8, "h0m"); - const HOUR24_MINUTE_SECOND_STR: TinyAsciiStr<8> = tinystr!(8, "h0ms"); - const AUTO_STR: TinyAsciiStr<8> = tinystr!(8, "a1"); + const HOUR: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("j"); + const HOUR_MINUTE: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("jm"); + const HOUR_MINUTE_SECOND: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("jms"); + const DAY_PERIOD_HOUR12: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("bh"); + const HOUR12: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("h"); + const DAY_PERIOD_HOUR12_MINUTE: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("bhm"); + const HOUR12_MINUTE: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("hm"); + const DAY_PERIOD_HOUR12_MINUTE_SECOND: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("bhms"); + const HOUR12_MINUTE_SECOND: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("hms"); + const HOUR24: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("h0"); + const HOUR24_MINUTE: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("h0m"); + const HOUR24_MINUTE_SECOND: &'static DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("h0ms"); + const AUTO: &'static DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("a1"); + + // For matching + const HOUR_STR: &'static str = Self::HOUR.as_str(); + const HOUR_MINUTE_STR: &'static str = Self::HOUR_MINUTE.as_str(); + const HOUR_MINUTE_SECOND_STR: &'static str = Self::HOUR_MINUTE_SECOND.as_str(); + const DAY_PERIOD_HOUR12_STR: &'static str = Self::DAY_PERIOD_HOUR12.as_str(); + const HOUR12_STR: &'static str = Self::HOUR12.as_str(); + const DAY_PERIOD_HOUR12_MINUTE_STR: &'static str = Self::DAY_PERIOD_HOUR12_MINUTE.as_str(); + const HOUR12_MINUTE_STR: &'static str = Self::HOUR12_MINUTE.as_str(); + const DAY_PERIOD_HOUR12_MINUTE_SECOND_STR: &'static str = + Self::DAY_PERIOD_HOUR12_MINUTE_SECOND.as_str(); + const HOUR12_MINUTE_SECOND_STR: &'static str = Self::HOUR12_MINUTE_SECOND.as_str(); + const HOUR24_STR: &'static str = Self::HOUR24.as_str(); + const HOUR24_MINUTE_STR: &'static str = Self::HOUR24_MINUTE.as_str(); + const HOUR24_MINUTE_SECOND_STR: &'static str = Self::HOUR24_MINUTE_SECOND.as_str(); + const AUTO_STR: &'static str = Self::AUTO.as_str(); /// Returns a stable string identifying this set of components. /// /// For details, see [`NeoDayComponents::id_str()`]. - pub const fn id_str(self) -> TinyAsciiStr<8> { + pub const fn id_str(self) -> &'static DataMarkerAttributes { match self { - Self::Hour => Self::HOUR_STR, - Self::HourMinute => Self::HOUR_MINUTE_STR, - Self::HourMinuteSecond => Self::HOUR_MINUTE_SECOND_STR, - Self::DayPeriodHour12 => Self::DAY_PERIOD_HOUR12_STR, - Self::Hour12 => Self::HOUR12_STR, - Self::DayPeriodHour12Minute => Self::DAY_PERIOD_HOUR12_MINUTE_STR, - Self::Hour12Minute => Self::HOUR12_MINUTE_STR, - Self::DayPeriodHour12MinuteSecond => Self::DAY_PERIOD_HOUR12_MINUTE_SECOND_STR, - Self::Hour12MinuteSecond => Self::HOUR12_MINUTE_SECOND_STR, - Self::Hour24 => Self::HOUR24_STR, - Self::Hour24Minute => Self::HOUR24_MINUTE_STR, - Self::Hour24MinuteSecond => Self::HOUR24_MINUTE_SECOND_STR, - Self::Auto => Self::AUTO_STR, + Self::Hour => Self::HOUR, + Self::HourMinute => Self::HOUR_MINUTE, + Self::HourMinuteSecond => Self::HOUR_MINUTE_SECOND, + Self::DayPeriodHour12 => Self::DAY_PERIOD_HOUR12, + Self::Hour12 => Self::HOUR12, + Self::DayPeriodHour12Minute => Self::DAY_PERIOD_HOUR12_MINUTE, + Self::Hour12Minute => Self::HOUR12_MINUTE, + Self::DayPeriodHour12MinuteSecond => Self::DAY_PERIOD_HOUR12_MINUTE_SECOND, + Self::Hour12MinuteSecond => Self::HOUR12_MINUTE_SECOND, + Self::Hour24 => Self::HOUR24, + Self::Hour24Minute => Self::HOUR24_MINUTE, + Self::Hour24MinuteSecond => Self::HOUR24_MINUTE_SECOND, + Self::Auto => Self::AUTO, } } /// Returns the set of components for the given stable string. /// /// For details, see [`NeoDayComponents::from_id_str()`]. - pub const fn from_id_str(id_str: TinyAsciiStr<8>) -> Option { - match id_str { + pub fn from_id_str(id_str: &DataMarkerAttributes) -> Option { + match &**id_str { Self::HOUR_STR => Some(Self::Hour), Self::HOUR_MINUTE_STR => Some(Self::HourMinute), Self::HOUR_MINUTE_SECOND_STR => Some(Self::HourMinuteSecond), diff --git a/components/datetime/src/provider/date_time.rs b/components/datetime/src/provider/date_time.rs index 9e5a20e70a6..acea905a090 100644 --- a/components/datetime/src/provider/date_time.rs +++ b/components/datetime/src/provider/date_time.rs @@ -126,7 +126,7 @@ where { Ok(data_provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload @@ -236,7 +236,7 @@ where let time_patterns_data = self .data_provider .load(DataRequest { - locale: self.locale, + id: DataIdentifierBorrowed::for_locale(self.locale), ..Default::default() }) .map_err(PatternForLengthError::Data)? @@ -374,7 +374,7 @@ where self.data_provider .load(DataRequest { - locale: &locale, + id: DataIdentifierBorrowed::for_locale(&locale), ..Default::default() }) .map(|r| r.payload) diff --git a/components/datetime/src/provider/neo.rs b/components/datetime/src/provider/neo.rs index e8b680741c0..3df5129f5d1 100644 --- a/components/datetime/src/provider/neo.rs +++ b/components/datetime/src/provider/neo.rs @@ -27,32 +27,69 @@ use core::ops::Range; #[allow(missing_docs)] pub mod marker_attrs { use crate::pattern::CoarseHourCycle; - use tinystr::{tinystr, TinyAsciiStr}; - - pub const NUMERIC: TinyAsciiStr<8> = tinystr!(8, "1"); - pub const ABBR: TinyAsciiStr<8> = tinystr!(8, "3"); - pub const NARROW: TinyAsciiStr<8> = tinystr!(8, "4"); - pub const WIDE: TinyAsciiStr<8> = tinystr!(8, "5"); - pub const SHORT: TinyAsciiStr<8> = tinystr!(8, "6"); - pub const ABBR_STANDALONE: TinyAsciiStr<8> = tinystr!(8, "3s"); - pub const NARROW_STANDALONE: TinyAsciiStr<8> = tinystr!(8, "4s"); - pub const WIDE_STANDALONE: TinyAsciiStr<8> = tinystr!(8, "5s"); - pub const SHORT_STANDALONE: TinyAsciiStr<8> = tinystr!(8, "6s"); - - pub const PATTERN_FULL: TinyAsciiStr<8> = tinystr!(8, "f"); - pub const PATTERN_LONG: TinyAsciiStr<8> = tinystr!(8, "l"); - pub const PATTERN_MEDIUM: TinyAsciiStr<8> = tinystr!(8, "m"); - pub const PATTERN_SHORT: TinyAsciiStr<8> = tinystr!(8, "s"); - - pub const PATTERN_FULL12: TinyAsciiStr<8> = tinystr!(8, "f12"); - pub const PATTERN_LONG12: TinyAsciiStr<8> = tinystr!(8, "l12"); - pub const PATTERN_MEDIUM12: TinyAsciiStr<8> = tinystr!(8, "m12"); - pub const PATTERN_SHORT12: TinyAsciiStr<8> = tinystr!(8, "s12"); - - pub const PATTERN_FULL24: TinyAsciiStr<8> = tinystr!(8, "f24"); - pub const PATTERN_LONG24: TinyAsciiStr<8> = tinystr!(8, "l24"); - pub const PATTERN_MEDIUM24: TinyAsciiStr<8> = tinystr!(8, "m24"); - pub const PATTERN_SHORT24: TinyAsciiStr<8> = tinystr!(8, "s24"); + use icu_provider::DataMarkerAttributes; + + pub const NUMERIC: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("1"); + pub const ABBR: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("3"); + pub const NARROW: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("4"); + pub const WIDE: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("5"); + pub const SHORT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("6"); + pub const ABBR_STANDALONE: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("3s"); + pub const NARROW_STANDALONE: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("4s"); + pub const WIDE_STANDALONE: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("5s"); + pub const SHORT_STANDALONE: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("6s"); + + pub const PATTERN_FULL: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("f"); + pub const PATTERN_LONG: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("l"); + pub const PATTERN_MEDIUM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("m"); + pub const PATTERN_SHORT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("s"); + + pub const PATTERN_FULL12: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("f12"); + pub const PATTERN_LONG12: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("l12"); + pub const PATTERN_MEDIUM12: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("m12"); + pub const PATTERN_SHORT12: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("s12"); + + pub const PATTERN_FULL24: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("f24"); + pub const PATTERN_LONG24: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("l24"); + pub const PATTERN_MEDIUM24: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("m24"); + pub const PATTERN_SHORT24: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("s24"); + + pub const NUMERIC_STR: &str = NUMERIC.as_str(); + pub const ABBR_STR: &str = ABBR.as_str(); + pub const NARROW_STR: &str = NARROW.as_str(); + pub const WIDE_STR: &str = WIDE.as_str(); + pub const SHORT_STR: &str = SHORT.as_str(); + pub const ABBR_STANDALONE_STR: &str = ABBR_STANDALONE.as_str(); + pub const NARROW_STANDALONE_STR: &str = NARROW_STANDALONE.as_str(); + pub const WIDE_STANDALONE_STR: &str = WIDE_STANDALONE.as_str(); + pub const SHORT_STANDALONE_STR: &str = SHORT_STANDALONE.as_str(); + + pub const PATTERN_FULL_STR: &str = PATTERN_FULL.as_str(); + pub const PATTERN_LONG_STR: &str = PATTERN_LONG.as_str(); + pub const PATTERN_MEDIUM_STR: &str = PATTERN_MEDIUM.as_str(); + pub const PATTERN_SHORT_STR: &str = PATTERN_SHORT.as_str(); + + pub const PATTERN_FULL12_STR: &str = PATTERN_FULL12.as_str(); + pub const PATTERN_LONG12_STR: &str = PATTERN_LONG12.as_str(); + pub const PATTERN_MEDIUM12_STR: &str = PATTERN_MEDIUM12.as_str(); + pub const PATTERN_SHORT12_STR: &str = PATTERN_SHORT12.as_str(); + + pub const PATTERN_FULL24_STR: &str = PATTERN_FULL24.as_str(); + pub const PATTERN_LONG24_STR: &str = PATTERN_LONG24.as_str(); + pub const PATTERN_MEDIUM24_STR: &str = PATTERN_MEDIUM24.as_str(); + pub const PATTERN_SHORT24_STR: &str = PATTERN_SHORT24.as_str(); /// Field lengths supported in data marker attribute. /// @@ -120,18 +157,20 @@ pub mod marker_attrs { /// including in SemVer minor releases. While the serde representation of data structs is guaranteed /// to be stable, their Rust representation might not be. Use with caution. /// - pub fn symbol_marker_attr_info(marker_attr: TinyAsciiStr<8>) -> Option<(Context, Length)> { + pub fn symbol_marker_attr_info( + marker_attr: &DataMarkerAttributes, + ) -> Option<(Context, Length)> { use {Context::*, Length::*}; - match marker_attr { - NUMERIC => Some((Format, Numeric)), - ABBR => Some((Format, Abbr)), - NARROW => Some((Format, Narrow)), - WIDE => Some((Format, Wide)), - SHORT => Some((Format, Short)), - ABBR_STANDALONE => Some((Standalone, Abbr)), - NARROW_STANDALONE => Some((Standalone, Narrow)), - WIDE_STANDALONE => Some((Standalone, Wide)), - SHORT_STANDALONE => Some((Standalone, Short)), + match &**marker_attr { + NUMERIC_STR => Some((Format, Numeric)), + ABBR_STR => Some((Format, Abbr)), + NARROW_STR => Some((Format, Narrow)), + WIDE_STR => Some((Format, Wide)), + SHORT_STR => Some((Format, Short)), + ABBR_STANDALONE_STR => Some((Standalone, Abbr)), + NARROW_STANDALONE_STR => Some((Standalone, Narrow)), + WIDE_STANDALONE_STR => Some((Standalone, Wide)), + SHORT_STANDALONE_STR => Some((Standalone, Short)), _ => None, } } @@ -144,24 +183,24 @@ pub mod marker_attrs { /// to be stable, their Rust representation might not be. Use with caution. /// pub fn pattern_marker_attr_info( - marker_attr: TinyAsciiStr<8>, + marker_attr: &DataMarkerAttributes, ) -> Option<(PatternLength, Option)> { use {CoarseHourCycle::*, PatternLength::*}; - match marker_attr { - PATTERN_FULL => Some((Full, None)), - PATTERN_LONG => Some((Long, None)), - PATTERN_MEDIUM => Some((Medium, None)), - PATTERN_SHORT => Some((Short, None)), - - PATTERN_FULL12 => Some((Full, Some(H11H12))), - PATTERN_LONG12 => Some((Long, Some(H11H12))), - PATTERN_MEDIUM12 => Some((Medium, Some(H11H12))), - PATTERN_SHORT12 => Some((Short, Some(H11H12))), - - PATTERN_FULL24 => Some((Full, Some(H23H24))), - PATTERN_LONG24 => Some((Long, Some(H23H24))), - PATTERN_MEDIUM24 => Some((Medium, Some(H23H24))), - PATTERN_SHORT24 => Some((Short, Some(H23H24))), + match &**marker_attr { + PATTERN_FULL_STR => Some((Full, None)), + PATTERN_LONG_STR => Some((Long, None)), + PATTERN_MEDIUM_STR => Some((Medium, None)), + PATTERN_SHORT_STR => Some((Short, None)), + + PATTERN_FULL12_STR => Some((Full, Some(H11H12))), + PATTERN_LONG12_STR => Some((Long, Some(H11H12))), + PATTERN_MEDIUM12_STR => Some((Medium, Some(H11H12))), + PATTERN_SHORT12_STR => Some((Short, Some(H11H12))), + + PATTERN_FULL24_STR => Some((Full, Some(H23H24))), + PATTERN_LONG24_STR => Some((Long, Some(H23H24))), + PATTERN_MEDIUM24_STR => Some((Medium, Some(H23H24))), + PATTERN_SHORT24_STR => Some((Short, Some(H23H24))), _ => None, } } @@ -173,7 +212,7 @@ pub mod marker_attrs { /// including in SemVer minor releases. While the serde representation of data structs is guaranteed /// to be stable, their Rust representation might not be. Use with caution. /// - pub fn symbol_attr_for(context: Context, length: Length) -> TinyAsciiStr<8> { + pub fn symbol_attr_for(context: Context, length: Length) -> &'static DataMarkerAttributes { use {Context::*, Length::*}; match (context, length) { (Format, Numeric) => NUMERIC, @@ -199,7 +238,7 @@ pub mod marker_attrs { pub fn pattern_marker_attr_for( length: PatternLength, hour_cycle: Option, - ) -> TinyAsciiStr<8> { + ) -> &'static DataMarkerAttributes { use {CoarseHourCycle::*, PatternLength::*}; match (length, hour_cycle) { (Full, None) => PATTERN_FULL, diff --git a/components/datetime/src/provider/neo/adapter.rs b/components/datetime/src/provider/neo/adapter.rs index 7a4b005c3a0..257302dd937 100644 --- a/components/datetime/src/provider/neo/adapter.rs +++ b/components/datetime/src/provider/neo/adapter.rs @@ -9,15 +9,26 @@ use icu_calendar::types::MonthCode; use icu_provider::prelude::*; mod key_attr_consts { - use tinystr::{tinystr, TinyAsciiStr}; - pub const STADLN_ABBR: TinyAsciiStr<8> = tinystr!(8, "3s"); - pub const STADLN_WIDE: TinyAsciiStr<8> = tinystr!(8, "4s"); - pub const STADLN_NARW: TinyAsciiStr<8> = tinystr!(8, "5s"); - pub const STADLN_SHRT: TinyAsciiStr<8> = tinystr!(8, "6s"); - pub const FORMAT_ABBR: TinyAsciiStr<8> = tinystr!(8, "3"); - pub const FORMAT_WIDE: TinyAsciiStr<8> = tinystr!(8, "4"); - pub const FORMAT_NARW: TinyAsciiStr<8> = tinystr!(8, "5"); - pub const FORMAT_SHRT: TinyAsciiStr<8> = tinystr!(8, "6"); + use super::*; + + pub const STADLN_ABBR: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("3s"); + pub const STADLN_WIDE: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("4s"); + pub const STADLN_NARW: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("5s"); + pub const STADLN_SHRT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("6s"); + pub const FORMAT_ABBR: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("3"); + pub const FORMAT_WIDE: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("4"); + pub const FORMAT_NARW: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("5"); + pub const FORMAT_SHRT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("6"); + + /// Used for matching + pub const STADLN_ABBR_STR: &str = STADLN_ABBR.as_str(); + pub const STADLN_WIDE_STR: &str = STADLN_WIDE.as_str(); + pub const STADLN_NARW_STR: &str = STADLN_NARW.as_str(); + pub const STADLN_SHRT_STR: &str = STADLN_SHRT.as_str(); + pub const FORMAT_ABBR_STR: &str = FORMAT_ABBR.as_str(); + pub const FORMAT_WIDE_STR: &str = FORMAT_WIDE.as_str(); + pub const FORMAT_NARW_STR: &str = FORMAT_NARW.as_str(); + pub const FORMAT_SHRT_STR: &str = FORMAT_SHRT.as_str(); } fn month_symbols_map_project_cloned( @@ -28,29 +39,25 @@ where M: DataMarker>, P: DataMarker>, { - let attribute = req - .marker_attributes - .single() - .ok_or_else(|| DataError::custom("TODO"))?; let new_payload = payload.try_map_project_cloned(|payload, _| { use key_attr_consts::*; - let result = match attribute { - STADLN_ABBR => payload.months.stand_alone_abbreviated(), - STADLN_WIDE => payload.months.stand_alone_wide(), - STADLN_NARW => payload.months.stand_alone_narrow(), + let result = match req.id.marker_attributes.as_str() { + STADLN_ABBR_STR => payload.months.stand_alone_abbreviated(), + STADLN_WIDE_STR => payload.months.stand_alone_wide(), + STADLN_NARW_STR => payload.months.stand_alone_narrow(), _ => None, }; if let Some(result) = result { return Ok(result.into()); } - let result = match attribute { - STADLN_ABBR | FORMAT_ABBR => &payload.months.format.abbreviated, - STADLN_WIDE | FORMAT_WIDE => &payload.months.format.wide, - STADLN_NARW | FORMAT_NARW => &payload.months.format.narrow, + let result = match req.id.marker_attributes.as_str() { + STADLN_ABBR_STR | FORMAT_ABBR_STR => &payload.months.format.abbreviated, + STADLN_WIDE_STR | FORMAT_WIDE_STR => &payload.months.format.wide, + STADLN_NARW_STR | FORMAT_NARW_STR => &payload.months.format.narrow, _ => { return Err(DataError::custom("Unknown marker attribute") .with_marker(M::INFO) - .with_display_context(&attribute)) + .with_display_context(req.id.marker_attributes.as_str())) } }; Ok(result.into()) @@ -69,39 +76,35 @@ where M: DataMarker>, P: DataMarker>, { - let attribute = req - .marker_attributes - .single() - .ok_or_else(|| DataError::custom("TODO"))?; let new_payload = payload.try_map_project_cloned(|payload, _| { use key_attr_consts::*; - let result = match attribute { - STADLN_ABBR => payload.weekdays.stand_alone_abbreviated(), - STADLN_WIDE => payload.weekdays.stand_alone_wide(), - STADLN_NARW => payload.weekdays.stand_alone_narrow(), - STADLN_SHRT => payload.weekdays.stand_alone_short(), + let result = match req.id.marker_attributes.as_str() { + STADLN_ABBR_STR => payload.weekdays.stand_alone_abbreviated(), + STADLN_WIDE_STR => payload.weekdays.stand_alone_wide(), + STADLN_NARW_STR => payload.weekdays.stand_alone_narrow(), + STADLN_SHRT_STR => payload.weekdays.stand_alone_short(), _ => None, }; if let Some(result) = result { return Ok(result.into()); } - let result = match attribute { - STADLN_SHRT | FORMAT_SHRT => payload.weekdays.format.short.as_ref(), + let result = match req.id.marker_attributes.as_str() { + STADLN_SHRT_STR | FORMAT_SHRT_STR => payload.weekdays.format.short.as_ref(), _ => None, }; if let Some(result) = result { return Ok(result.into()); } - let result = match attribute { - STADLN_ABBR | FORMAT_ABBR | STADLN_SHRT | FORMAT_SHRT => { + let result = match req.id.marker_attributes.as_str() { + STADLN_ABBR_STR | FORMAT_ABBR_STR | STADLN_SHRT_STR | FORMAT_SHRT_STR => { &payload.weekdays.format.abbreviated } - STADLN_WIDE | FORMAT_WIDE => &payload.weekdays.format.wide, - STADLN_NARW | FORMAT_NARW => &payload.weekdays.format.narrow, + STADLN_WIDE_STR | FORMAT_WIDE_STR => &payload.weekdays.format.wide, + STADLN_NARW_STR | FORMAT_NARW_STR => &payload.weekdays.format.narrow, _ => { return Err(DataError::custom("Unknown marker attribute") .with_marker(M::INFO) - .with_display_context(&attribute)) + .with_display_context(req.id.marker_attributes.as_str())) } }; Ok(result.into()) @@ -120,20 +123,16 @@ where M: DataMarker>, P: DataMarker>, { - let attribute = req - .marker_attributes - .single() - .ok_or_else(|| DataError::custom("TODO"))?; let new_payload = payload.try_map_project_cloned(|payload, _| { use key_attr_consts::*; - let result = match attribute { - FORMAT_ABBR => &payload.eras.abbr, - FORMAT_WIDE => &payload.eras.names, - FORMAT_NARW => &payload.eras.narrow, + let result = match req.id.marker_attributes.as_str() { + FORMAT_ABBR_STR => &payload.eras.abbr, + FORMAT_WIDE_STR => &payload.eras.names, + FORMAT_NARW_STR => &payload.eras.narrow, _ => { return Err(DataError::custom("Unknown marker attribute") .with_marker(M::INFO) - .with_display_context(&attribute)) + .with_display_context(req.id.marker_attributes.as_str())) } }; Ok(YearNamesV1::Eras(result.clone())) @@ -152,29 +151,25 @@ where M: DataMarker>, P: DataMarker>, { - let attribute = req - .marker_attributes - .single() - .ok_or_else(|| DataError::custom("TODO"))?; let new_payload = payload.try_map_project_cloned(|payload, _| { use key_attr_consts::*; - let result = match attribute { - STADLN_ABBR => payload.day_periods.stand_alone_abbreviated(), - STADLN_WIDE => payload.day_periods.stand_alone_wide(), - STADLN_NARW => payload.day_periods.stand_alone_narrow(), + let result = match req.id.marker_attributes.as_str() { + STADLN_ABBR_STR => payload.day_periods.stand_alone_abbreviated(), + STADLN_WIDE_STR => payload.day_periods.stand_alone_wide(), + STADLN_NARW_STR => payload.day_periods.stand_alone_narrow(), _ => None, }; if let Some(result) = result { return Ok(result.into()); } - let result = match attribute { - STADLN_ABBR | FORMAT_ABBR => &payload.day_periods.format.abbreviated, - STADLN_WIDE | FORMAT_WIDE => &payload.day_periods.format.wide, - STADLN_NARW | FORMAT_NARW => &payload.day_periods.format.narrow, + let result = match req.id.marker_attributes.as_str() { + STADLN_ABBR_STR | FORMAT_ABBR_STR => &payload.day_periods.format.abbreviated, + STADLN_WIDE_STR | FORMAT_WIDE_STR => &payload.day_periods.format.wide, + STADLN_NARW_STR | FORMAT_NARW_STR => &payload.day_periods.format.narrow, _ => { return Err(DataError::custom("Unknown marker attribute") .with_marker(M::INFO) - .with_display_context(&attribute)) + .with_display_context(req.id.marker_attributes.as_str())) } }; Ok(result.into()) @@ -466,15 +461,17 @@ mod tests { fn test_adapter_months_numeric() { let symbols: DataPayload = crate::provider::Baked .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap() .payload; let neo_month_abbreviated: DataPayload = symbols .load(DataRequest { - locale: &"en".parse().unwrap(), - marker_attributes: &"3".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("3"), + &"en".parse().unwrap(), + ), ..Default::default() }) .unwrap() @@ -490,15 +487,17 @@ mod tests { fn test_adapter_months_map() { let symbols: DataPayload = crate::provider::Baked .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap() .payload; let neo_month_abbreviated: DataPayload = symbols .load(DataRequest { - locale: &"en".parse().unwrap(), - marker_attributes: &"3".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("3"), + &"en".parse().unwrap(), + ), ..Default::default() }) .unwrap() @@ -514,15 +513,17 @@ mod tests { fn test_adapter_weekdays_abbreviated() { let symbols: DataPayload = crate::provider::Baked .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap() .payload; let neo_weekdays_abbreviated: DataPayload = symbols .load(DataRequest { - locale: &"en".parse().unwrap(), - marker_attributes: &"3".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("3"), + &"en".parse().unwrap(), + ), ..Default::default() }) .unwrap() @@ -538,15 +539,17 @@ mod tests { fn test_adapter_weekdays_short() { let symbols: DataPayload = crate::provider::Baked .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap() .payload; let neo_weekdays_short: DataPayload = symbols .load(DataRequest { - locale: &"en".parse().unwrap(), - marker_attributes: &"6s".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("6s"), + &"en".parse().unwrap(), + ), ..Default::default() }) .unwrap() @@ -562,15 +565,17 @@ mod tests { fn test_adapter_eras() { let symbols: DataPayload = crate::provider::Baked .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap() .payload; let neo_eras_wide: DataPayload = symbols .load(DataRequest { - locale: &"en".parse().unwrap(), - marker_attributes: &"4".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("4"), + &"en".parse().unwrap(), + ), ..Default::default() }) .unwrap() @@ -586,15 +591,17 @@ mod tests { fn test_adapter_dayperiods() { let symbols: DataPayload = crate::provider::Baked .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap() .payload; let neo_dayperiods_abbreviated: DataPayload = symbols .load(DataRequest { - locale: &"en".parse().unwrap(), - marker_attributes: &"3s".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("3s"), + &"en".parse().unwrap(), + ), ..Default::default() }) .unwrap() diff --git a/components/datetime/src/raw/datetime.rs b/components/datetime/src/raw/datetime.rs index faf83991af7..4d9f4e97c18 100644 --- a/components/datetime/src/raw/datetime.rs +++ b/components/datetime/src/raw/datetime.rs @@ -61,7 +61,7 @@ impl TimeFormatter { Some( crate::provider::Baked .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload, @@ -102,7 +102,7 @@ impl TimeFormatter { Some( provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload, @@ -243,7 +243,7 @@ impl DateFormatter { (*DataProvider::::load( provider, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, )? @@ -387,7 +387,7 @@ impl DateTimeFormatter { .map_err(|field| DateTimeError::UnsupportedField(field.symbol))?; let req = DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }; @@ -450,7 +450,7 @@ impl DateTimeFormatter { .map_err(|field| DateTimeError::UnsupportedField(field.symbol))?; let req = DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }; @@ -459,7 +459,7 @@ impl DateTimeFormatter { (*DataProvider::::load( provider, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, )? diff --git a/components/datetime/src/raw/neo.rs b/components/datetime/src/raw/neo.rs index cc911638aac..dd92cf9d712 100644 --- a/components/datetime/src/raw/neo.rs +++ b/components/datetime/src/raw/neo.rs @@ -93,8 +93,10 @@ impl DatePatternSelectionData { ) -> Result { let payload = provider .load_bound(DataRequest { - locale, - marker_attributes: &DataMarkerAttributes::from_tinystr(components.id_str()), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + components.id_str(), + locale, + ), ..Default::default() })? .payload @@ -138,8 +140,10 @@ impl TimePatternSelectionData { ) -> Result { let payload = provider .load_bound(DataRequest { - locale, - marker_attributes: &DataMarkerAttributes::from_tinystr(components.id_str()), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + components.id_str(), + locale, + ), ..Default::default() })? .payload @@ -220,8 +224,7 @@ impl DateTimeGluePatternSelectionData { )?; let glue = glue_provider .load_bound(DataRequest { - locale, - marker_attributes: &DataMarkerAttributes::from_tinystr( + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( marker_attrs::pattern_marker_attr_for( // According to UTS 35, use the date length here: use the glue // pattern "whose type matches the type of the date pattern" @@ -232,6 +235,7 @@ impl DateTimeGluePatternSelectionData { }, None, // no hour cycle for date patterns ), + locale, ), ..Default::default() })? diff --git a/components/datetime/src/raw/zoned_datetime.rs b/components/datetime/src/raw/zoned_datetime.rs index d80d075418f..13095707671 100644 --- a/components/datetime/src/raw/zoned_datetime.rs +++ b/components/datetime/src/raw/zoned_datetime.rs @@ -48,7 +48,7 @@ impl ZonedDateTimeFormatter { .map_err(|field| DateTimeError::UnsupportedField(field.symbol))?; let req = DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }; @@ -133,7 +133,7 @@ impl ZonedDateTimeFormatter { .map_err(|field| DateTimeError::UnsupportedField(field.symbol))?; let req = DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }; @@ -142,7 +142,7 @@ impl ZonedDateTimeFormatter { (*DataProvider::::load( provider, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, )? diff --git a/components/datetime/src/skeleton/mod.rs b/components/datetime/src/skeleton/mod.rs index 289f6e54a91..7b1f7c7aad1 100644 --- a/components/datetime/src/skeleton/mod.rs +++ b/components/datetime/src/skeleton/mod.rs @@ -39,7 +39,7 @@ mod test { ) { let locale = "en-u-ca-gregory".parse().unwrap(); let req = DataRequest { - locale: &locale, + id: DataIdentifierBorrowed::for_locale(&locale), ..Default::default() }; let patterns = crate::provider::Baked diff --git a/components/datetime/src/time_zone.rs b/components/datetime/src/time_zone.rs index f9527fffac2..35e14858134 100644 --- a/components/datetime/src/time_zone.rs +++ b/components/datetime/src/time_zone.rs @@ -37,7 +37,7 @@ where *destination = Some( provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload, @@ -219,7 +219,7 @@ impl TimeZoneFormatter { let data_payloads = TimeZoneDataPayloads { zone_formats: zone_provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload, @@ -475,7 +475,7 @@ impl TimeZoneFormatter { let data_payloads = TimeZoneDataPayloads { zone_formats: provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload, diff --git a/components/datetime/tests/datetime.rs b/components/datetime/tests/datetime.rs index c0c8a292b00..8a831c9aa80 100644 --- a/components/datetime/tests/datetime.rs +++ b/components/datetime/tests/datetime.rs @@ -395,7 +395,7 @@ fn test_dayperiod_patterns() { .set(key!("ca"), value!("gregory")); let mut data_locale = DataLocale::from(&locale); let req = DataRequest { - locale: &data_locale, + id: DataIdentifierBorrowed::for_locale(&data_locale), ..Default::default() }; let mut date_patterns_data: DataResponse = @@ -420,7 +420,7 @@ fn test_dayperiod_patterns() { data_locale.retain_unicode_ext(|_| false); let decimal_data: DataResponse = icu_decimal::provider::Baked .load(DataRequest { - locale: &data_locale, + id: DataIdentifierBorrowed::for_locale(&data_locale), ..Default::default() }) .unwrap(); @@ -570,7 +570,7 @@ fn test_time_zone_patterns() { .set(key!("ca"), value!("gregory")); let data_locale = DataLocale::from(&locale); let req = DataRequest { - locale: &data_locale, + id: DataIdentifierBorrowed::for_locale(&data_locale), ..Default::default() }; let mut config = test.config; diff --git a/components/decimal/src/lib.rs b/components/decimal/src/lib.rs index 2160316bd9b..15d06c5e844 100644 --- a/components/decimal/src/lib.rs +++ b/components/decimal/src/lib.rs @@ -138,7 +138,7 @@ impl FixedDecimalFormatter { ) -> Result { let symbols = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; diff --git a/components/experimental/src/compactdecimal/compactdecimal.rs b/components/experimental/src/compactdecimal/compactdecimal.rs index 7ad39de1290..fc2376f3a36 100644 --- a/components/experimental/src/compactdecimal/compactdecimal.rs +++ b/components/experimental/src/compactdecimal/compactdecimal.rs @@ -132,7 +132,7 @@ impl CompactDecimalFormatter { compact_data: DataProvider::::load( &crate::provider::Baked, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, )? @@ -174,7 +174,7 @@ impl CompactDecimalFormatter { compact_data: DataProvider::::load( provider, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, )? @@ -216,7 +216,7 @@ impl CompactDecimalFormatter { compact_data: DataProvider::::load( &crate::provider::Baked, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, )? @@ -258,7 +258,7 @@ impl CompactDecimalFormatter { compact_data: DataProvider::::load( provider, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }, )? diff --git a/components/experimental/src/dimension/currency/formatter.rs b/components/experimental/src/dimension/currency/formatter.rs index 7a490b24453..deff34ed188 100644 --- a/components/experimental/src/dimension/currency/formatter.rs +++ b/components/experimental/src/dimension/currency/formatter.rs @@ -72,7 +72,7 @@ impl CurrencyFormatter { })?; let essential = crate::provider::Baked .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; @@ -106,7 +106,7 @@ impl CurrencyFormatter { })?; let essential = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; diff --git a/components/experimental/src/displaynames/displaynames.rs b/components/experimental/src/displaynames/displaynames.rs index 6391efff2a3..85cb862286d 100644 --- a/components/experimental/src/displaynames/displaynames.rs +++ b/components/experimental/src/displaynames/displaynames.rs @@ -63,7 +63,7 @@ impl RegionDisplayNames { ) -> Result { let region_data = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; @@ -136,7 +136,7 @@ impl ScriptDisplayNames { ) -> Result { let script_data = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; @@ -210,7 +210,7 @@ impl VariantDisplayNames { ) -> Result { let variant_data = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; @@ -277,7 +277,7 @@ impl LanguageDisplayNames { ) -> Result { let language_data = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; @@ -376,7 +376,7 @@ impl LocaleDisplayNamesFormatter { + ?Sized, { let req = DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() }; diff --git a/components/experimental/src/personnames/formatter.rs b/components/experimental/src/personnames/formatter.rs index de7676e7bd4..b14edf8e366 100644 --- a/components/experimental/src/personnames/formatter.rs +++ b/components/experimental/src/personnames/formatter.rs @@ -87,7 +87,7 @@ impl PersonNamesFormatter { let data: DataResponse = provider .load(DataRequest { - locale: &DataLocale::from(effective_locale), + id: DataIdentifierBorrowed::for_locale(&DataLocale::from(effective_locale)), ..Default::default() }) .map_err(PersonNamesFormatterError::Data)?; diff --git a/components/experimental/src/relativetime/relativetime.rs b/components/experimental/src/relativetime/relativetime.rs index effba5ea75c..aff1919b019 100644 --- a/components/experimental/src/relativetime/relativetime.rs +++ b/components/experimental/src/relativetime/relativetime.rs @@ -130,7 +130,7 @@ macro_rules! constructor { )?; let rt: DataResponse<$marker> = crate::provider::Baked .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })?; let rt = rt.payload.cast(); @@ -175,7 +175,7 @@ macro_rules! constructor { )?; let rt: DataResponse<$marker> = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })?; let rt = rt.payload.cast(); diff --git a/components/experimental/src/transliterate/compile/mod.rs b/components/experimental/src/transliterate/compile/mod.rs index 9cd22e313ff..bdae41f1698 100644 --- a/components/experimental/src/transliterate/compile/mod.rs +++ b/components/experimental/src/transliterate/compile/mod.rs @@ -310,13 +310,13 @@ where ) -> Result, DataError> { let mut exclusive_data = self.collection.data.borrow_mut(); - if let Some(response) = exclusive_data.1.get(req.marker_attributes as &str) { + if let Some(response) = exclusive_data.1.get(req.id.marker_attributes.as_str()) { return response.clone(); }; let result = || -> Result, DataError> { let Some((source, reverse, visible)) = - exclusive_data.0.remove(req.marker_attributes as &str) + exclusive_data.0.remove(req.id.marker_attributes as &str) else { return Err( DataErrorKind::MissingLocale.with_req(TransliteratorRulesV1Marker::INFO, req) @@ -372,7 +372,7 @@ where exclusive_data .1 - .insert(req.marker_attributes.to_string(), result.clone()); + .insert(req.id.marker_attributes.to_string(), result.clone()); result } @@ -466,14 +466,18 @@ where + DataProvider, NP: ?Sized, { - fn iter_requests(&self) -> Result, DataError> { + fn iter_ids(&self) -> Result, DataError> { let exclusive_data = self.collection.data.borrow(); Ok(exclusive_data .0 .keys() .cloned() .chain(exclusive_data.1.keys().cloned()) - .filter_map(|s| Some((Default::default(), s.parse().ok()?))) + .filter_map(|s| { + Some(DataIdentifierCow::from_marker_attributes_owned( + DataMarkerAttributes::try_from_string(s).ok()?, + )) + }) .collect()) } } @@ -640,7 +644,9 @@ mod tests { let forward: DataResponse = collection .as_provider() .load(DataRequest { - marker_attributes: &"fwd".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes( + DataMarkerAttributes::from_str_or_panic("fwd"), + ), ..Default::default() }) .unwrap(); @@ -648,7 +654,10 @@ mod tests { let reverse: DataResponse = collection .as_provider() .load(DataRequest { - marker_attributes: &"rev".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes( + DataMarkerAttributes::from_str_or_panic("rev"), + ), + ..Default::default() }) .unwrap(); diff --git a/components/experimental/src/transliterate/compile/pass1.rs b/components/experimental/src/transliterate/compile/pass1.rs index 841bc2fdf8f..d178fbc5898 100644 --- a/components/experimental/src/transliterate/compile/pass1.rs +++ b/components/experimental/src/transliterate/compile/pass1.rs @@ -49,11 +49,11 @@ //! ### Cursors (`|@@@ rest`, `rest @@@|`, `rest | rest`) //! There are three kinds of cursors. //! * `|@@@ rest`: The cursor is at the beginning of the replacement and has placeholders. We call -//! this a cursor with _right_ placeholders, because they are on the right of the cursor. +//! this a cursor with _right_ placeholders, because they are on the right of the cursor. //! * `rest @@@|`: The cursor is at the end of the replacement and has placeholders. We call this -//! a cursor with _left_ placeholders, because they are on the left of the cursor. +//! a cursor with _left_ placeholders, because they are on the left of the cursor. //! * `rest | rest`: The cursor is in the middle of the replacement and has no placeholders. We call -//! this a pure cursor. +//! this a pure cursor. //! //! Pure cursors get a reserved code point that is inlined into the replacement encoding where the //! cursor appears. Cursors with placeholders are encoded like back references, so two more diff --git a/components/experimental/src/transliterate/transliterator/mod.rs b/components/experimental/src/transliterate/transliterator/mod.rs index 40fbf082c04..8e0e08fd85e 100644 --- a/components/experimental/src/transliterate/transliterator/mod.rs +++ b/components/experimental/src/transliterate/transliterator/mod.rs @@ -260,7 +260,7 @@ impl Transliterator { { let payload = Transliterator::load_rbt( #[allow(clippy::unwrap_used)] // infallible - &locale.to_string().to_ascii_lowercase().parse().unwrap(), + DataMarkerAttributes::try_from_str(&locale.to_string().to_ascii_lowercase()).unwrap(), transliterator_provider, )?; let rbt = payload.get(); @@ -316,7 +316,7 @@ impl Transliterator { .unwrap_or_else(|| { let rbt = Transliterator::load_rbt( #[allow(clippy::unwrap_used)] // infallible - &dep.to_ascii_lowercase().parse().unwrap(), + DataMarkerAttributes::try_from_str(&dep.to_ascii_lowercase()).unwrap(), transliterator_provider, )?; Ok(InternalTransliterator::RuleBased(rbt)) @@ -407,7 +407,7 @@ impl Transliterator { P: DataProvider + ?Sized, { let req = DataRequest { - marker_attributes, + id: DataIdentifierBorrowed::for_marker_attributes(marker_attributes), ..Default::default() }; let payload = provider.load(req)?.payload; diff --git a/components/experimental/src/unicodeset_parse/parse.rs b/components/experimental/src/unicodeset_parse/parse.rs index 9cea5f70c47..ce44697b9ef 100644 --- a/components/experimental/src/unicodeset_parse/parse.rs +++ b/components/experimental/src/unicodeset_parse/parse.rs @@ -1430,9 +1430,9 @@ where /// # Limitations /// /// * Currently, we only support the [ECMA-262 properties](https://tc39.es/ecma262/#table-nonbinary-unicode-properties). -/// The property names must match the exact spelling listed in ECMA-262. Note that we do support UTS35 syntax for elided `General_Category` -/// and `Script` property names, i.e., `[:Latn:]` and `[:Ll:]` are both valid, with the former implying the `Script` property, and the latter the -/// `General_Category` property. +/// The property names must match the exact spelling listed in ECMA-262. Note that we do support UTS35 syntax for elided `General_Category` +/// and `Script` property names, i.e., `[:Latn:]` and `[:Ll:]` are both valid, with the former implying the `Script` property, and the latter the +/// `General_Category` property. /// * We do not support `\N{Unicode code point name}` character escaping. Use any other escape method described in UTS35. /// /// ✨ *Enabled with the `compiled_data` Cargo feature.* diff --git a/components/experimental/src/units/ratio.rs b/components/experimental/src/units/ratio.rs index e6df5961bbd..6034e609427 100644 --- a/components/experimental/src/units/ratio.rs +++ b/components/experimental/src/units/ratio.rs @@ -348,7 +348,6 @@ impl FromStr for IcuRatio { /// - Simple fractions: e.g., "1/2". /// - Decimals: e.g., "1.5". /// - Mixed fractions: e.g., "1.5/6", "1.4/5.6". - /// /// Note: Scientific notation is not supported. fn parse_fraction(decimal: &str) -> Result { let mut components = decimal.split('/'); @@ -376,13 +375,10 @@ impl FromStr for IcuRatio { } /// Parses a decimal number from a string input. - /// /// Accepts input in various decimal formats, including: /// - Whole numbers: "1" /// - Decimal numbers: "1.5" - /// /// An empty string input is interpreted as "0". - /// /// Note: Fractional inputs are not supported in this context. fn parse_decimal(decimal: &str) -> Result { let is_negative = decimal.starts_with('-'); diff --git a/components/list/src/list_formatter.rs b/components/list/src/list_formatter.rs index 29a22d562f5..47cecc3b8cb 100644 --- a/components/list/src/list_formatter.rs +++ b/components/list/src/list_formatter.rs @@ -48,7 +48,7 @@ macro_rules! constructor { ) -> Result { let data = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload diff --git a/components/plurals/benches/parser.rs b/components/plurals/benches/parser.rs index b320a748658..2eda50016d7 100644 --- a/components/plurals/benches/parser.rs +++ b/components/plurals/benches/parser.rs @@ -21,7 +21,7 @@ fn parser(c: &mut Criterion) { let response: DataResponse = icu_plurals::provider::Baked .load(DataRequest { - locale: &langid.into(), + id: DataIdentifierBorrowed::for_locale(&langid.into()), ..Default::default() }) .unwrap(); diff --git a/components/plurals/src/lib.rs b/components/plurals/src/lib.rs index 9af588c4143..74ed56a16ab 100644 --- a/components/plurals/src/lib.rs +++ b/components/plurals/src/lib.rs @@ -370,7 +370,7 @@ impl PluralRules { Ok(Self( provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload @@ -430,7 +430,7 @@ impl PluralRules { Ok(Self( provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload @@ -767,7 +767,7 @@ where ) -> Result { let ranges = provider .load(DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; diff --git a/components/plurals/tests/plurals.rs b/components/plurals/tests/plurals.rs index 8575d0b7c77..915dd5aed7b 100644 --- a/components/plurals/tests/plurals.rs +++ b/components/plurals/tests/plurals.rs @@ -21,7 +21,7 @@ fn test_static_load_works() { DataProvider::::load( &icu_plurals::provider::Baked, DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }, ) diff --git a/components/properties/src/exemplar_chars.rs b/components/properties/src/exemplar_chars.rs index ab7ae25cbbf..6a2b8b5ccb0 100644 --- a/components/properties/src/exemplar_chars.rs +++ b/components/properties/src/exemplar_chars.rs @@ -55,7 +55,7 @@ macro_rules! make_exemplar_chars_unicode_set_property { Ok(UnicodeSetData::from_data( provider.load( DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload @@ -70,7 +70,7 @@ macro_rules! make_exemplar_chars_unicode_set_property { DataProvider::<$data_marker>::load( &crate::provider::Baked, DataRequest { - locale, + id: DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload diff --git a/components/segmenter/src/complex/dictionary.rs b/components/segmenter/src/complex/dictionary.rs index 3d8f3eb7087..578ddc881c6 100644 --- a/components/segmenter/src/complex/dictionary.rs +++ b/components/segmenter/src/complex/dictionary.rs @@ -197,7 +197,9 @@ mod tests { fn cj_dictionary_test() { let response: DataResponse = crate::provider::Baked .load(DataRequest { - marker_attributes: &"cjdict".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes( + DataMarkerAttributes::from_str_or_panic("cjdict"), + ), ..Default::default() }) .unwrap(); diff --git a/components/segmenter/src/complex/lstm/mod.rs b/components/segmenter/src/complex/lstm/mod.rs index 2dc1b5c2a13..16a3b9d32cd 100644 --- a/components/segmenter/src/complex/lstm/mod.rs +++ b/components/segmenter/src/complex/lstm/mod.rs @@ -348,7 +348,11 @@ mod tests { fn segment_file_by_lstm() { let lstm: DataResponse = crate::provider::Baked .load(DataRequest { - marker_attributes: &"Thai_codepoints_exclusive_model4_heavy".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes( + DataMarkerAttributes::from_str_or_panic( + "Thai_codepoints_exclusive_model4_heavy", + ), + ), ..Default::default() }) .unwrap(); diff --git a/components/segmenter/src/complex/mod.rs b/components/segmenter/src/complex/mod.rs index 70ab4bf5554..8a0187f7634 100644 --- a/components/segmenter/src/complex/mod.rs +++ b/components/segmenter/src/complex/mod.rs @@ -40,6 +40,25 @@ pub(crate) struct ComplexPayloads { ja: Option>, } +#[cfg(feature = "lstm")] +const MY_LSTM: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("Burmese_codepoints_exclusive_model4_heavy"); +#[cfg(feature = "lstm")] +const KM_LSTM: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("Khmer_codepoints_exclusive_model4_heavy"); +#[cfg(feature = "lstm")] +const LO_LSTM: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("Lao_codepoints_exclusive_model4_heavy"); +#[cfg(feature = "lstm")] +const TH_LSTM: &DataMarkerAttributes = + DataMarkerAttributes::from_str_or_panic("Thai_codepoints_exclusive_model4_heavy"); + +const MY_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("burmesedict"); +const KM_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("khmerdict"); +const LO_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("laodict"); +const TH_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("thaidict"); +const CJ_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("cjdict"); + impl ComplexPayloads { fn select(&self, language: Language) -> Option { const ERR: DataError = DataError::custom("No segmentation model for language"); @@ -77,34 +96,22 @@ impl ComplexPayloads { grapheme: DataPayload::from_static_ref( crate::provider::Baked::SINGLETON_GRAPHEME_CLUSTER_BREAK_DATA_V1_MARKER, ), - my: try_load::( - &crate::provider::Baked, - "Burmese_codepoints_exclusive_model4_heavy", - ) - .unwrap() - .map(DataPayload::cast) - .map(Err), - km: try_load::( - &crate::provider::Baked, - "Khmer_codepoints_exclusive_model4_heavy", - ) - .unwrap() - .map(DataPayload::cast) - .map(Err), - lo: try_load::( - &crate::provider::Baked, - "Lao_codepoints_exclusive_model4_heavy", - ) - .unwrap() - .map(DataPayload::cast) - .map(Err), - th: try_load::( - &crate::provider::Baked, - "Thai_codepoints_exclusive_model4_heavy", - ) - .unwrap() - .map(DataPayload::cast) - .map(Err), + my: try_load::(&crate::provider::Baked, MY_LSTM) + .unwrap() + .map(DataPayload::cast) + .map(Err), + km: try_load::(&crate::provider::Baked, KM_LSTM) + .unwrap() + .map(DataPayload::cast) + .map(Err), + lo: try_load::(&crate::provider::Baked, LO_LSTM) + .unwrap() + .map(DataPayload::cast) + .map(Err), + th: try_load::(&crate::provider::Baked, TH_LSTM) + .unwrap() + .map(DataPayload::cast) + .map(Err), ja: None, } } @@ -118,30 +125,18 @@ impl ComplexPayloads { { Ok(Self { grapheme: provider.load(Default::default())?.payload, - my: try_load::( - provider, - "Burmese_codepoints_exclusive_model4_heavy", - )? - .map(DataPayload::cast) - .map(Err), - km: try_load::( - provider, - "Khmer_codepoints_exclusive_model4_heavy", - )? - .map(DataPayload::cast) - .map(Err), - lo: try_load::( - provider, - "Lao_codepoints_exclusive_model4_heavy", - )? - .map(DataPayload::cast) - .map(Err), - th: try_load::( - provider, - "Thai_codepoints_exclusive_model4_heavy", - )? - .map(DataPayload::cast) - .map(Err), + my: try_load::(provider, MY_LSTM)? + .map(DataPayload::cast) + .map(Err), + km: try_load::(provider, KM_LSTM)? + .map(DataPayload::cast) + .map(Err), + lo: try_load::(provider, LO_LSTM)? + .map(DataPayload::cast) + .map(Err), + th: try_load::(provider, TH_LSTM)? + .map(DataPayload::cast) + .map(Err), ja: None, }) } @@ -156,33 +151,33 @@ impl ComplexPayloads { ), my: try_load::( &crate::provider::Baked, - "burmesedict", + MY_DICT, ) .unwrap() .map(DataPayload::cast) .map(Ok), km: try_load::( &crate::provider::Baked, - "khmerdict", + KM_DICT, ) .unwrap() .map(DataPayload::cast) .map(Ok), lo: try_load::( &crate::provider::Baked, - "laodict", + LO_DICT, ) .unwrap() .map(DataPayload::cast) .map(Ok), th: try_load::( &crate::provider::Baked, - "thaidict", + TH_DICT, ) .unwrap() .map(DataPayload::cast) .map(Ok), - ja: try_load::(&crate::provider::Baked, "cjdict") + ja: try_load::(&crate::provider::Baked, CJ_DICT) .unwrap() .map(DataPayload::cast), } @@ -197,19 +192,19 @@ impl ComplexPayloads { { Ok(Self { grapheme: provider.load(Default::default())?.payload, - my: try_load::(provider, "burmesedict")? + my: try_load::(provider, MY_DICT)? .map(DataPayload::cast) .map(Ok), - km: try_load::(provider, "khmerdict")? + km: try_load::(provider, KM_DICT)? .map(DataPayload::cast) .map(Ok), - lo: try_load::(provider, "laodict")? + lo: try_load::(provider, LO_DICT)? .map(DataPayload::cast) .map(Ok), - th: try_load::(provider, "thaidict")? + th: try_load::(provider, TH_DICT)? .map(DataPayload::cast) .map(Ok), - ja: try_load::(provider, "cjdict")? + ja: try_load::(provider, CJ_DICT)? .map(DataPayload::cast), }) } @@ -223,35 +218,23 @@ impl ComplexPayloads { grapheme: DataPayload::from_static_ref( crate::provider::Baked::SINGLETON_GRAPHEME_CLUSTER_BREAK_DATA_V1_MARKER, ), - my: try_load::( - &crate::provider::Baked, - "Burmese_codepoints_exclusive_model4_heavy", - ) - .unwrap() - .map(DataPayload::cast) - .map(Err), - km: try_load::( - &crate::provider::Baked, - "Khmer_codepoints_exclusive_model4_heavy", - ) - .unwrap() - .map(DataPayload::cast) - .map(Err), - lo: try_load::( - &crate::provider::Baked, - "Lao_codepoints_exclusive_model4_heavy", - ) - .unwrap() - .map(DataPayload::cast) - .map(Err), - th: try_load::( - &crate::provider::Baked, - "Thai_codepoints_exclusive_model4_heavy", - ) - .unwrap() - .map(DataPayload::cast) - .map(Err), - ja: try_load::(&crate::provider::Baked, "cjdict") + my: try_load::(&crate::provider::Baked, MY_LSTM) + .unwrap() + .map(DataPayload::cast) + .map(Err), + km: try_load::(&crate::provider::Baked, KM_LSTM) + .unwrap() + .map(DataPayload::cast) + .map(Err), + lo: try_load::(&crate::provider::Baked, LO_LSTM) + .unwrap() + .map(DataPayload::cast) + .map(Err), + th: try_load::(&crate::provider::Baked, TH_LSTM) + .unwrap() + .map(DataPayload::cast) + .map(Err), + ja: try_load::(&crate::provider::Baked, CJ_DICT) .unwrap() .map(DataPayload::cast), } @@ -267,31 +250,19 @@ impl ComplexPayloads { { Ok(Self { grapheme: provider.load(Default::default())?.payload, - my: try_load::( - provider, - "Burmese_codepoints_exclusive_model4_heavy", - )? - .map(DataPayload::cast) - .map(Err), - km: try_load::( - provider, - "Khmer_codepoints_exclusive_model4_heavy", - )? - .map(DataPayload::cast) - .map(Err), - lo: try_load::( - provider, - "Lao_codepoints_exclusive_model4_heavy", - )? - .map(DataPayload::cast) - .map(Err), - th: try_load::( - provider, - "Thai_codepoints_exclusive_model4_heavy", - )? - .map(DataPayload::cast) - .map(Err), - ja: try_load::(provider, "cjdict")? + my: try_load::(provider, MY_LSTM)? + .map(DataPayload::cast) + .map(Err), + km: try_load::(provider, KM_LSTM)? + .map(DataPayload::cast) + .map(Err), + lo: try_load::(provider, LO_LSTM)? + .map(DataPayload::cast) + .map(Err), + th: try_load::(provider, TH_LSTM)? + .map(DataPayload::cast) + .map(Err), + ja: try_load::(provider, CJ_DICT)? .map(DataPayload::cast), }) } @@ -306,28 +277,28 @@ impl ComplexPayloads { ), my: try_load::( &crate::provider::Baked, - "burmesedict", + MY_DICT, ) .unwrap() .map(DataPayload::cast) .map(Ok), km: try_load::( &crate::provider::Baked, - "khmerdict", + KM_DICT, ) .unwrap() .map(DataPayload::cast) .map(Ok), lo: try_load::( &crate::provider::Baked, - "laodict", + LO_DICT, ) .unwrap() .map(DataPayload::cast) .map(Ok), th: try_load::( &crate::provider::Baked, - "thaidict", + TH_DICT, ) .unwrap() .map(DataPayload::cast) @@ -344,16 +315,16 @@ impl ComplexPayloads { { Ok(Self { grapheme: provider.load(Default::default())?.payload, - my: try_load::(provider, "burmesedict")? + my: try_load::(provider, MY_DICT)? .map(DataPayload::cast) .map(Ok), - km: try_load::(provider, "khmerdict")? + km: try_load::(provider, KM_DICT)? .map(DataPayload::cast) .map(Ok), - lo: try_load::(provider, "laodict")? + lo: try_load::(provider, LO_DICT)? .map(DataPayload::cast) .map(Ok), - th: try_load::(provider, "thaidict")? + th: try_load::(provider, TH_DICT)? .map(DataPayload::cast) .map(Ok), ja: None, @@ -363,16 +334,15 @@ impl ComplexPayloads { fn try_load + ?Sized>( provider: &P, - model: &'static str, + model: &'static DataMarkerAttributes, ) -> Result>, DataError> { match provider.load(DataRequest { - marker_attributes: &model.parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes(model), metadata: { let mut m = DataRequestMetadata::default(); m.silent = true; m }, - ..Default::default() }) { Ok(response) => Ok(Some(response.payload)), Err(DataError { diff --git a/provider/adapters/src/either.rs b/provider/adapters/src/either.rs index 29f66537e98..990a54891cf 100644 --- a/provider/adapters/src/either.rs +++ b/provider/adapters/src/either.rs @@ -68,14 +68,14 @@ impl< > IterableDynamicDataProvider for EitherProvider { #[inline] - fn iter_requests_for_marker( + fn iter_ids_for_marker( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { + ) -> Result, DataError> { use EitherProvider::*; match self { - A(p) => p.iter_requests_for_marker(marker), - B(p) => p.iter_requests_for_marker(marker), + A(p) => p.iter_ids_for_marker(marker), + B(p) => p.iter_ids_for_marker(marker), } } } @@ -85,13 +85,11 @@ impl, P1: IterableDataProvider> IterableDataProvider for EitherProvider { #[inline] - fn iter_requests( - &self, - ) -> Result, DataError> { + fn iter_ids(&self) -> Result, DataError> { use EitherProvider::*; match self { - A(p) => p.iter_requests(), - B(p) => p.iter_requests(), + A(p) => p.iter_ids(), + B(p) => p.iter_ids(), } } } diff --git a/provider/adapters/src/empty.rs b/provider/adapters/src/empty.rs index 1fba0c8f988..d354b331046 100644 --- a/provider/adapters/src/empty.rs +++ b/provider/adapters/src/empty.rs @@ -91,9 +91,7 @@ impl IterableDataProvider for EmptyDataProvider where M: DataMarker, { - fn iter_requests( - &self, - ) -> Result, DataError> { + fn iter_ids(&self) -> Result, DataError> { Ok(Default::default()) } } @@ -103,10 +101,10 @@ impl IterableDynamicDataProvider for EmptyDataProvider where M: DynamicDataMarker, { - fn iter_requests_for_marker( + fn iter_ids_for_marker( &self, _: DataMarkerInfo, - ) -> Result, DataError> { + ) -> Result, DataError> { Ok(Default::default()) } } diff --git a/provider/adapters/src/fallback/mod.rs b/provider/adapters/src/fallback/mod.rs index d99e72cdfaf..30630390959 100644 --- a/provider/adapters/src/fallback/mod.rs +++ b/provider/adapters/src/fallback/mod.rs @@ -26,13 +26,13 @@ use icu_provider::prelude::*; /// # } /// # } /// -/// let req = DataRequest { -/// locale: &langid!("ja-JP").into(), -/// ..Default::default() -/// }; +/// let id = DataIdentifierCow::from_locale(langid!("ja-JP").into()); /// /// // The provider does not have data for "ja-JP": -/// DataProvider::::load(&provider, req).expect_err("No fallback"); +/// DataProvider::::load(&provider, DataRequest { +/// id: id.as_borrowed(), +/// ..Default::default() +/// }).expect_err("No fallback"); /// /// // But if we wrap the provider in a fallback provider... /// let provider = LocaleFallbackProvider::try_new_unstable(provider) @@ -40,7 +40,10 @@ use icu_provider::prelude::*; /// /// // ...then we can load "ja-JP" based on "ja" data /// let response = -/// DataProvider::::load(&provider, req).expect("successful with vertical fallback"); +/// DataProvider::::load(&provider, DataRequest { +/// id: id.as_borrowed(), +/// ..Default::default() +/// }).expect("successful with vertical fallback"); /// /// assert_eq!( /// response.metadata.locale.unwrap(), @@ -130,13 +133,13 @@ impl

LocaleFallbackProvider

{ /// /// let provider = HelloWorldProvider; /// - /// let req = DataRequest { - /// locale: &langid!("de-CH").into(), - /// ..Default::default() - /// }; + /// let id = DataIdentifierCow::from_locale(langid!("de-CH").into()); /// /// // There is no "de-CH" data in the `HelloWorldProvider` - /// DataProvider::::load(&provider, req) + /// DataProvider::::load(&provider, DataRequest { + /// id: id.as_borrowed(), + /// ..Default::default() + /// }) /// .expect_err("No data for de-CH"); /// /// // `HelloWorldProvider` does not contain fallback data, @@ -149,7 +152,10 @@ impl

LocaleFallbackProvider

{ /// /// // Now we can load the "de-CH" data via fallback to "de". /// let german_hello_world: DataResponse = provider - /// .load(req) + /// .load(DataRequest { + /// id: id.as_borrowed(), + /// ..Default::default() + /// }) /// .expect("Loading should succeed"); /// /// assert_eq!("Hallo Welt", german_hello_world.payload.get().message); @@ -200,11 +206,14 @@ impl

LocaleFallbackProvider

{ let mut fallback_iterator = self .fallbacker .for_config(marker.fallback_config) - .fallback_for(base_req.locale.clone()); + .fallback_for(base_req.id.locale.clone()); let base_silent = core::mem::replace(&mut base_req.metadata.silent, true); loop { let result = f1(DataRequest { - locale: fallback_iterator.get(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + base_req.id.marker_attributes, + fallback_iterator.get(), + ), ..base_req }); if !result_is_err_missing_locale(&result) { diff --git a/provider/adapters/src/filter/impls.rs b/provider/adapters/src/filter/impls.rs index 31f19387813..7c954b7d81d 100644 --- a/provider/adapters/src/filter/impls.rs +++ b/provider/adapters/src/filter/impls.rs @@ -35,21 +35,26 @@ where /// .filter_by_langid(|langid| langid.language != language!("en")); /// /// // German requests should succeed: - /// let req_de = DataRequest { - /// locale: &langid!("de").into(), - /// ..Default::default() - /// }; + /// let de = DataIdentifierCow::from_locale(langid!("de").into()); /// let response: Result, _> = - /// provider.load(req_de); + /// provider.load(DataRequest { + /// id: de.as_borrowed(), + /// ..Default::default() + /// }); /// assert!(matches!(response, Ok(_))); /// /// // English requests should fail: - /// let req_en = DataRequest { - /// locale: &langid!("en-US").into(), + /// let en = DataIdentifierCow::from_locale(langid!("en-US").into()); + /// let response: Result, _> = + /// provider.load(DataRequest { + /// id: en.as_borrowed(), /// ..Default::default() - /// }; + /// }); /// let response: Result, _> = - /// provider.load(req_en); + /// provider.load(DataRequest { + /// id: en.as_borrowed(), + /// ..Default::default() + /// }); /// assert!(matches!( /// response, /// Err(DataError { @@ -59,14 +64,11 @@ where /// )); /// /// // English should not appear in the iterator result: - /// let available_langids = provider - /// .iter_requests() - /// .expect("Should successfully make an iterator of supported locales") - /// .into_iter() - /// .map(|(locale, _)| locale.get_langid()) - /// .collect::>(); - /// assert!(available_langids.contains(&langid!("de"))); - /// assert!(!available_langids.contains(&langid!("en"))); + /// let available_ids = provider + /// .iter_ids() + /// .expect("Should successfully make an iterator of supported locales"); + /// assert!(available_ids.contains(&DataIdentifierCow::from_locale(langid!("de").into()))); + /// assert!(!available_ids.contains(&DataIdentifierCow::from_locale(langid!("en").into()))); /// ``` pub fn filter_by_langid<'a>( self, @@ -82,7 +84,7 @@ where if !(old_predicate)(request) { return false; } - predicate(&request.locale.get_langid()) + predicate(&request.id.locale.get_langid()) }), filter_name: self.filter_name, } @@ -111,21 +113,22 @@ where /// .filter_by_langid_allowlist_strict(&allowlist); /// /// // German requests should succeed: - /// let req_de = DataRequest { - /// locale: &langid!("de").into(), - /// ..Default::default() - /// }; + /// let de = DataIdentifierCow::from_locale(langid!("de").into()); /// let response: Result, _> = - /// provider.load(req_de); + /// provider.load(DataRequest { + /// id: de.as_borrowed(), + /// ..Default::default() + /// }); /// assert!(matches!(response, Ok(_))); /// /// // English requests should fail: - /// let req_en = DataRequest { - /// locale: &langid!("en-US").into(), - /// ..Default::default() - /// }; + /// let en = DataIdentifierCow::from_locale(langid!("en-US").into()); + /// let response: Result, _> = - /// provider.load(req_en); + /// provider.load(DataRequest { + /// id: en.as_borrowed(), + /// ..Default::default() + /// }); /// assert!(matches!( /// response, /// Err(DataError { @@ -152,7 +155,8 @@ where if !(old_predicate)(request) { return false; } - request.locale.is_langid_und() || allowlist.contains(&request.locale.get_langid()) + request.id.locale.is_langid_und() + || allowlist.contains(&request.id.locale.get_langid()) }), filter_name: self.filter_name, } @@ -172,22 +176,21 @@ where /// .filterable("Demo require-langid filter") /// .require_langid(); /// - /// // Requests with a langid should succeed: - /// let req_with_langid = DataRequest { - /// locale: &langid!("de").into(), - /// ..Default::default() - /// }; + /// // Requests with a data id should succeed: + /// let id = DataIdentifierCow::from_locale(langid!("de").into()); /// let response: Result, _> = - /// provider.load(req_with_langid); + /// provider.load(DataRequest { + /// id: id.as_borrowed(), + /// ..Default::default() + /// }); /// assert!(matches!(response, Ok(_))); /// - /// // Requests without a langid should fail: - /// let req_no_langid = DataRequest { - /// locale: Default::default(), - /// ..Default::default() - /// }; + /// // Requests without a data should fail: /// let response: Result, _> = - /// provider.load(req_no_langid); + /// provider.load(DataRequest { + /// id: Default::default(), + /// ..Default::default() + /// }); /// assert!(matches!( /// response, /// Err(DataError { @@ -207,7 +210,7 @@ where if !(old_predicate)(request) { return false; } - !request.locale.is_langid_und() + !request.id.locale.is_langid_und() }), filter_name: self.filter_name, } diff --git a/provider/adapters/src/filter/mod.rs b/provider/adapters/src/filter/mod.rs index dd8ef046d5e..50ec5fa4219 100644 --- a/provider/adapters/src/filter/mod.rs +++ b/provider/adapters/src/filter/mod.rs @@ -40,7 +40,7 @@ use icu_provider::prelude::*; /// /// Data requests that are rejected by the filter will return a [`DataError`] with kind /// [`FilteredResource`](DataErrorKind::FilteredResource), and they will not be returned -/// by [`IterableDynamicDataProvider::iter_requests_for_marker`]. +/// by [`IterableDynamicDataProvider::iter_ids_for_marker`]. /// /// Although this struct can be created directly, the traits in this module provide helper /// functions for common filtering patterns. @@ -122,17 +122,16 @@ where F: Fn(DataRequest) -> bool, D: IterableDynamicDataProvider, { - fn iter_requests_for_marker( + fn iter_ids_for_marker( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { - self.inner.iter_requests_for_marker(marker).map(|set| { + ) -> Result, DataError> { + self.inner.iter_ids_for_marker(marker).map(|set| { // Use filter_map instead of filter to avoid cloning the locale set.into_iter() - .filter(|(locale, marker_attributes)| { + .filter(|id| { (self.predicate)(DataRequest { - locale, - marker_attributes, + id: id.as_borrowed(), ..Default::default() }) }) @@ -148,16 +147,13 @@ where F: Fn(DataRequest) -> bool, D: IterableDataProvider, { - fn iter_requests( - &self, - ) -> Result, DataError> { - self.inner.iter_requests().map(|vec| { + fn iter_ids(&self) -> Result, DataError> { + self.inner.iter_ids().map(|vec| { // Use filter_map instead of filter to avoid cloning the locale vec.into_iter() - .filter(|(locale, marker_attributes)| { + .filter(|id| { (self.predicate)(DataRequest { - locale, - marker_attributes, + id: id.as_borrowed(), ..Default::default() }) }) diff --git a/provider/adapters/src/fork/by_error.rs b/provider/adapters/src/fork/by_error.rs index 6d69f32152c..7e9ffb82092 100644 --- a/provider/adapters/src/fork/by_error.rs +++ b/provider/adapters/src/fork/by_error.rs @@ -106,17 +106,17 @@ where P1: IterableDynamicDataProvider, F: ForkByErrorPredicate, { - fn iter_requests_for_marker( + fn iter_ids_for_marker( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { - let result = self.0.iter_requests_for_marker(marker); + ) -> Result, DataError> { + let result = self.0.iter_ids_for_marker(marker); match result { Ok(ok) => return Ok(ok), Err(err) if !self.2.test(marker, None, err) => return Err(err), _ => (), }; - self.1.iter_requests_for_marker(marker) + self.1.iter_ids_for_marker(marker) } } @@ -236,13 +236,13 @@ where P: IterableDynamicDataProvider, F: ForkByErrorPredicate, { - fn iter_requests_for_marker( + fn iter_ids_for_marker( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { + ) -> Result, DataError> { let mut last_error = F::UNIT_ERROR.with_marker(marker); for provider in self.providers.iter() { - let result = provider.iter_requests_for_marker(marker); + let result = provider.iter_ids_for_marker(marker); match result { Ok(ok) => return Ok(ok), Err(err) if !self.predicate.test(marker, None, err) => return Err(err), diff --git a/provider/adapters/src/fork/mod.rs b/provider/adapters/src/fork/mod.rs index 04f9020506e..ba2d9a08065 100644 --- a/provider/adapters/src/fork/mod.rs +++ b/provider/adapters/src/fork/mod.rs @@ -84,7 +84,7 @@ use predicates::MissingDataMarkerPredicate; /// /// let german_hello_world: DataResponse = provider /// .load(DataRequest { -/// locale: &langid!("de").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); @@ -118,7 +118,7 @@ use predicates::MissingDataMarkerPredicate; /// // Chinese is the first provider, so this succeeds /// let chinese_hello_world = provider /// .load(DataRequest { -/// locale: &langid!("zh").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("zh").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); @@ -128,7 +128,7 @@ use predicates::MissingDataMarkerPredicate; /// // German is shadowed by Chinese, so this fails /// provider /// .load(DataRequest { -/// locale: &langid!("de").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), /// ..Default::default() /// }) /// .expect_err("Should stop at the first provider, even though the second has data"); @@ -187,7 +187,7 @@ impl ForkByMarkerProvider { /// // Chinese is the first provider, so this succeeds /// let chinese_hello_world = provider /// .load(DataRequest { -/// locale: &langid!("zh").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("zh").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); @@ -197,7 +197,7 @@ impl ForkByMarkerProvider { /// // German is shadowed by Chinese, so this fails /// provider /// .load(DataRequest { -/// locale: &langid!("de").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), /// ..Default::default() /// }) /// .expect_err("Should stop at the first provider, even though the second has data"); diff --git a/provider/adapters/src/fork/predicates.rs b/provider/adapters/src/fork/predicates.rs index 8495b6d2e84..efeb0647a44 100644 --- a/provider/adapters/src/fork/predicates.rs +++ b/provider/adapters/src/fork/predicates.rs @@ -71,7 +71,7 @@ impl ForkByErrorPredicate for MissingDataMarkerPredicate { /// struct SingleLocaleProvider(icu_locale::LanguageIdentifier); /// impl DataProvider for SingleLocaleProvider { /// fn load(&self, req: DataRequest) -> Result, DataError> { -/// if req.locale.get_langid() != self.0 { +/// if req.id.locale.get_langid() != self.0 { /// return Err(DataErrorKind::MissingLocale.with_req(HelloWorldV1Marker::INFO, req)); /// } /// HelloWorldProvider.load(req) @@ -92,7 +92,7 @@ impl ForkByErrorPredicate for MissingDataMarkerPredicate { /// /// let german_hello_world: DataResponse = provider /// .load(DataRequest { -/// locale: &langid!("de").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); @@ -101,7 +101,7 @@ impl ForkByErrorPredicate for MissingDataMarkerPredicate { /// /// let romanian_hello_world: DataResponse = provider /// .load(DataRequest { -/// locale: &langid!("ro").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("ro").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); @@ -113,7 +113,7 @@ impl ForkByErrorPredicate for MissingDataMarkerPredicate { /// DataProvider::::load( /// &provider, /// DataRequest { -/// locale: &langid!("en").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), /// ..Default::default() /// } /// ) diff --git a/provider/baked/src/binary_search.rs b/provider/baked/src/binary_search.rs index fe8929c83e6..bb503f2d095 100644 --- a/provider/baked/src/binary_search.rs +++ b/provider/baked/src/binary_search.rs @@ -11,15 +11,20 @@ use icu_provider::prelude::*; #[cfg(feature = "export")] pub fn bake( marker_bake: &TokenStream, - reqs_to_idents: Vec<((DataLocale, DataMarkerAttributes), proc_macro2::Ident)>, + reqs_to_idents: Vec<(DataIdentifierCow, proc_macro2::Ident)>, idents_to_bakes: Vec<(proc_macro2::Ident, TokenStream)>, ) -> TokenStream { - let mut reqs_to_idents = reqs_to_idents + let mut ids_to_idents = reqs_to_idents .into_iter() - .map(|((l, a), i)| ((a.to_string(), l.to_string()), quote!(#i))) + .map(|(id, ident)| { + ( + (id.marker_attributes.to_string(), id.locale.to_string()), + quote!(#ident), + ) + }) .collect::>(); - reqs_to_idents.sort_by(|(a, _), (b, _)| a.cmp(b)); + ids_to_idents.sort_by(|(a, _), (b, _)| a.cmp(b)); let idents_to_bakes = idents_to_bakes.into_iter().map(|(ident, bake)| { quote! { @@ -27,20 +32,20 @@ pub fn bake( } }); - let (ty, reqs_to_idents) = if reqs_to_idents.iter().all(|((a, _), _)| a.is_empty()) { + let (ty, reqs_to_idents) = if ids_to_idents.iter().all(|((a, _), _)| a.is_empty()) { // Only DataLocales ( quote! { icu_provider_baked::binary_search::Locale }, - reqs_to_idents + ids_to_idents .iter() .map(|((_, l), i)| quote!((#l, #i))) .collect::>(), ) - } else if reqs_to_idents.iter().all(|((_, l), _)| *l == "und") { + } else if ids_to_idents.iter().all(|((_, l), _)| *l == "und") { // Only marker attributes ( quote! { icu_provider_baked::binary_search::Attributes }, - reqs_to_idents + ids_to_idents .iter() .map(|((a, _), i)| quote!((#a, #i))) .collect(), @@ -48,7 +53,7 @@ pub fn bake( } else { ( quote! { icu_provider_baked::binary_search::AttributesAndLocale }, - reqs_to_idents + ids_to_idents .iter() .map(|((a, l), i)| quote!(((#a, #l), #i))) .collect(), @@ -67,27 +72,27 @@ pub fn bake( pub struct Data(pub &'static [(K::Type, &'static M::Yokeable)]); impl super::DataStore for Data { - fn get(&self, req: DataRequest) -> Option<&'static M::Yokeable> { + fn get(&self, id: DataIdentifierBorrowed) -> Option<&'static M::Yokeable> { self.0 - .binary_search_by(|&(k, _)| K::cmp(k, req)) + .binary_search_by(|&(k, _)| K::cmp(k, id)) .map(|i| unsafe { self.0.get_unchecked(i) }.1) .ok() } type IterReturn = core::iter::Map< core::slice::Iter<'static, (K::Type, &'static M::Yokeable)>, - fn(&'static (K::Type, &'static M::Yokeable)) -> (DataLocale, DataMarkerAttributes), + fn(&'static (K::Type, &'static M::Yokeable)) -> DataIdentifierCow<'static>, >; fn iter(&self) -> Self::IterReturn { - self.0.iter().map(|&(k, _)| K::to_req(k)) + self.0.iter().map(|&(k, _)| K::to_id(k)) } } pub trait BinarySearchKey: 'static { type Type: Ord + Copy + 'static; - fn cmp(k: Self::Type, req: DataRequest) -> core::cmp::Ordering; - fn to_req(k: Self::Type) -> (DataLocale, DataMarkerAttributes); + fn cmp(k: Self::Type, id: DataIdentifierBorrowed) -> core::cmp::Ordering; + fn to_id(k: Self::Type) -> DataIdentifierCow<'static>; } pub struct Locale; @@ -95,12 +100,12 @@ pub struct Locale; impl BinarySearchKey for Locale { type Type = &'static str; - fn cmp(locale: Self::Type, req: DataRequest) -> core::cmp::Ordering { - req.locale.strict_cmp(locale.as_bytes()).reverse() + fn cmp(locale: Self::Type, id: DataIdentifierBorrowed) -> core::cmp::Ordering { + id.locale.strict_cmp(locale.as_bytes()).reverse() } - fn to_req(locale: Self::Type) -> (DataLocale, DataMarkerAttributes) { - (locale.parse().unwrap(), Default::default()) + fn to_id(locale: Self::Type) -> DataIdentifierCow<'static> { + DataIdentifierCow::from_locale(locale.parse().unwrap()) } } @@ -109,12 +114,14 @@ pub struct Attributes; impl BinarySearchKey for Attributes { type Type = &'static str; - fn cmp(attributes: Self::Type, req: DataRequest) -> core::cmp::Ordering { - attributes.cmp(&**req.marker_attributes) + fn cmp(attributes: Self::Type, id: DataIdentifierBorrowed) -> core::cmp::Ordering { + attributes.cmp(id.marker_attributes) } - fn to_req(attributes: Self::Type) -> (DataLocale, DataMarkerAttributes) { - (Default::default(), attributes.parse().unwrap()) + fn to_id(attributes: Self::Type) -> DataIdentifierCow<'static> { + DataIdentifierCow::from_marker_attributes(DataMarkerAttributes::from_str_or_panic( + attributes, + )) } } @@ -123,13 +130,16 @@ pub struct AttributesAndLocale; impl BinarySearchKey for AttributesAndLocale { type Type = (&'static str, &'static str); - fn cmp((attributes, locale): Self::Type, req: DataRequest) -> core::cmp::Ordering { + fn cmp((attributes, locale): Self::Type, id: DataIdentifierBorrowed) -> core::cmp::Ordering { attributes - .cmp(&**req.marker_attributes) - .then_with(|| req.locale.strict_cmp(locale.as_bytes()).reverse()) + .cmp(id.marker_attributes) + .then_with(|| id.locale.strict_cmp(locale.as_bytes()).reverse()) } - fn to_req((attributes, locale): Self::Type) -> (DataLocale, DataMarkerAttributes) { - (locale.parse().unwrap(), attributes.parse().unwrap()) + fn to_id((attributes, locale): Self::Type) -> DataIdentifierCow<'static> { + DataIdentifierCow::from_borrowed_and_owned( + DataMarkerAttributes::from_str_or_panic(attributes), + locale.parse().unwrap(), + ) } } diff --git a/provider/baked/src/export.rs b/provider/baked/src/export.rs index 80d74661502..8732e2093f6 100644 --- a/provider/baked/src/export.rs +++ b/provider/baked/src/export.rs @@ -156,10 +156,7 @@ pub struct BakedExporter { use_internal_fallback: bool, // Temporary storage for put_payload: marker -> (bake -> {(locale, marker_attributes)}) data: Mutex< - HashMap< - DataMarkerInfo, - BTreeMap>, - >, + HashMap>>>, >, /// (marker, file name) pairs to wire up in mod.rs. This is populated by `flush` and consumed by `close`. impl_data: Mutex>, @@ -355,8 +352,7 @@ impl DataExporter for BakedExporter { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, payload: &DataPayload, ) -> Result<(), DataError> { let payload = payload.tokenize(&self.dependencies); @@ -368,7 +364,7 @@ impl DataExporter for BakedExporter { .or_default() .entry(payload) .or_default() - .insert((locale.clone(), marker_attributes.clone())); + .insert(id.into_owned()); Ok(()) } @@ -410,7 +406,7 @@ impl DataExporter for BakedExporter { &self, req: icu_provider::DataRequest, ) -> Result, icu_provider::DataError> { - if req.locale.is_empty() { + if req.id.locale.is_empty() { Ok(icu_provider::DataResponse { payload: icu_provider::DataPayload::from_static_ref(Self::#singleton_ident), metadata: Default::default(), @@ -423,7 +419,7 @@ impl DataExporter for BakedExporter { }, quote! { #maybe_msrv impl icu_provider::IterableDataProvider<#marker_bake> for $provider { - fn iter_requests(&self) -> Result, icu_provider::DataError> { + fn iter_ids(&self) -> Result>, icu_provider::DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -459,7 +455,7 @@ impl DataExporter for BakedExporter { quote! { #maybe_msrv impl icu_provider::IterableDataProvider<#marker_bake> for $provider { - fn iter_requests(&self) -> Result, icu_provider::DataError> { + fn iter_ids(&self) -> Result>, icu_provider::DataError> { Ok(Default::default()) } } @@ -468,15 +464,15 @@ impl DataExporter for BakedExporter { } else { let mut idents_to_bakes = Vec::new(); - let reqs_to_idents = deduplicated_values + let ids_to_idents = deduplicated_values .iter() - .flat_map(|(bake, reqs)| { + .flat_map(|(bake, ids)| { let bake = bake.parse::().unwrap(); - let mut idents = reqs + let mut idents = ids .iter() - .map(|(locale, marker_attributes)| { - format!("_{}_{}", marker_attributes as &str, locale) + .map(|id| { + format!("_{}_{}", id.marker_attributes.as_str(), id.locale) .chars() .map(|ch| { if ch == '-' { @@ -492,7 +488,7 @@ impl DataExporter for BakedExporter { let ident = proc_macro2::Ident::new(&idents[0], proc_macro2::Span::call_site()); idents_to_bakes.push((ident.clone(), bake)); - reqs.iter().map(move |la| (la.clone(), ident.clone())) + ids.iter().map(move |id| (id.clone(), ident.clone())) }) .collect(); @@ -511,16 +507,16 @@ impl DataExporter for BakedExporter { .parse::() .unwrap(); - let data = crate::binary_search::bake(&marker_bake, reqs_to_idents, idents_to_bakes); + let data = crate::binary_search::bake(&marker_bake, ids_to_idents, idents_to_bakes); let search = if !self.use_internal_fallback || deduplicated_values .iter() - .all(|(_, reqs)| reqs.iter().all(|(l, _)| l.is_und())) + .all(|(_, ids)| ids.iter().all(|id| id.locale.is_und())) { quote! { let metadata = Default::default(); - let Some(payload) = icu_provider_baked::DataStore::get(&Self::#data_ident, req) else { + let Some(payload) = icu_provider_baked::DataStore::get(&Self::#data_ident, req.id) else { return Err(icu_provider::DataErrorKind::MissingLocale.with_req(<#marker_bake as icu_provider::DataMarker>::INFO, req)) }; } @@ -529,15 +525,15 @@ impl DataExporter for BakedExporter { quote! { let mut metadata = icu_provider::DataResponseMetadata::default(); - let payload = if let Some(payload) = icu_provider_baked::DataStore::get(&Self::#data_ident, req) { + let payload = if let Some(payload) = icu_provider_baked::DataStore::get(&Self::#data_ident, req.id) { payload } else { const FALLBACKER: icu_locale::fallback::LocaleFallbackerWithConfig<'static> = icu_locale::fallback::LocaleFallbacker::new() .for_config(<#marker_bake as icu_provider::DataMarker>::INFO.fallback_config); - let mut fallback_iterator = FALLBACKER.fallback_for(req.locale.clone()); + let mut fallback_iterator = FALLBACKER.fallback_for(req.id.locale.clone()); loop { - if let Some(payload) = icu_provider_baked::DataStore::get(&Self::#data_ident, icu_provider::DataRequest { locale: fallback_iterator.get(), ..req }) { + if let Some(payload) = icu_provider_baked::DataStore::get(&Self::#data_ident, icu_provider::DataIdentifierBorrowed::for_marker_attributes_and_locale(req.id.marker_attributes, fallback_iterator.get())) { metadata.locale = Some(fallback_iterator.take()); break payload; } @@ -576,7 +572,7 @@ impl DataExporter for BakedExporter { quote! { #maybe_msrv impl icu_provider::IterableDataProvider<#marker_bake> for $provider { - fn iter_requests(&self) -> Result, icu_provider::DataError> { + fn iter_ids(&self) -> Result>, icu_provider::DataError> { Ok(icu_provider_baked::DataStore::iter(&Self::#data_ident).collect()) } } diff --git a/provider/baked/src/lib.rs b/provider/baked/src/lib.rs index bfc27a975e7..fcdede45e6e 100644 --- a/provider/baked/src/lib.rs +++ b/provider/baked/src/lib.rs @@ -6,6 +6,8 @@ #![cfg_attr(not(feature = "export"), no_std)] +extern crate alloc; + #[cfg(feature = "export")] pub mod export; @@ -14,8 +16,8 @@ pub use icu_provider::prelude::*; pub mod binary_search; pub trait DataStore { - fn get(&self, req: DataRequest) -> Option<&'static M::Yokeable>; + fn get(&self, req: DataIdentifierBorrowed) -> Option<&'static M::Yokeable>; - type IterReturn: Iterator; + type IterReturn: Iterator>; fn iter(&self) -> Self::IterReturn; } diff --git a/provider/baked/tests/data/hello_world_v1_marker.rs.data b/provider/baked/tests/data/hello_world_v1_marker.rs.data index c011785b243..f791e4be93c 100644 --- a/provider/baked/tests/data/hello_world_v1_marker.rs.data +++ b/provider/baked/tests/data/hello_world_v1_marker.rs.data @@ -46,7 +46,7 @@ macro_rules! __impl_hello_world_v1_marker { impl icu_provider::DataProvider for $provider { fn load(&self, req: icu_provider::DataRequest) -> Result, icu_provider::DataError> { let metadata = Default::default(); - let Some(payload) = icu_provider_baked::DataStore::get(&Self::DATA_HELLO_WORLD_V1_MARKER, req) else { return Err(icu_provider::DataErrorKind::MissingLocale.with_req(::INFO, req)) }; + let Some(payload) = icu_provider_baked::DataStore::get(&Self::DATA_HELLO_WORLD_V1_MARKER, req.id) else { return Err(icu_provider::DataErrorKind::MissingLocale.with_req(::INFO, req)) }; Ok(icu_provider::DataResponse { payload: icu_provider::DataPayload::from_static_ref(payload), metadata }) } } @@ -55,7 +55,7 @@ macro_rules! __impl_hello_world_v1_marker { __impl_hello_world_v1_marker!($provider); #[clippy::msrv = "1.70"] impl icu_provider::IterableDataProvider for $provider { - fn iter_requests(&self) -> Result, icu_provider::DataError> { + fn iter_ids(&self) -> Result>, icu_provider::DataError> { Ok(icu_provider_baked::DataStore::iter(&Self::DATA_HELLO_WORLD_V1_MARKER).collect()) } } diff --git a/provider/baked/tests/test-baked-source.rs b/provider/baked/tests/test-baked-source.rs index 1d903846c61..b791516c715 100644 --- a/provider/baked/tests/test-baked-source.rs +++ b/provider/baked/tests/test-baked-source.rs @@ -22,7 +22,7 @@ fn iter() { use icu_provider::hello_world::HelloWorldProvider; use icu_provider::prelude::*; - assert_eq!(Baked.iter_requests(), HelloWorldProvider.iter_requests(),) + assert_eq!(Baked.iter_ids(), HelloWorldProvider.iter_ids(),) } #[test] @@ -30,10 +30,9 @@ fn load() { use icu_provider::hello_world::HelloWorldProvider; use icu_provider::prelude::*; - for (locale, marker_attributes) in HelloWorldProvider.iter_requests().unwrap().iter() { + for id in HelloWorldProvider.iter_ids().unwrap().iter() { let req = DataRequest { - locale, - marker_attributes, + id: id.as_borrowed(), ..Default::default() }; let expected = HelloWorldProvider.load(req).unwrap().payload; diff --git a/provider/bikeshed/src/calendar/chinese_based.rs b/provider/bikeshed/src/calendar/chinese_based.rs index e4a1ae77a0d..7ede22aaf49 100644 --- a/provider/bikeshed/src/calendar/chinese_based.rs +++ b/provider/bikeshed/src/calendar/chinese_based.rs @@ -41,17 +41,13 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/calendar/islamic.rs b/provider/bikeshed/src/calendar/islamic.rs index 270f1e1939d..b105e52cba6 100644 --- a/provider/bikeshed/src/calendar/islamic.rs +++ b/provider/bikeshed/src/calendar/islamic.rs @@ -36,9 +36,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -58,9 +56,7 @@ impl DataProvider for crate::DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/calendar/japanese.rs b/provider/bikeshed/src/calendar/japanese.rs index d98603e3486..d038ffda1d5 100644 --- a/provider/bikeshed/src/calendar/japanese.rs +++ b/provider/bikeshed/src/calendar/japanese.rs @@ -179,17 +179,13 @@ fn era_to_code(original: &str, year: i32) -> Result { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/characters/mod.rs b/provider/bikeshed/src/characters/mod.rs index b2be9fba695..4f2019f916d 100644 --- a/provider/bikeshed/src/characters/mod.rs +++ b/provider/bikeshed/src/characters/mod.rs @@ -24,7 +24,7 @@ macro_rules! exemplar_chars_impls { impl DataProvider<$data_marker_name> for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::<$data_marker_name>(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let data: &cldr_serde::exemplar_chars::Resource = self .cldr()? @@ -48,14 +48,12 @@ macro_rules! exemplar_chars_impls { } impl IterableDataProviderCached<$data_marker_name> for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .misc() .list_langs()? - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } diff --git a/provider/bikeshed/src/cldr_serde/currencies/data.rs b/provider/bikeshed/src/cldr_serde/currencies/data.rs index 11dc4778649..518b724afba 100644 --- a/provider/bikeshed/src/cldr_serde/currencies/data.rs +++ b/provider/bikeshed/src/cldr_serde/currencies/data.rs @@ -9,7 +9,6 @@ use serde::Deserialize; use std::collections::BTreeMap; -use tinystr::UnvalidatedTinyAsciiStr; #[derive(PartialEq, Debug, Deserialize)] pub(crate) struct CurrencyPatterns { @@ -43,7 +42,7 @@ pub(crate) struct CurrencyPatterns { #[derive(PartialEq, Debug, Deserialize)] pub(crate) struct Numbers { - pub(crate) currencies: BTreeMap, CurrencyPatterns>, + pub(crate) currencies: BTreeMap, } #[derive(PartialEq, Debug, Deserialize)] diff --git a/provider/bikeshed/src/collator/mod.rs b/provider/bikeshed/src/collator/mod.rs index 1c4525b6d20..7713a11a4f4 100644 --- a/provider/bikeshed/src/collator/mod.rs +++ b/provider/bikeshed/src/collator/mod.rs @@ -106,9 +106,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -184,7 +182,7 @@ macro_rules! collation_provider { .read_and_parse_toml(&format!( "collation/{}/{}{}.toml", self.collation_han_database(), - locale_to_file_name(&req.locale), + locale_to_file_name(&req.id.locale), $suffix )) .map_err(|e| match e.kind { @@ -206,7 +204,7 @@ macro_rules! collation_provider { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .icuexport()? .list(&format!( @@ -221,7 +219,7 @@ macro_rules! collation_provider { }) }) .filter_map(|s| file_name_to_locale(&s)) - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } diff --git a/provider/bikeshed/src/currency/essentials.rs b/provider/bikeshed/src/currency/essentials.rs index ca0384c1896..94079079db9 100644 --- a/provider/bikeshed/src/currency/essentials.rs +++ b/provider/bikeshed/src/currency/essentials.rs @@ -10,6 +10,7 @@ use crate::IterableDataProviderCached; use std::borrow::Cow; use icu_pattern::DoublePlaceholderPattern; +use tinystr::TinyAsciiStr; use std::collections::BTreeMap; use std::collections::HashMap; @@ -81,7 +82,7 @@ impl DataProvider for DatagenProvider { req: DataRequest, ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let currencies_resource: &cldr_serde::currencies::data::Resource = self.cldr()? @@ -103,14 +104,12 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .numbers() .list_langs()? - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -152,7 +151,7 @@ fn extract_currency_essentials<'data>( let short_placeholder_value = currency_pattern.short.as_ref().map(|short_placeholder| { if let Some(&index) = placeholders_checker_map.get(short_placeholder.as_str()) { PlaceholderValue::Index(index) - } else if short_placeholder == iso.try_into_tinystr().unwrap().as_str() { + } else if short_placeholder == iso { PlaceholderValue::ISO } else { let index = placeholders.len() as u16; @@ -165,7 +164,7 @@ fn extract_currency_essentials<'data>( let narrow_placeholder_value = currency_pattern.narrow.as_ref().map(|narrow_placeholder| { if let Some(&index) = placeholders_checker_map.get(narrow_placeholder.as_str()) { PlaceholderValue::Index(index) - } else if narrow_placeholder == iso.try_into_tinystr().unwrap().as_str() { + } else if narrow_placeholder == iso { PlaceholderValue::ISO } else { let index = placeholders.len() as u16; @@ -191,7 +190,6 @@ fn extract_currency_essentials<'data>( } } - let iso_string = iso.try_into_tinystr().unwrap().to_string(); let determine_pattern_selection = |placeholder_index: Option| -> Result { if standard_alpha_next_to_number.is_empty() { @@ -201,7 +199,7 @@ fn extract_currency_essentials<'data>( Some(PlaceholderValue::Index(index)) => { placeholders.get(index as usize).unwrap() } - Some(PlaceholderValue::ISO) | None => iso_string.as_str(), + Some(PlaceholderValue::ISO) | None => iso.as_str(), }; currency_pattern_selection(provider, standard, placeholder_value) } @@ -219,20 +217,21 @@ fn extract_currency_essentials<'data>( narrow_placeholder_value, }; + let iso = TinyAsciiStr::try_from_str(iso).unwrap().to_unvalidated(); match (short_pattern_selection, narrow_pattern_selection) { (PatternSelection::Standard, PatternSelection::Standard) if short_placeholder_value.is_none() && narrow_placeholder_value.is_none() => { - currency_patterns_standard_none.insert(*iso, currency_patterns); + currency_patterns_standard_none.insert(iso, currency_patterns); } ( PatternSelection::StandardAlphaNextToNumber, PatternSelection::StandardAlphaNextToNumber, ) if short_placeholder_value.is_none() && narrow_placeholder_value.is_none() => { - currency_patterns_standard_next_to_num.insert(*iso, currency_patterns); + currency_patterns_standard_next_to_num.insert(iso, currency_patterns); } _ => { - currency_patterns_map.insert(*iso, currency_patterns); + currency_patterns_map.insert(iso, currency_patterns); } } } @@ -354,7 +353,7 @@ fn test_basic() { let en: DataResponse = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap(); @@ -391,7 +390,7 @@ fn test_basic() { let ar_eg: DataResponse = provider .load(DataRequest { - locale: &langid!("ar-EG").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ar-EG").into()), ..Default::default() }) .unwrap(); diff --git a/provider/bikeshed/src/currency/extended.rs b/provider/bikeshed/src/currency/extended.rs index 64a636dbdef..e935039bb85 100644 --- a/provider/bikeshed/src/currency/extended.rs +++ b/provider/bikeshed/src/currency/extended.rs @@ -10,7 +10,6 @@ use std::collections::BTreeMap; use std::collections::HashSet; use icu::experimental::dimension::provider::extended_currency::Count; -use tinystr::TinyAsciiStr; use icu::experimental::dimension::provider::extended_currency::*; use icu_provider::prelude::*; @@ -23,22 +22,18 @@ impl DataProvider for crate::DatagenProvider { ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let currencies_resource: &cldr_serde::currencies::data::Resource = self.cldr()? .numbers() .read_and_parse(&langid, "currencies.json")?; - let aux = req - .marker_attributes - .parse::>() - .map_err(|_| DataError::custom("failed to parse aux key into tinystr"))?; let currency = currencies_resource .main .value .numbers .currencies - .get(&aux.to_unvalidated()) + .get(req.id.marker_attributes.as_str()) .ok_or(DataError::custom("No currency associated with the aux key"))?; let mut placeholders: BTreeMap = BTreeMap::new(); @@ -80,9 +75,7 @@ impl DataProvider for crate::DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { // TODO: This is a temporary implementation until we have a better way to handle large number of json files. let currencies_to_support: HashSet<_> = ["USD", "CAD", "EUR", "GBP", "EGP"].into_iter().collect(); @@ -98,21 +91,16 @@ impl crate::IterableDataProviderCached for Datagen let currencies = ¤cies_resource.main.value.numbers.currencies; for key in currencies.keys() { - let key_string = key - .try_into_tinystr() - .map_err(|_| DataError::custom("failed to parse currency code into tinystr"))? - .parse::() - .map_err(|_| DataError::custom("failed to parse currency code into string"))?; - if !currencies_to_support.contains(key_string.as_str()) { + if !currencies_to_support.contains(key.as_str()) { continue; } - let key = key - .try_into_tinystr() - .map_err(|_| DataError::custom("failed to parse currency code into tinystr"))?; - - let attributes = DataMarkerAttributes::from_tinystr(key.resize()); - result.insert((DataLocale::from(&langid), attributes)); + if let Ok(attributes) = DataMarkerAttributes::try_from_string(key.clone()) { + result.insert(DataIdentifierCow::from_owned( + attributes, + DataLocale::from(langid.clone()), + )); + } } } @@ -127,8 +115,10 @@ fn test_basic() { let provider = DatagenProvider::new_testing(); let en: DataPayload = provider .load(DataRequest { - locale: &langid!("en").into(), - marker_attributes: &"USD".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("USD"), + &langid!("en").into(), + ), ..Default::default() }) .unwrap() @@ -144,8 +134,10 @@ fn test_basic() { let fr: DataPayload = provider .load(DataRequest { - locale: &langid!("fr").into(), - marker_attributes: &"USD".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("USD"), + &langid!("fr").into(), + ), ..Default::default() }) .unwrap() diff --git a/provider/bikeshed/src/datetime/mod.rs b/provider/bikeshed/src/datetime/mod.rs index 5c902a68a3e..ff736b493d2 100644 --- a/provider/bikeshed/src/datetime/mod.rs +++ b/provider/bikeshed/src/datetime/mod.rs @@ -204,10 +204,11 @@ macro_rules! impl_data_provider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::<$marker>(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let calendar = if DateSkeletonPatternsV1Marker::INFO == $marker::INFO { - req.locale + req.id + .locale .get_unicode_ext(&key!("ca")) .ok_or_else(|| DataErrorKind::NeedsLocale.into_error())? } else { @@ -225,16 +226,14 @@ macro_rules! impl_data_provider { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { let mut r = HashSet::new(); if DateSkeletonPatternsV1Marker::INFO == $marker::INFO { for (cal_value, cldr_cal) in supported_cals() { r.extend(self.cldr()?.dates(cldr_cal).list_langs()?.map(|lid| { let mut locale = DataLocale::from(lid); locale.set_unicode_ext(key!("ca"), cal_value.clone()); - (locale, Default::default()) + DataIdentifierCow::from_locale(locale) })); } } else { @@ -245,15 +244,15 @@ macro_rules! impl_data_provider { self.cldr()? .dates(cldr_cal) .list_langs()? - .map(|l| (DataLocale::from(l), Default::default())), + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))), ); } // TODO(#3212): Remove if $marker::INFO == TimeLengthsV1Marker::INFO { - r.retain(|(l, _)| { - l.get_langid() != icu::locale::langid!("byn") - && l.get_langid() != icu::locale::langid!("ssy") + r.retain(|id| { + id.locale.get_langid() != icu::locale::langid!("byn") + && id.locale.get_langid() != icu::locale::langid!("ssy") }); } @@ -402,7 +401,7 @@ mod test { let cs_dates: DataResponse = provider .load(DataRequest { - locale: &langid!("cs").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("cs").into()), ..Default::default() }) .expect("Failed to load payload"); @@ -416,7 +415,7 @@ mod test { let cs_dates: DataResponse = provider .load(DataRequest { - locale: &langid!("haw").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("haw").into()), ..Default::default() }) .expect("Failed to load payload"); @@ -436,7 +435,7 @@ mod test { let skeletons: DataResponse = provider .load(DataRequest { - locale: &"fil-u-ca-gregory".parse().unwrap(), + id: DataIdentifierBorrowed::for_locale(&"fil-u-ca-gregory".parse().unwrap()), ..Default::default() }) .expect("Failed to load payload"); @@ -477,7 +476,7 @@ mod test { let cs_dates: DataResponse = provider .load(DataRequest { - locale: &langid!("cs").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("cs").into()), ..Default::default() }) .unwrap(); @@ -514,7 +513,7 @@ mod test { let cs_dates: DataResponse = provider .load(DataRequest { - locale: &langid!("cs").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("cs").into()), ..Default::default() }) .unwrap(); diff --git a/provider/bikeshed/src/datetime/neo.rs b/provider/bikeshed/src/datetime/neo.rs index bd4287fd38c..51e202db47b 100644 --- a/provider/bikeshed/src/datetime/neo.rs +++ b/provider/bikeshed/src/datetime/neo.rs @@ -26,7 +26,7 @@ use zerovec::ule::UnvalidatedStr; /// Most keys don't have short symbols (except weekdays) /// /// We may further investigate and kick out standalone for some keys -const NORMAL_KEY_LENGTHS: &[TinyAsciiStr<8>] = &[ +const NORMAL_KEY_LENGTHS: &[&DataMarkerAttributes] = &[ marker_attrs::ABBR, marker_attrs::NARROW, marker_attrs::WIDE, @@ -36,7 +36,7 @@ const NORMAL_KEY_LENGTHS: &[TinyAsciiStr<8>] = &[ ]; /// Lengths for month data (NORMAL_KEY_LENGTHS + numeric) -const NUMERIC_MONTHS_KEY_LENGTHS: &[TinyAsciiStr<8>] = &[ +const NUMERIC_MONTHS_KEY_LENGTHS: &[&DataMarkerAttributes] = &[ marker_attrs::ABBR, marker_attrs::NARROW, marker_attrs::WIDE, @@ -47,11 +47,11 @@ const NUMERIC_MONTHS_KEY_LENGTHS: &[TinyAsciiStr<8>] = &[ ]; /// Lengths for year data (does not do standalone formatting) -const YEARS_KEY_LENGTHS: &[TinyAsciiStr<8>] = +const YEARS_KEY_LENGTHS: &[&DataMarkerAttributes] = &[marker_attrs::ABBR, marker_attrs::NARROW, marker_attrs::WIDE]; /// All possible non-numeric lengths -const FULL_KEY_LENGTHS: &[TinyAsciiStr<8>] = &[ +const FULL_KEY_LENGTHS: &[&DataMarkerAttributes] = &[ marker_attrs::ABBR, marker_attrs::NARROW, marker_attrs::WIDE, @@ -63,7 +63,7 @@ const FULL_KEY_LENGTHS: &[TinyAsciiStr<8>] = &[ ]; /// Lengths for normal patterns (not counting hour cycle stuff) -const NORMAL_PATTERN_KEY_LENGTHS: &[TinyAsciiStr<8>] = &[ +const NORMAL_PATTERN_KEY_LENGTHS: &[&DataMarkerAttributes] = &[ marker_attrs::PATTERN_FULL, marker_attrs::PATTERN_LONG, marker_attrs::PATTERN_MEDIUM, @@ -99,23 +99,16 @@ impl DatagenProvider { &self, req: DataRequest, calendar: &Value, - conversion: impl FnOnce( - &LanguageIdentifier, - &ca::Dates, - TinyAsciiStr<8>, - ) -> Result, + conversion: impl FnOnce(DataIdentifierBorrowed, &ca::Dates) -> Result, ) -> Result, DataError> where Self: IterableDataProviderCached, { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let data = self.load_calendar_dates(&langid, calendar)?; - let attr = req.marker_attributes.single().ok_or_else(|| { - DataError::custom("Key attributes for datetime names must be single subtag") - })?; - let data = conversion(&langid, data, attr)?; + let data = conversion(req.id, data)?; #[allow(clippy::redundant_closure_call)] Ok(DataResponse { @@ -140,11 +133,23 @@ impl DatagenProvider { where Self: IterableDataProviderCached, { - self.load_neo_key(req, &calendar, |langid, data, aux| { - let Some((context, length)) = marker_attrs::symbol_marker_attr_info(aux) else { - panic!("Found unexpected auxiliary subtag {}", aux) + self.load_neo_key(req, &calendar, |id, data| { + let Some((context, length)) = + marker_attrs::symbol_marker_attr_info(id.marker_attributes) + else { + panic!( + "Found unexpected marker attributes {}", + id.marker_attributes.as_str() + ) }; - conversion(self, langid, data, &calendar, context, length) + conversion( + self, + &id.locale.get_langid(), + data, + &calendar, + context, + length, + ) }) } @@ -161,19 +166,23 @@ impl DatagenProvider { where Self: IterableDataProviderCached, { - self.load_neo_key(req, &calendar, |_langid, data, aux| { - let Some((length, hc)) = marker_attrs::pattern_marker_attr_info(aux) else { - panic!("Found unexpected auxiliary subtag {}", aux) + self.load_neo_key(req, &calendar, |id, data| { + let Some((length, hc)) = marker_attrs::pattern_marker_attr_info(id.marker_attributes) + else { + panic!( + "Found unexpected marker attributes {}", + id.marker_attributes.as_str() + ) }; conversion(data, length, hc) }) } - fn iter_requests_neo( + fn iter_ids_neo( &self, calendar: Value, - keylengths: &'static [TinyAsciiStr<8>], - ) -> Result, DataError> { + keylengths: &'static [&DataMarkerAttributes], + ) -> Result>, DataError> { let cldr_cal = supported_cals() .get(&calendar) .ok_or_else(|| DataErrorKind::MissingLocale.into_error())?; @@ -183,9 +192,9 @@ impl DatagenProvider { .list_langs()? .flat_map(|lid| { keylengths.iter().map(move |&length| { - ( + DataIdentifierCow::from_borrowed_and_owned( + length, DataLocale::from(lid.clone()), - DataMarkerAttributes::from_tinystr(length), ) }) }) @@ -638,9 +647,9 @@ fn timepattern_convert( /// non-default one fn nondefault_subtag( time_pattern: &ca::LengthPattern, - subtag12: TinyAsciiStr<8>, - subtag24: TinyAsciiStr<8>, -) -> TinyAsciiStr<8> { + subtag12: &'static DataMarkerAttributes, + subtag24: &'static DataMarkerAttributes, +) -> &'static DataMarkerAttributes { let pattern = time_pattern.get_pattern(); let pattern = pattern @@ -673,9 +682,7 @@ impl DataProvider for DatagenProvider { // and we can use a union of the H12/H24 key lengths arrays, instead checking for preferred hc // in timepattern_convert impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { let calendar = value!("gregory"); let cldr_cal = supported_cals() @@ -719,9 +726,9 @@ impl IterableDataProviderCached for DatagenProvider { ), ]; keylengths.into_iter().map(move |length| { - ( + DataIdentifierCow::from_borrowed_and_owned( + length, DataLocale::from(lid.clone()), - DataMarkerAttributes::from_tinystr(length), ) }) }) @@ -738,10 +745,8 @@ macro_rules! impl_symbols_datagen { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - self.iter_requests_neo(value!($calendar), $lengths) + fn iter_ids_cached(&self) -> Result>, DataError> { + self.iter_ids_neo(value!($calendar), $lengths) } } }; @@ -756,10 +761,8 @@ macro_rules! impl_pattern_datagen { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - self.iter_requests_neo(value!($calendar), $lengths) + fn iter_ids_cached(&self) -> Result>, DataError> { + self.iter_ids_neo(value!($calendar), $lengths) } } }; diff --git a/provider/bikeshed/src/datetime/neo_skeleton.rs b/provider/bikeshed/src/datetime/neo_skeleton.rs index d5b60d2e196..3ded452a06c 100644 --- a/provider/bikeshed/src/datetime/neo_skeleton.rs +++ b/provider/bikeshed/src/datetime/neo_skeleton.rs @@ -17,7 +17,6 @@ use icu::datetime::DateTimeFormatterOptions; use icu::locale::extensions::unicode::{value, Value}; use icu::locale::LanguageIdentifier; use icu_provider::prelude::*; -use tinystr::TinyAsciiStr; use super::supported_cals; @@ -26,7 +25,7 @@ impl DatagenProvider { &self, req: DataRequest, calendar: Either<&Value, &str>, - from_id_str: impl Fn(TinyAsciiStr<8>) -> Option, + from_id_str: impl Fn(&DataMarkerAttributes) -> Option, to_components_bag: impl Fn(NeoSkeletonLength, &C) -> DateTimeFormatterOptions, ) -> Result, DataError> where @@ -34,14 +33,10 @@ impl DatagenProvider { Self: crate::IterableDataProviderCached, { self.check_req::(req)?; - let id_str = req - .marker_attributes - .single() - .expect("Skeleton data provider called without key attribute"); - let neo_components = - from_id_str(id_str).expect("Skeleton data provider called with unknown skeleton"); + let neo_components = from_id_str(req.id.marker_attributes) + .expect("Skeleton data provider called with unknown skeleton"); let packed_skeleton_data = self.make_packed_skeleton_data( - &req.locale.get_langid(), + &req.id.locale.get_langid(), calendar, neo_components, to_components_bag, @@ -130,7 +125,7 @@ impl DatagenProvider { fn neo_time_skeleton_supported_locales( &self, - ) -> Result, DataError> { + ) -> Result>, DataError> { Ok(self .cldr()? .dates("generic") @@ -146,10 +141,10 @@ impl DatagenProvider { }) .copied() .map(NeoTimeComponents::id_str) - .map(move |id| { - ( - DataLocale::from(&langid), - DataMarkerAttributes::from_tinystr(id), + .map(move |attrs| { + DataIdentifierCow::from_borrowed_and_owned( + attrs, + DataLocale::from(langid.clone()), ) }) }) @@ -159,7 +154,7 @@ impl DatagenProvider { fn neo_date_skeleton_supported_locales( &self, calendar: &Value, - ) -> Result, DataError> { + ) -> Result>, DataError> { let cldr_cal = supported_cals() .get(calendar) .ok_or_else(|| DataErrorKind::MissingLocale.into_error())?; @@ -177,10 +172,10 @@ impl DatagenProvider { }) .copied() .map(NeoDateComponents::id_str) - .map(move |id| { - ( - DataLocale::from(&langid), - DataMarkerAttributes::from_tinystr(id), + .map(move |attrs| { + DataIdentifierCow::from_borrowed_and_owned( + attrs, + DataLocale::from(langid.clone()), ) }) }) @@ -206,9 +201,7 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.neo_time_skeleton_supported_locales() } } @@ -230,9 +223,7 @@ macro_rules! impl_neo_skeleton_datagen { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.neo_date_skeleton_supported_locales(&value!($calendar)) } } diff --git a/provider/bikeshed/src/datetime/week_data.rs b/provider/bikeshed/src/datetime/week_data.rs index f3f5903cb65..19f6644cf73 100644 --- a/provider/bikeshed/src/datetime/week_data.rs +++ b/provider/bikeshed/src/datetime/week_data.rs @@ -16,9 +16,7 @@ use icu_provider::prelude::*; use std::collections::HashSet; impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { let week_data: &cldr_serde::week_data::Resource = self .cldr()? .core() @@ -34,7 +32,7 @@ impl IterableDataProviderCached for DatagenProvider { _ => None, }) .map(LanguageIdentifier::from) - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -43,6 +41,7 @@ impl DataProvider for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::(req)?; let territory = req + .id .locale .region() .map(|v| -> Result { Ok(Territory::Region(v)) }) @@ -96,7 +95,7 @@ fn basic_cldr_week_data() { let fr_week_data: DataResponse = provider .load(DataRequest { - locale: &langid!("und-FR").into(), + id: DataIdentifierCow::from_locale(langid!("und-FR").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -105,7 +104,7 @@ fn basic_cldr_week_data() { let iq_week_data: DataResponse = provider .load(DataRequest { - locale: &langid!("und-IQ").into(), + id: DataIdentifierCow::from_locale(langid!("und-IQ").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -121,7 +120,7 @@ fn basic_cldr_week_data() { let gg_week_data: DataResponse = provider .load(DataRequest { - locale: &langid!("und-GG").into(), + id: DataIdentifierCow::from_locale(langid!("und-GG").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -137,6 +136,7 @@ impl DataProvider for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::(req)?; let territory = req + .id .locale .region() .map(Territory::Region) @@ -197,10 +197,8 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - IterableDataProviderCached::::iter_requests_cached(self) + fn iter_ids_cached(&self) -> Result>, DataError> { + IterableDataProviderCached::::iter_ids_cached(self) } } @@ -223,7 +221,7 @@ fn test_basic_cldr_week_data_v2() { let fr_week_data: DataResponse = provider .load(DataRequest { - locale: &langid!("und-FR").into(), + id: DataIdentifierCow::from_locale(langid!("und-FR").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -236,7 +234,7 @@ fn test_basic_cldr_week_data_v2() { let iq_week_data: DataResponse = provider .load(DataRequest { - locale: &langid!("und-IQ").into(), + id: DataIdentifierCow::from_locale(langid!("und-IQ").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -253,7 +251,7 @@ fn test_basic_cldr_week_data_v2() { let gg_week_data: DataResponse = provider .load(DataRequest { - locale: &langid!("und-GG").into(), + id: DataIdentifierCow::from_locale(langid!("und-GG").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -270,7 +268,7 @@ fn test_basic_cldr_week_data_v2() { let ir_week_data: DataResponse = provider .load(DataRequest { - locale: &langid!("und-IR").into(), + id: DataIdentifierCow::from_locale(langid!("und-IR").into()).as_borrowed(), ..Default::default() }) .unwrap(); diff --git a/provider/bikeshed/src/decimal/compact.rs b/provider/bikeshed/src/decimal/compact.rs index 01f1a64bf50..2b25e735fc6 100644 --- a/provider/bikeshed/src/decimal/compact.rs +++ b/provider/bikeshed/src/decimal/compact.rs @@ -17,7 +17,7 @@ impl DataProvider for DatagenProvider { req: DataRequest, ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let resource: &cldr_serde::numbers::Resource = self .cldr()? @@ -26,7 +26,7 @@ impl DataProvider for DatagenProvider { let numbers = &resource.main.value.numbers; - let nsname = match req.locale.get_unicode_ext(&key!("nu")) { + let nsname = match req.id.locale.get_unicode_ext(&key!("nu")) { Some(v) => *v.get_subtag(0).expect("expecting subtag if key is present"), None => Subtag::from_tinystr_unvalidated(numbers.default_numbering_system), }; @@ -62,7 +62,7 @@ impl DataProvider for DatagenProvider { req: DataRequest, ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let resource: &cldr_serde::numbers::Resource = self .cldr()? @@ -71,7 +71,7 @@ impl DataProvider for DatagenProvider { let numbers = &resource.main.value.numbers; - let nsname = match req.locale.get_unicode_ext(&key!("nu")) { + let nsname = match req.id.locale.get_unicode_ext(&key!("nu")) { Some(v) => *v.get_subtag(0).expect("expecting subtag if key is present"), None => Subtag::from_tinystr_unvalidated(numbers.default_numbering_system), }; @@ -102,18 +102,14 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - self.iter_requests_for_numbers() + fn iter_ids_cached(&self) -> Result>, DataError> { + self.iter_ids_for_numbers() } } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - self.iter_requests_for_numbers() + fn iter_ids_cached(&self) -> Result>, DataError> { + self.iter_ids_for_numbers() } } @@ -133,7 +129,7 @@ mod tests { let fr_compact_long: DataPayload = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap() @@ -198,7 +194,7 @@ mod tests { let ja_compact_short: DataResponse = provider .load(DataRequest { - locale: &langid!("ja").into(), + id: DataIdentifierCow::from_locale(langid!("ja").into()).as_borrowed(), ..Default::default() }) .unwrap(); diff --git a/provider/bikeshed/src/decimal/mod.rs b/provider/bikeshed/src/decimal/mod.rs index b064bccaf0a..de1b29a7e38 100644 --- a/provider/bikeshed/src/decimal/mod.rs +++ b/provider/bikeshed/src/decimal/mod.rs @@ -77,9 +77,7 @@ impl DatagenProvider { .collect()) } - fn iter_requests_for_numbers( - &self, - ) -> Result, DataError> { + fn iter_ids_for_numbers(&self) -> Result>, DataError> { Ok(self .cldr()? .numbers() @@ -96,10 +94,11 @@ impl DatagenProvider { Value::try_from_str(&nsname) .expect("CLDR should have valid numbering system names"), ); - (data_locale, Default::default()) + data_locale }) - .chain([(last, Default::default())]) + .chain([last]) }) + .map(DataIdentifierCow::from_locale) .collect()) } } diff --git a/provider/bikeshed/src/decimal/symbols.rs b/provider/bikeshed/src/decimal/symbols.rs index eb11dafa54b..c74b09ae974 100644 --- a/provider/bikeshed/src/decimal/symbols.rs +++ b/provider/bikeshed/src/decimal/symbols.rs @@ -15,7 +15,7 @@ use std::convert::TryFrom; impl DataProvider for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let resource: &cldr_serde::numbers::Resource = self .cldr()? @@ -24,7 +24,7 @@ impl DataProvider for DatagenProvider { let numbers = &resource.main.value.numbers; - let nsname = match req.locale.get_unicode_ext(&key!("nu")) { + let nsname = match req.id.locale.get_unicode_ext(&key!("nu")) { Some(v) => *v.get_subtag(0).expect("expecting subtag if key is present"), None => Subtag::from_tinystr_unvalidated(numbers.default_numbering_system), }; @@ -46,10 +46,8 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - self.iter_requests_for_numbers() + fn iter_ids_cached(&self) -> Result>, DataError> { + self.iter_ids_for_numbers() } } @@ -102,7 +100,7 @@ fn test_basic() { let ar_decimal: DataResponse = provider .load(DataRequest { - locale: &langid!("ar-EG").into(), + id: DataIdentifierCow::from_locale(langid!("ar-EG").into()).as_borrowed(), ..Default::default() }) .unwrap(); diff --git a/provider/bikeshed/src/displaynames/language.rs b/provider/bikeshed/src/displaynames/language.rs index bf31a2f0deb..5d81fca50cc 100644 --- a/provider/bikeshed/src/displaynames/language.rs +++ b/provider/bikeshed/src/displaynames/language.rs @@ -18,7 +18,7 @@ impl DataProvider for DatagenProvider { req: DataRequest, ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let data: &cldr_serde::displaynames::language::Resource = self .cldr()? @@ -37,7 +37,7 @@ impl DataProvider for DatagenProvider { req: DataRequest, ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let data: &cldr_serde::displaynames::language::Resource = self .cldr()? @@ -52,9 +52,7 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .displaynames() @@ -67,15 +65,13 @@ impl IterableDataProviderCached for DatagenProvide .file_exists(langid, "languages.json") .unwrap_or_default() }) - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .displaynames() @@ -88,7 +84,7 @@ impl IterableDataProviderCached for DatagenProvider .file_exists(langid, "languages.json") .unwrap_or_default() }) - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -213,7 +209,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() @@ -234,7 +230,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() @@ -255,7 +251,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() @@ -276,7 +272,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() @@ -297,7 +293,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() diff --git a/provider/bikeshed/src/displaynames/region.rs b/provider/bikeshed/src/displaynames/region.rs index a14a2f4e8b5..de1fb4af1aa 100644 --- a/provider/bikeshed/src/displaynames/region.rs +++ b/provider/bikeshed/src/displaynames/region.rs @@ -17,7 +17,7 @@ impl DataProvider for DatagenProvider { req: DataRequest, ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let data: &cldr_serde::displaynames::region::Resource = self .cldr()? @@ -34,9 +34,7 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .displaynames() @@ -49,7 +47,7 @@ impl IterableDataProviderCached for DatagenProvider .file_exists(langid, "territories.json") .unwrap_or_default() }) - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -98,7 +96,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() @@ -119,7 +117,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() diff --git a/provider/bikeshed/src/displaynames/script.rs b/provider/bikeshed/src/displaynames/script.rs index 92b97fb1bc5..fb8e04c3244 100644 --- a/provider/bikeshed/src/displaynames/script.rs +++ b/provider/bikeshed/src/displaynames/script.rs @@ -17,7 +17,7 @@ impl DataProvider for DatagenProvider { req: DataRequest, ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let data: &cldr_serde::displaynames::script::Resource = self .cldr()? @@ -34,9 +34,7 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .displaynames() @@ -49,7 +47,7 @@ impl IterableDataProviderCached for DatagenProvider .file_exists(langid, "scripts.json") .unwrap_or_default() }) - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -106,7 +104,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() @@ -127,7 +125,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() diff --git a/provider/bikeshed/src/displaynames/variant.rs b/provider/bikeshed/src/displaynames/variant.rs index ffb6b2aa7dc..84f89978486 100644 --- a/provider/bikeshed/src/displaynames/variant.rs +++ b/provider/bikeshed/src/displaynames/variant.rs @@ -17,7 +17,7 @@ impl DataProvider for DatagenProvider { req: DataRequest, ) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let data: &cldr_serde::displaynames::variant::Resource = self .cldr()? @@ -34,9 +34,7 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .displaynames() @@ -49,7 +47,7 @@ impl IterableDataProviderCached for DatagenProvider .file_exists(langid, "variants.json") .unwrap_or_default() }) - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -93,7 +91,7 @@ mod tests { let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() }) .unwrap() diff --git a/provider/bikeshed/src/fallback/mod.rs b/provider/bikeshed/src/fallback/mod.rs index febbe8b34ab..9814ba00d39 100644 --- a/provider/bikeshed/src/fallback/mod.rs +++ b/provider/bikeshed/src/fallback/mod.rs @@ -52,17 +52,13 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/hello_world/mod.rs b/provider/bikeshed/src/hello_world/mod.rs index 376139548a3..ff8cdf39780 100644 --- a/provider/bikeshed/src/hello_world/mod.rs +++ b/provider/bikeshed/src/hello_world/mod.rs @@ -15,9 +15,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - HelloWorldProvider.iter_requests() + fn iter_ids_cached(&self) -> Result>, DataError> { + HelloWorldProvider.iter_ids() } } diff --git a/provider/bikeshed/src/lib.rs b/provider/bikeshed/src/lib.rs index 94e7c6797cf..1fb542e515e 100644 --- a/provider/bikeshed/src/lib.rs +++ b/provider/bikeshed/src/lib.rs @@ -21,7 +21,6 @@ use cldr_cache::CldrCache; use elsa::sync::FrozenMap; use icu_provider::prelude::*; use source::{AbstractFs, SerdeCache}; -use std::borrow::Cow; use std::collections::HashSet; use std::fmt::Debug; use std::path::PathBuf; @@ -87,14 +86,7 @@ pub struct DatagenProvider { requests_cache: Arc< FrozenMap< DataMarkerInfo, - Box< - OnceLock< - Result< - HashSet<(Cow<'static, DataLocale>, Cow<'static, DataMarkerAttributes>)>, - DataError, - >, - >, - >, + Box>, DataError>>>, >, >, } @@ -329,15 +321,12 @@ impl DatagenProvider { DatagenProvider: IterableDataProviderCached, { if ::INFO.is_singleton { - if !req.locale.is_empty() { + if !req.id.locale.is_empty() { Err(DataErrorKind::ExtraneousLocale) } else { Ok(()) } - } else if !self.populate_requests_cache()?.contains(&( - Cow::Borrowed(req.locale), - Cow::Borrowed(req.marker_attributes), - )) { + } else if !self.populate_requests_cache()?.contains(&req.id.as_cow()) { Err(DataErrorKind::MissingLocale) } else { Ok(()) @@ -354,7 +343,7 @@ fn test_missing_locale() { assert!(DataProvider::::load( &provider, DataRequest { - locale: &langid!("fi").into(), + id: DataIdentifierCow::from_locale(langid!("fi").into()).as_borrowed(), ..Default::default() } ) @@ -362,7 +351,7 @@ fn test_missing_locale() { assert!(DataProvider::::load( &provider, DataRequest { - locale: &langid!("arc").into(), + id: DataIdentifierCow::from_locale(langid!("arc").into()).as_borrowed(), ..Default::default() } ) @@ -370,29 +359,21 @@ fn test_missing_locale() { } trait IterableDataProviderCached: DataProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError>; + fn iter_ids_cached(&self) -> Result>, DataError>; } impl DatagenProvider { #[allow(clippy::type_complexity)] // not as complex as it appears fn populate_requests_cache( &self, - ) -> Result<&HashSet<(Cow<'static, DataLocale>, Cow<'static, DataMarkerAttributes>)>, DataError> + ) -> Result<&HashSet, DataError> where DatagenProvider: IterableDataProviderCached, { self.requests_cache .insert_with(M::INFO, || Box::new(OnceLock::new())) - // write lock gets dropped here, `iter_requests_cached` might be expensive - .get_or_init(|| { - self.iter_requests_cached().map(|m| { - m.into_iter() - .map(|(k, v)| (Cow::Owned(k), Cow::Owned(v))) - .collect() - }) - }) + // write lock gets dropped here, `iter_ids_cached` might be expensive + .get_or_init(|| self.iter_ids_cached()) .as_ref() .map_err(|&e| e) } @@ -402,14 +383,11 @@ impl IterableDataProvider for DatagenProvider where DatagenProvider: IterableDataProviderCached, { - fn iter_requests(&self) -> Result, DataError> { + fn iter_ids(&self) -> Result, DataError> { Ok(if ::INFO.is_singleton { [Default::default()].into_iter().collect() } else { - self.populate_requests_cache()? - .iter() - .map(|(k, v)| (k.clone().into_owned(), v.clone().into_owned())) - .collect() + self.populate_requests_cache()?.clone() }) } } diff --git a/provider/bikeshed/src/list/mod.rs b/provider/bikeshed/src/list/mod.rs index 47a56658a90..9a228ff6e6a 100644 --- a/provider/bikeshed/src/list/mod.rs +++ b/provider/bikeshed/src/list/mod.rs @@ -16,7 +16,7 @@ fn load>>( selff: &DatagenProvider, req: DataRequest, ) -> Result, DataError> { - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let resource: &cldr_serde::list_patterns::Resource = selff .cldr()? @@ -130,14 +130,12 @@ macro_rules! implement { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .misc() .list_langs()? - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } diff --git a/provider/bikeshed/src/locale_canonicalizer/aliases.rs b/provider/bikeshed/src/locale_canonicalizer/aliases.rs index dffceb4abbc..77ced23799c 100644 --- a/provider/bikeshed/src/locale_canonicalizer/aliases.rs +++ b/provider/bikeshed/src/locale_canonicalizer/aliases.rs @@ -29,9 +29,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -51,9 +49,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/locale_canonicalizer/directionality.rs b/provider/bikeshed/src/locale_canonicalizer/directionality.rs index 3198c4f5e4e..59d1515e0eb 100644 --- a/provider/bikeshed/src/locale_canonicalizer/directionality.rs +++ b/provider/bikeshed/src/locale_canonicalizer/directionality.rs @@ -23,9 +23,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/locale_canonicalizer/likely_subtags.rs b/provider/bikeshed/src/locale_canonicalizer/likely_subtags.rs index 56582480951..1b172447f73 100644 --- a/provider/bikeshed/src/locale_canonicalizer/likely_subtags.rs +++ b/provider/bikeshed/src/locale_canonicalizer/likely_subtags.rs @@ -24,9 +24,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -47,9 +45,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -69,9 +65,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -91,9 +85,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/normalizer/mod.rs b/provider/bikeshed/src/normalizer/mod.rs index ad9e6959954..7ebdbb2dd61 100644 --- a/provider/bikeshed/src/normalizer/mod.rs +++ b/provider/bikeshed/src/normalizer/mod.rs @@ -35,9 +35,7 @@ macro_rules! normalization_provider { } impl crate::IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/percent/mod.rs b/provider/bikeshed/src/percent/mod.rs index a8e3a6467f4..839c9de346d 100644 --- a/provider/bikeshed/src/percent/mod.rs +++ b/provider/bikeshed/src/percent/mod.rs @@ -16,7 +16,7 @@ use tinystr::tinystr; impl DataProvider for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let numbers_resource: &cldr_serde::numbers::Resource = self .cldr()? @@ -33,14 +33,12 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .numbers() .list_langs()? - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -116,7 +114,7 @@ fn test_basic() { let en: DataResponse = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierCow::from_locale(langid!("en").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -137,7 +135,7 @@ fn test_basic() { let fr: DataResponse = provider .load(DataRequest { - locale: &langid!("fr").into(), + id: DataIdentifierCow::from_locale(langid!("fr").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -158,7 +156,7 @@ fn test_basic() { let tr: DataResponse = provider .load(DataRequest { - locale: &langid!("tr").into(), + id: DataIdentifierCow::from_locale(langid!("tr").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -179,7 +177,7 @@ fn test_basic() { let ar_eg: DataResponse = provider .load(DataRequest { - locale: &langid!("ar-EG").into(), + id: DataIdentifierCow::from_locale(langid!("ar-EG").into()).as_borrowed(), ..Default::default() }) .unwrap(); diff --git a/provider/bikeshed/src/personnames/person_names_format_data_providers.rs b/provider/bikeshed/src/personnames/person_names_format_data_providers.rs index 2e1e5d8ba0d..8468c271f38 100644 --- a/provider/bikeshed/src/personnames/person_names_format_data_providers.rs +++ b/provider/bikeshed/src/personnames/person_names_format_data_providers.rs @@ -15,7 +15,7 @@ use crate::IterableDataProviderCached; impl DataProvider for crate::DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let data: &Resource = self .cldr()? @@ -33,9 +33,7 @@ impl DataProvider for crate::DatagenProvider { } impl IterableDataProviderCached for crate::DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .personnames() @@ -48,7 +46,7 @@ impl IterableDataProviderCached for crate::DatagenPro .file_exists(langid, "personNames.json") .unwrap_or_default() }) - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -164,7 +162,7 @@ mod tests { let data_payload: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() })? .payload; @@ -187,7 +185,7 @@ mod tests { let data_payload: DataPayload = provider .load(DataRequest { - locale: &langid!("en-001").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en-001").into()), ..Default::default() })? .payload; @@ -235,7 +233,7 @@ mod tests { let data_payload: DataPayload = provider .load(DataRequest { - locale: &langid!("es").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("es").into()), ..Default::default() })? .payload; diff --git a/provider/bikeshed/src/plurals/mod.rs b/provider/bikeshed/src/plurals/mod.rs index e933d1a9d9f..f31354a14c8 100644 --- a/provider/bikeshed/src/plurals/mod.rs +++ b/provider/bikeshed/src/plurals/mod.rs @@ -60,7 +60,7 @@ macro_rules! implement { payload: DataPayload::from_owned(PluralRulesV1::from( self.get_rules_for(<$marker>::INFO)? .0 - .get(&req.locale.get_langid()) + .get(&req.id.locale.get_langid()) .ok_or(DataErrorKind::MissingLocale.into_error())?, )), }) @@ -68,14 +68,12 @@ macro_rules! implement { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .get_rules_for(<$marker>::INFO)? .0 .keys() - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -105,7 +103,7 @@ impl From<&cldr_serde::plurals::LocalePluralRules> for PluralRulesV1<'static> { impl DataProvider for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::(req)?; - if req.locale.is_und() { + if req.id.locale.is_und() { Ok(DataResponse { metadata: Default::default(), payload: DataPayload::from_owned(PluralRangesV1 { @@ -118,7 +116,7 @@ impl DataProvider for DatagenProvider { payload: DataPayload::from_owned(PluralRangesV1::from( self.get_plural_ranges()? .0 - .get(&req.locale.get_langid()) + .get(&req.id.locale.get_langid()) .ok_or(DataErrorKind::MissingLocale.into_error())?, )), }) @@ -127,14 +125,12 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .get_plural_ranges()? .0 .keys() - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .chain([Default::default()]) // `und` is not included in the locales of plural ranges. .collect()) } @@ -184,7 +180,7 @@ fn test_basic() { // Spot-check locale 'cs' since it has some interesting entries let cs_rules: DataResponse = provider .load(DataRequest { - locale: &langid!("cs").into(), + id: DataIdentifierCow::from_locale(langid!("cs").into()).as_borrowed(), ..Default::default() }) .unwrap(); @@ -214,7 +210,7 @@ fn test_ranges() { // locale 'sl' seems to have a lot of interesting cases. let plural_ranges: DataResponse = provider .load(DataRequest { - locale: &langid!("sl").into(), + id: DataIdentifierCow::from_locale(langid!("sl").into()).as_borrowed(), ..Default::default() }) .unwrap(); diff --git a/provider/bikeshed/src/properties/bidi_data.rs b/provider/bikeshed/src/properties/bidi_data.rs index a778b5c4360..3004cc3944c 100644 --- a/provider/bikeshed/src/properties/bidi_data.rs +++ b/provider/bikeshed/src/properties/bidi_data.rs @@ -111,9 +111,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/properties/bin_cp_set.rs b/provider/bikeshed/src/properties/bin_cp_set.rs index efd9bd7cdd4..68123521cc9 100644 --- a/provider/bikeshed/src/properties/bin_cp_set.rs +++ b/provider/bikeshed/src/properties/bin_cp_set.rs @@ -53,7 +53,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_binary_prop_for_code_point_set($prop_name)?; Ok(HashSet::from_iter([Default::default()])) diff --git a/provider/bikeshed/src/properties/bin_uniset.rs b/provider/bikeshed/src/properties/bin_uniset.rs index cfbe4d015eb..a17cb0cbab1 100644 --- a/provider/bikeshed/src/properties/bin_uniset.rs +++ b/provider/bikeshed/src/properties/bin_uniset.rs @@ -60,7 +60,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_binary_prop_for_unicodeset($prop_name)?; Ok(HashSet::from_iter([Default::default()])) diff --git a/provider/bikeshed/src/properties/enum_codepointtrie.rs b/provider/bikeshed/src/properties/enum_codepointtrie.rs index 072b028e0c9..32adc542239 100644 --- a/provider/bikeshed/src/properties/enum_codepointtrie.rs +++ b/provider/bikeshed/src/properties/enum_codepointtrie.rs @@ -244,7 +244,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_enumerated_prop($prop_name)?; Ok(HashSet::from_iter([Default::default()])) } @@ -267,7 +267,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker_n2e> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_enumerated_prop($prop_name)?; Ok(HashSet::from_iter([Default::default()])) } @@ -283,7 +283,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker_e2sns> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_enumerated_prop($prop_name)?; Ok(HashSet::from_iter([Default::default()])) } @@ -298,7 +298,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker_e2lns> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_enumerated_prop($prop_name)?; Ok(HashSet::from_iter([Default::default()])) } @@ -315,7 +315,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker_e2snl> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_enumerated_prop($prop_name)?; Ok(HashSet::from_iter([Default::default()])) } @@ -330,7 +330,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker_e2lnl> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_enumerated_prop($prop_name)?; Ok(HashSet::from_iter([Default::default()])) } @@ -347,7 +347,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker_e2snl4> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_enumerated_prop($prop_name)?; Ok(HashSet::from_iter([Default::default()])) } @@ -363,7 +363,7 @@ macro_rules! expand { } impl crate::IterableDataProviderCached<$marker_e2lnl4> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_enumerated_prop($prop_name)?; Ok(HashSet::from_iter([Default::default()])) } @@ -404,9 +404,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { self.get_mask_prop("gcm")?; Ok(HashSet::from_iter([Default::default()])) } diff --git a/provider/bikeshed/src/properties/script.rs b/provider/bikeshed/src/properties/script.rs index 5c6453b05cc..bbea9a38875 100644 --- a/provider/bikeshed/src/properties/script.rs +++ b/provider/bikeshed/src/properties/script.rs @@ -64,9 +64,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/relativetime/mod.rs b/provider/bikeshed/src/relativetime/mod.rs index b142bfff328..2bd542f021b 100644 --- a/provider/bikeshed/src/relativetime/mod.rs +++ b/provider/bikeshed/src/relativetime/mod.rs @@ -86,7 +86,7 @@ macro_rules! make_data_provider { impl DataProvider<$marker> for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::<$marker>(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let resource: &cldr_serde::date_fields::Resource = self .cldr()? .dates("gregorian") @@ -109,12 +109,12 @@ macro_rules! make_data_provider { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(self .cldr()? .dates("gregorian") .list_langs()? - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -204,7 +204,7 @@ mod tests { let provider = DatagenProvider::new_testing(); let data: DataPayload = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap() @@ -223,7 +223,7 @@ mod tests { let provider = DatagenProvider::new_testing(); let data: DataPayload = provider .load(DataRequest { - locale: &langid!("ar").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ar").into()), ..Default::default() }) .unwrap() diff --git a/provider/bikeshed/src/segmenter/dictionary.rs b/provider/bikeshed/src/segmenter/dictionary.rs index 9f8824f4cac..7d29cce4826 100644 --- a/provider/bikeshed/src/segmenter/dictionary.rs +++ b/provider/bikeshed/src/segmenter/dictionary.rs @@ -25,7 +25,7 @@ impl DatagenProvider { ) -> Result, DataError> { let filename = format!( "segmenter/dictionary/{}.toml", - req.marker_attributes as &str + req.id.marker_attributes as &str ); let toml_data = self @@ -39,7 +39,7 @@ impl DatagenProvider { } macro_rules! implement { - ($marker:ident, $supported:expr) => { + ($marker:ident, [$($supported:expr),*]) => { impl DataProvider<$marker> for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::<$marker>(req)?; @@ -52,12 +52,14 @@ macro_rules! implement { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached( + fn iter_ids_cached( &self, - ) -> Result, DataError> { - Ok($supported - .into_iter() - .map(|m| (Default::default(), m.parse().unwrap())) + ) -> Result>, DataError> { + const SUPPORTED: &[&DataMarkerAttributes] = &[$(DataMarkerAttributes::from_str_or_panic($supported)),*]; + Ok(SUPPORTED + .iter() + .copied() + .map(DataIdentifierCow::from_marker_attributes) .collect()) } } diff --git a/provider/bikeshed/src/segmenter/lstm.rs b/provider/bikeshed/src/segmenter/lstm.rs index b3c23858f3b..e57c2e167ff 100644 --- a/provider/bikeshed/src/segmenter/lstm.rs +++ b/provider/bikeshed/src/segmenter/lstm.rs @@ -195,7 +195,7 @@ impl DataProvider for DatagenProvider { .segmenter_lstm()? .read_and_parse_json::(&format!( "{}/weights.json", - req.marker_attributes as &str + req.id.marker_attributes as &str )) .map_err(|_| DataErrorKind::MissingLocale.into_error())?; @@ -209,18 +209,17 @@ impl DataProvider for DatagenProvider { } impl IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - Ok([ - "Burmese_codepoints_exclusive_model4_heavy", - "Khmer_codepoints_exclusive_model4_heavy", - "Lao_codepoints_exclusive_model4_heavy", - "Thai_codepoints_exclusive_model4_heavy", - ] - .into_iter() - .map(|m| (Default::default(), m.parse().unwrap())) - .collect()) + fn iter_ids_cached(&self) -> Result>, DataError> { + const SUPPORTED: [&DataMarkerAttributes; 4] = [ + DataMarkerAttributes::from_str_or_panic("Burmese_codepoints_exclusive_model4_heavy"), + DataMarkerAttributes::from_str_or_panic("Khmer_codepoints_exclusive_model4_heavy"), + DataMarkerAttributes::from_str_or_panic("Lao_codepoints_exclusive_model4_heavy"), + DataMarkerAttributes::from_str_or_panic("Thai_codepoints_exclusive_model4_heavy"), + ]; + Ok(SUPPORTED + .into_iter() + .map(DataIdentifierCow::from_marker_attributes) + .collect()) } } diff --git a/provider/bikeshed/src/segmenter/mod.rs b/provider/bikeshed/src/segmenter/mod.rs index 89e2b254309..fefaa0094d3 100644 --- a/provider/bikeshed/src/segmenter/mod.rs +++ b/provider/bikeshed/src/segmenter/mod.rs @@ -603,7 +603,7 @@ macro_rules! implement { } impl crate::IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/tests/make_testdata.rs b/provider/bikeshed/src/tests/make_testdata.rs index b807ebdd088..c371ed2eaf3 100644 --- a/provider/bikeshed/src/tests/make_testdata.rs +++ b/provider/bikeshed/src/tests/make_testdata.rs @@ -99,8 +99,7 @@ impl DataExporter for ZeroCopyCheckExporter { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, payload_before: &DataPayload, ) -> Result<(), DataError> { use postcard::{ @@ -149,19 +148,19 @@ impl DataExporter for ZeroCopyCheckExporter { if payload_before != &payload_after { self.rountrip_errors.lock().expect("poison").insert(( marker, - locale.to_string() - + if marker_attributes.is_empty() { + id.locale.to_string() + + if id.marker_attributes.is_empty() { "" } else { "-x" } - + marker_attributes, + + id.marker_attributes.as_str(), )); } if deallocated != allocated { if !EXPECTED_VIOLATIONS.contains(&marker) { - eprintln!("Zerocopy violation {marker:?} {locale}: {allocated}B allocated, {deallocated}B deallocated"); + eprintln!("Zerocopy violation {marker:?} {id:?}: {allocated}B allocated, {deallocated}B deallocated"); } self.zero_copy_violations .lock() @@ -169,7 +168,7 @@ impl DataExporter for ZeroCopyCheckExporter { .insert(marker); } else if allocated > 0 { if !EXPECTED_TRANSIENT_VIOLATIONS.contains(&marker) { - eprintln!("Transient zerocopy violation {marker:?} {locale}: {allocated}B allocated/deallocated"); + eprintln!("Transient zerocopy violation {marker:?} {id:?}: {allocated}B allocated/deallocated"); } self.zero_copy_transient_violations .lock() diff --git a/provider/bikeshed/src/time_zones/mod.rs b/provider/bikeshed/src/time_zones/mod.rs index 8bcd4fdaba3..edf12eb1bb2 100644 --- a/provider/bikeshed/src/time_zones/mod.rs +++ b/provider/bikeshed/src/time_zones/mod.rs @@ -33,7 +33,7 @@ macro_rules! impl_data_provider { impl DataProvider<$marker> for DatagenProvider { fn load(&self, req: DataRequest) -> Result, DataError> { self.check_req::<$marker>(req)?; - let langid = req.locale.get_langid(); + let langid = req.id.locale.get_langid(); let resource: &cldr_serde::time_zones::time_zone_names::Resource = self .cldr()? @@ -72,7 +72,7 @@ macro_rules! impl_data_provider { } impl IterableDataProviderCached<$marker> for DatagenProvider { - fn iter_requests_cached(&self) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { if <$marker>::INFO == MetazonePeriodV1Marker::INFO { // MetazonePeriodV1 does not require localized time zone data Ok([Default::default()].into_iter().collect()) @@ -81,7 +81,7 @@ macro_rules! impl_data_provider { .cldr()? .dates("gregorian") .list_langs()? - .map(|l| (DataLocale::from(l), Default::default())) + .map(|l| DataIdentifierCow::from_locale(DataLocale::from(l))) .collect()) } } @@ -115,7 +115,7 @@ mod tests { let time_zone_formats: DataResponse = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap(); @@ -123,7 +123,7 @@ mod tests { let exemplar_cities: DataResponse = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap(); @@ -139,7 +139,7 @@ mod tests { let generic_names_long: DataResponse = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap(); @@ -164,7 +164,7 @@ mod tests { let specific_names_long: DataResponse = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap(); @@ -192,7 +192,7 @@ mod tests { let generic_names_short: DataResponse = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap(); @@ -217,7 +217,7 @@ mod tests { let specific_names_short: DataResponse = provider .load(DataRequest { - locale: &langid!("en").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), ..Default::default() }) .unwrap(); diff --git a/provider/bikeshed/src/time_zones/names.rs b/provider/bikeshed/src/time_zones/names.rs index d16ba8f29a5..f55e6e1d671 100644 --- a/provider/bikeshed/src/time_zones/names.rs +++ b/provider/bikeshed/src/time_zones/names.rs @@ -58,9 +58,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -114,9 +112,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -148,9 +144,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/transforms/mod.rs b/provider/bikeshed/src/transforms/mod.rs index d782b3372a1..7af36ec4419 100644 --- a/provider/bikeshed/src/transforms/mod.rs +++ b/provider/bikeshed/src/transforms/mod.rs @@ -129,15 +129,17 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { - self.cldr()? + fn iter_ids_cached(&self) -> Result>, DataError> { + Ok(self + .cldr()? .transforms()? .lock() .expect("poison") .as_provider_unstable(self, self)? - .iter_requests() + .iter_ids()? + .into_iter() + .map(|id| id.as_borrowed().into_owned()) + .collect()) } } @@ -151,7 +153,10 @@ mod tests { let _data: DataPayload = provider .load(DataRequest { - marker_attributes: &"de-t-de-d0-ascii".parse().unwrap(), + id: DataIdentifierCow::from_marker_attributes( + DataMarkerAttributes::from_str_or_panic("de-t-de-d0-ascii"), + ) + .as_borrowed(), ..Default::default() }) .unwrap() @@ -164,7 +169,10 @@ mod tests { let _data: DataPayload = provider .load(DataRequest { - marker_attributes: &"und-latn-t-s0-ascii".parse().unwrap(), + id: DataIdentifierCow::from_marker_attributes( + DataMarkerAttributes::from_str_or_panic("und-latn-t-s0-ascii"), + ) + .as_borrowed(), ..Default::default() }) .unwrap() diff --git a/provider/bikeshed/src/ucase/mod.rs b/provider/bikeshed/src/ucase/mod.rs index 4bf7c1ba79c..ac9bbcdf1a8 100644 --- a/provider/bikeshed/src/ucase/mod.rs +++ b/provider/bikeshed/src/ucase/mod.rs @@ -49,9 +49,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -78,9 +76,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } diff --git a/provider/bikeshed/src/units/data.rs b/provider/bikeshed/src/units/data.rs index d8c2adde690..a839c976f9c 100644 --- a/provider/bikeshed/src/units/data.rs +++ b/provider/bikeshed/src/units/data.rs @@ -22,8 +22,8 @@ impl DataProvider for DatagenProvider { self.check_req::(req)?; // Get langid and the unit. - let langid = req.locale.get_langid(); - let unit = match req.marker_attributes.parse::() { + let langid = req.id.locale.get_langid(); + let unit = match req.id.marker_attributes.parse::() { Ok(aux_keys) => aux_keys, Err(_) => return Err(DataError::custom("Failed to get aux keys")), }; @@ -92,18 +92,19 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { fn make_request_element( langid: &LanguageIdentifier, unit: &str, - ) -> Result<(DataLocale, DataMarkerAttributes), DataError> { + ) -> Result, DataError> { let data_locale = DataLocale::from(langid); - let attribute = unit.parse().map_err(|_| { + let attribute = DataMarkerAttributes::try_from_str(unit).map_err(|_| { DataError::custom("Failed to parse the attribute").with_debug_context(unit) })?; - Ok((data_locale, attribute)) + Ok(DataIdentifierCow::from_owned( + attribute.to_owned(), + data_locale, + )) } let mut data_locales = HashSet::new(); @@ -149,8 +150,10 @@ fn test_basic() { let us_locale: DataPayload = provider .load(DataRequest { - locale: &langid!("en").into(), - marker_attributes: &"meter".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("meter"), + &langid!("en").into(), + ), ..Default::default() }) .unwrap() @@ -166,8 +169,10 @@ fn test_basic() { let ar_eg_locale: DataPayload = provider .load(DataRequest { - locale: &langid!("ar-EG").into(), - marker_attributes: &"meter".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("meter"), + &langid!("ar-EG").into(), + ), ..Default::default() }) .unwrap() @@ -183,8 +188,10 @@ fn test_basic() { let fr_locale: DataPayload = provider .load(DataRequest { - locale: &langid!("fr").into(), - marker_attributes: &"meter".parse().unwrap(), + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::from_str_or_panic("meter"), + &langid!("fr").into(), + ), ..Default::default() }) .unwrap() diff --git a/provider/bikeshed/src/units/info.rs b/provider/bikeshed/src/units/info.rs index 740b191f26c..e7e9be56acf 100644 --- a/provider/bikeshed/src/units/info.rs +++ b/provider/bikeshed/src/units/info.rs @@ -95,9 +95,7 @@ impl DataProvider for DatagenProvider { } impl crate::IterableDataProviderCached for DatagenProvider { - fn iter_requests_cached( - &self, - ) -> Result, DataError> { + fn iter_ids_cached(&self) -> Result>, DataError> { Ok(HashSet::from_iter([Default::default()])) } } @@ -117,7 +115,7 @@ fn test_basic() { let und: DataResponse = provider .load(DataRequest { - locale: &langid!("und").into(), + id: DataIdentifierCow::from_locale(langid!("und").into()).as_borrowed(), ..Default::default() }) .unwrap(); diff --git a/provider/blob/benches/auxkey_bench.rs b/provider/blob/benches/auxkey_bench.rs index 4fc83774aa6..47e06deefd3 100644 --- a/provider/blob/benches/auxkey_bench.rs +++ b/provider/blob/benches/auxkey_bench.rs @@ -35,10 +35,7 @@ macro_rules! implement { } } impl IterableDataProvider<$marker> for Baked { - fn iter_requests( - &self, - ) -> Result, DataError> - { + fn iter_ids(&self) -> Result, DataError> { const LOCALES: &[LanguageIdentifier] = &[ langid!("af"), langid!("am"), @@ -217,16 +214,27 @@ macro_rules! implement { langid!("zh"), langid!("zu"), ]; - const ATTRS: &[&str] = &[ - "a1", "a1e", "de", "gy", "gym0", "gym0d", "gym0de", "m0d", "m0de", "y0w", - "ym0", "ym0d", "ym0de", + const ATTRS: &[&DataMarkerAttributes] = &[ + DataMarkerAttributes::from_str_or_panic("a1"), + DataMarkerAttributes::from_str_or_panic("a1e"), + DataMarkerAttributes::from_str_or_panic("de"), + DataMarkerAttributes::from_str_or_panic("gy"), + DataMarkerAttributes::from_str_or_panic("gym0"), + DataMarkerAttributes::from_str_or_panic("gym0d"), + DataMarkerAttributes::from_str_or_panic("gym0de"), + DataMarkerAttributes::from_str_or_panic("m0d"), + DataMarkerAttributes::from_str_or_panic("m0de"), + DataMarkerAttributes::from_str_or_panic("y0w"), + DataMarkerAttributes::from_str_or_panic("ym0"), + DataMarkerAttributes::from_str_or_panic("ym0d"), + DataMarkerAttributes::from_str_or_panic("ym0de"), ]; Ok(LOCALES .iter() .flat_map(|l| { - ATTRS - .iter() - .map(|a| (DataLocale::from(l.clone()), a.parse().unwrap())) + ATTRS.iter().map(|a| { + DataIdentifierCow::from_borrowed_and_owned(a, l.clone().into()) + }) }) .collect()) } @@ -244,18 +252,16 @@ where Baked: IterableDataProvider, ExportMarker: UpcastDataPayload, { - for (locale, marker_attributes) in &IterableDataProvider::::iter_requests(&Baked).unwrap() { + for id in &IterableDataProvider::::iter_ids(&Baked).unwrap() { let req = DataRequest { - locale, - marker_attributes, + id: id.as_borrowed(), ..Default::default() }; let res = DataProvider::::load(&Baked, req).unwrap(); exporter .put_payload( M::INFO, - locale, - marker_attributes, + id.as_borrowed(), &ExportMarker::upcast(res.payload), ) .unwrap(); @@ -311,10 +317,12 @@ fn auxkey_bench_for_version(c: &mut Criterion, blob: &[u8], version_id: &str) { for (locale_str, attr_str) in [("sr-Latn", "ym0d"), ("sr-ME", "ym0d")] { let locale = locale_str.parse::().unwrap(); - let attrs = attr_str.parse::().unwrap(); + let marker_attributes = DataMarkerAttributes::from_str_or_panic(attr_str); let req = DataRequest { - locale: &locale, - marker_attributes: &attrs, + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + marker_attributes, + &locale, + ), metadata: Default::default(), }; diff --git a/provider/blob/benches/blob_version_bench.rs b/provider/blob/benches/blob_version_bench.rs index a1c06a97420..ac0f2e4e40e 100644 --- a/provider/blob/benches/blob_version_bench.rs +++ b/provider/blob/benches/blob_version_bench.rs @@ -21,18 +21,17 @@ fn blob_version_bench(c: &mut Criterion) { }); let hello_world_provider = HelloWorldProvider; - let locales = hello_world_provider.iter_requests().unwrap(); + let locales = hello_world_provider.iter_ids().unwrap(); c.bench_function("provider/read/v1", |b| { let provider = BlobDataProvider::try_new_from_static_blob(black_box(BLOB_V1)).unwrap(); b.iter(|| { - for (locale, marker_attributes) in black_box(&locales).iter() { + for id in black_box(&locales).iter() { black_box(&provider) .load_data( HelloWorldV1Marker::INFO, DataRequest { - locale, - marker_attributes, + id: id.as_borrowed(), ..Default::default() }, ) @@ -43,13 +42,12 @@ fn blob_version_bench(c: &mut Criterion) { c.bench_function("provider/read/v2", |b| { let provider = BlobDataProvider::try_new_from_static_blob(black_box(BLOB_V2)).unwrap(); b.iter(|| { - for (locale, marker_attributes) in black_box(&locales).iter() { + for id in black_box(&locales).iter() { black_box(&provider) .load_data( HelloWorldV1Marker::INFO, DataRequest { - locale, - marker_attributes, + id: id.as_borrowed(), ..Default::default() }, ) diff --git a/provider/blob/src/blob_data_provider.rs b/provider/blob/src/blob_data_provider.rs index 738c22625c3..2d76cb3fcc7 100644 --- a/provider/blob/src/blob_data_provider.rs +++ b/provider/blob/src/blob_data_provider.rs @@ -203,7 +203,9 @@ mod test { provider.load_data( HelloSingletonV1Marker::INFO, DataRequest { - locale: &icu_locale_core::langid!("de").into(), + id: DataIdentifierBorrowed::for_locale( + &icu_locale_core::langid!("de").into() + ), ..Default::default() } ), diff --git a/provider/blob/src/blob_schema.rs b/provider/blob/src/blob_schema.rs index 0255854f2df..78c6a64ff0f 100644 --- a/provider/blob/src/blob_schema.rs +++ b/provider/blob/src/blob_schema.rs @@ -51,7 +51,7 @@ impl<'data> BlobSchema<'data> { pub fn list_requests( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { + ) -> Result, DataError> { match self { BlobSchema::V001(s) => s.list_requests(marker), BlobSchema::V002(s) => s.list_requests(marker), @@ -100,7 +100,7 @@ impl<'data> BlobSchemaV1<'data> { .get0(&marker.path.hashed()) .ok_or(DataErrorKind::MissingDataMarker) .and_then(|cursor| { - if marker.is_singleton && !req.locale.is_empty() { + if marker.is_singleton && !req.id.locale.is_empty() { return Err(DataErrorKind::ExtraneousLocale); } cursor @@ -119,7 +119,7 @@ impl<'data> BlobSchemaV1<'data> { Ok(()) } } - Comparator(req.locale, req.marker_attributes) + Comparator(req.id.locale, req.id.marker_attributes) .writeable_cmp_bytes(&k.0) .reverse() }) @@ -135,7 +135,7 @@ impl<'data> BlobSchemaV1<'data> { pub fn list_requests( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { + ) -> Result, DataError> { Ok(self .markers .get0(&marker.path.hashed()) @@ -144,9 +144,12 @@ impl<'data> BlobSchemaV1<'data> { .filter_map(|(s, _)| std::str::from_utf8(&s.0).ok()) .filter_map(|s| { if let Some((locale, attrs)) = s.split_once(REQUEST_SEPARATOR) { - Some((locale.parse().ok()?, attrs.parse().ok()?)) + Some(DataIdentifierCow::from_owned( + DataMarkerAttributes::try_from_str(attrs).ok()?.to_owned(), + locale.parse().ok()?, + )) } else { - Some((s.parse().ok()?, Default::default())) + Some(DataIdentifierCow::from_locale(s.parse().ok()?)) } }) .collect()) @@ -221,7 +224,7 @@ impl<'data, LocaleVecFormat: VarZeroVecFormat> BlobSchemaV2<'data, LocaleVecForm .binary_search(&marker.path.hashed()) .ok() .ok_or_else(|| DataErrorKind::MissingDataMarker.with_req(marker, req))?; - if marker.is_singleton && !req.locale.is_empty() { + if marker.is_singleton && !req.id.locale.is_empty() { return Err(DataErrorKind::ExtraneousLocale.with_req(marker, req)); } let zerotrie = self @@ -229,10 +232,11 @@ impl<'data, LocaleVecFormat: VarZeroVecFormat> BlobSchemaV2<'data, LocaleVecForm .get(marker_index) .ok_or_else(|| DataError::custom("Invalid blob bytes").with_req(marker, req))?; let mut cursor = ZeroTrieSimpleAscii::from_store(zerotrie).into_cursor(); - let _infallible_ascii = req.locale.write_to(&mut cursor); - if !req.marker_attributes.is_empty() { + let _infallible_ascii = req.id.locale.write_to(&mut cursor); + if !req.id.marker_attributes.is_empty() { let _infallible_ascii = cursor.write_char(REQUEST_SEPARATOR); - req.marker_attributes + req.id + .marker_attributes .write_to(&mut cursor) .map_err(|_| DataErrorKind::MissingLocale.with_req(marker, req))?; } @@ -250,7 +254,7 @@ impl<'data, LocaleVecFormat: VarZeroVecFormat> BlobSchemaV2<'data, LocaleVecForm pub fn list_requests( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { + ) -> Result, DataError> { let marker_index = self .markers .binary_search(&marker.path.hashed()) @@ -264,9 +268,12 @@ impl<'data, LocaleVecFormat: VarZeroVecFormat> BlobSchemaV2<'data, LocaleVecForm .iter() .filter_map(|(s, _)| { if let Some((locale, attrs)) = s.split_once(REQUEST_SEPARATOR) { - Some((locale.parse().ok()?, attrs.parse().ok()?)) + Some(DataIdentifierCow::from_owned( + DataMarkerAttributes::try_from_str(attrs).ok()?.to_owned(), + locale.parse().ok()?, + )) } else { - Some((s.parse().ok()?, Default::default())) + Some(DataIdentifierCow::from_locale(s.parse().ok()?)) } }) .collect()) diff --git a/provider/blob/src/export/blob_exporter.rs b/provider/blob/src/export/blob_exporter.rs index 7e8fddd30f3..23f80314af5 100644 --- a/provider/blob/src/export/blob_exporter.rs +++ b/provider/blob/src/export/blob_exporter.rs @@ -85,8 +85,7 @@ impl DataExporter for BlobExporter<'_> { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, payload: &DataPayload, ) -> Result<(), DataError> { let mut serializer = postcard::Serializer { @@ -109,10 +108,10 @@ impl DataExporter for BlobExporter<'_> { .entry(marker.path.hashed()) .or_default() .entry({ - let mut key = locale.to_string(); - if !marker_attributes.is_empty() { + let mut key = id.locale.to_string(); + if !id.marker_attributes.is_empty() { key.push(crate::blob_schema::REQUEST_SEPARATOR); - key.push_str(marker_attributes); + key.push_str(id.marker_attributes); } key.into_bytes() }) diff --git a/provider/blob/src/export/iter.rs b/provider/blob/src/export/iter.rs index 29002d9f3d7..ba68ba04c9b 100644 --- a/provider/blob/src/export/iter.rs +++ b/provider/blob/src/export/iter.rs @@ -8,10 +8,10 @@ use crate::BlobDataProvider; use icu_provider::prelude::*; impl IterableDynamicDataProvider for BlobDataProvider { - fn iter_requests_for_marker( + fn iter_ids_for_marker( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { + ) -> Result, DataError> { self.data.get().list_requests(marker) } } diff --git a/provider/blob/tests/test_versions.rs b/provider/blob/tests/test_versions.rs index f881e075dcb..31e154d45df 100644 --- a/provider/blob/tests/test_versions.rs +++ b/provider/blob/tests/test_versions.rs @@ -19,18 +19,16 @@ fn run_driver(mut exporter: BlobExporter, provider: &impl IterableDataProvider, { - for (locale, marker_attributes) in &provider.iter_requests().unwrap() { + for id in &provider.iter_ids().unwrap() { let req = DataRequest { - locale, - marker_attributes, + id: id.as_borrowed(), ..Default::default() }; let res = DataProvider::::load(provider, req).unwrap(); exporter .put_payload( HelloWorldV1Marker::INFO, - locale, - marker_attributes, + id.as_borrowed(), &ExportMarker::upcast(res.payload), ) .unwrap(); @@ -41,24 +39,22 @@ where fn check_hello_world(blob_provider: impl DataProvider) { let hello_world_provider = HelloWorldProvider; - for (locale, marker_attributes) in hello_world_provider.iter_requests().unwrap() { + for id in hello_world_provider.iter_ids().unwrap() { let blob_result = blob_provider .load(DataRequest { - locale: &locale, - marker_attributes: &marker_attributes, + id: id.as_borrowed(), ..Default::default() }) .unwrap() .payload; let expected_result = hello_world_provider .load(DataRequest { - locale: &locale, - marker_attributes: &marker_attributes, + id: id.as_borrowed(), ..Default::default() }) .unwrap() .payload; - assert_eq!(blob_result, expected_result, "{locale:?}"); + assert_eq!(blob_result, expected_result, "{:?}", id); } } @@ -127,7 +123,7 @@ fn test_v2_bigger() { let blob_result = DataProvider::::load( &blob_provider, DataRequest { - locale: &loc.parse().expect("locale must parse"), + id: DataIdentifierBorrowed::for_locale(&loc.parse().expect("locale must parse")), ..Default::default() }, ) @@ -144,7 +140,7 @@ impl DataProvider for ManyLocalesProvider { Ok(DataResponse { metadata: Default::default(), payload: DataPayload::from_owned(HelloWorldV1 { - message: format!("Hello {}!", req.locale).into(), + message: format!("Hello {}!", req.id.locale).into(), }), }) } @@ -153,7 +149,7 @@ impl DataProvider for ManyLocalesProvider { const LOWERCASE: core::ops::RangeInclusive = b'a'..=b'z'; impl IterableDataProvider for ManyLocalesProvider { - fn iter_requests(&self) -> Result, DataError> { + fn iter_ids(&self) -> Result, DataError> { let mut r = HashSet::new(); let mut bytes = [ b'a', b'a', b'a', b'-', b'L', b'a', b't', b'n', b'-', b'0', b'0', b'1', @@ -166,7 +162,7 @@ impl IterableDataProvider for ManyLocalesProvider { bytes[2] = i2; let locale = LanguageIdentifier::try_from_utf8(&bytes).expect("locale must parse"); - r.insert((locale.into(), Default::default())); + r.insert(DataIdentifierCow::from_locale(locale.into())); } } } diff --git a/provider/core/src/any.rs b/provider/core/src/any.rs index 44c7bc39821..09da780a523 100644 --- a/provider/core/src/any.rs +++ b/provider/core/src/any.rs @@ -293,18 +293,17 @@ where /// use icu_provider::hello_world::*; /// use icu_provider::prelude::*; /// use std::borrow::Cow; +/// use icu_locale_core::langid; /// /// let any_provider = HelloWorldProvider.as_any_provider(); /// -/// let req = DataRequest { -/// locale: &icu_locale_core::langid!("de").into(), -/// ..Default::default() -/// }; -/// /// // Downcasting manually /// assert_eq!( /// any_provider -/// .load_any(HelloWorldV1Marker::INFO, req) +/// .load_any(HelloWorldV1Marker::INFO, DataRequest { +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), +/// ..Default::default() +/// }) /// .expect("load should succeed") /// .downcast::() /// .expect("types should match") @@ -321,7 +320,10 @@ where /// /// assert_eq!( /// downcasting_provider -/// .load(req) +/// .load(DataRequest { +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), +/// ..Default::default() +/// }) /// .expect("load should succeed") /// .payload /// .get(), diff --git a/provider/core/src/buf.rs b/provider/core/src/buf.rs index 6b103d55238..669ed83aba2 100644 --- a/provider/core/src/buf.rs +++ b/provider/core/src/buf.rs @@ -49,16 +49,14 @@ impl DynamicDataMarker for BufferMarker { /// /// let buffer_provider = HelloWorldProvider.into_json_provider(); /// -/// let req = DataRequest { -/// locale: &langid!("de").into(), -/// ..Default::default() -/// }; -/// /// // Deserializing manually /// assert_eq!( /// serde_json::from_slice::( /// buffer_provider -/// .load_data(HelloWorldV1Marker::INFO, req) +/// .load_data(HelloWorldV1Marker::INFO, DataRequest { +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), +/// ..Default::default() +/// }) /// .expect("load should succeed") /// .payload /// .get() @@ -75,7 +73,10 @@ impl DynamicDataMarker for BufferMarker { /// /// assert_eq!( /// deserializing_provider -/// .load(req) +/// .load(DataRequest { +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), +/// ..Default::default() +/// }) /// .expect("load should succeed") /// .payload /// .get(), diff --git a/provider/core/src/data_provider.rs b/provider/core/src/data_provider.rs index 4e8a6af68a8..aa844688aee 100644 --- a/provider/core/src/data_provider.rs +++ b/provider/core/src/data_provider.rs @@ -53,15 +53,13 @@ where } #[cfg(feature = "std")] -/// A [`DataProvider`] that can iterate over all supported [`DataLocale`] for a certain marker. +/// A [`DataProvider`] that can iterate over all supported [`DataIdentifierCow`]s. /// -/// The provider is not allowed to return `Ok` for requests that were not returned by `iter_requests`, +/// The provider is not allowed to return `Ok` for requests that were not returned by `iter_ids`, /// and must not fail with a [`DataErrorKind::MissingLocale`] for requests that were returned. pub trait IterableDataProvider: DataProvider { - /// Returns a list of [`DataLocale`]. - fn iter_requests( - &self, - ) -> Result, DataError>; + /// Returns a set of [`DataIdentifierCow`]. + fn iter_ids(&self) -> Result, DataError>; } #[cfg(target_has_atomic = "ptr")] @@ -159,16 +157,16 @@ where } #[cfg(feature = "std")] -/// A [`DynamicDataProvider`] that can iterate over all supported [`DataLocale`] for a certain marker. +/// A [`DynamicDataProvider`] that can iterate over all supported [`DataIdentifierCow`]s for a certain marker. /// -/// The provider is not allowed to return `Ok` for requests that were not returned by `iter_requests`, +/// The provider is not allowed to return `Ok` for requests that were not returned by `iter_ids`, /// and must not fail with a [`DataErrorKind::MissingLocale`] for requests that were returned. pub trait IterableDynamicDataProvider: DynamicDataProvider { - /// Given a [`DataMarkerInfo`], returns a list of [`DataLocale`]. - fn iter_requests_for_marker( + /// Given a [`DataMarkerInfo`], returns a set of [`DataIdentifierCow`]. + fn iter_ids_for_marker( &self, marker: DataMarkerInfo, - ) -> Result, DataError>; + ) -> Result, DataError>; } #[cfg(feature = "std")] @@ -177,11 +175,11 @@ where M: DynamicDataMarker, P: IterableDynamicDataProvider + ?Sized, { - fn iter_requests_for_marker( + fn iter_ids_for_marker( &self, marker: DataMarkerInfo, - ) -> Result, DataError> { - (**self).iter_requests_for_marker(marker) + ) -> Result, DataError> { + (**self).iter_ids_for_marker(marker) } } diff --git a/provider/core/src/dynutil.rs b/provider/core/src/dynutil.rs index 3b16b69f1f1..827cb8dbb77 100644 --- a/provider/core/src/dynutil.rs +++ b/provider/core/src/dynutil.rs @@ -96,6 +96,7 @@ pub use __impl_casting_upcast as impl_casting_upcast; /// use icu_provider::prelude::*; /// use icu_provider::hello_world::*; /// use icu_provider::marker::NeverMarker; +/// use icu_locale_core::langid; /// # /// # // Duplicating HelloWorldProvider because the real one already implements DynamicDataProvider /// # struct HelloWorldProvider; @@ -111,13 +112,11 @@ pub use __impl_casting_upcast as impl_casting_upcast; /// // Implement DynamicDataProvider on HelloWorldProvider: DataProvider /// icu_provider::dynutil::impl_dynamic_data_provider!(HelloWorldProvider, [HelloWorldV1Marker,], AnyMarker); /// -/// let req = DataRequest { -/// locale: &icu_locale_core::langid!("de").into(), -/// ..Default::default() -/// }; -/// /// // Successful because the marker matches: -/// HelloWorldProvider.load_data(HelloWorldV1Marker::INFO, req).unwrap(); +/// HelloWorldProvider.load_data(HelloWorldV1Marker::INFO, DataRequest { +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), +/// ..Default::default() +/// }).unwrap(); /// /// # struct DummyMarker; /// # impl DynamicDataMarker for DummyMarker { @@ -128,7 +127,10 @@ pub use __impl_casting_upcast as impl_casting_upcast; /// # } /// // MissingDataMarker error as the marker does not match: /// assert_eq!( -/// HelloWorldProvider.load_data(DummyMarker::INFO, req).unwrap_err().kind, +/// HelloWorldProvider.load_data(DummyMarker::INFO, DataRequest { +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), +/// ..Default::default() +/// }).unwrap_err().kind, /// DataErrorKind::MissingDataMarker, /// ); /// ``` @@ -142,6 +144,7 @@ pub use __impl_casting_upcast as impl_casting_upcast; /// use icu_provider::prelude::*; /// use icu_provider::hello_world::*; /// use icu_provider::any::*; +/// use icu_locale_core::langid; /// # /// # struct HelloWorldProvider; /// # impl DynamicDataProvider for HelloWorldProvider { @@ -159,13 +162,11 @@ pub use __impl_casting_upcast as impl_casting_upcast; /// _ => HelloWorldV1Marker, /// }, AnyMarker); /// -/// let req = DataRequest { -/// locale: &icu_locale_core::langid!("de").into(), -/// ..Default::default() -/// }; -/// /// // Successful because the marker matches: -/// HelloWorldProvider.as_any_provider().load_any(HelloWorldV1Marker::INFO, req).unwrap(); +/// HelloWorldProvider.as_any_provider().load_any(HelloWorldV1Marker::INFO, DataRequest { +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), +/// ..Default::default() +/// }).unwrap(); /// /// // Because of the wildcard, any marker actually works: /// struct DummyMarker; @@ -175,7 +176,10 @@ pub use __impl_casting_upcast as impl_casting_upcast; /// impl DataMarker for DummyMarker { /// const INFO: DataMarkerInfo = DataMarkerInfo::from_path(icu_provider::marker::data_marker_path!("dummy@1")); /// } -/// HelloWorldProvider.as_any_provider().load_any(DummyMarker::INFO, req).unwrap(); +/// HelloWorldProvider.as_any_provider().load_any(DummyMarker::INFO, DataRequest { +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), +/// ..Default::default() +/// }).unwrap(); /// ``` /// /// [`DynamicDataProvider`]: crate::DynamicDataProvider @@ -276,12 +280,12 @@ pub use __impl_dynamic_data_provider as impl_dynamic_data_provider; macro_rules! __impl_iterable_dynamic_data_provider { ($provider:ty, [ $($(#[$cfg:meta])? $struct_m:ty),+, ], $dyn_m:path) => { impl $crate::IterableDynamicDataProvider<$dyn_m> for $provider { - fn iter_requests_for_marker(&self, marker: $crate::DataMarkerInfo) -> Result, $crate::DataError> { + fn iter_ids_for_marker(&self, marker: $crate::DataMarkerInfo) -> Result, $crate::DataError> { match marker.path.hashed() { $( $(#[$cfg])? h if h == <$struct_m as $crate::DataMarker>::INFO.path.hashed() => { - $crate::IterableDataProvider::<$struct_m>::iter_requests(self) + $crate::IterableDataProvider::<$struct_m>::iter_ids(self) } )+, _ => Err($crate::DataErrorKind::MissingDataMarker.with_marker(marker)) diff --git a/provider/core/src/error.rs b/provider/core/src/error.rs index 6a5ba803623..db759e01eca 100644 --- a/provider/core/src/error.rs +++ b/provider/core/src/error.rs @@ -211,7 +211,7 @@ impl DataError { } // Don't write out a log for MissingDataMarker since there is no context to add if !self.silent && self.kind != DataErrorKind::MissingDataMarker { - log::warn!("{self} (marker: {marker:?}, request: {req})"); + log::warn!("{self} (marker: {marker:?}, request: {})", req.id); } self.with_marker(marker) } diff --git a/provider/core/src/export/mod.rs b/provider/core/src/export/mod.rs index 2657c1e9ca9..32fbcc6ded4 100644 --- a/provider/core/src/export/mod.rs +++ b/provider/core/src/export/mod.rs @@ -22,8 +22,7 @@ pub trait DataExporter: Sync { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, payload: &DataPayload, ) -> Result<(), DataError>; @@ -34,7 +33,7 @@ pub trait DataExporter: Sync { marker: DataMarkerInfo, payload: &DataPayload, ) -> Result<(), DataError> { - self.put_payload(marker, Default::default(), Default::default(), payload)?; + self.put_payload(marker, Default::default(), payload)?; self.flush(marker) } @@ -58,11 +57,10 @@ impl DataExporter for Box { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, payload: &DataPayload, ) -> Result<(), DataError> { - (**self).put_payload(marker, locale, marker_attributes, payload) + (**self).put_payload(marker, id, payload) } fn flush_singleton( @@ -164,13 +162,12 @@ impl DataExporter for MultiExporter { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, payload: &DataPayload, ) -> Result<(), DataError> { self.0 .iter() - .try_for_each(|e| e.put_payload(marker, locale, marker_attributes, payload)) + .try_for_each(|e| e.put_payload(marker, id, payload)) } fn flush_singleton( diff --git a/provider/core/src/hello_world.rs b/provider/core/src/hello_world.rs index 7f3786182c5..96fd9292121 100644 --- a/provider/core/src/hello_world.rs +++ b/provider/core/src/hello_world.rs @@ -66,7 +66,7 @@ impl DataMarker for HelloWorldV1Marker { /// let german_hello_world: DataResponse = /// HelloWorldProvider /// .load(DataRequest { -/// locale: &langid!("de").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); @@ -79,12 +79,12 @@ impl DataMarker for HelloWorldV1Marker { /// ``` /// use icu_provider::hello_world::*; /// use icu_provider::prelude::*; +/// use icu_locale_core::langid; /// /// let reverse_hello_world: DataResponse = /// HelloWorldProvider /// .load(DataRequest { -/// locale: &"en".parse().unwrap(), -/// marker_attributes: &"reverse".parse().unwrap(), +/// id: DataIdentifierBorrowed::for_marker_attributes_and_locale(DataMarkerAttributes::from_str_or_panic("reverse"), &langid!("en").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); @@ -146,7 +146,8 @@ impl DataProvider for HelloWorldProvider { let data = Self::DATA .iter() .find(|(l, a, _)| { - req.locale.strict_cmp(l.as_bytes()).is_eq() && **a == **req.marker_attributes + req.id.locale.strict_cmp(l.as_bytes()).is_eq() + && *a == req.id.marker_attributes.as_str() }) .map(|(_, _, v)| v) .ok_or_else(|| DataErrorKind::MissingLocale.with_req(HelloWorldV1Marker::INFO, req))?; @@ -187,7 +188,7 @@ icu_provider::dynutil::impl_dynamic_data_provider!( /// let german_hello_world = HelloWorldProvider /// .into_json_provider() /// .load_data(HelloWorldV1Marker::INFO, DataRequest { -/// locale: &langid!("de").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); @@ -223,13 +224,16 @@ impl DynamicDataProvider for HelloWorldJsonProvider { #[cfg(feature = "std")] impl IterableDataProvider for HelloWorldProvider { - fn iter_requests( - &self, - ) -> Result, DataError> { + fn iter_ids(&self) -> Result, DataError> { #[allow(clippy::unwrap_used)] // hello-world Ok(Self::DATA .iter() - .map(|(l, a, _)| (l.parse().unwrap(), a.parse().unwrap())) + .map(|(l, a, _)| { + DataIdentifierCow::from_borrowed_and_owned( + DataMarkerAttributes::from_str_or_panic(a), + l.parse().unwrap(), + ) + }) .collect()) } } @@ -293,7 +297,7 @@ impl HelloWorldFormatter { { let data = provider .load(DataRequest { - locale, + id: crate::request::DataIdentifierBorrowed::for_locale(locale), ..Default::default() })? .payload; @@ -338,35 +342,41 @@ fn test_iter() { use std::collections::HashSet; assert_eq!( - HelloWorldProvider.iter_requests().unwrap(), + HelloWorldProvider.iter_ids().unwrap(), HashSet::from_iter([ - (locale!("bn").into(), Default::default()), - (locale!("cs").into(), Default::default()), - (locale!("de").into(), Default::default()), - (locale!("de-AT").into(), Default::default()), - (locale!("el").into(), Default::default()), - (locale!("en").into(), Default::default()), - (locale!("en-001").into(), Default::default()), - (locale!("en-002").into(), Default::default()), - (locale!("en-019").into(), Default::default()), - (locale!("en-142").into(), Default::default()), - (locale!("en-GB").into(), Default::default()), - (locale!("en-GB-u-sd-gbeng").into(), Default::default()), - (locale!("en").into(), "reverse".parse().unwrap()), - (locale!("eo").into(), Default::default()), - (locale!("fa").into(), Default::default()), - (locale!("fi").into(), Default::default()), - (locale!("is").into(), Default::default()), - (locale!("ja").into(), Default::default()), - (locale!("ja").into(), "reverse".parse().unwrap()), - (locale!("la").into(), Default::default()), - (locale!("pt").into(), Default::default()), - (locale!("ro").into(), Default::default()), - (locale!("ru").into(), Default::default()), - (locale!("sr").into(), Default::default()), - (locale!("sr-Latn").into(), Default::default()), - (locale!("vi").into(), Default::default()), - (locale!("zh").into(), Default::default()), + DataIdentifierCow::from_locale(locale!("bn").into()), + DataIdentifierCow::from_locale(locale!("cs").into()), + DataIdentifierCow::from_locale(locale!("de").into()), + DataIdentifierCow::from_locale(locale!("de-AT").into()), + DataIdentifierCow::from_locale(locale!("el").into()), + DataIdentifierCow::from_locale(locale!("en").into()), + DataIdentifierCow::from_locale(locale!("en-001").into()), + DataIdentifierCow::from_locale(locale!("en-002").into()), + DataIdentifierCow::from_locale(locale!("en-019").into()), + DataIdentifierCow::from_locale(locale!("en-142").into()), + DataIdentifierCow::from_locale(locale!("en-GB").into()), + DataIdentifierCow::from_locale(locale!("en-GB-u-sd-gbeng").into()), + DataIdentifierCow::from_borrowed_and_owned( + DataMarkerAttributes::from_str_or_panic("reverse"), + locale!("en").into() + ), + DataIdentifierCow::from_locale(locale!("eo").into()), + DataIdentifierCow::from_locale(locale!("fa").into()), + DataIdentifierCow::from_locale(locale!("fi").into()), + DataIdentifierCow::from_locale(locale!("is").into()), + DataIdentifierCow::from_locale(locale!("ja").into()), + DataIdentifierCow::from_borrowed_and_owned( + DataMarkerAttributes::from_str_or_panic("reverse"), + locale!("ja").into() + ), + DataIdentifierCow::from_locale(locale!("la").into()), + DataIdentifierCow::from_locale(locale!("pt").into()), + DataIdentifierCow::from_locale(locale!("ro").into()), + DataIdentifierCow::from_locale(locale!("ru").into()), + DataIdentifierCow::from_locale(locale!("sr").into()), + DataIdentifierCow::from_locale(locale!("sr-Latn").into()), + DataIdentifierCow::from_locale(locale!("vi").into()), + DataIdentifierCow::from_locale(locale!("zh").into()), ]) ); } diff --git a/provider/core/src/lib.rs b/provider/core/src/lib.rs index ae3a4addf1c..e6fecfb67de 100644 --- a/provider/core/src/lib.rs +++ b/provider/core/src/lib.rs @@ -144,7 +144,7 @@ pub use error::{DataError, DataErrorKind}; pub use icu_provider_macros::data_struct; mod request; -pub use request::{DataLocale, DataMarkerAttributes, DataRequest, DataRequestMetadata}; +pub use request::{DataLocale, DataMarkerAttributes, DataRequest, DataRequestMetadata, *}; mod response; #[doc(hidden)] // TODO(#4467): establish this as an internal API @@ -176,6 +176,7 @@ pub mod prelude { pub use crate::buf::AsDeserializingBufferProvider; #[doc(no_inline)] pub use crate::buf::{BufferMarker, BufferProvider}; + pub use crate::request::*; #[doc(no_inline)] pub use crate::{ BoundDataProvider, DataError, DataErrorKind, DataLocale, DataMarker, DataMarkerAttributes, diff --git a/provider/core/src/marker.rs b/provider/core/src/marker.rs index 4864b7db4d2..da8cd833c97 100644 --- a/provider/core/src/marker.rs +++ b/provider/core/src/marker.rs @@ -117,7 +117,7 @@ pub trait DataMarker: DynamicDataMarker { /// let result = DataProvider::>>::load( /// &buffer_provider.as_deserializing(), /// DataRequest { -/// locale: &langid!("en").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("en").into()), /// ..Default::default() /// }, /// ); @@ -166,7 +166,7 @@ where /// let result = DataProvider::>>::load( /// &MyProvider, /// DataRequest { -/// locale: &langid!("und").into(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("und").into()), /// ..Default::default() /// }, /// ); diff --git a/provider/core/src/request.rs b/provider/core/src/request.rs index 59c6e1e8372..09bf62be555 100644 --- a/provider/core/src/request.rs +++ b/provider/core/src/request.rs @@ -2,46 +2,36 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). +use alloc::borrow::Cow; +use alloc::borrow::ToOwned; +use alloc::boxed::Box; +use alloc::string::String; use core::cmp::Ordering; use core::default::Default; use core::fmt; use core::fmt::Debug; use core::hash::Hash; +use core::ops::Deref; use core::str::FromStr; use icu_locale_core::extensions::unicode as unicode_ext; use icu_locale_core::subtags::{Language, Region, Script, Variants}; use icu_locale_core::{LanguageIdentifier, Locale, ParseError}; use writeable::{LengthHint, Writeable}; - -use alloc::string::String; -use core::ops::Deref; -use icu_locale_core::extensions::private::Subtag; -use tinystr::TinyAsciiStr; - -#[cfg(doc)] -use icu_locale_core::subtags::Variant; +use zerovec::ule::VarULE; /// The request type passed into all data provider implementations. #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] #[allow(clippy::exhaustive_structs)] // this type is stable pub struct DataRequest<'a> { - /// The locale for which to load data. + /// The data identifier for which to load data. /// - /// If locale fallback is enabled, the resulting data may be from a different locale + /// If locale fallback is enabled, the resulting data may be from a different identifier /// than the one requested here. - pub locale: &'a DataLocale, - /// Key-specific request attributes - pub marker_attributes: &'a DataMarkerAttributes, + pub id: DataIdentifierBorrowed<'a>, /// Metadata that may affect the behavior of the data provider. pub metadata: DataRequestMetadata, } -impl fmt::Display for DataRequest<'_> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&self.locale, f) - } -} - /// Metadata for data requests. This is currently empty, but it may be extended with options /// for tuning locale fallback, buffer layout, and so forth. #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -51,6 +41,155 @@ pub struct DataRequestMetadata { pub silent: bool, } +/// TODO +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct DataIdentifierBorrowed<'a> { + /// Marker-specific request attributes + pub marker_attributes: &'a DataMarkerAttributes, + /// The CLDR locale + pub locale: &'a DataLocale, +} + +impl fmt::Display for DataIdentifierBorrowed<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(self.locale, f)?; + if !self.marker_attributes.is_empty() { + write!(f, "/{}", self.marker_attributes.as_str())?; + } + Ok(()) + } +} + +impl<'a> DataIdentifierBorrowed<'a> { + /// TODO + pub fn for_locale(locale: &'a DataLocale) -> Self { + Self { + locale, + ..Default::default() + } + } + + /// TODO + pub fn for_marker_attributes(marker_attributes: &'a DataMarkerAttributes) -> Self { + Self { + marker_attributes, + ..Default::default() + } + } + + /// TODO + pub fn for_marker_attributes_and_locale( + marker_attributes: &'a DataMarkerAttributes, + locale: &'a DataLocale, + ) -> Self { + Self { + marker_attributes, + locale, + } + } + + /// TODO + pub fn into_owned(&self) -> DataIdentifierCow<'static> { + DataIdentifierCow { + marker_attributes: Cow::Owned(self.marker_attributes.to_owned()), + locale: Cow::Owned(self.locale.clone()), + } + } + + /// TODO + pub fn as_cow(&self) -> DataIdentifierCow<'a> { + DataIdentifierCow { + marker_attributes: Cow::Borrowed(self.marker_attributes), + locale: Cow::Borrowed(self.locale), + } + } +} + +/// TODO +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +#[non_exhaustive] +pub struct DataIdentifierCow<'a> { + /// Marker-specific request attributes + pub marker_attributes: Cow<'a, DataMarkerAttributes>, + /// The CLDR locale + pub locale: Cow<'a, DataLocale>, +} + +impl fmt::Display for DataIdentifierCow<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&*self.locale, f)?; + if !self.marker_attributes.is_empty() { + write!(f, "/{}", self.marker_attributes.as_str())?; + } + Ok(()) + } +} + +impl<'a> DataIdentifierCow<'a> { + /// TODO + pub fn as_borrowed(&'a self) -> DataIdentifierBorrowed<'a> { + DataIdentifierBorrowed { + marker_attributes: &self.marker_attributes, + locale: &self.locale, + } + } +} + +impl DataIdentifierCow<'static> { + /// TODO + pub fn from_locale(locale: DataLocale) -> Self { + Self { + marker_attributes: Cow::Borrowed(DataMarkerAttributes::empty()), + locale: Cow::Owned(locale), + } + } + + /// TODO + pub fn from_marker_attributes(marker_attributes: &'static DataMarkerAttributes) -> Self { + Self { + marker_attributes: Cow::Borrowed(marker_attributes), + locale: Cow::Borrowed(Default::default()), + } + } + + /// TODO + pub fn from_marker_attributes_owned(marker_attributes: Box) -> Self { + Self { + marker_attributes: Cow::Owned(marker_attributes), + locale: Cow::Borrowed(Default::default()), + } + } + + /// TODO + pub fn from_owned(marker_attributes: Box, locale: DataLocale) -> Self { + Self { + marker_attributes: Cow::Owned(marker_attributes), + locale: Cow::Owned(locale), + } + } + + /// TODO + pub fn from_borrowed_and_owned( + marker_attributes: &'static DataMarkerAttributes, + locale: DataLocale, + ) -> Self { + Self { + marker_attributes: Cow::Borrowed(marker_attributes), + locale: Cow::Owned(locale), + } + } +} + +impl Default for DataIdentifierCow<'_> { + fn default() -> Self { + Self { + marker_attributes: Cow::Borrowed(Default::default()), + locale: Cow::Borrowed(Default::default()), + } + } +} + /// A locale type optimized for use in fallbacking and the ICU4X data pipeline. /// /// [`DataLocale`] contains less functionality than [`Locale`] but more than @@ -397,26 +536,6 @@ impl DataLocale { /// /// If you have ownership over the `DataLocale`, use [`DataLocale::into_locale()`] /// and then access the `id` field. - /// - /// # Examples - /// - /// ``` - /// use icu_locale_core::langid; - /// use icu_provider::prelude::*; - /// - /// let req_no_langid = DataRequest { - /// locale: &Default::default(), - /// ..Default::default() - /// }; - /// - /// let req_with_langid = DataRequest { - /// locale: &langid!("ar-EG").into(), - /// ..Default::default() - /// }; - /// - /// assert_eq!(req_no_langid.locale.get_langid(), langid!("und")); - /// assert_eq!(req_with_langid.locale.get_langid(), langid!("ar-EG")); - /// ``` pub fn get_langid(&self) -> LanguageIdentifier { self.langid.clone() } @@ -495,7 +614,7 @@ impl DataLocale { self.langid.region = region; } - /// Returns whether there are any [`Variant`] subtags in this [`DataLocale`]. + /// Returns whether there are any [`Variant`](icu_locale_core::subtags::Variant) subtags in this [`DataLocale`]. #[inline] pub fn has_variants(&self) -> bool { !self.langid.variants.is_empty() @@ -507,7 +626,7 @@ impl DataLocale { self.langid.variants = variants; } - /// Removes all [`Variant`] subtags in this [`DataLocale`]. + /// Removes all [`Variant`](icu_locale_core::subtags::Variant) subtags in this [`DataLocale`]. #[inline] pub fn clear_variants(&mut self) -> Variants { self.langid.variants.clear() @@ -579,27 +698,16 @@ impl DataLocale { } /// TODO -#[derive(Clone, Default)] +#[derive(PartialEq, Eq, Ord, PartialOrd, Hash)] +#[repr(transparent)] pub struct DataMarkerAttributes { - value: DataMarkerAttributesInner, -} - -#[derive(Clone, Default)] -enum DataMarkerAttributesInner { - #[default] - Empty, - Boxed(alloc::boxed::Box), - Stack(TinyAsciiStr<23>), - // NOTE: In the future, a `Static` variant could be added to allow `data_locale!("...")` - // Static(&'static str), + // Validated to be non-empty ASCII alphanumeric + hyphen + underscore + value: str, } impl<'a> Default for &'a DataMarkerAttributes { fn default() -> Self { - static DEFAULT: DataMarkerAttributes = DataMarkerAttributes { - value: DataMarkerAttributesInner::Empty, - }; - &DEFAULT + DataMarkerAttributes::empty() } } @@ -607,142 +715,77 @@ impl Deref for DataMarkerAttributes { type Target = str; #[inline] fn deref(&self) -> &Self::Target { - match &self.value { - DataMarkerAttributesInner::Empty => "", - DataMarkerAttributesInner::Boxed(s) => s.deref(), - DataMarkerAttributesInner::Stack(s) => s.as_str(), - } + &self.value } } -impl PartialEq for DataMarkerAttributes { +impl Debug for DataMarkerAttributes { #[inline] - fn eq(&self, other: &Self) -> bool { - self.deref() == other.deref() + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.value.fmt(f) } } -impl Eq for DataMarkerAttributes {} - -impl PartialOrd for DataMarkerAttributes { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} +/// TODO +#[derive(Debug)] +#[non_exhaustive] +pub struct AttributeParseError; -impl Ord for DataMarkerAttributes { - fn cmp(&self, other: &Self) -> Ordering { - self.deref().cmp(other.deref()) +impl DataMarkerAttributes { + const fn validate(s: &str) -> Result<(), AttributeParseError> { + let mut i = 0; + while i < s.len() { + #[allow(clippy::indexing_slicing)] // duh + if !matches!(s.as_bytes()[i], b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_') { + return Err(AttributeParseError); + } + i += 1; + } + Ok(()) } -} -impl Debug for DataMarkerAttributes { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.deref().fmt(f) - } -} + /// TODO + pub const fn try_from_str(s: &str) -> Result<&Self, AttributeParseError> { + let Ok(()) = Self::validate(s) else { + return Err(AttributeParseError); + }; -impl Hash for DataMarkerAttributes { - #[inline] - fn hash(&self, state: &mut H) { - self.deref().hash(state) + Ok(unsafe { &*(s as *const str as *const DataMarkerAttributes) }) } -} -impl FromStr for DataMarkerAttributes { - type Err = core::convert::Infallible; + /// TODO + pub fn try_from_string(s: String) -> Result, AttributeParseError> { + let Ok(()) = Self::validate(&s) else { + return Err(AttributeParseError); + }; - fn from_str(s: &str) -> Result { - if s.is_empty() { - Ok(Self { - value: DataMarkerAttributesInner::Empty, - }) - } else if let Ok(tiny) = s.parse() { - Ok(Self { - value: DataMarkerAttributesInner::Stack(tiny), - }) - } else { - Ok(Self { - value: DataMarkerAttributesInner::Boxed(s.into()), - }) - } + Ok(unsafe { core::mem::transmute(s.into_boxed_str()) }) } -} -impl DataMarkerAttributes { - /// Creates a [`DataMarkerAttributes`] from an iterator of individual keys. - /// - /// # Examples - /// - /// ``` - /// use icu_locale_core::extensions::private::subtag; - /// use icu_provider::prelude::*; - /// - /// // Single auxiliary key: - /// let a = DataMarkerAttributes::try_from_iter([subtag!("abc")]); - /// let b = "abc".parse::().unwrap(); - /// assert_eq!(a, b); - /// - /// // Multiple auxiliary keys: - /// let a = DataMarkerAttributes::try_from_iter([subtag!("abc"), subtag!("defg")]); - /// let b = "abc-defg".parse::().unwrap(); - /// assert_eq!(a, b); - /// ``` - pub fn try_from_iter(iter: impl IntoIterator) -> Self { - // TODO: Avoid the allocation when possible - let mut builder = String::new(); - for item in iter { - if !builder.is_empty() { - builder.push('-'); - } - builder.push_str(item.as_str()) - } - Self { - value: if builder.is_empty() { - DataMarkerAttributesInner::Empty - } else if builder.len() <= 23 { - #[allow(clippy::unwrap_used)] // we just checked that the string is ascii - DataMarkerAttributesInner::Stack(builder.parse().unwrap()) - } else { - DataMarkerAttributesInner::Boxed(builder.into()) - }, - } + /// TODO + pub const fn from_str_or_panic(s: &str) -> &Self { + let Ok(r) = Self::try_from_str(s) else { + panic!("Invalid marker attribute syntax") + }; + r } - /// Creates a [`DataMarkerAttributes`] from a single subtag. - /// - /// # Examples - /// - /// ``` - /// use icu_locale_core::extensions::private::subtag; - /// use icu_provider::prelude::*; - /// - /// // Single auxiliary key: - /// let a = DataMarkerAttributes::from_tinystr("abc".parse().unwrap()); - /// let b = "abc".parse::().unwrap(); - /// assert_eq!(a, b); - /// ``` - pub const fn from_tinystr(input: TinyAsciiStr<8>) -> Self { - Self { - value: DataMarkerAttributesInner::Stack(input.resize()), - } + /// TODO + pub const fn empty() -> &'static Self { + unsafe { &*("" as *const str as *const DataMarkerAttributes) } } /// TODO - pub fn single(&self) -> Option> { - let mut iter = self.split('-').filter_map(|x| match x.parse() { - Ok(x) => Some(x), - Err(_) => { - debug_assert!(false, "failed to convert to subtag: {x}"); - None - } - }); - let subtag = iter.next()?; - if iter.next().is_some() { - return None; - } - Some(subtag) + pub const fn as_str(&self) -> &str { + &self.value + } +} + +impl ToOwned for DataMarkerAttributes { + type Owned = Box; + fn to_owned(&self) -> Self::Owned { + let str = unsafe { &*(self as *const DataMarkerAttributes as *const str) }; + unsafe { core::mem::transmute(str.to_boxed()) } } } diff --git a/provider/core/src/response.rs b/provider/core/src/response.rs index 869ddc9df10..1eff44716f8 100644 --- a/provider/core/src/response.rs +++ b/provider/core/src/response.rs @@ -87,10 +87,11 @@ pub struct DataPayload(pub(crate) DataPayloadInner); /// use icu_provider::hello_world::*; /// use icu_provider::prelude::*; /// use icu_provider::DataPayloadOr; +/// use icu_locale_core::langid; /// /// let response: DataResponse = HelloWorldProvider /// .load(DataRequest { -/// locale: &"de".parse().unwrap(), +/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()), /// ..Default::default() /// }) /// .expect("Loading should succeed"); diff --git a/provider/datagen/src/driver.rs b/provider/datagen/src/driver.rs index 917c0410306..8ac51e00dbb 100644 --- a/provider/datagen/src/driver.rs +++ b/provider/datagen/src/driver.rs @@ -549,23 +549,25 @@ impl DatagenDriver { } ); - let load_with_fallback = |marker, locale: &_, marker_attributes: &_| { - log::trace!("Generating marker/locale: {marker:?}/{locale}"); + let load_with_fallback = |marker, id: DataIdentifierBorrowed<'_>| { + log::trace!("Generating marker/locale: {marker:?}/{}", id.locale); let mut metadata = DataRequestMetadata::default(); metadata.silent = true; // Lazy-compute the fallback iterator so that we don't always require CLDR data let mut locale_iter: Option = None; loop { let req = DataRequest { - locale: locale_iter.as_ref().map(|i| i.get()).unwrap_or(locale), - marker_attributes, + id: DataIdentifierBorrowed::for_marker_attributes_and_locale( + id.marker_attributes, + locale_iter.as_ref().map(|i| i.get()).unwrap_or(id.locale), + ), metadata, }; match provider.load_data(marker, req) { Ok(data_response) => { if let Some(iter) = locale_iter.as_ref() { - if iter.get().is_und() && !locale.is_und() { - log::debug!("Falling back to und: {marker:?}/{locale}"); + if iter.get().is_und() && !id.locale.is_und() { + log::debug!("Falling back to und: {marker:?}/{}", id.locale); } } return Some(Ok(data_response.payload)); @@ -576,7 +578,7 @@ impl DatagenDriver { }) => { if let Some(iter) = locale_iter.as_mut() { if iter.get().is_und() { - log::debug!("Could not find data for: {marker:?}/{locale}"); + log::debug!("Could not find data for: {marker:?}/{}", id.locale); return None; } iter.step(); @@ -584,7 +586,7 @@ impl DatagenDriver { locale_iter = Some( fallbacker .for_config(marker.fallback_config) - .fallback_for(locale.clone()), + .fallback_for(id.locale.clone()), ) } } @@ -598,8 +600,7 @@ impl DatagenDriver { let instant1 = Instant::now(); if marker.is_singleton { - if provider.iter_requests_for_marker(marker)? - != HashSet::from_iter([Default::default()]) + if provider.iter_ids_for_marker(marker)? != HashSet::from_iter([Default::default()]) { return Err(DataError::custom( "Invalid supported locales for singleton marker", @@ -647,13 +648,10 @@ impl DatagenDriver { DeduplicationStrategy::Maximal => { let payloads = locales_to_export .into_par_iter() - .filter_map(|(locale, marker_attributes)| { + .filter_map(|id| { let instant2 = Instant::now(); - load_with_fallback(marker, &locale, &marker_attributes).map(|r| { - r.map(|payload| { - ((locale, marker_attributes), (payload, instant2.elapsed())) - }) - }) + load_with_fallback(marker, id.as_borrowed()) + .map(|r| r.map(|payload| (id, (payload, instant2.elapsed())))) }) .collect::, _>>()?; deduplicate_payloads::(marker, &payloads, &fallbacker, sink)? @@ -661,35 +659,31 @@ impl DatagenDriver { DeduplicationStrategy::RetainBaseLanguages => { let payloads = locales_to_export .into_par_iter() - .filter_map(|(locale, marker_attributes)| { + .filter_map(|id| { let instant2 = Instant::now(); - load_with_fallback(marker, &locale, &marker_attributes).map(|r| { - r.map(|payload| { - ((locale, marker_attributes), (payload, instant2.elapsed())) - }) - }) + load_with_fallback(marker, id.as_borrowed()) + .map(|r| r.map(|payload| (id, (payload, instant2.elapsed())))) }) .collect::, _>>()?; deduplicate_payloads::(marker, &payloads, &fallbacker, sink)? } DeduplicationStrategy::None => locales_to_export .into_par_iter() - .filter_map(|(locale, marker_attributes)| { + .filter_map(|id| { let instant2 = Instant::now(); - let result = load_with_fallback(marker, &locale, &marker_attributes)?; + let result = load_with_fallback(marker, id.as_borrowed())?; let result = result .and_then(|payload| { - sink.put_payload(marker, &locale, &marker_attributes, &payload) + sink.put_payload(marker, id.as_borrowed(), &payload) }) // Note: in Hybrid mode the elapsed time includes sink.put_payload. // In Runtime mode the elapsed time is only load_with_fallback. - .map(|_| (instant2.elapsed(), locale.write_to_string().into_owned())) + .map(|_| (instant2.elapsed(), id.locale.write_to_string().into_owned())) .map_err(|e| { e.with_req( marker, DataRequest { - locale: &locale, - marker_attributes: &marker_attributes, + id: id.as_borrowed(), ..Default::default() }, ) @@ -729,47 +723,55 @@ impl DatagenDriver { /// Selects the maximal set of locales to export based on a [`DataMarkerInfo`] and this datagen /// provider's options bag. The locales may be later optionally deduplicated for fallback. -fn select_locales_for_marker( - provider: &dyn ExportableProvider, +fn select_locales_for_marker<'a>( + provider: &'a dyn ExportableProvider, marker: DataMarkerInfo, requested_families: &HashMap, include_full: bool, additional_collations: &HashSet, segmenter_models: &[String], fallbacker: &LocaleFallbacker, -) -> Result, DataError> { +) -> Result>, DataError> { // Map from all supported LanguageIdentifiers to their // corresponding supported DataLocales. - let mut supported_map = - HashMap::>::new(); - for (locale, marker_attributes) in provider - .iter_requests_for_marker(marker) + let mut supported_map = HashMap::>>::new(); + for id in provider + .iter_ids_for_marker(marker) .map_err(|e| e.with_marker(marker))? { supported_map - .entry(locale.get_langid()) + .entry(id.locale.get_langid()) .or_default() - .insert((locale, marker_attributes)); + .insert(id); } if marker.path.as_str().starts_with("segmenter/dictionary/") { - supported_map.retain(|_, locales| { - locales.retain(|(_, attrs)| segmenter_models.iter().any(|m| **m == **attrs)); - !locales.is_empty() + supported_map.retain(|_, ids| { + ids.retain(|id| { + segmenter_models + .iter() + .any(|m| **m == **id.marker_attributes) + }); + !ids.is_empty() }); // Don't perform additional locale filtering return Ok(supported_map.into_values().flatten().collect()); } else if marker.path.as_str().starts_with("segmenter/lstm/") { supported_map.retain(|_, locales| { - locales.retain(|(_, attrs)| segmenter_models.iter().any(|m| **m == **attrs)); + locales.retain(|id| { + segmenter_models + .iter() + .any(|m| **m == **id.marker_attributes) + }); !locales.is_empty() }); // Don't perform additional locale filtering return Ok(supported_map.into_values().flatten().collect()); } else if marker.path.as_str().starts_with("collator/") { - supported_map.retain(|_, locales| { - locales.retain(|(locale, _)| { - let Some(collation) = locale + supported_map.retain(|_, ids| { + ids.retain(|id| { + let Some(collation) = id + .locale .get_unicode_ext(&key!("co")) .and_then(|co| co.into_single_subtag()) else { @@ -782,7 +784,7 @@ fn select_locales_for_marker( !["big5han", "gb2312"].contains(&collation.as_str()) } }); - !locales.is_empty() + !ids.is_empty() }); } @@ -801,7 +803,7 @@ fn select_locales_for_marker( // Compute a map from LanguageIdentifiers to DataLocales, including inherited auxiliary keys // and extensions. Also resolve the ancestors and descendants while building this map. let mut selected_langids = requested_families.keys().cloned().collect::>(); - let expansion_map: HashMap<&LanguageIdentifier, HashSet<(DataLocale, DataMarkerAttributes)>> = + let expansion_map: HashMap<&LanguageIdentifier, HashSet> = all_candidate_langids .into_iter() .map(|current_langid| { @@ -829,7 +831,7 @@ fn select_locales_for_marker( loop { // Inherit aux keys and extension keywords from parent locales let parent_langid: LanguageIdentifier = iter.get().get_langid(); - let maybe_parent_locales = supported_map.get(&parent_langid); + let maybe_parent_ids = supported_map.get(&parent_langid); let include_descendants = requested_families .get(&parent_langid) .map(|family| family.include_descendants) @@ -847,17 +849,21 @@ fn select_locales_for_marker( ); selected_langids.insert(parent_langid); } - if let Some(parent_locales) = maybe_parent_locales { - for morphed_req in parent_locales.iter() { + if let Some(parent_ids) = maybe_parent_ids { + for morphed_id in parent_ids.iter() { // Special case: don't pull extensions or aux keys up from the root. - if morphed_req.0.is_langid_und() - && !(morphed_req.0.is_empty() && morphed_req.1.is_empty()) + if morphed_id.locale.is_langid_und() + && !(morphed_id.locale.is_empty() + && morphed_id.marker_attributes.is_empty()) { continue; } - let mut morphed_req = morphed_req.clone(); - morphed_req.0.set_langid(current_langid.clone()); - expansion.insert(morphed_req); + let mut morphed_id = morphed_id.clone(); + morphed_id + .locale + .to_mut() + .set_langid(current_langid.clone()); + expansion.insert(morphed_id); } } if iter.get().is_und() { @@ -879,79 +885,83 @@ fn select_locales_for_marker( fn deduplicate_payloads( marker: DataMarkerInfo, - payloads: &HashMap<(DataLocale, DataMarkerAttributes), (DataPayload, Duration)>, + payloads: &HashMap, Duration)>, fallbacker: &LocaleFallbacker, sink: &dyn DataExporter, ) -> Result, DataError> { let fallbacker_with_config = fallbacker.for_config(marker.fallback_config); - payloads - .iter() - .try_for_each(|((locale, marker_attributes), (payload, _duration))| { - // Always export `und`. This prevents calling `step` on an empty locale. - if locale.is_und() { - return sink - .put_payload(marker, locale, marker_attributes, payload) - .map_err(|e| { - e.with_req( - marker, - DataRequest { - locale, - ..Default::default() - }, - ) - }); - } - let mut iter = fallbacker_with_config.fallback_for(locale.clone()); - loop { - if !MAXIMAL { - // To retain base languages, preemptively step to the - // parent locale. This should retain the locale if - // the next parent is `und`. - iter.step(); - } - if iter.get().is_und() { - break; - } - if MAXIMAL { - iter.step(); - } - - if let Some((inherited_payload, _duration)) = - payloads.get(&(iter.get().clone(), marker_attributes.clone())) - { - if inherited_payload == payload { - // Found a match: don't need to write anything - log::trace!( - "Deduplicating {marker:?}/{locale} (inherits from {})", - iter.get() - ); - return Ok(()); - } else { - // Not a match: we must include this - break; - } - } - } - // Did not find a match: export this payload - sink.put_payload(marker, locale, marker_attributes, payload) + payloads.iter().try_for_each(|(id, (payload, _duration))| { + // Always export `und`. This prevents calling `step` on an empty locale. + if id.locale.is_und() { + return sink + .put_payload(marker, id.as_borrowed(), payload) .map_err(|e| { e.with_req( marker, DataRequest { - locale, + id: id.as_borrowed(), ..Default::default() }, ) - }) - })?; + }); + } + let mut iter = fallbacker_with_config.fallback_for(id.locale.clone().into_owned()); + loop { + if !MAXIMAL { + // To retain base languages, preemptively step to the + // parent locale. This should retain the locale if + // the next parent is `und`. + iter.step(); + } + if iter.get().is_und() { + break; + } + if MAXIMAL { + iter.step(); + } + + if let Some((inherited_payload, _duration)) = payloads.get( + &DataIdentifierBorrowed::for_marker_attributes_and_locale( + &id.marker_attributes, + iter.get(), + ) + .as_cow(), + ) { + if inherited_payload == payload { + // Found a match: don't need to write anything + log::trace!( + "Deduplicating {:?}/{} (inherits from {})", + id.locale, + id.marker_attributes.as_str(), + iter.get() + ); + return Ok(()); + } else { + // Not a match: we must include this + break; + } + } + } + // Did not find a match: export this payload + sink.put_payload(marker, id.as_borrowed(), payload) + .map_err(|e| { + e.with_req( + marker, + DataRequest { + id: id.as_borrowed(), + ..Default::default() + }, + ) + }) + })?; // Slowest locale calculation: Ok(payloads .iter() - .map(|((locale, marker_attributes), (_payload, duration))| { + .map(|(id, (_payload, duration))| { ( *duration, - locale.write_to_string().into_owned() + "/" + marker_attributes, + id.locale.write_to_string().into_owned() + "/" + id.marker_attributes.as_str(), ) }) .max()) @@ -992,7 +1002,7 @@ fn test_collation_filtering() { } impl IterableDataProvider for Provider { - fn iter_requests(&self) -> Result, DataError> { + fn iter_ids(&self) -> Result, DataError> { Ok(HashSet::from_iter( [ locale!("ko-u-co-search"), @@ -1011,7 +1021,7 @@ fn test_collation_filtering() { locale!("zh"), ] .into_iter() - .map(|l| (l.into(), Default::default())), + .map(|l| DataIdentifierCow::from_locale(l.into())), )) } } @@ -1111,7 +1121,7 @@ fn test_collation_filtering() { ) .unwrap() .into_iter() - .map(|(l, _)| l.to_string()) + .map(|id| id.locale.to_string()) .collect::>(); let expected_locales = cas .expected diff --git a/provider/datagen/tests/test-options.rs b/provider/datagen/tests/test-options.rs index d8059dcbcbb..e862bee8cf0 100644 --- a/provider/datagen/tests/test-options.rs +++ b/provider/datagen/tests/test-options.rs @@ -59,8 +59,8 @@ impl DataProvider for TestingProvider { message: (*self .0 .get(&( - req.locale.to_string().as_str(), - req.marker_attributes as &str, + req.id.locale.to_string().as_str(), + req.id.marker_attributes as &str, )) .ok_or(DataErrorKind::MissingLocale.into_error())?) .into(), @@ -70,11 +70,16 @@ impl DataProvider for TestingProvider { } impl IterableDataProvider for TestingProvider { - fn iter_requests(&self) -> Result, DataError> { + fn iter_ids(&self) -> Result, DataError> { Ok(self .0 .keys() - .map(|(l, a)| (l.parse().unwrap(), a.parse().unwrap())) + .map(|(l, a)| { + DataIdentifierCow::from_owned( + DataMarkerAttributes::from_str_or_panic(a).to_owned(), + l.parse().unwrap(), + ) + }) .collect()) } } diff --git a/provider/datagen/tests/testutil.rs b/provider/datagen/tests/testutil.rs index 9d6f2352fa4..233db93c187 100644 --- a/provider/datagen/tests/testutil.rs +++ b/provider/datagen/tests/testutil.rs @@ -11,14 +11,13 @@ use icu_provider::prelude::*; use postcard::ser_flavors::{AllocVec, Flavor}; #[derive(Default)] -pub struct TestingExporter(FrozenMap<(DataLocale, DataMarkerAttributes), Vec>); +pub struct TestingExporter(FrozenMap, Vec>); impl DataExporter for &mut TestingExporter { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, payload: &DataPayload, ) -> Result<(), DataError> { let mut serializer = postcard::Serializer { @@ -29,9 +28,12 @@ impl DataExporter for &mut TestingExporter { .output .finalize() .expect("Failed to finalize serializer output"); - println!("Putting: {marker:?}/{}/{locale}", marker_attributes as &str); - self.0 - .insert((locale.clone(), marker_attributes.clone()), output); + println!( + "Putting: {marker:?}/{marker_attributes}/{locale}", + marker_attributes = id.marker_attributes.as_str(), + locale = id.locale + ); + self.0.insert(id.into_owned(), output); Ok(()) } } @@ -41,7 +43,7 @@ impl TestingExporter { self.0 .into_tuple_vec() .into_iter() - .map(|((locale, _marker_attributes), buffer)| (locale.to_string(), buffer)) + .map(|(id, buffer)| (id.locale.to_string(), buffer)) .collect() } } diff --git a/provider/fs/benches/provider_fs.rs b/provider/fs/benches/provider_fs.rs index 69595302113..29ccc53fbfb 100644 --- a/provider/fs/benches/provider_fs.rs +++ b/provider/fs/benches/provider_fs.rs @@ -18,7 +18,7 @@ fn overview_bench(c: &mut Criterion) { let _: DataResponse = black_box(&provider) .as_deserializing() .load(DataRequest { - locale: &langid!("ru").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ru").into()), ..Default::default() }) .expect("Loading was successful"); @@ -43,7 +43,7 @@ fn json_bench(c: &mut Criterion) { let _: DataResponse = black_box(&provider) .as_deserializing() .load(DataRequest { - locale: &langid!("ru").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ru").into()), ..Default::default() }) .expect("Loading was successful"); @@ -55,7 +55,7 @@ fn json_bench(c: &mut Criterion) { let _: DataResponse = black_box(&provider as &dyn BufferProvider) .as_deserializing() .load(DataRequest { - locale: &langid!("ru").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ru").into()), ..Default::default() }) .expect("Loading was successful"); @@ -73,7 +73,7 @@ fn bincode_bench(c: &mut Criterion) { let _: DataResponse = black_box(&provider) .as_deserializing() .load(DataRequest { - locale: &langid!("ru").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ru").into()), ..Default::default() }) .expect("Loading was successful"); @@ -85,7 +85,7 @@ fn bincode_bench(c: &mut Criterion) { let _: DataResponse = black_box(&provider as &dyn BufferProvider) .as_deserializing() .load(DataRequest { - locale: &langid!("ru").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ru").into()), ..Default::default() }) .expect("Loading was successful"); @@ -103,7 +103,7 @@ fn postcard_bench(c: &mut Criterion) { let _: DataResponse = black_box(&provider) .as_deserializing() .load(DataRequest { - locale: &langid!("ru").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ru").into()), ..Default::default() }) .expect("Loading was successful"); @@ -115,7 +115,7 @@ fn postcard_bench(c: &mut Criterion) { let _: DataResponse = black_box(&provider as &dyn BufferProvider) .as_deserializing() .load(DataRequest { - locale: &langid!("ru").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("ru").into()), ..Default::default() }) .expect("Loading was successful"); diff --git a/provider/fs/src/export/fs_exporter.rs b/provider/fs/src/export/fs_exporter.rs index c79ec3890fb..d627af13118 100644 --- a/provider/fs/src/export/fs_exporter.rs +++ b/provider/fs/src/export/fs_exporter.rs @@ -102,16 +102,15 @@ impl DataExporter for FilesystemExporter { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, obj: &DataPayload, ) -> Result<(), DataError> { let mut path_buf = self.root.clone().into_os_string(); write!(&mut path_buf, "/{}", marker.path.as_str()).expect("infallible"); - if !marker_attributes.is_empty() { - write!(&mut path_buf, "/{}", marker_attributes as &str).expect("infallible"); + if !id.marker_attributes.is_empty() { + write!(&mut path_buf, "/{}", id.marker_attributes.as_str()).expect("infallible"); } - write!(&mut path_buf, "/{locale}").expect("infallible"); + write!(&mut path_buf, "/{}", id.locale).expect("infallible"); write!(&mut path_buf, ".{}", self.manifest.file_extension).expect("infallible"); #[allow(clippy::unwrap_used)] // has parent by construction diff --git a/provider/fs/src/fs_data_provider.rs b/provider/fs/src/fs_data_provider.rs index cc276c27f4a..894aeddac6d 100644 --- a/provider/fs/src/fs_data_provider.rs +++ b/provider/fs/src/fs_data_provider.rs @@ -68,7 +68,7 @@ impl DynamicDataProvider for FsDataProvider { marker: DataMarkerInfo, req: DataRequest, ) -> Result, DataError> { - if marker.is_singleton && !req.locale.is_empty() { + if marker.is_singleton && !req.id.locale.is_empty() { return Err(DataErrorKind::ExtraneousLocale.with_req(marker, req)); } let mut path = self.root.clone().into_os_string(); @@ -76,10 +76,10 @@ impl DynamicDataProvider for FsDataProvider { if !Path::new(&path).exists() { return Err(DataErrorKind::MissingDataMarker.with_req(marker, req)); } - if !req.marker_attributes.is_empty() { - write!(&mut path, "/{}", req.marker_attributes as &str).expect("infallible"); + if !req.id.marker_attributes.is_empty() { + write!(&mut path, "/{}", req.id.marker_attributes as &str).expect("infallible"); } - write!(&mut path, "/{}", req.locale).expect("infallible"); + write!(&mut path, "/{}", req.id.locale).expect("infallible"); write!(&mut path, ".{}", self.manifest.file_extension).expect("infallible"); if !Path::new(&path).exists() { return Err(DataErrorKind::MissingLocale.with_req(marker, req)); diff --git a/provider/fs/tests/test_file_io.rs b/provider/fs/tests/test_file_io.rs index 1461d97f5d6..3d3525a0586 100644 --- a/provider/fs/tests/test_file_io.rs +++ b/provider/fs/tests/test_file_io.rs @@ -17,29 +17,28 @@ const PATHS: &[&str] = &[ fn test_provider() { for path in PATHS { let provider = FsDataProvider::try_new(path).unwrap(); - for (locale, marker_attributes) in HelloWorldProvider.iter_requests().unwrap() { + for id in HelloWorldProvider.iter_ids().unwrap() { let req = DataRequest { - locale: &locale, - marker_attributes: &marker_attributes, + id: id.as_borrowed(), ..Default::default() }; let expected = HelloWorldProvider .load(req) - .unwrap_or_else(|e| panic!("{e}: {req} ({path})")) + .unwrap_or_else(|e| panic!("{e}: {req:?} ({path})")) .payload; let actual: DataPayload = provider .as_deserializing() .load(req) - .unwrap_or_else(|e| panic!("{e}: {req} ({path})")) + .unwrap_or_else(|e| panic!("{e}: {req:?} ({path})")) .payload; assert_eq!(actual.get(), expected.get()); let actual: DataPayload = (&provider as &dyn BufferProvider) .as_deserializing() .load(req) - .unwrap_or_else(|e| panic!("{e}: {req} ({path})")) + .unwrap_or_else(|e| panic!("{e}: {req:?} ({path})")) .payload; assert_eq!(actual.get(), expected.get()); } @@ -53,7 +52,7 @@ fn test_errors() { let err: Result, DataError> = provider.as_deserializing().load(DataRequest { - locale: &langid!("zh-DE").into(), + id: DataIdentifierBorrowed::for_locale(&langid!("zh-DE").into()), ..Default::default() }); diff --git a/provider/icu4x-datagen/src/main.rs b/provider/icu4x-datagen/src/main.rs index 063f813a41a..958f3ca9cbb 100644 --- a/provider/icu4x-datagen/src/main.rs +++ b/provider/icu4x-datagen/src/main.rs @@ -675,9 +675,7 @@ where BlobDataProvider: AsDeserializingBufferProvider, for<'a> DeserializingBufferProvider<'a, BlobDataProvider>: DataProvider, { - fn iter_requests( - &self, - ) -> Result, DataError> { - self.0.iter_requests_for_marker(M::INFO) + fn iter_ids(&self) -> Result, DataError> { + self.0.iter_ids_for_marker(M::INFO) } } diff --git a/tools/bakeddata-scripts/src/main.rs b/tools/bakeddata-scripts/src/main.rs index 0b3e91dab18..8bd612c7803 100644 --- a/tools/bakeddata-scripts/src/main.rs +++ b/tools/bakeddata-scripts/src/main.rs @@ -192,8 +192,7 @@ impl DataExporter for StubExporter { fn put_payload( &self, _marker: DataMarkerInfo, - _locale: &DataLocale, - _marker_attributes: &DataMarkerAttributes, + _id: DataIdentifierBorrowed, _payload: &DataPayload, ) -> Result<(), DataError> { Ok(()) @@ -225,8 +224,7 @@ impl DataExporter for PostcardFingerprintExporter { fn put_payload( &self, marker: DataMarkerInfo, - locale: &DataLocale, - marker_attributes: &DataMarkerAttributes, + id: DataIdentifierBorrowed, payload_before: &DataPayload, ) -> Result<(), DataError> { let mut serialized = vec![]; @@ -248,15 +246,16 @@ impl DataExporter for PostcardFingerprintExporter { self.size_hash.lock().expect("poison").insert( ( marker, - if marker.is_singleton && locale.is_und() { + if marker.is_singleton && id.locale.is_und() { "".to_string() - } else if !marker_attributes.is_empty() { + } else if !id.marker_attributes.is_empty() { format!( "{locale}/{marker_attributes}", - marker_attributes = marker_attributes as &str + locale = id.locale, + marker_attributes = id.marker_attributes.as_str(), ) } else { - locale.to_string() + id.locale.to_string() }, ), (size, hash), diff --git a/tutorials/data_provider.md b/tutorials/data_provider.md index 4719e576b62..9ae3011e35e 100644 --- a/tutorials/data_provider.md +++ b/tutorials/data_provider.md @@ -32,7 +32,7 @@ struct SingleLocaleProvider(DataLocale); impl DataProvider for SingleLocaleProvider { fn load(&self, req: DataRequest) -> Result, DataError> { - if *req.locale != self.0 { + if *req.id.locale != self.0 { return Err(DataErrorKind::MissingLocale.with_req(HelloWorldV1Marker::INFO, req)); } HelloWorldProvider.load(req) @@ -115,7 +115,7 @@ where { // First lock: cache retrieval let mut cache = self.cache.lock().unwrap(); - let borrowed_cache_key = CacheKey(M::INFO, Cow::Borrowed(req.locale)); + let borrowed_cache_key = CacheKey(M::INFO, Cow::Borrowed(req.id.locale)); if let Some(any_res) = cache.get(&borrowed_cache_key) { // Note: Cloning a DataPayload is usually cheap, and it is necessary in order to // convert the short-lived cache object into one we can return. @@ -124,7 +124,7 @@ where } // Release the lock to invoke the inner provider let response = self.provider.load(req)?; - let owned_cache_key = CacheKeyWrap(CacheKey(M::INFO, Cow::Owned(req.locale.clone()))); + let owned_cache_key = CacheKeyWrap(CacheKey(M::INFO, Cow::Owned(req.id.locale.clone()))); // Second lock: cache storage self.cache.lock() .unwrap() @@ -217,7 +217,7 @@ where // Cast from `DataPayload` to `DataPayload` let mut any_payload = (&mut res.payload) as &mut dyn Any; if let Some(mut decimal_payload) = any_payload.downcast_mut::>() { - if req.locale.region() == Some(region!("CH")) { + if req.id.locale.region() == Some(region!("CH")) { decimal_payload.with_mut(|data| { // Change the grouping separator for all Swiss locales to '🐮' data.grouping_separator = Cow::Borrowed("🐮");