diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 7b24e91e441..6f9cc3dbb28 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -32,7 +32,7 @@ concurrency: jobs: # ci-job-check & ci-job-features - # Running at MSRV (1.67) + # Running at MSRV msrv: runs-on: ubuntu-latest # Defined as a matrix so that features can start immediately, but @@ -68,11 +68,13 @@ jobs: # Cargo-all-features boilerplate - name: Get cargo-all-features version + if: matrix.behavior != 'check' id: cargo-all-features-version run: | echo "hash=$(cargo search cargo-all-features | grep '^cargo-all-features =' | md5sum)" >> $GITHUB_OUTPUT shell: bash - name: Attempt to load cached cargo-all-features + if: matrix.behavior != 'check' uses: actions/cache@v3 id: cargo-all-features-cache with: @@ -81,7 +83,7 @@ jobs: ~/.cargo/bin/cargo-all-features.exe key: ${{ runner.os }}-all-features-${{ steps.cargo-all-features-version.outputs.hash }} - name: Install cargo-all-features - if: steps.cargo-all-features-cache.outputs.cache-hit != 'true' + if: matrix.behavior != 'check' && steps.cargo-all-features-cache.outputs.cache-hit != 'true' run: cargo +stable install cargo-all-features # No toolchain boilerplate as this runs on MSRV @@ -133,6 +135,41 @@ jobs: if: matrix.os == 'windows-latest' run: cargo make ci-job-test-windows + # ci-job-test-docs + test-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + # Cargo-make boilerplate + - name: Get cargo-make version + id: cargo-make-version + run: | + echo "hash=$(cargo search cargo-make | grep '^cargo-make =' | md5sum)" >> $GITHUB_OUTPUT + shell: bash + - name: Attempt to load cached cargo-make + uses: actions/cache@v3 + id: cargo-make-cache + with: + path: | + ~/.cargo/bin/cargo-make + ~/.cargo/bin/cargo-make.exe + key: ${{ runner.os }}-make-${{ steps.cargo-make-version.outputs.hash }} + - name: Install cargo-make + if: steps.cargo-make-cache.outputs.cache-hit != 'true' + run: cargo +stable install cargo-make + + + # Toolchain boilerplate + - name: Potentially override rust version with nightly + run: cargo make set-nightly-version-for-ci + - name: Show the selected Rust toolchain + run: rustup show + + # Actual job + - name: Run `cargo make ci-job-test-docs` + run: cargo make ci-job-test-docs + # ci-job-test-tutorials test-tutorials: runs-on: ubuntu-latest @@ -189,6 +226,48 @@ jobs: steps: - uses: actions/checkout@v3 + # Cargo-make boilerplate + - name: Get cargo-make version + id: cargo-make-version + run: | + echo "hash=$(cargo search cargo-make | grep '^cargo-make =' | md5sum)" >> $GITHUB_OUTPUT + shell: bash + - name: Attempt to load cached cargo-make + uses: actions/cache@v3 + id: cargo-make-cache + with: + path: | + ~/.cargo/bin/cargo-make + ~/.cargo/bin/cargo-make.exe + key: ${{ runner.os }}-make-${{ steps.cargo-make-version.outputs.hash }} + - name: Install cargo-make + if: steps.cargo-make-cache.outputs.cache-hit != 'true' + run: cargo +stable install cargo-make + + # Toolchain boilerplate + - name: Potentially override rust version with nightly + run: cargo make set-nightly-version-for-ci + - name: Show the selected Rust toolchain + run: rustup show + + # Job-specific dependencies + - name: Attempt to load download cache + uses: actions/cache@v3 + with: + key: download-cache + path: /tmp/icu4x-source-cache + + # Actual job + - name: Run `cargo make ci-job-testdata` + run: cargo make ci-job-testdata + + # ci-job-testdata-legacy + testdata-legacy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + # Cargo-make boilerplate - name: Get cargo-make version id: cargo-make-version @@ -239,20 +318,11 @@ jobs: path: /tmp/icu4x-source-cache # Actual job - - name: Run `cargo make testdata-check` - run: cargo make testdata-check - - - name: Run `cargo make testdata-legacy` - run: cargo make testdata-legacy - - # This is too slow for Windows and Mac - - name: Run `cargo make testdata-legacy-test` - if: matrix.os == 'ubuntu-latest' - run: cargo make testdata-legacy-test + - name: Run `cargo make ci-job-testdata-legacy` + run: cargo make ci-job-testdata-legacy - - # ci-job-test-docs - test-docs: + # ci-job-full-datagen + full-datagen: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -282,15 +352,32 @@ jobs: - name: Show the selected Rust toolchain run: rustup show + # Job-specific dependencies + - name: Attempt to load download cache + uses: actions/cache@v3 + with: + key: download-cache + path: /tmp/icu4x-source-cache + - name: Install rustfmt + run: rustup component add rustfmt + # Actual job - - name: Run `cargo make ci-job-test-docs` - run: cargo make ci-job-test-docs + - name: Run `cargo make ci-job-full-datagen` + run: cargo make ci-job-full-datagen - # ci-job-full-datagen - full-datagen: + # ci-job-test-c + test-c: runs-on: ubuntu-latest + container: + image: rust:bookworm + steps: - uses: actions/checkout@v3 + + # Software setup/installation for the container + - name: Install Rust toolchains + run: | + rustup toolchain install stable # Cargo-make boilerplate - name: Get cargo-make version @@ -305,7 +392,8 @@ jobs: path: | ~/.cargo/bin/cargo-make ~/.cargo/bin/cargo-make.exe - key: ${{ runner.os }}-make-${{ steps.cargo-make-version.outputs.hash }} + # Since this is inside a container, use a different key than the other jobs + key: ${{ runner.os }}-make-ffi-${{ steps.cargo-make-version.outputs.hash }} - name: Install cargo-make if: steps.cargo-make-cache.outputs.cache-hit != 'true' run: cargo +stable install cargo-make @@ -318,20 +406,29 @@ jobs: run: rustup show # Job-specific dependencies + - name: Run apt-get update + run: | + apt-get update + - name: Install Clang-15 + run: | + apt-get install -y clang-15 lld-15 llvm-15-dev libc++-15-dev + - name: Install GCC 11 + run: | + apt-get install -y gcc-11 g++-11 - name: Attempt to load download cache uses: actions/cache@v3 with: key: download-cache path: /tmp/icu4x-source-cache - - name: Install rustfmt - run: rustup component add rustfmt # Actual job - - name: Run `cargo make ci-job-full-datagen` - run: cargo make ci-job-full-datagen + - name: Run `cargo make ci-job-test-c` + run: cargo make ci-job-test-c + env: + CXX: "g++-11" - # ci-job-ffi - ffi: + # ci-job-test-js + test-js: runs-on: ubuntu-latest container: image: rust:bookworm @@ -340,21 +437,6 @@ jobs: - uses: actions/checkout@v3 # Software setup/installation for the container - - name: Run apt-get update - run: | - apt-get update - - name: Install Clang-15 - run: | - apt-get install -y clang-15 lld-15 llvm-15-dev libc++-15-dev - - name: Install GCC 11 - run: | - apt-get install -y gcc-11 g++-11 - - name: Install Node.js v16.18.0 - uses: actions/setup-node@v3 - with: - node-version: 16.18.0 - cache: 'npm' - cache-dependency-path: '**/package-lock.json' - name: Install Rust toolchains run: | rustup toolchain install stable @@ -386,24 +468,65 @@ jobs: run: rustup show # Job-specific dependencies + - name: Install Node.js v16.18.0 + uses: actions/setup-node@v3 + with: + node-version: 16.18.0 + cache: 'npm' + cache-dependency-path: '**/package-lock.json' + - name: Run apt-get update + run: | + apt-get update + - name: Install lld-15 + run: | + apt-get install -y lld-15 - name: Attempt to load download cache uses: actions/cache@v3 with: key: download-cache path: /tmp/icu4x-source-cache - - name: Warm up datagen - # We do this so its compilation doesn't go on the time bugdet of the - # first test that uses it. - run: cargo build -p icu_datagen # Actual job - - name: Run `cargo make ci-job-ffi` - run: cargo make ci-job-ffi - env: - CXX: "g++-11" + - name: Run `cargo make ci-job-test-js` + run: cargo make ci-job-test-js + + # ci-job-nostd + nostd: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + # Cargo-make boilerplate + - name: Get cargo-make version + id: cargo-make-version + run: | + echo "hash=$(cargo search cargo-make | grep '^cargo-make =' | md5sum)" >> $GITHUB_OUTPUT + shell: bash + - name: Attempt to load cached cargo-make + uses: actions/cache@v3 + id: cargo-make-cache + with: + path: | + ~/.cargo/bin/cargo-make + ~/.cargo/bin/cargo-make.exe + key: ${{ runner.os }}-make-${{ steps.cargo-make-version.outputs.hash }} + - name: Install cargo-make + if: steps.cargo-make-cache.outputs.cache-hit != 'true' + run: cargo +stable install cargo-make + + + # Toolchain boilerplate + - name: Potentially override rust version with nightly + run: cargo make set-nightly-version-for-ci + - name: Show the selected Rust toolchain + run: rustup show + + # Actual job + - name: Run `cargo make ci-job-nostd` + run: cargo make ci-job-nostd - # ci-job-verify-ffi - verify-ffi: + # ci-job-diplomat + diplomat: runs-on: [ ubuntu-latest ] steps: - uses: actions/checkout@v3 @@ -460,10 +583,10 @@ jobs: run: cargo +stable install --git https://github.com/rust-diplomat/diplomat.git --version ${{ steps.diplomat-version.outputs.rev }} diplomat-tool # Actual job - - name: Run `cargo make ci-job-verify-ffi` - run: cargo make ci-job-verify-ffi + - name: Run `cargo make ci-job-diplomat` + run: cargo make ci-job-diplomat - # ci-job-verify-gn, ci-job-gn + # ci-job-gn gn: runs-on: ubuntu-latest @@ -528,63 +651,9 @@ jobs: version: latest # Actual job - - name: Run `cargo make ci-job-verify-gn` - run: cargo make ci-job-verify-gn - name: Run `cargo make ci-job-gn` run: cargo make ci-job-gn - # ci-job-wasm - wasm: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - # Cargo-make boilerplate - - name: Get cargo-make version - id: cargo-make-version - run: | - echo "hash=$(cargo search cargo-make | grep '^cargo-make =' | md5sum)" >> $GITHUB_OUTPUT - shell: bash - - name: Attempt to load cached cargo-make - uses: actions/cache@v3 - id: cargo-make-cache - with: - path: | - ~/.cargo/bin/cargo-make - ~/.cargo/bin/cargo-make.exe - key: ${{ runner.os }}-make-${{ steps.cargo-make-version.outputs.hash }} - - name: Install cargo-make - if: steps.cargo-make-cache.outputs.cache-hit != 'true' - run: cargo +stable install cargo-make - - - # Toolchain boilerplate - - name: Potentially override rust version with nightly - run: cargo make set-nightly-version-for-ci - - name: Show the selected Rust toolchain - run: rustup show - - # Job-specific dependencies - - name: Install emsdk - run: | - cd ~ - git clone https://github.com/emscripten-core/emsdk.git --branch 2.0.27 - cd emsdk - ./emsdk install latest - ./emsdk activate latest - - name: Install Node.js v16.18.0 - uses: actions/setup-node@v3 - with: - node-version: 16.18.0 - cache: 'npm' - cache-dependency-path: '**/package-lock.json' - # Actual job - - name: Run emscripten/wasm test - run: | - . ~/emsdk/emsdk_env.sh - cargo make ci-job-wasm - # ci-job-fmt fmt: runs-on: ubuntu-latest @@ -760,8 +829,7 @@ jobs: # Notify on slack notify-slack: - # Skipped tasks: bench-perf, bench-memory, bench-binsize, bench-datasize - needs: [msrv, test, testdata, test-docs, full-datagen, ffi, verify-ffi, gn, wasm, fmt, tidy, clippy, doc] + needs: [msrv, test, testdata, testdata-legacy, test-docs, full-datagen, test-c, test-js, nostd, diplomat, gn, fmt, tidy, clippy, doc] if: ${{ always() && contains(needs.*.result, 'failure') && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_name == 'main')) }} runs-on: ubuntu-latest steps: diff --git a/Makefile.toml b/Makefile.toml index d312fb24bde..8f59f0f5d79 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -98,6 +98,12 @@ description = "Run all tests for the CI 'testdata' job" category = "CI" dependencies = [ "testdata-check", +] + +[tasks.ci-job-testdata-legacy] +description = "Run all tests for the CI 'testdata' job" +category = "CI" +dependencies = [ "testdata-legacy", "testdata-legacy-test", ] @@ -116,32 +122,46 @@ dependencies = [ "bakeddata-check", ] -[tasks.ci-job-ffi] -description = "Run all tests for the CI 'ffi' job" +[tasks.ci-job-test-c] +description = "Run all tests for the CI 'test-c' job" category = "CI" dependencies = [ - "test-ffi", + "test-c", + "test-c-tiny", + "test-cpp", ] -[tasks.ci-job-verify-ffi] -description = "Run all tests for the CI 'verify-ffi' job" +[tasks.ci-job-test-js] +description = "Run all tests for the CI 'test-js' job" category = "CI" dependencies = [ - "verify-ffi", + "test-js", + "test-tinywasm", ] -[tasks.ci-job-gn] -description = "Run all tests for the CI 'gn' job" +[tasks.ci-job-nostd] +description = "Run all tests for the CI 'test-nostd' job" category = "CI" dependencies = [ - "gn-run", + "check-nostd", + "check-freertos-wearos", +] + + +[tasks.ci-job-diplomat] +description = "Run all tests for the CI 'diplomat' job" +category = "CI" +dependencies = [ + "verify-diplomat-gen", + "verify-diplomat-coverage", ] -[tasks.ci-job-verify-gn] -description = "Run all tests for the CI 'verify-gn' job" +[tasks.ci-job-gn] +description = "Run all tests for the CI 'gn' job" category = "CI" dependencies = [ "verify-gn-gen", + "gn-run", ] [tasks.ci-job-msrv-features-1] @@ -188,13 +208,6 @@ dependencies = [ "depcheck", ] -[tasks.ci-job-wasm] -description = "Run all tests for the CI 'wasm' job" -category = "CI" -dependencies = [ - "test-cpp-emscripten", -] - [tasks.ci-job-clippy] description = "Run all tests for the CI 'clippy' job" category = "CI" @@ -224,15 +237,16 @@ dependencies = [ # Get a coffee "ci-job-test", + "ci-job-nostd", "ci-job-test-docs", "ci-job-testdata", "ci-job-msrv-features", "ci-job-full-datagen", # Needs tools other than Cargo to be installed - "ci-job-ffi", - "ci-job-wasm", - "ci-job-verify-ffi", + "ci-job-diplomat", + "ci-job-test-c", + "ci-job-test-js", "ci-job-gn", # benchmarking and coverage jobs not included diff --git a/components/plurals/src/lib.rs b/components/plurals/src/lib.rs index 1552297d30b..4c050e30510 100644 --- a/components/plurals/src/lib.rs +++ b/components/plurals/src/lib.rs @@ -88,6 +88,8 @@ pub use operands::PluralOperands; use provider::CardinalV1Marker; use provider::ErasedPluralRulesV1Marker; use provider::OrdinalV1Marker; +use provider::PluralRangesV1Marker; +use provider::UnvalidatedPluralRange; use rules::runtime::test_rule; #[doc(no_inline)] @@ -550,3 +552,313 @@ impl PluralRules { .chain(Some(PluralCategory::Other)) } } + +/// A [`PluralRules`] that also has the ability to retrieve an appropriate [`Plural Category`] for a +/// range. +/// +/// # Examples +/// +/// ``` +/// use icu::locid::locale; +/// use icu::plurals::{PluralCategory, PluralOperands}; +/// use icu::plurals::{PluralRuleType, PluralRulesWithRanges}; +/// use std::convert::TryFrom; +/// +/// let ranges = PluralRulesWithRanges::try_new( +/// &locale!("ar").into(), +/// PluralRuleType::Cardinal, +/// ) +/// .expect("locale should be present"); +/// +/// let operands = PluralOperands::from(1_usize); +/// let operands2: PluralOperands = +/// "2.0".parse().expect("parsing to operands should succeed"); +/// +/// assert_eq!( +/// ranges.category_for_range(operands, operands2), +/// PluralCategory::Other +/// ); +/// ``` +/// +/// [`Plural Category`]: PluralCategory +#[derive(Debug)] +pub struct PluralRulesWithRanges { + rules: PluralRules, + ranges: DataPayload, +} + +impl PluralRulesWithRanges { + icu_provider::gen_any_buffer_data_constructors!( + locale: include, + rule_type: PluralRuleType, + error: PluralsError, + /// Constructs a new `PluralRulesWithRanges` for a given locale using compiled data. + /// + /// ✨ *Enabled with the `compiled_data` Cargo feature.* + /// + /// [📚 Help choosing a constructor](icu_provider::constructors) + /// + /// # Examples + /// + /// ``` + /// use icu::locid::locale; + /// use icu::plurals::{PluralRuleType, PluralRulesWithRanges}; + /// + /// let _ = PluralRulesWithRanges::try_new( + /// &locale!("en").into(), + /// PluralRuleType::Cardinal, + /// ).expect("locale should be present"); + /// ``` + ); + + #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::try_new)] + pub fn try_new_unstable( + provider: &(impl DataProvider + + DataProvider + + DataProvider + + ?Sized), + locale: &DataLocale, + rule_type: PluralRuleType, + ) -> Result { + match rule_type { + PluralRuleType::Cardinal => Self::try_new_cardinal_unstable(provider, locale), + PluralRuleType::Ordinal => Self::try_new_ordinal_unstable(provider, locale), + } + } + + icu_provider::gen_any_buffer_data_constructors!( + locale: include, + options: skip, + error: PluralsError, + /// Constructs a new `PluralRulesWithRanges` for a given locale for cardinal numbers using + /// compiled data. + /// + /// See [`PluralRules::try_new_cardinal`] for more information. + /// + /// ✨ *Enabled with the `compiled_data` Cargo feature.* + /// + /// [📚 Help choosing a constructor](icu_provider::constructors) + /// + /// # Examples + /// + /// ``` + /// use icu::locid::locale; + /// use icu::plurals::{PluralCategory, PluralRulesWithRanges}; + /// + /// let rules = PluralRulesWithRanges::try_new_cardinal(&locale!("ru").into()) + /// .expect("locale should be present"); + /// + /// assert_eq!(rules.category_for_range(0_usize, 2_usize), PluralCategory::Few); + /// ``` + functions: [ + try_new_cardinal, + try_new_cardinal_with_any_provider, + try_new_cardinal_with_buffer_provider, + try_new_cardinal_unstable, + Self, + ] + ); + + #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::try_new_cardinal)] + pub fn try_new_cardinal_unstable( + provider: &(impl DataProvider + DataProvider + ?Sized), + locale: &DataLocale, + ) -> Result { + let rules = PluralRules::try_new_cardinal_unstable(provider, locale)?; + let ranges = provider + .load(DataRequest { + locale, + metadata: Default::default(), + })? + .take_payload()?; + + Ok(Self { rules, ranges }) + } + + icu_provider::gen_any_buffer_data_constructors!( + locale: include, + options: skip, + error: PluralsError, + /// Constructs a new `PluralRulesWithRanges` for a given locale for ordinal numbers using + /// compiled data. + /// + /// See [`PluralRules::try_new_ordinal`] for more information. + /// + /// ✨ *Enabled with the `compiled_data` Cargo feature.* + /// + /// [📚 Help choosing a constructor](icu_provider::constructors) + /// + /// # Examples + /// + /// ``` + /// use icu::locid::locale; + /// use icu::plurals::{PluralCategory, PluralRulesWithRanges}; + /// + /// let rules = PluralRulesWithRanges::try_new_ordinal( + /// &locale!("ru").into(), + /// ) + /// .expect("locale should be present"); + /// + /// assert_eq!(rules.category_for_range(0_usize, 2_usize), PluralCategory::Other); + /// ``` + functions: [ + try_new_ordinal, + try_new_ordinal_with_any_provider, + try_new_ordinal_with_buffer_provider, + try_new_ordinal_unstable, + Self, + ] + ); + + #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::try_new_ordinal)] + pub fn try_new_ordinal_unstable( + provider: &(impl DataProvider + DataProvider + ?Sized), + locale: &DataLocale, + ) -> Result { + let rules = PluralRules::try_new_ordinal_unstable(provider, locale)?; + let ranges = provider + .load(DataRequest { + locale, + metadata: Default::default(), + })? + .take_payload()?; + + Ok(Self { rules, ranges }) + } + + /// Returns the [`Plural Category`] appropriate for the given number. + /// + /// See [`PluralRules::category_for`] for more information. + /// + /// # Examples + /// + /// ``` + /// use icu::locid::locale; + /// use icu::plurals::{PluralCategory, PluralRuleType, PluralRulesWithRanges}; + /// + /// let pr = PluralRulesWithRanges::try_new( + /// &locale!("en").into(), + /// PluralRuleType::Cardinal, + /// ) + /// .expect("locale should be present"); + /// + /// match pr.category_for(1_usize) { + /// PluralCategory::One => "One item", + /// PluralCategory::Other => "Many items", + /// _ => unreachable!(), + /// }; + /// ``` + /// + /// [`Plural Category`]: PluralCategory + pub fn category_for>(&self, input: I) -> PluralCategory { + self.rules.category_for(input) + } + + /// Returns all [`Plural Categories`] appropriate for a [`PluralRulesWithRanges`] object + /// based on the [`LanguageIdentifier`](icu::locid::{LanguageIdentifier}) and [`PluralRuleType`]. + /// + /// See [`PluralRules::categories`] for more information. + /// + /// # Examples + /// + /// ``` + /// use icu::locid::locale; + /// use icu::plurals::{PluralCategory, PluralRuleType, PluralRulesWithRanges}; + /// + /// let pr = PluralRulesWithRanges::try_new( + /// &locale!("es").into(), + /// PluralRuleType::Cardinal, + /// ) + /// .expect("locale should be present"); + /// + /// let mut categories = pr.categories(); + /// assert_eq!(categories.next(), Some(PluralCategory::One)); + /// assert_eq!(categories.next(), Some(PluralCategory::Many)); + /// assert_eq!(categories.next(), Some(PluralCategory::Other)); + /// assert_eq!(categories.next(), None); + /// ``` + /// + /// [`Plural Categories`]: PluralCategory + pub fn categories(&self) -> impl Iterator + '_ { + self.rules.categories() + } + + /// Returns the [`Plural Category`] appropriate for a range. + /// + /// Note that the returned category is correct only if the range fulfills the following requirements: + /// - The start value is strictly less than the end value. + /// - Both values are positive. + /// + /// # Examples + /// + /// ``` + /// use icu::locid::locale; + /// use icu::plurals::{ + /// PluralCategory, PluralOperands, PluralRuleType, PluralRulesWithRanges, + /// }; + /// + /// let ranges = PluralRulesWithRanges::try_new( + /// &locale!("ro").into(), + /// PluralRuleType::Cardinal, + /// ) + /// .expect("locale should be present"); + /// let operands: PluralOperands = + /// "1.5".parse().expect("parsing to operands should succeed"); + /// let operands2 = PluralOperands::from(1_usize); + /// + /// assert_eq!( + /// ranges.category_for_range(operands, operands2), + /// PluralCategory::Few + /// ); + /// ``` + /// + /// [`Plural Category`]: PluralCategory + pub fn category_for_range, E: Into>( + &self, + start: S, + end: E, + ) -> PluralCategory { + let start = self.rules.category_for(start); + let end = self.rules.category_for(end); + + self.resolve_range(start, end) + } + + /// Returns the [`Plural Category`] appropriate for a range from the categories of its endpoints. + /// + /// Note that the returned category is correct only if the original numeric range fulfills the + /// following requirements: + /// - The start value is strictly less than the end value. + /// - Both values are positive. + /// + /// # Examples + /// + /// ``` + /// use icu::locid::locale; + /// use icu::plurals::{PluralCategory, PluralRuleType, PluralRulesWithRanges}; + /// + /// let ranges = PluralRulesWithRanges::try_new( + /// &locale!("sl").into(), + /// PluralRuleType::Ordinal, + /// ) + /// .expect("locale should be present"); + /// + /// assert_eq!( + /// ranges.resolve_range(PluralCategory::Other, PluralCategory::One), + /// PluralCategory::Few + /// ); + /// ``` + /// + /// [`Plural Category`]: PluralCategory + pub fn resolve_range(&self, start: PluralCategory, end: PluralCategory) -> PluralCategory { + self.ranges + .get() + .ranges + .get_copied(&UnvalidatedPluralRange::from_range( + start.into(), + end.into(), + )) + .map(PluralCategory::from) + .unwrap_or(end) + } +} diff --git a/components/plurals/src/provider.rs b/components/plurals/src/provider.rs index 6ff2340ae33..4643785feab 100644 --- a/components/plurals/src/provider.rs +++ b/components/plurals/src/provider.rs @@ -147,6 +147,19 @@ impl RawPluralCategory { } } +impl From for PluralCategory { + fn from(value: RawPluralCategory) -> Self { + match value { + RawPluralCategory::Other => PluralCategory::Other, + RawPluralCategory::Zero => PluralCategory::Zero, + RawPluralCategory::One => PluralCategory::One, + RawPluralCategory::Two => PluralCategory::Two, + RawPluralCategory::Few => PluralCategory::Few, + RawPluralCategory::Many => PluralCategory::Many, + } + } +} + impl From for RawPluralCategory { fn from(value: PluralCategory) -> Self { match value { diff --git a/components/plurals/tests/ranges.rs b/components/plurals/tests/ranges.rs new file mode 100644 index 00000000000..e374eae46b8 --- /dev/null +++ b/components/plurals/tests/ranges.rs @@ -0,0 +1,46 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +use icu_locid::locale; +use icu_plurals::{PluralCategory, PluralOperands, PluralRuleType, PluralRulesWithRanges}; + +#[test] +fn test_plural_ranges_raw() { + assert_eq!( + PluralRulesWithRanges::try_new_cardinal(&locale!("he").into()) + .unwrap() + .resolve_range(PluralCategory::One, PluralCategory::Two), + PluralCategory::Other + ); +} + +#[test] +fn test_plural_ranges_optimized_data() { + assert_eq!( + PluralRulesWithRanges::try_new_ordinal(&locale!("en").into()) + .unwrap() + .resolve_range(PluralCategory::One, PluralCategory::Other), + PluralCategory::Other + ); +} + +#[test] +fn test_plural_ranges_missing_data_fallback() { + assert_eq!( + PluralRulesWithRanges::try_new_cardinal(&locale!("nl").into()) + .unwrap() + .resolve_range(PluralCategory::Two, PluralCategory::Many), + PluralCategory::Many + ); +} + +#[test] +fn test_plural_ranges_full() { + let ranges = + PluralRulesWithRanges::try_new(&locale!("sl").into(), PluralRuleType::Cardinal).unwrap(); + let start: PluralOperands = "0.5".parse().unwrap(); // PluralCategory::Other + let end: PluralOperands = PluralOperands::try_from(1).unwrap(); // PluralCategory::One + + assert_eq!(ranges.category_for_range(start, end), PluralCategory::Few) +} diff --git a/docs/tutorials/Cargo.lock b/docs/tutorials/Cargo.lock index 4a1c4f47079..03742b89f0f 100644 --- a/docs/tutorials/Cargo.lock +++ b/docs/tutorials/Cargo.lock @@ -399,7 +399,6 @@ dependencies = [ "databake-derive", "proc-macro2", "quote", - "syn 2.0.37", ] [[package]] @@ -705,7 +704,7 @@ dependencies = [ [[package]] name = "icu" -version = "1.3.0" +version = "1.3.2" dependencies = [ "icu_calendar", "icu_casemap", @@ -730,7 +729,7 @@ dependencies = [ [[package]] name = "icu_calendar" -version = "1.3.0" +version = "1.3.2" dependencies = [ "calendrical_calculations", "databake", @@ -747,11 +746,11 @@ dependencies = [ [[package]] name = "icu_calendar_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_casemap" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -767,11 +766,11 @@ dependencies = [ [[package]] name = "icu_casemap_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_codepointtrie_builder" -version = "0.3.5" +version = "0.3.6" dependencies = [ "icu_collections", "once_cell", @@ -782,7 +781,7 @@ dependencies = [ [[package]] name = "icu_collator" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -802,11 +801,11 @@ dependencies = [ [[package]] name = "icu_collator_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_collections" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -818,7 +817,7 @@ dependencies = [ [[package]] name = "icu_compactdecimal" -version = "0.2.1" +version = "0.2.2" dependencies = [ "displaydoc", "fixed_decimal", @@ -833,11 +832,11 @@ dependencies = [ [[package]] name = "icu_compactdecimal_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_datagen" -version = "1.3.0" +version = "1.3.2" dependencies = [ "clap", "crlify", @@ -869,12 +868,12 @@ dependencies = [ "memchr", "ndarray", "once_cell", + "proc-macro2", "rayon", "serde", "serde-aux", "serde_json", "simple_logger", - "syn 2.0.37", "tinystr", "toml", "twox-hash", @@ -887,7 +886,7 @@ dependencies = [ [[package]] name = "icu_datetime" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -911,11 +910,11 @@ dependencies = [ [[package]] name = "icu_datetime_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_decimal" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -930,11 +929,11 @@ dependencies = [ [[package]] name = "icu_decimal_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_displaynames" -version = "0.11.0" +version = "0.11.1" dependencies = [ "icu_displaynames_data", "icu_locid", @@ -947,11 +946,11 @@ dependencies = [ [[package]] name = "icu_displaynames_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_list" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "deduplicating_array", @@ -966,11 +965,11 @@ dependencies = [ [[package]] name = "icu_list_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_locid" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -983,7 +982,7 @@ dependencies = [ [[package]] name = "icu_locid_transform" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -997,11 +996,11 @@ dependencies = [ [[package]] name = "icu_locid_transform_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_normalizer" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -1019,11 +1018,11 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_plurals" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -1038,11 +1037,11 @@ dependencies = [ [[package]] name = "icu_plurals_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_properties" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -1057,11 +1056,11 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_provider" -version = "1.3.0" +version = "1.3.2" dependencies = [ "bincode", "databake", @@ -1083,7 +1082,7 @@ dependencies = [ [[package]] name = "icu_provider_adapters" -version = "1.3.0" +version = "1.3.2" dependencies = [ "icu_locid", "icu_locid_transform", @@ -1095,7 +1094,7 @@ dependencies = [ [[package]] name = "icu_provider_blob" -version = "1.3.0" +version = "1.3.2" dependencies = [ "icu_provider", "log", @@ -1107,7 +1106,7 @@ dependencies = [ [[package]] name = "icu_provider_fs" -version = "1.3.0" +version = "1.3.2" dependencies = [ "bincode", "crlify", @@ -1122,7 +1121,7 @@ dependencies = [ [[package]] name = "icu_provider_macros" -version = "1.3.0" +version = "1.3.2" dependencies = [ "proc-macro2", "quote", @@ -1131,7 +1130,7 @@ dependencies = [ [[package]] name = "icu_relativetime" -version = "0.1.2" +version = "0.1.3" dependencies = [ "displaydoc", "fixed_decimal", @@ -1147,11 +1146,11 @@ dependencies = [ [[package]] name = "icu_relativetime_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_segmenter" -version = "1.3.0" +version = "1.3.2" dependencies = [ "core_maths", "databake", @@ -1167,11 +1166,11 @@ dependencies = [ [[package]] name = "icu_segmenter_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_timezone" -version = "1.3.0" +version = "1.3.2" dependencies = [ "databake", "displaydoc", @@ -1187,11 +1186,11 @@ dependencies = [ [[package]] name = "icu_timezone_data" -version = "1.3.0" +version = "1.3.2" [[package]] name = "icu_transliterate" -version = "0.1.0" +version = "0.1.1" dependencies = [ "displaydoc", "icu_collections", @@ -1207,7 +1206,7 @@ dependencies = [ [[package]] name = "icu_unicodeset_parse" -version = "0.1.0" +version = "0.1.1" dependencies = [ "icu_collections", "icu_properties", @@ -2122,7 +2121,7 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.3" +version = "0.7.4" dependencies = [ "databake", "displaydoc", @@ -2976,7 +2975,7 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.1.0" +version = "0.1.1" dependencies = [ "databake", "displaydoc", @@ -3025,7 +3024,7 @@ version = "0.2.1" [[patch.unused]] name = "icu4x_ecma402" -version = "0.8.0" +version = "0.8.1" [[patch.unused]] name = "icu_benchmark_macros" @@ -3033,23 +3032,23 @@ version = "0.0.0" [[patch.unused]] name = "icu_capi" -version = "1.3.0" +version = "1.3.2" [[patch.unused]] name = "icu_capi_cdylib" -version = "1.3.0" +version = "1.3.2" [[patch.unused]] name = "icu_capi_staticlib" -version = "1.3.0" +version = "1.3.2" [[patch.unused]] name = "icu_freertos" -version = "1.3.0" +version = "1.3.2" [[patch.unused]] name = "icu_harfbuzz" -version = "0.1.0" +version = "0.1.1" [[patch.unused]] name = "icu_pattern" @@ -3065,7 +3064,7 @@ version = "0.0.0" [[patch.unused]] name = "icu_singlenumberformatter_data" -version = "1.3.0" +version = "1.3.2" [[patch.unused]] name = "icu_unitsconversion" diff --git a/ffi/capi/c/include/ICU4XPluralRulesWithRanges.h b/ffi/capi/c/include/ICU4XPluralRulesWithRanges.h new file mode 100644 index 00000000000..a8f47178859 --- /dev/null +++ b/ffi/capi/c/include/ICU4XPluralRulesWithRanges.h @@ -0,0 +1,45 @@ +#ifndef ICU4XPluralRulesWithRanges_H +#define ICU4XPluralRulesWithRanges_H +#include +#include +#include +#include +#include "diplomat_runtime.h" + +#ifdef __cplusplus +namespace capi { +#endif + +typedef struct ICU4XPluralRulesWithRanges ICU4XPluralRulesWithRanges; +#ifdef __cplusplus +} // namespace capi +#endif +#include "ICU4XDataProvider.h" +#include "ICU4XLocale.h" +#include "diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError.h" +#include "ICU4XPluralOperands.h" +#include "ICU4XPluralCategory.h" +#include "ICU4XPluralCategories.h" +#ifdef __cplusplus +namespace capi { +extern "C" { +#endif + +diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError ICU4XPluralRulesWithRanges_create_cardinal(const ICU4XDataProvider* provider, const ICU4XLocale* locale); + +diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError ICU4XPluralRulesWithRanges_create_ordinal(const ICU4XDataProvider* provider, const ICU4XLocale* locale); + +ICU4XPluralCategory ICU4XPluralRulesWithRanges_category_for(const ICU4XPluralRulesWithRanges* self, const ICU4XPluralOperands* op); + +ICU4XPluralCategories ICU4XPluralRulesWithRanges_categories(const ICU4XPluralRulesWithRanges* self); + +ICU4XPluralCategory ICU4XPluralRulesWithRanges_category_for_range(const ICU4XPluralRulesWithRanges* self, const ICU4XPluralOperands* start, const ICU4XPluralOperands* end); + +ICU4XPluralCategory ICU4XPluralRulesWithRanges_resolve_range(const ICU4XPluralRulesWithRanges* self, ICU4XPluralCategory start, ICU4XPluralCategory end); +void ICU4XPluralRulesWithRanges_destroy(ICU4XPluralRulesWithRanges* self); + +#ifdef __cplusplus +} // extern "C" +} // namespace capi +#endif +#endif diff --git a/ffi/capi/c/include/diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError.h b/ffi/capi/c/include/diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError.h new file mode 100644 index 00000000000..3a68cf517e3 --- /dev/null +++ b/ffi/capi/c/include/diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError.h @@ -0,0 +1,26 @@ +#ifndef diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError_H +#define diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError_H +#include +#include +#include +#include +#include "diplomat_runtime.h" + +typedef struct ICU4XPluralRulesWithRanges ICU4XPluralRulesWithRanges; +#include "ICU4XError.h" +#ifdef __cplusplus +namespace capi { +extern "C" { +#endif +typedef struct diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError { + union { + ICU4XPluralRulesWithRanges* ok; + ICU4XError err; + }; + bool is_ok; +} diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError; +#ifdef __cplusplus +} // extern "C" +} // namespace capi +#endif +#endif diff --git a/ffi/capi/cpp/docs/source/pluralrules_ffi.rst b/ffi/capi/cpp/docs/source/pluralrules_ffi.rst index 80a01d258a0..4ac7d7b7dd9 100644 --- a/ffi/capi/cpp/docs/source/pluralrules_ffi.rst +++ b/ffi/capi/cpp/docs/source/pluralrules_ffi.rst @@ -94,3 +94,52 @@ See the `Rust documentation for categories `__ for more information. + +.. cpp:class:: ICU4XPluralRulesWithRanges + + FFI version of ``PluralRulesWithRanges``. + + See the `Rust documentation for PluralRulesWithRanges `__ for more information. + + + .. cpp:function:: static diplomat::result create_cardinal(const ICU4XDataProvider& provider, const ICU4XLocale& locale) + + Construct an :cpp:class:`ICU4XPluralRulesWithRanges` for the given locale, for cardinal numbers + + See the `Rust documentation for try_new_cardinal `__ for more information. + + + .. cpp:function:: static diplomat::result create_ordinal(const ICU4XDataProvider& provider, const ICU4XLocale& locale) + + Construct an :cpp:class:`ICU4XPluralRulesWithRanges` for the given locale, for ordinal numbers + + See the `Rust documentation for try_new_ordinal `__ for more information. + + + .. cpp:function:: ICU4XPluralCategory category_for(const ICU4XPluralOperands& op) const + + Get the category for a given number represented as operands + + See the `Rust documentation for category_for `__ for more information. + + + .. cpp:function:: ICU4XPluralCategories categories() const + + Get all of the categories needed in the current locale + + See the `Rust documentation for categories `__ for more information. + + + .. cpp:function:: ICU4XPluralCategory category_for_range(const ICU4XPluralOperands& start, const ICU4XPluralOperands& end) const + + Get the appropriate category for a numeric range. + + See the `Rust documentation for category_for_range `__ for more information. + + + .. cpp:function:: ICU4XPluralCategory resolve_range(ICU4XPluralCategory start, ICU4XPluralCategory end) const + + Get the appropriate category for a numeric range from the categories of its endpoints. + + See the `Rust documentation for resolve_range `__ for more information. + diff --git a/ffi/capi/cpp/include/ICU4XPluralRulesWithRanges.h b/ffi/capi/cpp/include/ICU4XPluralRulesWithRanges.h new file mode 100644 index 00000000000..a8f47178859 --- /dev/null +++ b/ffi/capi/cpp/include/ICU4XPluralRulesWithRanges.h @@ -0,0 +1,45 @@ +#ifndef ICU4XPluralRulesWithRanges_H +#define ICU4XPluralRulesWithRanges_H +#include +#include +#include +#include +#include "diplomat_runtime.h" + +#ifdef __cplusplus +namespace capi { +#endif + +typedef struct ICU4XPluralRulesWithRanges ICU4XPluralRulesWithRanges; +#ifdef __cplusplus +} // namespace capi +#endif +#include "ICU4XDataProvider.h" +#include "ICU4XLocale.h" +#include "diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError.h" +#include "ICU4XPluralOperands.h" +#include "ICU4XPluralCategory.h" +#include "ICU4XPluralCategories.h" +#ifdef __cplusplus +namespace capi { +extern "C" { +#endif + +diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError ICU4XPluralRulesWithRanges_create_cardinal(const ICU4XDataProvider* provider, const ICU4XLocale* locale); + +diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError ICU4XPluralRulesWithRanges_create_ordinal(const ICU4XDataProvider* provider, const ICU4XLocale* locale); + +ICU4XPluralCategory ICU4XPluralRulesWithRanges_category_for(const ICU4XPluralRulesWithRanges* self, const ICU4XPluralOperands* op); + +ICU4XPluralCategories ICU4XPluralRulesWithRanges_categories(const ICU4XPluralRulesWithRanges* self); + +ICU4XPluralCategory ICU4XPluralRulesWithRanges_category_for_range(const ICU4XPluralRulesWithRanges* self, const ICU4XPluralOperands* start, const ICU4XPluralOperands* end); + +ICU4XPluralCategory ICU4XPluralRulesWithRanges_resolve_range(const ICU4XPluralRulesWithRanges* self, ICU4XPluralCategory start, ICU4XPluralCategory end); +void ICU4XPluralRulesWithRanges_destroy(ICU4XPluralRulesWithRanges* self); + +#ifdef __cplusplus +} // extern "C" +} // namespace capi +#endif +#endif diff --git a/ffi/capi/cpp/include/ICU4XPluralRulesWithRanges.hpp b/ffi/capi/cpp/include/ICU4XPluralRulesWithRanges.hpp new file mode 100644 index 00000000000..25276015689 --- /dev/null +++ b/ffi/capi/cpp/include/ICU4XPluralRulesWithRanges.hpp @@ -0,0 +1,128 @@ +#ifndef ICU4XPluralRulesWithRanges_HPP +#define ICU4XPluralRulesWithRanges_HPP +#include +#include +#include +#include +#include +#include +#include +#include "diplomat_runtime.hpp" + +#include "ICU4XPluralRulesWithRanges.h" + +class ICU4XDataProvider; +class ICU4XLocale; +class ICU4XPluralRulesWithRanges; +#include "ICU4XError.hpp" +class ICU4XPluralOperands; +#include "ICU4XPluralCategory.hpp" +struct ICU4XPluralCategories; + +/** + * A destruction policy for using ICU4XPluralRulesWithRanges with std::unique_ptr. + */ +struct ICU4XPluralRulesWithRangesDeleter { + void operator()(capi::ICU4XPluralRulesWithRanges* l) const noexcept { + capi::ICU4XPluralRulesWithRanges_destroy(l); + } +}; + +/** + * FFI version of `PluralRulesWithRanges`. + * + * See the [Rust documentation for `PluralRulesWithRanges`](https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html) for more information. + */ +class ICU4XPluralRulesWithRanges { + public: + + /** + * Construct an [`ICU4XPluralRulesWithRanges`] for the given locale, for cardinal numbers + * + * See the [Rust documentation for `try_new_cardinal`](https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.try_new_cardinal) for more information. + */ + static diplomat::result create_cardinal(const ICU4XDataProvider& provider, const ICU4XLocale& locale); + + /** + * Construct an [`ICU4XPluralRulesWithRanges`] for the given locale, for ordinal numbers + * + * See the [Rust documentation for `try_new_ordinal`](https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.try_new_ordinal) for more information. + */ + static diplomat::result create_ordinal(const ICU4XDataProvider& provider, const ICU4XLocale& locale); + + /** + * Get the category for a given number represented as operands + * + * See the [Rust documentation for `category_for`](https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.category_for) for more information. + */ + ICU4XPluralCategory category_for(const ICU4XPluralOperands& op) const; + + /** + * Get all of the categories needed in the current locale + * + * See the [Rust documentation for `categories`](https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.categories) for more information. + */ + ICU4XPluralCategories categories() const; + + /** + * Get the appropriate category for a numeric range. + * + * See the [Rust documentation for `category_for_range`](https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.category_for_range) for more information. + */ + ICU4XPluralCategory category_for_range(const ICU4XPluralOperands& start, const ICU4XPluralOperands& end) const; + + /** + * Get the appropriate category for a numeric range from the categories of its endpoints. + * + * See the [Rust documentation for `resolve_range`](https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.resolve_range) for more information. + */ + ICU4XPluralCategory resolve_range(ICU4XPluralCategory start, ICU4XPluralCategory end) const; + inline const capi::ICU4XPluralRulesWithRanges* AsFFI() const { return this->inner.get(); } + inline capi::ICU4XPluralRulesWithRanges* AsFFIMut() { return this->inner.get(); } + inline ICU4XPluralRulesWithRanges(capi::ICU4XPluralRulesWithRanges* i) : inner(i) {} + ICU4XPluralRulesWithRanges() = default; + ICU4XPluralRulesWithRanges(ICU4XPluralRulesWithRanges&&) noexcept = default; + ICU4XPluralRulesWithRanges& operator=(ICU4XPluralRulesWithRanges&& other) noexcept = default; + private: + std::unique_ptr inner; +}; + +#include "ICU4XDataProvider.hpp" +#include "ICU4XLocale.hpp" +#include "ICU4XPluralOperands.hpp" +#include "ICU4XPluralCategories.hpp" + +inline diplomat::result ICU4XPluralRulesWithRanges::create_cardinal(const ICU4XDataProvider& provider, const ICU4XLocale& locale) { + auto diplomat_result_raw_out_value = capi::ICU4XPluralRulesWithRanges_create_cardinal(provider.AsFFI(), locale.AsFFI()); + diplomat::result diplomat_result_out_value; + if (diplomat_result_raw_out_value.is_ok) { + diplomat_result_out_value = diplomat::Ok(ICU4XPluralRulesWithRanges(diplomat_result_raw_out_value.ok)); + } else { + diplomat_result_out_value = diplomat::Err(static_cast(diplomat_result_raw_out_value.err)); + } + return diplomat_result_out_value; +} +inline diplomat::result ICU4XPluralRulesWithRanges::create_ordinal(const ICU4XDataProvider& provider, const ICU4XLocale& locale) { + auto diplomat_result_raw_out_value = capi::ICU4XPluralRulesWithRanges_create_ordinal(provider.AsFFI(), locale.AsFFI()); + diplomat::result diplomat_result_out_value; + if (diplomat_result_raw_out_value.is_ok) { + diplomat_result_out_value = diplomat::Ok(ICU4XPluralRulesWithRanges(diplomat_result_raw_out_value.ok)); + } else { + diplomat_result_out_value = diplomat::Err(static_cast(diplomat_result_raw_out_value.err)); + } + return diplomat_result_out_value; +} +inline ICU4XPluralCategory ICU4XPluralRulesWithRanges::category_for(const ICU4XPluralOperands& op) const { + return static_cast(capi::ICU4XPluralRulesWithRanges_category_for(this->inner.get(), op.AsFFI())); +} +inline ICU4XPluralCategories ICU4XPluralRulesWithRanges::categories() const { + capi::ICU4XPluralCategories diplomat_raw_struct_out_value = capi::ICU4XPluralRulesWithRanges_categories(this->inner.get()); + return ICU4XPluralCategories{ .zero = std::move(diplomat_raw_struct_out_value.zero), .one = std::move(diplomat_raw_struct_out_value.one), .two = std::move(diplomat_raw_struct_out_value.two), .few = std::move(diplomat_raw_struct_out_value.few), .many = std::move(diplomat_raw_struct_out_value.many), .other = std::move(diplomat_raw_struct_out_value.other) }; +} +inline ICU4XPluralCategory ICU4XPluralRulesWithRanges::category_for_range(const ICU4XPluralOperands& start, const ICU4XPluralOperands& end) const { + return static_cast(capi::ICU4XPluralRulesWithRanges_category_for_range(this->inner.get(), start.AsFFI(), end.AsFFI())); +} +inline ICU4XPluralCategory ICU4XPluralRulesWithRanges::resolve_range(ICU4XPluralCategory start, ICU4XPluralCategory end) const { + return static_cast(capi::ICU4XPluralRulesWithRanges_resolve_range(this->inner.get(), static_cast(start), static_cast(end))); +} +#endif diff --git a/ffi/capi/cpp/include/diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError.h b/ffi/capi/cpp/include/diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError.h new file mode 100644 index 00000000000..3a68cf517e3 --- /dev/null +++ b/ffi/capi/cpp/include/diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError.h @@ -0,0 +1,26 @@ +#ifndef diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError_H +#define diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError_H +#include +#include +#include +#include +#include "diplomat_runtime.h" + +typedef struct ICU4XPluralRulesWithRanges ICU4XPluralRulesWithRanges; +#include "ICU4XError.h" +#ifdef __cplusplus +namespace capi { +extern "C" { +#endif +typedef struct diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError { + union { + ICU4XPluralRulesWithRanges* ok; + ICU4XError err; + }; + bool is_ok; +} diplomat_result_box_ICU4XPluralRulesWithRanges_ICU4XError; +#ifdef __cplusplus +} // extern "C" +} // namespace capi +#endif +#endif diff --git a/ffi/capi/js/docs/source/pluralrules_ffi.rst b/ffi/capi/js/docs/source/pluralrules_ffi.rst index ca29ec47e07..5d4880d98c3 100644 --- a/ffi/capi/js/docs/source/pluralrules_ffi.rst +++ b/ffi/capi/js/docs/source/pluralrules_ffi.rst @@ -82,3 +82,52 @@ See the `Rust documentation for categories `__ for more information. + +.. js:class:: ICU4XPluralRulesWithRanges + + FFI version of ``PluralRulesWithRanges``. + + See the `Rust documentation for PluralRulesWithRanges `__ for more information. + + + .. js:function:: create_cardinal(provider, locale) + + Construct an :js:class:`ICU4XPluralRulesWithRanges` for the given locale, for cardinal numbers + + See the `Rust documentation for try_new_cardinal `__ for more information. + + + .. js:function:: create_ordinal(provider, locale) + + Construct an :js:class:`ICU4XPluralRulesWithRanges` for the given locale, for ordinal numbers + + See the `Rust documentation for try_new_ordinal `__ for more information. + + + .. js:method:: category_for(op) + + Get the category for a given number represented as operands + + See the `Rust documentation for category_for `__ for more information. + + + .. js:method:: categories() + + Get all of the categories needed in the current locale + + See the `Rust documentation for categories `__ for more information. + + + .. js:method:: category_for_range(start, end) + + Get the appropriate category for a numeric range. + + See the `Rust documentation for category_for_range `__ for more information. + + + .. js:method:: resolve_range(start, end) + + Get the appropriate category for a numeric range from the categories of its endpoints. + + See the `Rust documentation for resolve_range `__ for more information. + diff --git a/ffi/capi/js/include/ICU4XPluralRulesWithRanges.d.ts b/ffi/capi/js/include/ICU4XPluralRulesWithRanges.d.ts new file mode 100644 index 00000000000..00ee34dbc5c --- /dev/null +++ b/ffi/capi/js/include/ICU4XPluralRulesWithRanges.d.ts @@ -0,0 +1,66 @@ +import { FFIError } from "./diplomat-runtime" +import { ICU4XDataProvider } from "./ICU4XDataProvider"; +import { ICU4XError } from "./ICU4XError"; +import { ICU4XLocale } from "./ICU4XLocale"; +import { ICU4XPluralCategories } from "./ICU4XPluralCategories"; +import { ICU4XPluralCategory } from "./ICU4XPluralCategory"; +import { ICU4XPluralOperands } from "./ICU4XPluralOperands"; + +/** + + * FFI version of `PluralRulesWithRanges`. + + * See the {@link https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html Rust documentation for `PluralRulesWithRanges`} for more information. + */ +export class ICU4XPluralRulesWithRanges { + + /** + + * Construct an {@link ICU4XPluralRulesWithRanges `ICU4XPluralRulesWithRanges`} for the given locale, for cardinal numbers + + * See the {@link https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.try_new_cardinal Rust documentation for `try_new_cardinal`} for more information. + * @throws {@link FFIError}<{@link ICU4XError}> + */ + static create_cardinal(provider: ICU4XDataProvider, locale: ICU4XLocale): ICU4XPluralRulesWithRanges | never; + + /** + + * Construct an {@link ICU4XPluralRulesWithRanges `ICU4XPluralRulesWithRanges`} for the given locale, for ordinal numbers + + * See the {@link https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.try_new_ordinal Rust documentation for `try_new_ordinal`} for more information. + * @throws {@link FFIError}<{@link ICU4XError}> + */ + static create_ordinal(provider: ICU4XDataProvider, locale: ICU4XLocale): ICU4XPluralRulesWithRanges | never; + + /** + + * Get the category for a given number represented as operands + + * See the {@link https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.category_for Rust documentation for `category_for`} for more information. + */ + category_for(op: ICU4XPluralOperands): ICU4XPluralCategory; + + /** + + * Get all of the categories needed in the current locale + + * See the {@link https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.categories Rust documentation for `categories`} for more information. + */ + categories(): ICU4XPluralCategories; + + /** + + * Get the appropriate category for a numeric range. + + * See the {@link https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.category_for_range Rust documentation for `category_for_range`} for more information. + */ + category_for_range(start: ICU4XPluralOperands, end: ICU4XPluralOperands): ICU4XPluralCategory; + + /** + + * Get the appropriate category for a numeric range from the categories of its endpoints. + + * See the {@link https://docs.rs/icu/latest/icu/plurals/struct.PluralRulesWithRanges.html#method.resolve_range Rust documentation for `resolve_range`} for more information. + */ + resolve_range(start: ICU4XPluralCategory, end: ICU4XPluralCategory): ICU4XPluralCategory; +} diff --git a/ffi/capi/js/include/ICU4XPluralRulesWithRanges.js b/ffi/capi/js/include/ICU4XPluralRulesWithRanges.js new file mode 100644 index 00000000000..176b42af9a0 --- /dev/null +++ b/ffi/capi/js/include/ICU4XPluralRulesWithRanges.js @@ -0,0 +1,76 @@ +import wasm from "./diplomat-wasm.mjs" +import * as diplomatRuntime from "./diplomat-runtime.js" +import { ICU4XError_js_to_rust, ICU4XError_rust_to_js } from "./ICU4XError.js" +import { ICU4XPluralCategories } from "./ICU4XPluralCategories.js" +import { ICU4XPluralCategory_js_to_rust, ICU4XPluralCategory_rust_to_js } from "./ICU4XPluralCategory.js" + +const ICU4XPluralRulesWithRanges_box_destroy_registry = new FinalizationRegistry(underlying => { + wasm.ICU4XPluralRulesWithRanges_destroy(underlying); +}); + +export class ICU4XPluralRulesWithRanges { + #lifetimeEdges = []; + constructor(underlying, owned, edges) { + this.underlying = underlying; + this.#lifetimeEdges.push(...edges); + if (owned) { + ICU4XPluralRulesWithRanges_box_destroy_registry.register(this, underlying); + } + } + + static create_cardinal(arg_provider, arg_locale) { + return (() => { + const diplomat_receive_buffer = wasm.diplomat_alloc(5, 4); + wasm.ICU4XPluralRulesWithRanges_create_cardinal(diplomat_receive_buffer, arg_provider.underlying, arg_locale.underlying); + const is_ok = diplomatRuntime.resultFlag(wasm, diplomat_receive_buffer, 4); + if (is_ok) { + const ok_value = new ICU4XPluralRulesWithRanges(diplomatRuntime.ptrRead(wasm, diplomat_receive_buffer), true, []); + wasm.diplomat_free(diplomat_receive_buffer, 5, 4); + return ok_value; + } else { + const throw_value = ICU4XError_rust_to_js[diplomatRuntime.enumDiscriminant(wasm, diplomat_receive_buffer)]; + wasm.diplomat_free(diplomat_receive_buffer, 5, 4); + throw new diplomatRuntime.FFIError(throw_value); + } + })(); + } + + static create_ordinal(arg_provider, arg_locale) { + return (() => { + const diplomat_receive_buffer = wasm.diplomat_alloc(5, 4); + wasm.ICU4XPluralRulesWithRanges_create_ordinal(diplomat_receive_buffer, arg_provider.underlying, arg_locale.underlying); + const is_ok = diplomatRuntime.resultFlag(wasm, diplomat_receive_buffer, 4); + if (is_ok) { + const ok_value = new ICU4XPluralRulesWithRanges(diplomatRuntime.ptrRead(wasm, diplomat_receive_buffer), true, []); + wasm.diplomat_free(diplomat_receive_buffer, 5, 4); + return ok_value; + } else { + const throw_value = ICU4XError_rust_to_js[diplomatRuntime.enumDiscriminant(wasm, diplomat_receive_buffer)]; + wasm.diplomat_free(diplomat_receive_buffer, 5, 4); + throw new diplomatRuntime.FFIError(throw_value); + } + })(); + } + + category_for(arg_op) { + return ICU4XPluralCategory_rust_to_js[wasm.ICU4XPluralRulesWithRanges_category_for(this.underlying, arg_op.underlying)]; + } + + categories() { + return (() => { + const diplomat_receive_buffer = wasm.diplomat_alloc(6, 1); + wasm.ICU4XPluralRulesWithRanges_categories(diplomat_receive_buffer, this.underlying); + const out = new ICU4XPluralCategories(diplomat_receive_buffer); + wasm.diplomat_free(diplomat_receive_buffer, 6, 1); + return out; + })(); + } + + category_for_range(arg_start, arg_end) { + return ICU4XPluralCategory_rust_to_js[wasm.ICU4XPluralRulesWithRanges_category_for_range(this.underlying, arg_start.underlying, arg_end.underlying)]; + } + + resolve_range(arg_start, arg_end) { + return ICU4XPluralCategory_rust_to_js[wasm.ICU4XPluralRulesWithRanges_resolve_range(this.underlying, ICU4XPluralCategory_js_to_rust[arg_start], ICU4XPluralCategory_js_to_rust[arg_end])]; + } +} diff --git a/ffi/capi/js/include/index.d.ts b/ffi/capi/js/include/index.d.ts index 9e966117926..9a4911392aa 100644 --- a/ffi/capi/js/include/index.d.ts +++ b/ffi/capi/js/include/index.d.ts @@ -93,6 +93,7 @@ export { ICU4XPluralCategories } from './ICU4XPluralCategories.js'; export { ICU4XPluralCategory } from './ICU4XPluralCategory.js'; export { ICU4XPluralOperands } from './ICU4XPluralOperands.js'; export { ICU4XPluralRules } from './ICU4XPluralRules.js'; +export { ICU4XPluralRulesWithRanges } from './ICU4XPluralRulesWithRanges.js'; export { ICU4XPropertyValueNameToEnumMapper } from './ICU4XPropertyValueNameToEnumMapper.js'; export { ICU4XRegionDisplayNames } from './ICU4XRegionDisplayNames.js'; export { ICU4XReorderedIndexMap } from './ICU4XReorderedIndexMap.js'; diff --git a/ffi/capi/js/include/index.js b/ffi/capi/js/include/index.js index eed06fd21e7..e438b2f787c 100644 --- a/ffi/capi/js/include/index.js +++ b/ffi/capi/js/include/index.js @@ -93,6 +93,7 @@ export { ICU4XPluralCategories } from './ICU4XPluralCategories.js'; export { ICU4XPluralCategory } from './ICU4XPluralCategory.js'; export { ICU4XPluralOperands } from './ICU4XPluralOperands.js'; export { ICU4XPluralRules } from './ICU4XPluralRules.js'; +export { ICU4XPluralRulesWithRanges } from './ICU4XPluralRulesWithRanges.js'; export { ICU4XPropertyValueNameToEnumMapper } from './ICU4XPropertyValueNameToEnumMapper.js'; export { ICU4XRegionDisplayNames } from './ICU4XRegionDisplayNames.js'; export { ICU4XReorderedIndexMap } from './ICU4XReorderedIndexMap.js'; diff --git a/ffi/capi/src/pluralrules.rs b/ffi/capi/src/pluralrules.rs index 569ccc0050f..006f416654d 100644 --- a/ffi/capi/src/pluralrules.rs +++ b/ffi/capi/src/pluralrules.rs @@ -9,7 +9,7 @@ pub mod ffi { use alloc::boxed::Box; use fixed_decimal::FixedDecimal; - use icu_plurals::{PluralCategory, PluralOperands, PluralRules}; + use icu_plurals::{PluralCategory, PluralOperands, PluralRules, PluralRulesWithRanges}; use crate::{locale::ffi::ICU4XLocale, provider::ffi::ICU4XDataProvider}; @@ -91,27 +91,7 @@ pub mod ffi { /// Get all of the categories needed in the current locale #[diplomat::rust_link(icu::plurals::PluralRules::categories, FnInStruct)] pub fn categories(&self) -> ICU4XPluralCategories { - self.0.categories().fold( - ICU4XPluralCategories { - zero: false, - one: false, - two: false, - few: false, - many: false, - other: false, - }, - |mut categories, category| { - match category { - PluralCategory::Zero => categories.zero = true, - PluralCategory::One => categories.one = true, - PluralCategory::Two => categories.two = true, - PluralCategory::Few => categories.few = true, - PluralCategory::Many => categories.many = true, - PluralCategory::Other => categories.other = true, - }; - categories - }, - ) + ICU4XPluralCategories::from_iter(self.0.categories()) } } @@ -132,6 +112,81 @@ pub mod ffi { } } + /// FFI version of `PluralRulesWithRanges`. + #[diplomat::opaque] + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges, Struct)] + pub struct ICU4XPluralRulesWithRanges(pub icu_plurals::PluralRulesWithRanges); + + impl ICU4XPluralRulesWithRanges { + /// Construct an [`ICU4XPluralRulesWithRanges`] for the given locale, for cardinal numbers + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges::try_new_cardinal, FnInStruct)] + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges::try_new, FnInStruct, hidden)] + #[diplomat::rust_link(icu::plurals::PluralRuleType, Enum, hidden)] + pub fn create_cardinal( + provider: &ICU4XDataProvider, + locale: &ICU4XLocale, + ) -> Result, ICU4XError> { + let locale = locale.to_datalocale(); + Ok(Box::new(ICU4XPluralRulesWithRanges(call_constructor!( + PluralRulesWithRanges::try_new_cardinal, + PluralRulesWithRanges::try_new_cardinal_with_any_provider, + PluralRulesWithRanges::try_new_cardinal_with_buffer_provider, + provider, + &locale + )?))) + } + + /// Construct an [`ICU4XPluralRulesWithRanges`] for the given locale, for ordinal numbers + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges::try_new_ordinal, FnInStruct)] + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges::try_new, FnInStruct, hidden)] + #[diplomat::rust_link(icu::plurals::PluralRuleType, Enum, hidden)] + pub fn create_ordinal( + provider: &ICU4XDataProvider, + locale: &ICU4XLocale, + ) -> Result, ICU4XError> { + let locale = locale.to_datalocale(); + Ok(Box::new(ICU4XPluralRulesWithRanges(call_constructor!( + PluralRulesWithRanges::try_new_ordinal, + PluralRulesWithRanges::try_new_ordinal_with_any_provider, + PluralRulesWithRanges::try_new_ordinal_with_buffer_provider, + provider, + &locale + )?))) + } + + /// Get the category for a given number represented as operands + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges::category_for, FnInStruct)] + pub fn category_for(&self, op: &ICU4XPluralOperands) -> ICU4XPluralCategory { + self.0.category_for(op.0).into() + } + + /// Get all of the categories needed in the current locale + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges::categories, FnInStruct)] + pub fn categories(&self) -> ICU4XPluralCategories { + ICU4XPluralCategories::from_iter(self.0.categories()) + } + + /// Get the appropriate category for a numeric range. + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges::category_for_range, FnInStruct)] + pub fn category_for_range( + &self, + start: &ICU4XPluralOperands, + end: &ICU4XPluralOperands, + ) -> ICU4XPluralCategory { + self.0.category_for_range(start.0, end.0).into() + } + + /// Get the appropriate category for a numeric range from the categories of its endpoints. + #[diplomat::rust_link(icu::plurals::PluralRulesWithRanges::resolve_range, FnInStruct)] + pub fn resolve_range( + &self, + start: ICU4XPluralCategory, + end: ICU4XPluralCategory, + ) -> ICU4XPluralCategory { + self.0.resolve_range(start.into(), end.into()).into() + } + } + /// FFI version of `PluralRules::categories()` data. pub struct ICU4XPluralCategories { pub zero: bool, @@ -141,4 +196,30 @@ pub mod ffi { pub many: bool, pub other: bool, } + + impl ICU4XPluralCategories { + fn from_iter(i: impl Iterator) -> Self { + i.fold( + ICU4XPluralCategories { + zero: false, + one: false, + two: false, + few: false, + many: false, + other: false, + }, + |mut categories, category| { + match category { + PluralCategory::Zero => categories.zero = true, + PluralCategory::One => categories.one = true, + PluralCategory::Two => categories.two = true, + PluralCategory::Few => categories.few = true, + PluralCategory::Many => categories.many = true, + PluralCategory::Other => categories.other = true, + }; + categories + }, + ) + } + } } diff --git a/provider/datagen/src/transform/cldr/characters/mod.rs b/provider/datagen/src/transform/cldr/characters/mod.rs index d56f2084fbd..68ea8b7e948 100644 --- a/provider/datagen/src/transform/cldr/characters/mod.rs +++ b/provider/datagen/src/transform/cldr/characters/mod.rs @@ -173,14 +173,9 @@ fn unescape_exemplar_chars(char_block: &str) -> String { return char_block.replace('\\', ""); } - // Unescape the escape sequences like \uXXXX and \UXXXXXXXX into the proper code points. - // Also, workaround errant extra backslash escaping. - // Because JSON does not support \UXXXXXXXX Unicode code point escaping, use the TOML parser. - let ch_for_json = format!("x=\"{char_block}\""); - - // Workaround for literal values like "\\-"" that cause problems for the TOML parser. + // Workaround for literal values like "\\-" that cause problems for the TOML parser. // In such cases, remove the '\\' character preceding the non-Unicode-escape-sequence character. - let mut ch_vec = ch_for_json.chars().collect::>(); + let mut ch_vec = char_block.chars().collect::>(); let mut ch_indices_to_remove: Vec = vec![]; for (idx, ch) in ch_vec.iter().enumerate().rev() { if ch == &'\\' { @@ -193,10 +188,27 @@ fn unescape_exemplar_chars(char_block: &str) -> String { for idx in ch_indices_to_remove { ch_vec.remove(idx); } - let ch_for_json = ch_vec.iter().collect::(); + let ch_for_toml = ch_vec.iter().collect::(); + + // Workaround for double quotation mark literal values, which can appear in a string as a backslash followed + // by the quotation mark itself (\"), but for the purposes of the TOML parser, should be escaped to be 2 + // slashes followed by the quotation mark (\\"). + // Start by removing all preceding backslashes from quotation marks, and finally add back 2 backslashes. + let mut ch_for_toml = ch_for_toml.to_string(); + // Remove up to 3 consecutive backslashes preceding a quotation mark. Preprocessing should have already + // removed 4-/6-/8-fold preceding backslashes before a character. + for _i in 1..=3 { + ch_for_toml = ch_for_toml.replace("\\\"", "\""); + } + ch_for_toml = ch_for_toml.replace('\"', "\\\""); + + // Unescape the escape sequences like \uXXXX and \UXXXXXXXX into the proper code points. + // Also, workaround errant extra backslash escaping. + // Because JSON does not support \UXXXXXXXX Unicode code point escaping, use the TOML parser. + let ch_for_toml = format!("x=\"{ch_for_toml}\""); let ch_lite_t_val: toml::Value = - toml::from_str(&ch_for_json).unwrap_or_else(|_| panic!("{char_block:?}")); + toml::from_str(&ch_for_toml).unwrap_or_else(|_| panic!("{char_block:?}")); let ch_lite = if let toml::Value::Table(t) = ch_lite_t_val { if let Some(toml::Value::String(s)) = t.get("x") { s.to_owned() @@ -463,6 +475,42 @@ mod tests { assert!(actual.contains("ক\u{09CD}ষ")); } + #[test] + fn test_parse_consecutive_main_chars_without_spaces() { + let en_aux = "[áàăâåäãā æ ç éèĕêëē íìĭîïī ñ óòŏôöøō œ úùŭûüū ÿ]"; + + let actual = parse_exemplar_char_string(en_aux); + + assert!(actual.contains("æ")); + assert!(actual.contains("ç")); + assert!(actual.contains("á")); + assert!(!actual.contains("áàăâåäãā")); + } + + #[test] + fn test_parse_consecutive_punct_chars_subset() { + let input = "[\"“”]"; + let actual = parse_exemplar_char_string(input); + assert!(actual.contains("\"")); + assert!(actual.contains("“")); + assert!(actual.contains("”")); + assert!(!actual.contains("\"“”")); + } + + #[test] + fn test_parse_all_punct_chars() { + let en_punct = "[\\- ‐‑ – — , ; \\: ! ? . … '‘’ \"“” ( ) \\[ \\] § @ * / \\& # † ‡ ′ ″]"; + + let actual = parse_exemplar_char_string(en_punct); + + assert!(actual.contains("‐")); + assert!(actual.contains("‑")); + assert!(actual.contains("'")); + assert!(actual.contains("‘")); + assert!(actual.contains("’")); + assert!(!actual.contains("'‘’")); + } + #[test] fn test_basic() { let provider = crate::DatagenProvider::new_testing(); diff --git a/provider/datagen/tests/make-testdata.rs b/provider/datagen/tests/make-testdata.rs index fad466a191f..f5f7e74ebaf 100644 --- a/provider/datagen/tests/make-testdata.rs +++ b/provider/datagen/tests/make-testdata.rs @@ -19,6 +19,7 @@ use std::sync::Mutex; include!("locales.rs.data"); #[test] +#[ignore] // has side effects, run manually fn generate_json_and_verify_postcard() { simple_logger::SimpleLogger::new() .env() diff --git a/tools/make/data.toml b/tools/make/data.toml index 2d1812ce815..084a720b07b 100644 --- a/tools/make/data.toml +++ b/tools/make/data.toml @@ -22,11 +22,12 @@ command = "cargo" args = [ "test", "-p=icu_datagen", + "--test=make-testdata", "--no-default-features", "--features=fs_exporter,use_wasm,rayon,experimental_components", - "generate_json_and_verify_postcard", "--", - "--nocapture" + "--ignored", + "--nocapture", ] [tasks.testdata-hello-world] @@ -80,6 +81,7 @@ command = "cargo" args = [ "run", "--bin=make-testdata-legacy", + "--release", "--manifest-path=tools/testdata-scripts/Cargo.toml", # avoid global feature resolution ] diff --git a/tools/make/ffi.toml b/tools/make/ffi.toml index 7d666d43d08..95cba3a4113 100644 --- a/tools/make/ffi.toml +++ b/tools/make/ffi.toml @@ -4,91 +4,7 @@ # This is a cargo-make file included in the toplevel Makefile.toml -[tasks.test-ffi] -description = "Run FFI tests" -category = "ICU4X Development" -dependencies = [ - "cargo-make-min-version", - "test-c", - "test-c-tiny", - "test-cpp", - "test-js", - "test-tinywasm", - "build-wearos-ffi", - "test-nostd", -] - -[tasks.verify-ffi] -description = "Run FFI tests that verify checked in code is up to date" -category = "ICU4X Development" -dependencies = [ - "cargo-make-min-version", - "verify-diplomat-gen", - "verify-diplomat-coverage", -] - -# Some tasks need a minimum version of cargo-make. Configs within cargo-make -# task stanzas such as this allow setting the minimum version of deps (including -# min version for cargo-make itself). But cargo-make might only be applying -# such dep upgrades defined in the `install_crate` field after the task script -# has run. -[tasks.cargo-make-min-version] -description = "Verify that the minimum version of cargo-make exists" -category = "ICU4X Development" -install_crate = { crate_name = "cargo-make", binary = "cargo", test_arg = ["make", "--version"], min_version = "0.36.2" } - -[tasks.diplomat-coverage] -description = "Produces the list of ICU APIs that are not exported through Diplomat" -category = "ICU4X Development" -dependencies = ["install-nightly"] -script_runner = "@duckscript" -script = ''' -exit_on_error true -if "${ICU4X_BUILDING_WITH_FORCED_NIGHTLY}" - echo "Skipping diplomat-coverage since ICU4X_BUILDING_WITH_FORCED_NIGHTLY is set" -else - exec --fail-on-error cargo run -p icu_ffi_coverage --all-features -- ffi/capi/tests/missing_apis.txt -end -''' - -[tasks.verify-diplomat-coverage] -description = "Verify that checked-in Diplomat coverage is up to date" -category = "ICU4X Development" -dependencies = [ - "diplomat-coverage", -] -script_runner = "@duckscript" -script = ''' -exit_on_error true -code = exec --get-exit-code git diff --exit-code -- ffi/capi/tests -if ${code} - trigger_error "Diplomat coverage dump need to be regenerated. Please run `cargo make diplomat-coverage` and commit." -end -''' - -[tasks.verify-diplomat-gen] -description = "Verify that checked-in Diplomat bindings are up to date" -category = "ICU4X Development" -dependencies = [ - "diplomat-gen", -] -script_runner = "@duckscript" -script = ''' -exit_on_error true -code = exec --get-exit-code git diff --exit-code -- ffi/capi -if ${code} - trigger_error "Diplomat bindings need to be regenerated. Please run `cargo make diplomat-install`, then `cargo make diplomat-gen`, and commit. (Testing against different Diplomat versions may omit install step.)" -end -''' - -[tasks.diplomat-gen] -description = "Regenerate Diplomat bindings" -category = "ICU4X Development" -dependencies = [ - "diplomat-gen-c", - "diplomat-gen-cpp", - "diplomat-gen-js", -] +# C/C++ tasks [tasks.test-c] description = "Run C API tests" @@ -96,21 +12,9 @@ category = "ICU4X Development" script_runner = "@duckscript" script = ''' exit_on_error true -cd ffi/capi/c/examples/pluralrules -exec --fail-on-error make -cd ../fixeddecimal -exec --fail-on-error make -cd ../locale -exec --fail-on-error make -''' - -[tasks.install-unknown-linux] -description = "Installs the unknown-linux target" -category = "ICU4X Development" -dependencies = ["install-nightly"] -script_runner = "@duckscript" -script = ''' -exec --fail-on-error rustup target add x86_64-unknown-linux-gnu --toolchain ${ICU4X_NIGHTLY_TOOLCHAIN} +exec --fail-on-error make -C ffi/capi/c/examples/pluralrules +exec --fail-on-error make -C ffi/capi/c/examples/fixeddecimal +exec --fail-on-error make -C ffi/capi/c/examples/locale ''' [tasks.test-c-tiny] @@ -123,12 +27,11 @@ exit_on_error true if "${ICU4X_BUILDING_WITH_FORCED_NIGHTLY}" echo "Skipping test-c-tiny since ICU4X_BUILDING_WITH_FORCED_NIGHTLY is set" else - cd ffi/capi/c/examples/fixeddecimal_tiny - exec --fail-on-error make - exec ls -l - cd ../segmenter_tiny - exec --fail-on-error make - exec ls -l + exec --fail-on-error make -C ffi/capi/c/examples/fixeddecimal_tiny + exec ls -l ffi/capi/c/examples/fixeddecimal_tiny + + exec --fail-on-error make -C ffi/capi/c/examples/segmenter_tiny + exec ls -l ffi/capi/c/examples/segmenter_tiny end ''' @@ -138,52 +41,18 @@ category = "ICU4X Development" script_runner = "@duckscript" script = ''' exit_on_error true -cd ffi/capi/cpp/examples/properties -exec --fail-on-error make -cd ../segmenter -exec --fail-on-error make -cd ../datetime -exec --fail-on-error make -cd ../fixeddecimal -exec --fail-on-error make -cd ../locale -exec --fail-on-error make -cd ../pluralrules -exec --fail-on-error make -cd ../bidi -exec --fail-on-error make -cd ../fixeddecimal_wasm -exec --fail-on-error make test-host +exec --fail-on-error make -C ffi/capi/cpp/examples/properties +exec --fail-on-error make -C ffi/capi/cpp/examples/segmenter +exec --fail-on-error make -C ffi/capi/cpp/examples/datetime +exec --fail-on-error make -C ffi/capi/cpp/examples/fixeddecimal +exec --fail-on-error make -C ffi/capi/cpp/examples/locale +exec --fail-on-error make -C ffi/capi/cpp/examples/pluralrules +exec --fail-on-error make -C ffi/capi/cpp/examples/bidi +exec --fail-on-error make -C ffi/capi/cpp/examples/fixeddecimal_wasm test-host ''' -[tasks.install-emscripten] -description = "Installs the emscripten target" -category = "ICU4X Development" -dependencies = ["install-nightly"] -script_runner = "@duckscript" -script = ''' -exec --fail-on-error rustup target add wasm32-unknown-emscripten --toolchain ${ICU4X_NIGHTLY_TOOLCHAIN} -''' - -[tasks.install-wasm] -description = "Installs the wasm target" -category = "ICU4X Development" -dependencies = ["install-nightly"] -script_runner = "@duckscript" -script = ''' -exec --fail-on-error rustup target add wasm32-unknown-unknown --toolchain ${ICU4X_NIGHTLY_TOOLCHAIN} -''' -[tasks.test-cpp-emscripten] -description = "Run the C++-emscripten test (needs emsdk)" -category = "ICU4X Development" -dependencies = ["install-nightly", "install-emscripten"] -script_runner = "@duckscript" -script = ''' -exit_on_error true -cd ffi/capi/cpp/examples/fixeddecimal_wasm -exec make test -''' +# JS/WASM tasks [tasks.test-js] description = "Run JS tests" @@ -198,14 +67,63 @@ exec --fail-on-error npm ci --foreground-scripts exec --fail-on-error npm test ''' -[tasks.test-cppdoc] -description = "Build the cpp tests" +[tasks.test-tinywasm] +description = "Test the Tiny WASM example" +category = "ICU4X Development" +dependencies = ["install-nightly", "install-wasm"] +script_runner = "@duckscript" +script = ''' +exit_on_error true +if "${ICU4X_BUILDING_WITH_FORCED_NIGHTLY}" + echo "Skipping test-wasm since ICU4X_BUILDING_WITH_FORCED_NIGHTLY is set" +else + cd ffi/capi/js/examples/tinywasm + + exec --fail-on-error make + exec --fail-on-error ls -l + exec --fail-on-error node tiny.mjs +end +''' + +# no_std tasks + +[tasks.check-freertos-wearos] +description = "Build ICU4X CAPI for Cortex" +category = "ICU4X FFI" +dependencies = ["install-nightly", "install-cortex-8"] +toolchain = "${ICU4X_NIGHTLY_TOOLCHAIN}" +env = { RUSTFLAGS = "-Ctarget-cpu=cortex-m33 -Cpanic=abort -Copt-level=s" } +command = "cargo" +args = ["check", "--package", "icu_freertos", + "--target", "thumbv8m.main-none-eabihf", + "--no-default-features", "--features=wearos", + "-Zbuild-std=core,alloc", "-Zbuild-std-features=panic_immediate_abort"] + + +# Diplomat gen + +[tasks.diplomat-gen] +description = "Regenerate Diplomat bindings" +category = "ICU4X Development" +dependencies = [ + "diplomat-gen-c", + "diplomat-gen-cpp", + "diplomat-gen-js", +] + +[tasks.verify-diplomat-gen] +description = "Verify that checked-in Diplomat bindings are up to date" category = "ICU4X Development" +dependencies = [ + "diplomat-gen", +] script_runner = "@duckscript" script = ''' exit_on_error true -cd ffi/capi/cpp/docs; -exec --fail-on-error make html +code = exec --get-exit-code git diff --exit-code -- ffi/capi +if ${code} + trigger_error "Diplomat bindings need to be regenerated. Please run `cargo make diplomat-install`, then `cargo make diplomat-gen`, and commit. (Testing against different Diplomat versions may omit install step.)" +end ''' [tasks.diplomat-gen-c] @@ -215,6 +133,7 @@ script_runner = "@duckscript" script = ''' exit_on_error true cd ffi/capi + # Duckscript doesn't support globs in rm, so we delete the dir rm -r ./c/include/ mkdir ./c/include @@ -229,7 +148,6 @@ script = ''' exit_on_error true cd ffi/capi - # Duckscript doesn't support globs in rm, so we delete the dir. # Preserve conf.py across the deletion. conf_py = readfile ./cpp/docs/source/conf.py @@ -261,63 +179,66 @@ echo "$conf_py" > ./js/docs/source/conf.py diplomat-tool js ./js/include/ --docs ./js/docs/source ''' -[tasks.install-cortex-8] -description = "Install the thumbv8m target" -category = "ICU4X FFI" +# Diplomat coverage + +[tasks.diplomat-coverage] +description = "Produces the list of ICU APIs that are not exported through Diplomat" +category = "ICU4X Development" dependencies = ["install-nightly"] -toolchain = "${ICU4X_NIGHTLY_TOOLCHAIN}" -command = "rustup" -args = [ - "target", "add", "thumbv8m.main-none-eabihf", +script_runner = "@duckscript" +script = ''' +exit_on_error true +if "${ICU4X_BUILDING_WITH_FORCED_NIGHTLY}" + echo "Skipping diplomat-coverage since ICU4X_BUILDING_WITH_FORCED_NIGHTLY is set" +else + exec --fail-on-error cargo run -p icu_ffi_coverage --all-features -- ffi/capi/tests/missing_apis.txt +end +''' + +[tasks.verify-diplomat-coverage] +description = "Verify that checked-in Diplomat coverage is up to date" +category = "ICU4X Development" +dependencies = [ + "diplomat-coverage", ] +script_runner = "@duckscript" +script = ''' +exit_on_error true +code = exec --get-exit-code git diff --exit-code -- ffi/capi/tests +if ${code} + trigger_error "Diplomat coverage dump need to be regenerated. Please run `cargo make diplomat-coverage` and commit." +end +''' -[tasks.build-wearos-ffi] -description = "Build ICU4X CAPI for Cortex" -category = "ICU4X FFI" -dependencies = ["install-nightly", "install-cortex-8"] -toolchain = "${ICU4X_NIGHTLY_TOOLCHAIN}" -env = { RUSTFLAGS = "-Ctarget-cpu=cortex-m33 -Cpanic=abort" } -command = "cargo" -args = ["build", "--package", "icu_freertos", - "--target", "thumbv8m.main-none-eabihf", - "--no-default-features", "--features=wearos", - "--profile=release-opt-size", - "-Zbuild-std", "-Zbuild-std=std,panic_abort", "-Zbuild-std-features=panic_immediate_abort"] -[tasks.install-cortex-7] -description = "Install the thumbv7m target" -category = "ICU4X FFI" -dependencies = ["install-nightly"] -toolchain = "${ICU4X_NIGHTLY_TOOLCHAIN}" -command = "rustup" -args = [ - "target", "add", "thumbv7m-none-eabi", -] +# Install tasks -[tasks.test-nostd] -description = "Ensure ICU4X core builds on no-std" -category = "ICU4X FFI" -dependencies = ["install-nightly", "install-cortex-7"] -toolchain = "${ICU4X_NIGHTLY_TOOLCHAIN}" -command = "cargo" -args = ["build", "--package", "icu", "--target", "thumbv7m-none-eabi"] +# These tasks need a minimum version of cargo-make. Configs within cargo-make +# task stanzas such as this allow setting the minimum version of deps (including +# min version for cargo-make itself). But cargo-make might only be applying +# such dep upgrades defined in the `install_crate` field after the task script +# has run. +[tasks.cargo-make-min-version] +description = "Verify that the minimum version of cargo-make exists" +category = "ICU4X Development" +install_crate = { crate_name = "cargo-make", binary = "cargo", test_arg = ["make", "--version"], min_version = "0.36.2" } -[tasks.test-tinywasm] -description = "Test the Tiny WASM example" +[tasks.diplomat-install] +description = "Install Diplomat at current Diplomat revision" category = "ICU4X Development" -dependencies = ["install-nightly", "install-wasm"] +dependencies = [ "cargo-make-min-version" ] script_runner = "@duckscript" script = ''' exit_on_error true -if "${ICU4X_BUILDING_WITH_FORCED_NIGHTLY}" - echo "Skipping test-wasm since ICU4X_BUILDING_WITH_FORCED_NIGHTLY is set" -else - echo "Skipping test-wasm since it's slow https://github.com/unicode-org/icu4x/issues/3582" - cd ffi/capi/js/examples/tinywasm +rev = exec --fail-on-error cargo make --loglevel error diplomat-get-rev +rev = trim ${rev.stdout} +if contains ${rev} "." + echo "Installing Diplomat version ${rev}" + exec --fail-on-error cargo install --version ${rev} diplomat-tool -f - exec --fail-on-error make - exec --fail-on-error ls -l - exec --fail-on-error node tiny.mjs +else + echo "Installing Diplomat rev ${rev}" + exec --fail-on-error cargo install --git https://github.com/rust-diplomat/diplomat.git --rev ${rev} diplomat-tool -f end ''' @@ -361,21 +282,51 @@ end release --recursive ${metadata} ''' -[tasks.diplomat-install] -description = "Install Diplomat at current Diplomat revision" +[tasks.install-unknown-linux] +description = "Installs the unknown-linux target" category = "ICU4X Development" -dependencies = [ "cargo-make-min-version" ] +dependencies = ["install-nightly"] script_runner = "@duckscript" script = ''' -exit_on_error true -rev = exec --fail-on-error cargo make --loglevel error diplomat-get-rev -rev = trim ${rev.stdout} -if contains ${rev} "." - echo "Installing Diplomat version ${rev}" - exec --fail-on-error cargo install --version ${rev} diplomat-tool -f +exec --fail-on-error rustup target add x86_64-unknown-linux-gnu --toolchain ${ICU4X_NIGHTLY_TOOLCHAIN} +''' -else - echo "Installing Diplomat rev ${rev}" - exec --fail-on-error cargo install --git https://github.com/rust-diplomat/diplomat.git --rev ${rev} diplomat-tool -f -end +[tasks.install-emscripten] +description = "Installs the emscripten target" +category = "ICU4X Development" +dependencies = ["install-nightly"] +script_runner = "@duckscript" +script = ''' +exec --fail-on-error rustup target add wasm32-unknown-emscripten --toolchain ${ICU4X_NIGHTLY_TOOLCHAIN} +''' + +[tasks.install-wasm] +description = "Installs the wasm target" +category = "ICU4X Development" +dependencies = ["install-nightly"] +script_runner = "@duckscript" +script = ''' +exec --fail-on-error rustup target add wasm32-unknown-unknown --toolchain ${ICU4X_NIGHTLY_TOOLCHAIN} +''' + +[tasks.install-cortex-8] +description = "Install the thumbv8m target" +category = "ICU4X FFI" +dependencies = ["install-nightly"] +toolchain = "${ICU4X_NIGHTLY_TOOLCHAIN}" +command = "rustup" +args = [ + "target", "add", "thumbv8m.main-none-eabihf", +] + +# Unused tasks + +[tasks.test-cpp-emscripten] +description = "Run the C++-emscripten test (needs emsdk)" +category = "ICU4X Development" +dependencies = ["install-nightly", "install-emscripten"] +script_runner = "@duckscript" +script = ''' +exit_on_error true +exec make -C ffi/capi/cpp/examples/fixeddecimal_wasm test ''' diff --git a/tools/make/tests.toml b/tools/make/tests.toml index 0c2482d48a4..8e24531f1b3 100644 --- a/tools/make/tests.toml +++ b/tools/make/tests.toml @@ -138,3 +138,18 @@ exec --fail-on-error make -C crates/custom_compiled set_env ICU4X_DATA_DIR ${project_dir}/docs/tutorials/crates/custom_compiled/baked_data exec --fail-on-error cargo run --release -p tutorial_custom_compiled ''' + +[tasks.install-cortex-7] +description = "Install the thumbv7m target" +category = "ICU4X Development" +command = "rustup" +args = [ + "target", "add", "thumbv7m-none-eabi", +] + +[tasks.check-nostd] +description = "Ensure ICU4X builds on no-std" +category = "ICU4X Development" +dependencies = ["install-cortex-7"] +command = "cargo" +args = ["check", "--package", "icu", "--target", "thumbv7m-none-eabi"]