Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft sketch of ResolvedLocaleAdapter #4607

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Draft sketch of ResolvedLocaleAdapter
sffc committed Feb 14, 2024
commit 83f19f8592ed4e3faedc99b47c35fca5d8f458e0
2 changes: 2 additions & 0 deletions provider/adapters/src/lib.rs
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
//! - Use the [`either`] module to choose between multiple provider types at runtime.
//! - Use the [`filter`] module to programmatically reject certain data requests.
//! - Use the [`fallback`] module to automatically resolve arbitrary locales for data loading.
//! - Use the [`resolved`] module to determine the supported or resolved locale for a data request.
// https://github.com/unicode-org/icu4x/blob/main/docs/process/boilerplate.md#library-annotations
#![cfg_attr(not(any(test, feature = "std")), no_std)]
@@ -34,3 +35,4 @@ pub mod fallback;
pub mod filter;
pub mod fork;
mod helpers;
pub mod resolved;
73 changes: 73 additions & 0 deletions provider/adapters/src/resolved.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use alloc::collections::{BTreeMap, BTreeSet};
use core::cell::RefCell;

use icu_provider::prelude::*;

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
struct ResolvedLocaleInfo {
pub requested_locale: DataLocale,
pub resolved_locale: Option<DataLocale>,
}

/// TODO: Docs
#[derive(Debug)]
pub struct ResolvedLocaleAdapter<P> {
inner: P,
resolved_locales: RefCell<BTreeMap<DataKey, ResolvedLocaleInfo>>,
drop_payloads: bool,
}

impl<P> ResolvedLocaleAdapter<P> {
pub fn into_inner(self) -> P {
self.inner
}

pub fn clear(&mut self) {
self.resolved_locales.borrow_mut().clear()
}

pub fn take_resolved_locale_for_key(&mut self, key: DataKey) -> Option<DataLocale> {
self.resolved_locales
.borrow_mut()
.remove(&key)
.and_then(|info| info.resolved_locale)
}

pub fn take_all_resolved_locales(&mut self) -> BTreeSet<DataLocale> {
let map = self.resolved_locales.take();
map.into_iter()
.filter_map(|(_, info)| info.resolved_locale)
.collect()
}

pub fn saw_last_resort_fallback(&self) -> bool {
self.resolved_locales.borrow().values().any(|info| {
info.resolved_locale
.as_ref()
.map(|l| l.is_langid_und())
.unwrap_or(false)
})
}
}

impl<P: BufferProvider> BufferProvider for ResolvedLocaleAdapter<P> {
fn load_buffer(
&self,
key: DataKey,
req: DataRequest,
) -> Result<DataResponse<BufferMarker>, DataError> {
let mut response = self.inner.load_buffer(key, req)?;
self.resolved_locales.borrow_mut().insert(
key,
ResolvedLocaleInfo {
requested_locale: req.locale.clone(),
resolved_locale: response.metadata.locale.take(),
},
);
Ok(response)
}
}
12 changes: 8 additions & 4 deletions provider/blob/src/blob_data_provider.rs
Original file line number Diff line number Diff line change
@@ -123,10 +123,14 @@ impl BufferProvider for BlobDataProvider {
metadata.buffer_format = Some(BufferFormat::Postcard1);
Ok(DataResponse {
metadata,
payload: Some(DataPayload::from_yoked_buffer(
self.data
.try_map_project_cloned(|blob, _| blob.load(key, req))?,
)),
payload: if req.metadata.drop_payload {
None
} else {
Some(DataPayload::from_yoked_buffer(
self.data
.try_map_project_cloned(|blob, _| blob.load(key, req))?,
))
},
})
}
}
24 changes: 16 additions & 8 deletions provider/core/src/any.rs
Original file line number Diff line number Diff line change
@@ -437,10 +437,7 @@ where
{
#[inline]
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
self.0
.load_any(M::KEY, req)?
.downcast()
.map_err(|e| e.with_req(M::KEY, req))
self.load_data(M::KEY, req)
}
}

@@ -454,10 +451,21 @@ where
{
#[inline]
fn load_data(&self, key: DataKey, req: DataRequest) -> Result<DataResponse<M>, DataError> {
self.0
.load_any(key, req)?
.downcast()
.map_err(|e| e.with_req(key, req))
let any_response = AnyProvider::load_any(self.0, key, req)?;
Ok(DataResponse {
metadata: any_response.metadata,
payload: any_response
.payload
.and_then(|p| {
if req.metadata.drop_payload {
None
} else {
Some(p.downcast())
}
})
.transpose()
.map_err(|e| e.with_req(key, req))?,
})
}
}

3 changes: 3 additions & 0 deletions provider/core/src/request.rs
Original file line number Diff line number Diff line change
@@ -52,6 +52,9 @@ impl fmt::Display for DataRequest<'_> {
pub struct DataRequestMetadata {
/// Silent requests do not log errors. This can be used for exploratory querying, such as fallbacks.
pub silent: bool,
/// Whether to drop the payload from the [`DataResponse`](crate::DataResponse). This can be used
/// for exploratory queries where the returned data is not of interest.
pub drop_payload: bool,
}

/// A locale type optimized for use in fallbacking and the ICU4X data pipeline.
8 changes: 7 additions & 1 deletion provider/core/src/serde/mod.rs
Original file line number Diff line number Diff line change
@@ -173,7 +173,13 @@ where
metadata: buffer_response.metadata,
payload: buffer_response
.payload
.map(|p| p.into_deserialized(buffer_format))
.and_then(|p| {
if req.metadata.drop_payload {
None
} else {
Some(p.into_deserialized(buffer_format))
}
})
.transpose()
.map_err(|e| e.with_req(key, req))?,
})
8 changes: 7 additions & 1 deletion provider/fs/src/fs_data_provider.rs
Original file line number Diff line number Diff line change
@@ -85,9 +85,15 @@ impl BufferProvider for FsDataProvider {
if !Path::new(&path).exists() {
return Err(DataErrorKind::MissingLocale.with_req(key, req));
}
let buffer = fs::read(&path).map_err(|e| DataError::from(e).with_path_context(&path))?;
let mut metadata = DataResponseMetadata::default();
metadata.buffer_format = Some(self.manifest.buffer_format);
if req.metadata.drop_payload {
return Ok(DataResponse {
metadata,
payload: None,
});
}
let buffer = fs::read(&path).map_err(|e| DataError::from(e).with_path_context(&path))?;
Ok(DataResponse {
metadata,
payload: Some(DataPayload::from_owned_buffer(buffer.into_boxed_slice())),