diff --git a/Cargo.lock b/Cargo.lock index 4676963c0f..c4d7351a0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5628,6 +5628,7 @@ version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ + "indexmap", "itoa", "ryu", "serde", @@ -6633,6 +6634,24 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "translation_merger" +version = "0.1.0" +dependencies = [ + "candid", + "canister_agent_utils", + "clap", + "ic-agent", + "ic-utils", + "itertools", + "serde", + "serde_json", + "tokio", + "translations_canister", + "translations_canister_client", + "types", +] + [[package]] name = "translations_canister" version = "0.1.0" @@ -6643,6 +6662,17 @@ dependencies = [ "types", ] +[[package]] +name = "translations_canister_client" +version = "0.1.0" +dependencies = [ + "candid", + "canister_client", + "ic-agent", + "translations_canister", + "types", +] + [[package]] name = "translations_canister_impl" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index ea3e11a865..34e7e1dd05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,6 +140,7 @@ members = [ "backend/notification_pusher/aws", "backend/notification_pusher/cli", "backend/notification_pusher/core", + "backend/translation_merger", ] resolver = "2" @@ -214,4 +215,4 @@ debug = false [patch.crates-io] ic-cdk-macros = { git = "https://github.com/hpeebles/cdk-rs", rev = "1833cb37ecfd6154cc9cc55b5de76329114e8c5f" } -pocket-ic = { git = "https://github.com/dfinity/ic", rev = "a7862784e8da4a97a1d608fd5b3db365de41a2d7" } \ No newline at end of file +pocket-ic = { git = "https://github.com/dfinity/ic", rev = "a7862784e8da4a97a1d608fd5b3db365de41a2d7" } diff --git a/backend/canisters/translations/client/Cargo.toml b/backend/canisters/translations/client/Cargo.toml new file mode 100644 index 0000000000..5c5a35a944 --- /dev/null +++ b/backend/canisters/translations/client/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "translations_canister_client" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +candid = { workspace = true } +canister_client = { path = "../../../libraries/canister_client" } +ic-agent = { workspace = true } +translations_canister = { path = "../api" } +types = { path = "../../../libraries/types" } diff --git a/backend/canisters/translations/client/src/lib.rs b/backend/canisters/translations/client/src/lib.rs new file mode 100644 index 0000000000..30dbe9821a --- /dev/null +++ b/backend/canisters/translations/client/src/lib.rs @@ -0,0 +1,8 @@ +use canister_client::{generate_query_call, generate_update_call}; +use translations_canister::*; + +// Queries +generate_query_call!(pending_deployment); + +// Updates +generate_update_call!(mark_deployed); diff --git a/backend/translation_merger/Cargo.toml b/backend/translation_merger/Cargo.toml new file mode 100644 index 0000000000..ae54b7a5b0 --- /dev/null +++ b/backend/translation_merger/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "translation_merger" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +candid = { workspace = true } +canister_agent_utils = { path = "../libraries/canister_agent_utils" } +clap = { workspace = true, features = ["derive"] } +ic-agent = { workspace = true } +ic-utils = { workspace = true } +itertools = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true, features = ["preserve_order"] } +tokio = { workspace = true, features = ["full"] } +translations_canister = { path = "../canisters/translations/api" } +translations_canister_client = { path = "../canisters/translations/client" } +types = { path = "../libraries/types" } diff --git a/backend/translation_merger/src/lib.rs b/backend/translation_merger/src/lib.rs new file mode 100644 index 0000000000..760242af21 --- /dev/null +++ b/backend/translation_merger/src/lib.rs @@ -0,0 +1,181 @@ +use canister_agent_utils::{build_ic_agent, get_dfx_identity}; +use clap::Parser; +use itertools::Itertools; +use serde_json::{Map, Value}; +use std::{ + collections::{hash_map::Entry, HashMap}, + error::Error, + fs, +}; +use translations_canister::pending_deployment::{Response, SuccessResponse, Translation}; +use types::{CanisterId, Empty, TimestampMillis}; + +#[derive(Parser, Debug)] +pub struct Config { + /// The id of the Translations canister + #[arg(long)] + translations_canister_id: CanisterId, + + /// IC URL + #[arg(long)] + url: String, + + /// The DFX identity of controller + #[arg(long)] + controller: String, + + /// The path to the translation files + #[arg(long)] + directory: String, +} + +pub async fn merge(config: Config) -> Result<(), Box> { + let path = config.directory.clone(); + let mut translations = read_translations_from_files(&path).await?; + + let corrections = read_latest_translation_corrections(config).await?; + + merge_translations(&mut translations, corrections.translations)?; + + write_translation_files(&path, translations).await?; + + write_latest_approval(&path, corrections.latest_approval).await?; + + Ok(()) +} + +async fn read_translations_from_files( + path: &str, +) -> Result>, Box> { + let mut files = HashMap::new(); + + let paths = fs::read_dir(path)?; + + for entry in paths { + let entry = entry?; + let meta = entry.metadata()?; + if meta.is_file() { + let filename = entry.file_name().into_string().unwrap(); + let parts: Vec<_> = filename.split('.').collect(); + if parts.len() == 2 && parts[1] == "json" { + let locale = parts[0]; + let path_buf = entry.path(); + let path = path_buf.to_str().unwrap(); + let file = read_translations_from_file(path).await?; + files.insert(locale.to_string(), file); + } + } + } + + Ok(files) +} + +async fn read_translations_from_file(path: &str) -> Result, Box> { + fn parse_object( + key_prefix: &str, + object: Map, + translations: &mut HashMap, + ) -> Result<(), Box> { + for (key, value) in object { + let full_key = if key_prefix.is_empty() { key } else { format!("{key_prefix}.{key}") }; + match value { + Value::String(s) => { + translations.insert(full_key, s); + } + Value::Object(o) => parse_object(&full_key, o, translations)?, + _ => return Err("Syntax error")?, + } + } + + Ok(()) + } + + let mut translations = HashMap::new(); + + let file = fs::read_to_string(path)?; + let json: Value = serde_json::from_str(&file)?; + + let Value::Object(object) = json else { + return Err("Syntax error")?; + }; + + parse_object("", object, &mut translations)?; + + Ok(translations) +} + +async fn read_latest_translation_corrections(config: Config) -> Result> { + let identity = get_dfx_identity(&config.controller); + let agent = build_ic_agent(config.url, identity).await; + + translations_canister_client::pending_deployment(&agent, &config.translations_canister_id, &Empty {}) + .await + .map(|response| match response { + Response::Success(result) => Ok(result), + })? +} + +fn merge_translations( + translations: &mut HashMap>, + corrections: Vec, +) -> Result<(), Box> { + for correction in corrections { + let locale = correction.locale; + let Some(file) = translations.get_mut(&locale) else { + Err(format!("Locale not found: {locale}"))? + }; + + match file.entry(correction.key) { + Entry::Occupied(mut o) => { + o.insert(correction.value); + } + Entry::Vacant(v) => { + let key = v.into_key(); + Err(format!("Key not found: {locale} {key}"))? + } + } + } + + Ok(()) +} + +async fn write_translation_files( + path: &str, + data: HashMap>, +) -> Result<(), Box> { + fn build_object(translations: Vec<&(String, String)>, depth: usize) -> Value { + let mut map = Map::new(); + + for (key, group) in &translations.into_iter().group_by(|(k, _)| k.split('.').nth(depth).unwrap()) { + let group_vec: Vec<_> = group.collect(); + let (k0, v0) = group_vec[0]; + + let value = if group_vec.len() == 1 && k0.matches('.').count() <= depth { + Value::String(v0.clone()) + } else { + build_object(group_vec, depth + 1) + }; + + map.insert(key.to_string(), value); + } + + Value::Object(map) + } + + for (locale, translations) in data { + let mut translations: Vec<_> = translations.into_iter().collect(); + translations.sort_by(|(k1, _), (k2, _)| k1.partial_cmp(k2).unwrap()); + + let object = build_object(translations.iter().collect(), 0); + + let text = serde_json::to_string_pretty(&object)?; + fs::write(format!("{path}/{locale}.json"), text)?; + } + Ok(()) +} + +async fn write_latest_approval(path: &str, timestamp: TimestampMillis) -> Result<(), Box> { + let text = serde_json::to_string_pretty(×tamp)?; + fs::write(format!("{path}/latest-approval.txt"), text)?; + Ok(()) +} diff --git a/backend/translation_merger/src/main.rs b/backend/translation_merger/src/main.rs new file mode 100644 index 0000000000..6bb5c0de1d --- /dev/null +++ b/backend/translation_merger/src/main.rs @@ -0,0 +1,14 @@ +use clap::Parser; +use std::process; +use translation_merger::merge; +use translation_merger::Config; + +#[tokio::main] +async fn main() { + let config = Config::parse(); + + if let Err(e) = merge(config).await { + eprintln!("Application error: {e}"); + process::exit(1); + } +} diff --git a/scripts/merge-latest-translations.sh b/scripts/merge-latest-translations.sh new file mode 100755 index 0000000000..826fe59a50 --- /dev/null +++ b/scripts/merge-latest-translations.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Pass in the dfx identity name +# eg './merge-latest-translations-local.sh openchat' + +IDENTITY=${1:-default} + +SCRIPT=$(readlink -f "$0") +SCRIPT_DIR=$(dirname "$SCRIPT") +cd $SCRIPT_DIR + +TRANSLATIONS_CANISTER_ID=$(dfx canister --network ic id translations) + +cargo run \ + --manifest-path ../backend/translation_merger/Cargo.toml -- \ + --translations-canister-id $TRANSLATIONS_CANISTER_ID \ + --url https://ic0.app/ \ + --controller $IDENTITY \ + --directory ../frontend/app/src/i18n \