Tool to merge translation corrections (#5331)

Cargo.toml

@@ -140,6 +140,7 @@ members = [
     "backend/notification_pusher/aws",
     "backend/notification_pusher/cli",
     "backend/notification_pusher/core",
+    "backend/translation_merger",
 ]
 resolver = "2"
 
@@ -214,4 +215,4 @@ debug = false
 
 [patch.crates-io]
 ic-cdk-macros = { git = "https://github.com/hpeebles/cdk-rs", rev = "1833cb37ecfd6154cc9cc55b5de76329114e8c5f" }
-pocket-ic = { git = "https://github.com/dfinity/ic", rev = "a7862784e8da4a97a1d608fd5b3db365de41a2d7" }
+pocket-ic = { git = "https://github.com/dfinity/ic", rev = "a7862784e8da4a97a1d608fd5b3db365de41a2d7" }

backend/canisters/translations/client/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "translations_canister_client"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+candid = { workspace = true }
+canister_client = { path = "../../../libraries/canister_client" }
+ic-agent = { workspace = true }
+translations_canister = { path = "../api" }
+types = { path = "../../../libraries/types" }

backend/canisters/translations/client/src/lib.rs

@@ -0,0 +1,8 @@
+use canister_client::{generate_query_call, generate_update_call};
+use translations_canister::*;
+
+// Queries
+generate_query_call!(pending_deployment);
+
+// Updates
+generate_update_call!(mark_deployed);

backend/translation_merger/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "translation_merger"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+candid = { workspace = true }
+canister_agent_utils = { path = "../libraries/canister_agent_utils" }
+clap = { workspace = true, features = ["derive"] }
+ic-agent = { workspace = true }
+ic-utils = { workspace = true }
+itertools = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true, features = ["preserve_order"] }
+tokio = { workspace = true, features = ["full"] }
+translations_canister = { path = "../canisters/translations/api" }
+translations_canister_client = { path = "../canisters/translations/client" }
+types = { path = "../libraries/types" }

backend/translation_merger/src/lib.rs

@@ -0,0 +1,181 @@
+use canister_agent_utils::{build_ic_agent, get_dfx_identity};
+use clap::Parser;
+use itertools::Itertools;
+use serde_json::{Map, Value};
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    error::Error,
+    fs,
+};
+use translations_canister::pending_deployment::{Response, SuccessResponse, Translation};
+use types::{CanisterId, Empty, TimestampMillis};
+
+#[derive(Parser, Debug)]
+pub struct Config {
+    /// The id of the Translations canister
+    #[arg(long)]
+    translations_canister_id: CanisterId,
+
+    /// IC URL
+    #[arg(long)]
+    url: String,
+
+    /// The DFX identity of controller
+    #[arg(long)]
+    controller: String,
+
+    /// The path to the translation files
+    #[arg(long)]
+    directory: String,
+}
+
+pub async fn merge(config: Config) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let path = config.directory.clone();
+    let mut translations = read_translations_from_files(&path).await?;
+
+    let corrections = read_latest_translation_corrections(config).await?;
+
+    merge_translations(&mut translations, corrections.translations)?;
+
+    write_translation_files(&path, translations).await?;
+
+    write_latest_approval(&path, corrections.latest_approval).await?;
+
+    Ok(())
+}
+
+async fn read_translations_from_files(
+    path: &str,
+) -> Result<HashMap<String, HashMap<String, String>>, Box<dyn Error + Send + Sync>> {
+    let mut files = HashMap::new();
+
+    let paths = fs::read_dir(path)?;
+
+    for entry in paths {
+        let entry = entry?;
+        let meta = entry.metadata()?;
+        if meta.is_file() {
+            let filename = entry.file_name().into_string().unwrap();
+            let parts: Vec<_> = filename.split('.').collect();
+            if parts.len() == 2 && parts[1] == "json" {
+                let locale = parts[0];
+                let path_buf = entry.path();
+                let path = path_buf.to_str().unwrap();
+                let file = read_translations_from_file(path).await?;
+                files.insert(locale.to_string(), file);
+            }
+        }
+    }
+
+    Ok(files)
+}
+
+async fn read_translations_from_file(path: &str) -> Result<HashMap<String, String>, Box<dyn Error + Send + Sync>> {
+    fn parse_object(
+        key_prefix: &str,
+        object: Map<String, Value>,
+        translations: &mut HashMap<String, String>,
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        for (key, value) in object {
+            let full_key = if key_prefix.is_empty() { key } else { format!("{key_prefix}.{key}") };
+            match value {
+                Value::String(s) => {
+                    translations.insert(full_key, s);
+                }
+                Value::Object(o) => parse_object(&full_key, o, translations)?,
+                _ => return Err("Syntax error")?,
+            }
+        }
+
+        Ok(())
+    }
+
+    let mut translations = HashMap::new();
+
+    let file = fs::read_to_string(path)?;
+    let json: Value = serde_json::from_str(&file)?;
+
+    let Value::Object(object) = json else {
+        return Err("Syntax error")?;
+    };
+
+    parse_object("", object, &mut translations)?;
+
+    Ok(translations)
+}
+
+async fn read_latest_translation_corrections(config: Config) -> Result<SuccessResponse, Box<dyn Error + Send + Sync>> {
+    let identity = get_dfx_identity(&config.controller);
+    let agent = build_ic_agent(config.url, identity).await;
+
+    translations_canister_client::pending_deployment(&agent, &config.translations_canister_id, &Empty {})
+        .await
+        .map(|response| match response {
+            Response::Success(result) => Ok(result),
+        })?
+}
+
+fn merge_translations(
+    translations: &mut HashMap<String, HashMap<String, String>>,
+    corrections: Vec<Translation>,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    for correction in corrections {
+        let locale = correction.locale;
+        let Some(file) = translations.get_mut(&locale) else {
+            Err(format!("Locale not found: {locale}"))?
+        };
+
+        match file.entry(correction.key) {
+            Entry::Occupied(mut o) => {
+                o.insert(correction.value);
+            }
+            Entry::Vacant(v) => {
+                let key = v.into_key();
+                Err(format!("Key not found: {locale} {key}"))?
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn write_translation_files(
+    path: &str,
+    data: HashMap<String, HashMap<String, String>>,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    fn build_object(translations: Vec<&(String, String)>, depth: usize) -> Value {
+        let mut map = Map::new();
+
+        for (key, group) in &translations.into_iter().group_by(|(k, _)| k.split('.').nth(depth).unwrap()) {
+            let group_vec: Vec<_> = group.collect();
+            let (k0, v0) = group_vec[0];
+
+            let value = if group_vec.len() == 1 && k0.matches('.').count() <= depth {
+                Value::String(v0.clone())
+            } else {
+                build_object(group_vec, depth + 1)
+            };
+
+            map.insert(key.to_string(), value);
+        }
+
+        Value::Object(map)
+    }
+
+    for (locale, translations) in data {
+        let mut translations: Vec<_> = translations.into_iter().collect();
+        translations.sort_by(|(k1, _), (k2, _)| k1.partial_cmp(k2).unwrap());
+
+        let object = build_object(translations.iter().collect(), 0);
+
+        let text = serde_json::to_string_pretty(&object)?;
+        fs::write(format!("{path}/{locale}.json"), text)?;
+    }
+    Ok(())
+}
+
+async fn write_latest_approval(path: &str, timestamp: TimestampMillis) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let text = serde_json::to_string_pretty(&timestamp)?;
+    fs::write(format!("{path}/latest-approval.txt"), text)?;
+    Ok(())
+}

backend/translation_merger/src/main.rs

@@ -0,0 +1,14 @@
+use clap::Parser;
+use std::process;
+use translation_merger::merge;
+use translation_merger::Config;
+
+#[tokio::main]
+async fn main() {
+    let config = Config::parse();
+
+    if let Err(e) = merge(config).await {
+        eprintln!("Application error: {e}");
+        process::exit(1);
+    }
+}

scripts/merge-latest-translations.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Pass in the dfx identity name
+# eg './merge-latest-translations-local.sh openchat'
+
+IDENTITY=${1:-default}
+
+SCRIPT=$(readlink -f "$0")
+SCRIPT_DIR=$(dirname "$SCRIPT")
+cd $SCRIPT_DIR
+
+TRANSLATIONS_CANISTER_ID=$(dfx canister --network ic id translations)
+
+cargo run \
+  --manifest-path ../backend/translation_merger/Cargo.toml -- \
+  --translations-canister-id $TRANSLATIONS_CANISTER_ID \
+  --url https://ic0.app/ \
+  --controller $IDENTITY \
+  --directory ../frontend/app/src/i18n \