From 30a2461030ad9cb6330376d528a24d876d957ecf Mon Sep 17 00:00:00 2001 From: Pablo Suarez <2323608+pavlops@users.noreply.github.com> Date: Thu, 26 Dec 2024 20:53:35 +0400 Subject: [PATCH] feat: DEVOPS-1781 persistence export cron job for checkpoint node (#2079) * feat: DEVOPS-1781 persistence export cron job for checkpoint node * feat: DEVOPS-1781 persistence export functions * feat: DEVOPS-1781 persistence export folder instead of zip file --- z2/resources/persistence_export.tera.sh | 57 ++++++++++++++++++++ z2/src/chain/node.rs | 70 ++++++++++++++++++++++--- 2 files changed, 119 insertions(+), 8 deletions(-) create mode 100644 z2/resources/persistence_export.tera.sh diff --git a/z2/resources/persistence_export.tera.sh b/z2/resources/persistence_export.tera.sh new file mode 100644 index 000000000..ec623e97c --- /dev/null +++ b/z2/resources/persistence_export.tera.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Directory where checkpoint files are located +CHECKPOINT_DIR="/data/{{ eth_chain_id }}/checkpoints" + +# GCS bucket where the persistence will be exported +GCS_BUCKET="gs://{{ network_name }}-persistence" + +# Log name in Google Cloud Logging +LOG_NAME="{{ network_name }}-persistence-export-cron-log" + +# Function to log messages to Google Cloud Logging +log_message() { + local message="$1" + logger -t "$LOG_NAME" "$message" +} + +# Function to check if directory is empty +is_dir_empty() { + local dir="$1" + [ -z "$(ls -A $dir)" ] +} + +# Start the persistence export process +start_time=$(date +%s) + +if is_dir_empty "$CHECKPOINT_DIR"; then + # Stop zilliqa service + if ! sudo systemctl stop zilliqa.service; then + log_message "Error: Failed to stop zilliqa service" + exit 1 + fi + + # Create persistence export folder name with timestamp + persistence_export_name="{{ network_name }}-$(date +%Y%m%d%H%M%S)-persistence" + log_message "Creating persistence export: $persistence_export_name" + + # Upload to GCS + if ! gsutil -m cp -r /data "$GCS_BUCKET/$persistence_export_name/"; then + log_message "Error: Failed to upload data to GCS" + sudo systemctl start zilliqa.service + exit 1 + fi + + # Start zilliqa service + if ! sudo systemctl start zilliqa.service; then + log_message "Error: Failed to start zilliqa service" + exit 1 + fi + + # Calculate and log total execution time + end_time=$(date +%s) + duration=$((end_time - start_time)) + log_message "Persistence export completed successfully in $duration seconds" +else + log_message "Checkpoint files present in $CHECKPOINT_DIR. Skipping persistence export." +fi \ No newline at end of file diff --git a/z2/src/chain/node.rs b/z2/src/chain/node.rs index 293df9605..bec85a099 100644 --- a/z2/src/chain/node.rs +++ b/z2/src/chain/node.rs @@ -475,6 +475,20 @@ impl ChainNode { self.machine .copy(&[checkpoint_cron_job], "/tmp/checkpoint_cron_job.sh") .await?; + + let temp_persistence_export_cron_job = NamedTempFile::new()?; + let persistence_export_cron_job = &self + .create_persistence_export_cron_job( + temp_persistence_export_cron_job.path().to_str().unwrap(), + ) + .await?; + + self.machine + .copy( + &[persistence_export_cron_job], + "/tmp/persistence_export_cron_job.sh", + ) + .await?; } println!("Configuration files imported in the node"); @@ -491,11 +505,24 @@ impl ChainNode { } if self.role == NodeRole::Checkpoint { - let cmd = "sudo rm -f /tmp/checkpoint_cron_job.sh"; - let output = self.machine.run(cmd, true).await?; - if !output.success { - println!("{:?}", output.stderr); - return Err(anyhow!("Error removing previous checkpoint cronjob")); + let cmd_checkpoint_cron_job = "sudo rm -f /tmp/checkpoint_cron_job.sh"; + let output_cmd_checkpoint_cron_job = + self.machine.run(cmd_checkpoint_cron_job, true).await?; + if !output_cmd_checkpoint_cron_job.success { + println!("{:?}", output_cmd_checkpoint_cron_job.stderr); + return Err(anyhow!("Error removing previous checkpoint cron job")); + } + + let cmd_persistence_export_cron_job = "sudo rm -f /tmp/persistence_export_cron_job.sh"; + let output_persistence_export_cron_job = self + .machine + .run(cmd_persistence_export_cron_job, true) + .await?; + if !output_persistence_export_cron_job.success { + println!("{:?}", output_persistence_export_cron_job.stderr); + return Err(anyhow!( + "Error removing previous persistence export cron job" + )); } } @@ -513,11 +540,17 @@ impl ChainNode { } if self.role == NodeRole::Checkpoint { - let cmd = "sudo chmod 777 /tmp/checkpoint_cron_job.sh && sudo mv /tmp/checkpoint_cron_job.sh /checkpoint_cron_job.sh && echo '*/5 * * * * /checkpoint_cron_job.sh' | sudo crontab -"; + let cmd = r#" + sudo chmod 777 /tmp/checkpoint_cron_job.sh && \ + sudo mv /tmp/checkpoint_cron_job.sh /checkpoint_cron_job.sh && \ + sudo chmod 777 /tmp/persistence_export_cron_job.sh && \ + sudo mv /tmp/persistence_export_cron_job.sh /persistence_export_cron_job.sh && \ + (echo '*/5 * * * * /checkpoint_cron_job.sh'; echo '0 */6 * * * /persistence_export_cron_job.sh') | sudo crontab -"#; + let output = self.machine.run(cmd, true).await?; if !output.success { println!("{:?}", output.stderr); - return Err(anyhow!("Error creating the checkpoint cronjob")); + return Err(anyhow!("Error setting up cron jobs")); } } @@ -545,7 +578,28 @@ impl ChainNode { let mut fh = File::create(filename).await?; fh.write_all(config_file.as_bytes()).await?; - println!("Cron job file created: {filename}"); + println!("Checkpoint cron job file created: {filename}"); + + Ok(filename.to_owned()) + } + + async fn create_persistence_export_cron_job(&self, filename: &str) -> Result { + let spec_config = include_str!("../../resources/persistence_export.tera.sh"); + + let chain_name = self.chain.name(); + let eth_chain_id = self.eth_chain_id.to_string(); + + let mut var_map = BTreeMap::<&str, &str>::new(); + var_map.insert("network_name", &chain_name); + var_map.insert("eth_chain_id", ð_chain_id); + + let ctx = Context::from_serialize(var_map)?; + let rendered_template = Tera::one_off(spec_config, &ctx, false)?; + let config_file = rendered_template.as_str(); + + let mut fh = File::create(filename).await?; + fh.write_all(config_file.as_bytes()).await?; + println!("Persistence export cron job file created: {filename}"); Ok(filename.to_owned()) }