feat: DEVOPS-1781 persistence export cron job for checkpoint node (#2079

)

* feat: DEVOPS-1781 persistence export cron job for checkpoint node

* feat: DEVOPS-1781 persistence export functions

* feat: DEVOPS-1781 persistence export folder instead of zip file

z2/resources/persistence_export.tera.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Directory where checkpoint files are located
+CHECKPOINT_DIR="/data/{{ eth_chain_id }}/checkpoints"
+
+# GCS bucket where the persistence will be exported
+GCS_BUCKET="gs://{{ network_name }}-persistence"
+
+# Log name in Google Cloud Logging
+LOG_NAME="{{ network_name }}-persistence-export-cron-log"
+
+# Function to log messages to Google Cloud Logging
+log_message() {
+    local message="$1"
+    logger -t "$LOG_NAME" "$message"
+}
+
+# Function to check if directory is empty
+is_dir_empty() {
+    local dir="$1"
+    [ -z "$(ls -A $dir)" ]
+}
+
+# Start the persistence export process
+start_time=$(date +%s)
+
+if is_dir_empty "$CHECKPOINT_DIR"; then  
+    # Stop zilliqa service
+    if ! sudo systemctl stop zilliqa.service; then
+        log_message "Error: Failed to stop zilliqa service"
+        exit 1
+    fi
+
+    # Create persistence export folder name with timestamp
+    persistence_export_name="{{ network_name }}-$(date +%Y%m%d%H%M%S)-persistence"
+    log_message "Creating persistence export: $persistence_export_name"
+
+    # Upload to GCS
+    if ! gsutil -m cp -r /data "$GCS_BUCKET/$persistence_export_name/"; then
+        log_message "Error: Failed to upload data to GCS"
+        sudo systemctl start zilliqa.service
+        exit 1
+    fi
+
+    # Start zilliqa service
+    if ! sudo systemctl start zilliqa.service; then
+        log_message "Error: Failed to start zilliqa service"
+        exit 1
+    fi
+
+    # Calculate and log total execution time
+    end_time=$(date +%s)
+    duration=$((end_time - start_time))
+    log_message "Persistence export completed successfully in $duration seconds"
+else
+    log_message "Checkpoint files present in $CHECKPOINT_DIR. Skipping persistence export."
+fi

z2/src/chain/node.rs

@@ -475,6 +475,20 @@ impl ChainNode {
             self.machine
                 .copy(&[checkpoint_cron_job], "/tmp/checkpoint_cron_job.sh")
                 .await?;
+
+            let temp_persistence_export_cron_job = NamedTempFile::new()?;
+            let persistence_export_cron_job = &self
+                .create_persistence_export_cron_job(
+                    temp_persistence_export_cron_job.path().to_str().unwrap(),
+                )
+                .await?;
+
+            self.machine
+                .copy(
+                    &[persistence_export_cron_job],
+                    "/tmp/persistence_export_cron_job.sh",
+                )
+                .await?;
         }
 
         println!("Configuration files imported in the node");
@@ -491,11 +505,24 @@ impl ChainNode {
         }
 
         if self.role == NodeRole::Checkpoint {
-            let cmd = "sudo rm -f /tmp/checkpoint_cron_job.sh";
-            let output = self.machine.run(cmd, true).await?;
-            if !output.success {
-                println!("{:?}", output.stderr);
-                return Err(anyhow!("Error removing previous checkpoint cronjob"));
+            let cmd_checkpoint_cron_job = "sudo rm -f /tmp/checkpoint_cron_job.sh";
+            let output_cmd_checkpoint_cron_job =
+                self.machine.run(cmd_checkpoint_cron_job, true).await?;
+            if !output_cmd_checkpoint_cron_job.success {
+                println!("{:?}", output_cmd_checkpoint_cron_job.stderr);
+                return Err(anyhow!("Error removing previous checkpoint cron job"));
+            }
+
+            let cmd_persistence_export_cron_job = "sudo rm -f /tmp/persistence_export_cron_job.sh";
+            let output_persistence_export_cron_job = self
+                .machine
+                .run(cmd_persistence_export_cron_job, true)
+                .await?;
+            if !output_persistence_export_cron_job.success {
+                println!("{:?}", output_persistence_export_cron_job.stderr);
+                return Err(anyhow!(
+                    "Error removing previous persistence export cron job"
+                ));
             }
         }
 
@@ -513,11 +540,17 @@ impl ChainNode {
         }
 
         if self.role == NodeRole::Checkpoint {
-            let cmd = "sudo chmod 777 /tmp/checkpoint_cron_job.sh && sudo mv /tmp/checkpoint_cron_job.sh /checkpoint_cron_job.sh && echo '*/5 * * * * /checkpoint_cron_job.sh' | sudo crontab -";
+            let cmd = r#"
+                sudo chmod 777 /tmp/checkpoint_cron_job.sh && \
+                sudo mv /tmp/checkpoint_cron_job.sh /checkpoint_cron_job.sh && \
+                sudo chmod 777 /tmp/persistence_export_cron_job.sh && \
+                sudo mv /tmp/persistence_export_cron_job.sh /persistence_export_cron_job.sh && \
+                (echo '*/5 * * * * /checkpoint_cron_job.sh'; echo '0 */6 * * * /persistence_export_cron_job.sh') | sudo crontab -"#;
+
             let output = self.machine.run(cmd, true).await?;
             if !output.success {
                 println!("{:?}", output.stderr);
-                return Err(anyhow!("Error creating the checkpoint cronjob"));
+                return Err(anyhow!("Error setting up cron jobs"));
             }
         }
 
@@ -545,7 +578,28 @@ impl ChainNode {
 
         let mut fh = File::create(filename).await?;
         fh.write_all(config_file.as_bytes()).await?;
-        println!("Cron job file created: {filename}");
+        println!("Checkpoint cron job file created: {filename}");
+
+        Ok(filename.to_owned())
+    }
+
+    async fn create_persistence_export_cron_job(&self, filename: &str) -> Result<String> {
+        let spec_config = include_str!("../../resources/persistence_export.tera.sh");
+
+        let chain_name = self.chain.name();
+        let eth_chain_id = self.eth_chain_id.to_string();
+
+        let mut var_map = BTreeMap::<&str, &str>::new();
+        var_map.insert("network_name", &chain_name);
+        var_map.insert("eth_chain_id", &eth_chain_id);
+
+        let ctx = Context::from_serialize(var_map)?;
+        let rendered_template = Tera::one_off(spec_config, &ctx, false)?;
+        let config_file = rendered_template.as_str();
+
+        let mut fh = File::create(filename).await?;
+        fh.write_all(config_file.as_bytes()).await?;
+        println!("Persistence export cron job file created: {filename}");
 
         Ok(filename.to_owned())
     }