From 70ae382af8d7e264f1c893a53bbabe8589fd187e Mon Sep 17 00:00:00 2001 From: Vehbi Sinan Tunalioglu Date: Mon, 18 Apr 2022 22:46:15 +0800 Subject: [PATCH] feat: make rrclone configuration self-sufficient BREAKING: We made rrclone configuration self-sufficient, ie. we no longer need rclone configuration present on the host. --- config.yaml.tmpl | 76 ++++++++++++++++++++++++++++++--------- src/config.rs | 45 +++++++++++++++++++++++ src/main.rs | 93 +++++++----------------------------------------- src/rclone.rs | 86 ++++++++++++++++++++++++++++++++++++++++++++ src/utils.rs | 9 +++++ 5 files changed, 213 insertions(+), 96 deletions(-) create mode 100644 src/config.rs create mode 100644 src/rclone.rs create mode 100644 src/utils.rs diff --git a/config.yaml.tmpl b/config.yaml.tmpl index b247988..6d53143 100644 --- a/config.yaml.tmpl +++ b/config.yaml.tmpl @@ -1,16 +1,60 @@ -- source_remote: source1 - source_path: / - target_remote: target1 - target_path: /data/backups/source1 - filters: - - "+ /data/**" - - "+ /home/**" - - "- *" -- source_remote: source2 - source_path: / - target_remote: target1 - target_path: /data/backups/source2 - filters: - - "+ /data/**" - - "+ /home/**" - - "- *" +tasks: + - name: "Backup remote to remote" + source: + backend: + type: "sftp" + vars: + host: "remote-source" + user: "user1" + md5sum_command: "md5sum" + sha1sum_command: "sha1sum" + path: "/" + filters: + - "+ /data/**" + - "+ /home/**" + - "- *" + target: + backend: + type: "sftp" + vars: + host: "remote-target" + user: "user2" + md5sum_command: "md5sum" + sha1sum_command: "sha1sum" + path: "/backups/remote-source/" + - name: "Backup remote to local" + source: + backend: + type: "sftp" + vars: + host: "remote-source" + user: "user1" + md5sum_command: "md5sum" + sha1sum_command: "sha1sum" + path: "/" + filters: + - "+ /data/**" + - "+ /home/**" + - "- *" + target: + backend: + type: "local" + path: "/backups/remote-source" + - name: "Backup local to remote" + source: + backend: + type: "local" + path: "/" + filters: + - "+ /data/**" + - "+ /home/**" + - "- *" + target: + backend: + type: "sftp" + vars: + host: "remote-target" + user: "user1" + md5sum_command: "md5sum" + sha1sum_command: "sha1sum" + path: "/backups/laptop" diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..4c149b7 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,45 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Config { + pub tasks: Vec, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Task { + pub name: String, + pub source: Source, + pub target: Target, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Source { + pub backend: Backend, + pub path: String, + pub filters: Vec, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Target { + pub backend: Backend, + pub path: String, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Backend { + #[serde(rename = "type")] + pub ctype: String, + pub vars: Option>, +} + +pub fn read_config(path: &String) -> Result { + match fs::read_to_string(path) { + Ok(content) => match serde_yaml::from_str(&content) { + Ok(config) => Ok(config), + Err(err) => Err(format!("Can not parse configuration. Error: {:?}", err)), + }, + Err(err) => Err(format!("Can not read file {}. Error: {}", path, err)), + } +} diff --git a/src/main.rs b/src/main.rs index eb016e2..4bc53ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,91 +1,24 @@ -use chrono::Utc; -use serde::{Deserialize, Serialize}; use std::env; -use std::fs; -use std::process::Command; -use std::process::Stdio; -#[derive(Debug, PartialEq, Serialize, Deserialize)] -struct Task { - source_remote: String, - source_path: String, - target_remote: String, - target_path: String, - filters: Vec, -} +mod config; +mod rclone; +mod utils; -fn main() -> Result<(), serde_yaml::Error> { +fn main() { let args: Vec = env::args().collect(); let dryrun = args.len() == 3 && &args[2] == "--dry-run"; - let contents = - fs::read_to_string(&args[1]).expect("Something went wrong while reading the YAML file"); - - let result: Result, _> = serde_yaml::from_str(&contents); - - match result { - Ok(tasks) => run_tasks(&tasks, dryrun), - Err(x) => log(format!("Something went wrong while decoding the YAML file {:?}:", x)), - } - - Ok(()) -} - -fn log(msg: String) -> () { - eprintln!( - "RRCLONE>> [{}] {}", - Utc::now().format("%Y-%m-%d %H:%M:%S").to_string(), - msg - ); -} - -fn run_tasks(ts: &Vec, dryrun: bool) -> () { - for t in ts { - run_task(t, dryrun) + match config::read_config(&args[1]) { + Ok(config) => run(&config, dryrun), + Err(err) => utils::log(format!( + "Something went wrong while reading configuration file. Error was: {}", + err + )), } } -fn run_task(t: &Task, dryrun: bool) -> () { - let tstart = Utc::now(); - let source = format!("{}:{}", t.source_remote, t.source_path); - let target = format!("{}:{}", t.target_remote, t.target_path); - - log(format!("Syncing from \"{}\" to \"{}\"", &source, &target)); - - let mut args = vec![ - "-v", - "--stats-log-level", - "NOTICE", - "--stats", - "10000m", - "sync", - &source, - &target, - ]; - - if dryrun { - args.push("--dry-run"); - } - - for f in &t.filters { - args.push("--filter"); - args.push(f); - } - - let mut child = Command::new("rclone") - .args(&args) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn() - .expect("Failed to execute process"); - - let output = child.wait().expect("Failed to read stdout from the task"); - - let elapsed = Utc::now() - tstart; - - match output.code() { - Some(0) => log(format!("Tasks finished successfully in {:?} second(s).", elapsed.num_seconds())), - Some(code) => log(format!("Task finished with errors in {:?} second(s). Error code: {}", elapsed.num_seconds(), code)), - None => log(format!("Task terminated by signal in {:?} second(s).", elapsed.num_seconds())), +fn run(config: &config::Config, dryrun: bool) -> () { + for task in &config.tasks { + rclone::run_task(&task, dryrun) } } diff --git a/src/rclone.rs b/src/rclone.rs new file mode 100644 index 0000000..0eab89e --- /dev/null +++ b/src/rclone.rs @@ -0,0 +1,86 @@ +use chrono::Utc; +use std::process::Command; +use std::process::Stdio; + +use crate::config; +use crate::utils; + +pub fn run_task(task: &config::Task, dryrun: bool) -> () { + let tstart = Utc::now(); + utils::log(format!("Running task: {}", &task.name)); + + let mut args = task_to_args(&task); + + if dryrun { + args.push("--dry-run".to_string()); + utils::log(format!("Running rclone with args {}", args.join(" "))) + } + + let mut child = Command::new("rclone") + .args(&args) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .expect("Failed to execute process"); + + let output = child.wait().expect("Failed to read stdout from the task"); + + let elapsed = (Utc::now() - tstart).num_seconds(); + + match output.code() { + Some(0) => utils::log(format!( + "Tasks finished successfully in {:?} second(s).", + elapsed + )), + Some(code) => utils::log(format!( + "Task finished with errors in {:?} second(s). Error code: {}", + elapsed, code + )), + None => utils::log(format!( + "Task terminated by signal in {:?} second(s).", + elapsed + )), + } +} + +pub fn task_to_args(task: &config::Task) -> Vec { + let mut args = vec![ + "-v".to_string(), // Run in verbose mode. + "--stats-log-level".to_string(), + "NOTICE".to_string(), + "--stats".to_string(), + "10000m".to_string(), + "sync".to_string(), + format!( + "{}{}", + backend_to_args(&task.source.backend), + &task.source.path + ), + format!( + "{}{}", + backend_to_args(&task.target.backend), + &task.target.path + ), + ]; + + for f in &task.source.filters { + args.push("--filter".to_string()); + args.push(f.to_string()); + } + + args +} + +pub fn backend_to_args(backend: &config::Backend) -> String { + let vars: Vec = match &backend.vars { + Some(xs) => xs.iter().map(|(x, y)| format!("{}={}", x, y)).collect(), + None => vec![], + }; + + return format!( + ":{}{}{}:", + backend.ctype, + if vars.len() == 0 { "" } else { "," }, + vars.join(","), + ); +} diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..ed79346 --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,9 @@ +use chrono::Utc; + +pub fn log(msg: String) -> () { + eprintln!("RRCLONE>> [{}] {}", isonow(), msg); +} + +pub fn isonow() -> String { + Utc::now().format("%Y-%m-%d %H:%M:%S").to_string() +}