diff --git a/.gitignore b/.gitignore index 4b8b7d8..16eb10c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ ideas.md templify-example dev/ -tpy-test-dir \ No newline at end of file +tpy-test-dir +command.md \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 591e789..427f4ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -671,6 +680,35 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + [[package]] name = "reqwest" version = "0.12.2" @@ -913,7 +951,9 @@ dependencies = [ name = "templify" version = "1.0.0" dependencies = [ + "base64", "chrono", + "regex", "reqwest", "self-replace", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index b076fef..dadbe98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,5 @@ reqwest = { version = "0.12", features = ["blocking", "json"] } self-replace = "1.3.6" serde_json = "1.0.1" chrono = "0.4.19" +base64 = "0.21.7" +regex = "1.11.1" diff --git a/src/commands/load.rs b/src/commands/load.rs index fbe70be..46ce5c8 100644 --- a/src/commands/load.rs +++ b/src/commands/load.rs @@ -17,7 +17,7 @@ pub(crate) fn definition() -> Command { "url".to_string(), 0, true, - "The url of the github repository.".to_string(), + "The url of the github or gitlab repository.".to_string(), )); load_command.add_flag(Flag::new_bool_flag( @@ -46,12 +46,6 @@ pub(crate) fn load(command: &Command) -> Status { } let url = command.get_argument("url").value.clone(); - if !url.starts_with("https://github.com") { - return Status::error(format!( - "Invalid url: {}\nOnly templates from GitHub are supported at the moment.", - url - )); - } let load_template = command.get_bool_flag("template"); if load_template { @@ -61,6 +55,7 @@ pub(crate) fn load(command: &Command) -> Status { format!(".templates/{}", name).as_str(), url.as_str(), command.get_bool_flag("force"), + None, ); if !st.is_ok { return st; diff --git a/src/types/load_types.rs b/src/types/load_types.rs new file mode 100644 index 0000000..92e4646 --- /dev/null +++ b/src/types/load_types.rs @@ -0,0 +1,6 @@ +/// This enum is used to define type of URL (.i.e.. GitHub, GitLab) +#[derive(Clone)] +pub(crate) enum URLType { + GitHub, + GitLab, +} diff --git a/src/types/mod.rs b/src/types/mod.rs index 0d2a42f..384c641 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -2,6 +2,7 @@ pub mod argument; pub mod command; pub mod flag; pub mod global_flag; +pub mod load_types; pub mod status; pub mod template_meta; pub mod var_placeholder; diff --git a/src/utils/template_handler.rs b/src/utils/template_handler.rs index 63ac26b..87fbc39 100644 --- a/src/utils/template_handler.rs +++ b/src/utils/template_handler.rs @@ -1,7 +1,10 @@ use crate::log; +use crate::types::load_types::URLType; use crate::types::status::Status; use crate::types::template_meta::TemplateMeta; use crate::utils::formater; +use base64::{engine::general_purpose::STANDARD, Engine as _}; +use regex::Regex; use std::io::Write; use std::path::Path; @@ -80,10 +83,13 @@ pub(crate) fn reload_template(name: String, strict: bool, reset: bool) -> Status std::fs::rename(&dir, &backup_dir).unwrap(); } + let url = meta.get_source(); + let st = load_remote_template( format!(".templates/{}", name).as_str(), - meta.get_source().as_str(), + url.as_str(), true, + None, ); if !st.is_ok { if reset { @@ -102,6 +108,11 @@ pub(crate) fn reload_template(name: String, strict: bool, reset: bool) -> Status /// Load a collection of templates from a remote repository pub(crate) fn load_remote_template_collection(path: &str, url: &str, force: bool) -> Status { + let url_type = match determine_url_type(url) { + Ok(url_type) => url_type, + Err(st) => return st, + }; + let response = rest::json_call(url); if response.is_err() { return Status::error(format!( @@ -118,9 +129,18 @@ pub(crate) fn load_remote_template_collection(path: &str, url: &str, force: bool } let response: serde_json::Value = response.unwrap(); - let items = response["payload"]["tree"]["items"].as_array().unwrap(); + let items = match url_type { + URLType::GitHub => response["payload"]["tree"]["items"].as_array().unwrap(), + URLType::GitLab => response.as_array().unwrap(), + }; + for item in items { - if item["contentType"] == "directory" { + let check_collection = match url_type { + URLType::GitHub => item["contentType"] == "directory", + URLType::GitLab => item["type"] == "tree", + }; + + if check_collection { let st = load_remote_template( format!("{}/{}", path, item["name"]) .replace('"', "") @@ -129,17 +149,24 @@ pub(crate) fn load_remote_template_collection(path: &str, url: &str, force: bool .replace('"', "") .as_str(), force, + Some(&url_type), ); if !st.is_ok { return st; } } } + Status::ok() } /// Load a template from a remote repository -pub(crate) fn load_remote_template(path: &str, url: &str, force: bool) -> Status { +pub(crate) fn load_remote_template( + path: &str, + url: &str, + force: bool, + url_type: Option<&URLType>, +) -> Status { if !force && Path::new(path).exists() { return Status::error(format!( "Template {} already exists...", @@ -147,6 +174,14 @@ pub(crate) fn load_remote_template(path: &str, url: &str, force: bool) -> Status )); } + let url_type_info: URLType = match url_type { + Some(ut) => ut.clone(), + None => match determine_url_type(url) { + Ok(ut) => ut, + Err(st) => return st, + }, + }; + if !Path::new(path).exists() { std::fs::create_dir(path).unwrap(); } @@ -167,37 +202,13 @@ pub(crate) fn load_remote_template(path: &str, url: &str, force: bool) -> Status } let response: serde_json::Value = response.unwrap(); - let items = response["payload"]["tree"]["items"].as_array().unwrap(); + let status = match url_type_info { + URLType::GitHub => load_github_template(response, path, url, force), + URLType::GitLab => load_gitlab_template(response, path, url, force), + }; - for item in items { - if item["contentType"] == "directory" { - let st = load_remote_template_dir( - format!("{}/{}", path, item["name"]) - .replace('"', "") - .as_str(), - format!("{}/{}", url, item["name"]) - .replace('"', "") - .as_str(), - force, - ); - if !st.is_ok { - return st; - } - continue; - } - - let st = load_remote_template_file( - format!("{}/{}", path, item["name"]) - .replace('"', "") - .as_str(), - format!("{}/{}", url, item["name"]) - .replace('"', "") - .as_str(), - force, - ); - if !st.is_ok { - return st; - } + if !status.is_ok { + return status; } let temp_file = format!("{}/.templify", path); @@ -228,6 +239,64 @@ pub(crate) fn load_remote_template(path: &str, url: &str, force: bool) -> Status Status::ok() } +/// Load a template from a gitlab repository +fn load_gitlab_template(response: serde_json::Value, path: &str, url: &str, force: bool) -> Status { + let items = response.as_array().unwrap(); + + for item in items { + let formatted_path = &format_path_or_url(path, item); + let formatted_url = &format_path_or_url(url, item); + + if item["type"] == "tree" { + let st = load_remote_gitlab_template_dir(formatted_path, formatted_url, force); + if !st.is_ok { + return st; + } + continue; + } + + let base_url = url.split("/tree").next().unwrap_or(""); + + if base_url.is_empty() { + return Status::error(format!("Invalid url: {}\n", url)); + } + + let formatted_blob_url = &format_blob_url(base_url, item); + + let st = load_remote_gitlab_template_file(formatted_path, formatted_blob_url, force); + if !st.is_ok { + return st; + } + } + + Status::ok() +} + +/// Load a template from a github repository +fn load_github_template(response: serde_json::Value, path: &str, url: &str, force: bool) -> Status { + let items = response["payload"]["tree"]["items"].as_array().unwrap(); + + for item in items { + let formatted_path = &format_path_or_url(path, item); + let formatted_url = &format_path_or_url(url, item); + + if item["contentType"] == "directory" { + let st = load_remote_template_dir(formatted_path, formatted_url, force); + if !st.is_ok { + return st; + } + continue; + } + + let st = load_remote_template_file(formatted_path, formatted_url, force); + if !st.is_ok { + return st; + } + } + + Status::ok() +} + /// Load a directory from a remote repository fn load_remote_template_dir(path: &str, url: &str, force: bool) -> Status { if !force && Path::new(path).exists() { @@ -288,6 +357,60 @@ fn load_remote_template_dir(path: &str, url: &str, force: bool) -> Status { Status::ok() } +/// Load Gitlab Template Directory +fn load_remote_gitlab_template_dir(path: &str, url: &str, force: bool) -> Status { + if !force && Path::new(path).exists() { + return Status::error(format!( + "Directory {} already exists...", + path.replace(".templates/", "") + )); + } + + if !Path::new(path).exists() { + std::fs::create_dir(path).unwrap(); + } + + let response = rest::json_call(url); + if response.is_err() { + return Status::error(format!( + "Failed to get template from {}: Request failed", + url + )); + } + let response = response.unwrap().json(); + if response.is_err() { + return Status::error(format!( + "Failed to get template from {}: JSON parse error", + url + )); + } + let response: serde_json::Value = response.unwrap(); + let items = response.as_array().unwrap(); + + for item in items { + let formatted_path = &format_path_or_url(path, item); + let formatted_url = &format_path_or_url(url, item); + + if item["type"] == "tree" { + let st = load_remote_gitlab_template_dir(formatted_path, formatted_url, force); + if !st.is_ok { + return st; + } + continue; + } + + let base_url = url.split("/tree").next().unwrap_or(""); + + if base_url.is_empty() { + return Status::error(format!("Invalid url: {}\n", url)); + } + let formatted_blob_url = &format_blob_url(base_url, item); + + load_remote_gitlab_template_file(formatted_path, formatted_blob_url, force); + } + Status::ok() +} + /// Load a file from a remote repository fn load_remote_template_file(path: &str, url: &str, force: bool) -> Status { if Path::new(path).exists() && !force { @@ -334,6 +457,73 @@ fn load_remote_template_file(path: &str, url: &str, force: bool) -> Status { Status::ok() } +/// Load a file from gitlab remote repository +fn load_remote_gitlab_template_file(path: &str, url: &str, force: bool) -> Status { + if Path::new(path).exists() && !force { + return Status::error(format!( + "File {} already exists...", + path.replace(".templates/", "") + )); + } + + let response = rest::json_call(url); + if response.is_err() { + return Status::error(format!( + "Failed to get template from {}: Request failed", + url + )); + } + let response = response.unwrap().json(); + if response.is_err() { + return Status::error(format!( + "Failed to get template from {}: JSON parse error", + url + )); + } + let response: serde_json::Value = response.unwrap(); + + let content = response["content"].as_str(); + let encoding = response["encoding"].as_str(); + + if encoding.unwrap_or("") != "base64" || content.is_none() { + return Status::error(format!( + "Failed to get template from {}: Decoding Error", + url + )); + } + + let mut text = match STANDARD.decode(content.unwrap()) { + Ok(decoded) => match String::from_utf8(decoded) { + Ok(message) => message, + Err(_e) => { + return Status::error(format!( + "Failed to get template from {}: Decoding Error", + url + )) + } + }, + Err(_e) => { + return Status::error(format!( + "Failed to get template from {}: Decoding Error", + url + )) + } + }; + + text = text.replace("\\n", "\n"); + + // create all subdirs if they don't exist + let path_dir = path.split('/').collect::>(); + let path_dir = path_dir[..path_dir.len() - 1].join("/"); + std::fs::create_dir_all(path_dir.clone()).unwrap(); + + let mut new_file = std::fs::File::create(path).unwrap(); + new_file.write_all(text.as_bytes()).unwrap(); + + log!("Created file {}", path); + Status::ok() +} + /// Generate a template from a template pub(crate) fn generate_template_dir( path: &str, @@ -420,3 +610,35 @@ pub(crate) fn generate_template_file( log!("Created file {}", abs_path.to_str().unwrap()); true } + +/// Format Path for loading Gitlab Template +fn format_path_or_url(path_or_url: &str, item: &serde_json::Value) -> String { + format!("{}/{}", path_or_url, item["name"].as_str().unwrap_or("")).replace('"', "") +} + +/// Format URL for loading Gitlab File Blob +fn format_blob_url(base_url: &str, item: &serde_json::Value) -> String { + format!("{}/blobs/{}", base_url, item["id"].as_str().unwrap_or("")).replace('"', "") +} + +/// Determine the URL Type (Gitlab , Github) from the url +fn determine_url_type(url: &str) -> Result { + let url_type = if url.starts_with("https://github.com") { + URLType::GitHub + } else if is_valid_gitlab_url(url) { + URLType::GitLab + } else { + return Err(Status::error(format!( + "Invalid url: {}\nOnly templates from GitHub and Gitlab are supported at the moment.", + url + ))); + }; + + Ok(url_type) +} + +/// Check for valid gitlab url +fn is_valid_gitlab_url(url: &str) -> bool { + let gitlab_url_pattern = Regex::new(r"^https:\/\/(?:[\w-]+\.)*gitlab\.com(?:\/.*)?$").unwrap(); + gitlab_url_pattern.is_match(url) +} diff --git a/tests/command_tests/load_test.rs b/tests/command_tests/load_test.rs index f78b4c0..bb54863 100644 --- a/tests/command_tests/load_test.rs +++ b/tests/command_tests/load_test.rs @@ -3,6 +3,44 @@ include!("../common/fs.rs"); include!("../common/log.rs"); pub fn test() { + check_github_load(); + check_gitlab_load(); +} + +pub fn check_gitlab_load() { + utils::init_tpy(); + + utils::run_successfully("tpy load https://gitlab.com/api/v4/projects/cophilot%2Ftemplify-vault/repository/tree?path=Test"); + check_gitlab_test_structure(); + utils::run_failure("tpy load https://gitlab.com/api/v4/projects/cophilot%2Ftemplify-vault/repository/tree?path=Test"); + + // check -force flag + utils::run_successfully( + "tpy load https://gitlab.com/api/v4/projects/cophilot%2Ftemplify-vault/repository/tree?path=Test -f", + ); + check_gitlab_test_structure(); + + utils::reset_dir(); + utils::init_tpy(); + + // check -template flag + utils::run_successfully( + "tpy load https://gitlab.com/api/v4/projects/cophilot%2Ftemplify-vault/repository/tree?path=Test/Test1 -t", + ); + check_gitlab_test_1_structure(); + fs::templates_dir().dir("Test2").check_not_exists(); + + utils::reset_dir(); + utils::init_tpy(); + + utils::run_successfully( + "tpy load https://gitlab.com/api/v4/projects/cophilot%2Ftemplify-vault/repository/tree?path=Test/Test2 -t", + ); + check_gitlab_test_2_structure(); + fs::templates_dir().dir("Test1").check_not_exists(); +} + +pub fn check_github_load() { utils::init_tpy(); utils::run_successfully("tpy load https://github.com/cophilot/templify-vault/tree/main/Test"); @@ -33,6 +71,16 @@ pub fn test() { ); check_test_2_structure(); fs::templates_dir().dir("Test1").check_not_exists(); + + utils::reset_dir(); + + utils::run_failure("tpy load https://my-gitlab.company.com"); +} + +pub fn check_gitlab_test_structure() { + check_gitlab_test_1_structure(); + check_gitlab_test_2_structure(); + check_gitlab_my_test_structure(); } pub fn check_test_structure() { @@ -89,3 +137,52 @@ pub fn check_my_test_structure() { ".source:https://github.com/cophilot/templify-vault/tree/main/Test/MyTest", ); } + +pub fn check_gitlab_test_1_structure() { + fs::templates_dir() + .dir("Test1") + .file("Test1$$name$$.txt") + .contains_string("$$name$$") + .contains_string( + "Paddington loves to eat marmalade sandwiches and he is a very polite bear.", + ); + fs::templates_dir() + .dir("Test1") + .file(".templify") + .contains_string("description:This is used to test templify") + .contains_string("path:src") + .contains_string(".source:https://gitlab.com/api/v4/projects/cophilot%2Ftemplify-vault/repository/tree?path=Test/Test1"); +} + +pub fn check_gitlab_test_2_structure() { + let mut base = fs::templates_dir().dir("Test2"); + + base.file(".templify") + .contains_string("description:This is used to test templify") + .contains_string("path:src") + .contains_string(".source:https://gitlab.com/api/v4/projects/cophilot%2Ftemplify-vault/repository/tree?path=Test/Test2"); + base.file("file.txt") + .contains_string("A elephant can eat 300 pounds of food in a day."); + base.dir("subdir") + .file("file.txt") + .contains_string("Apollo 11 started its journey to the moon on July 16, 1969."); + base.dir("subdir") + .dir("subdir") + .file(".tpykeep") + .check_all_exists(); +} + +pub fn check_gitlab_my_test_structure() { + fs::templates_dir() + .dir("MyTest") + .file("file.txt") + .contains_string("Nebraska has the largest indoor rainforest in the world."); + fs::templates_dir() + .dir("MyTest") + .file(".templify") + .contains_string("description:This is used to test templify") + .contains_string("path:src") + .contains_string( + ".source:https://gitlab.com/api/v4/projects/cophilot%2Ftemplify-vault/repository/tree?path=Test/MyTest", + ); +}