diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 2ba1703..3914d77 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -9,10 +9,10 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: clean: false - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache with: path: | diff --git a/Cargo.lock b/Cargo.lock index 0332730..65f6d32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,6 +208,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "either" version = "1.13.0" @@ -699,6 +705,7 @@ dependencies = [ name = "ua_generator" version = "0.5.2" dependencies = [ + "dotenv", "fastrand", "serde", "serde_json", diff --git a/ua_generator/Cargo.toml b/ua_generator/Cargo.toml index ecba24c..b0da95a 100644 --- a/ua_generator/Cargo.toml +++ b/ua_generator/Cargo.toml @@ -19,4 +19,5 @@ fastrand = "2" ureq = { version = "2", features = ["json", "charset", "brotli"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1" } -toml = { version = "0" } \ No newline at end of file +toml = { version = "0" } +dotenv = "0.15.0" diff --git a/ua_generator/build.rs b/ua_generator/build.rs index 0912142..77bf0f2 100644 --- a/ua_generator/build.rs +++ b/ua_generator/build.rs @@ -30,25 +30,29 @@ fn bump_version_in_cargo_toml() -> Result<(), Box> { let cargo_toml_path = Path::new("Cargo.toml"); let mut cargo_toml_content = String::new(); + crate::fs::OpenOptions::new() .read(true) .open(&cargo_toml_path)? .read_to_string(&mut cargo_toml_content)?; - let mut parsed_toml: Value = cargo_toml_content.parse()?; - if let Some(version) = parsed_toml - .get_mut("package") - .and_then(|pkg| pkg.get_mut("version")) - { - if let Some(version_str) = version.as_str() { - let new_version = increment_version(version_str); - *version = Value::String(new_version.clone()); - println!("Bumped version to: {}", new_version); + if !cargo_toml_content.is_empty() { + let mut parsed_toml: Value = cargo_toml_content.parse()?; + + if let Some(version) = parsed_toml + .get_mut("package") + .and_then(|pkg| pkg.get_mut("version")) + { + if let Some(version_str) = version.as_str() { + let new_version = increment_version(version_str); + *version = Value::String(new_version.clone()); + println!("Bumped version to: {}", new_version); + } } - } - let new_cargo_toml_content = toml::to_string(&parsed_toml)?; - fs::write(cargo_toml_path, new_cargo_toml_content)?; + let new_cargo_toml_content = toml::to_string(&parsed_toml)?; + fs::write(cargo_toml_path, new_cargo_toml_content)?; + } Ok(()) } @@ -71,14 +75,15 @@ pub fn get_agent(url: &str, token: &String) -> String { /// build entry for setting required agents fn main() -> Result<(), Box> { - let build_enabled = env::var("BUILD_ENABLED").map(|v| v == "1").unwrap_or(false); + let build_enabled = dotenv::var("BUILD_ENABLED") + .map(|v| v == "1" || v == "true") + .unwrap_or(false); if build_enabled { let base_api = - env::var("API_URL").unwrap_or("https://api.spider.cloud/data/user_agents".into()); - + dotenv::var("API_URL").unwrap_or("https://api.spider.cloud/data/user_agents".into()); // fetch the latest ua and parse to files. - let token: String = match env::var("APILAYER_KEY") { + let token: String = match dotenv::var("APILAYER_KEY") { Ok(key) => key, Err(_) => { println!("You need a valid {} API key to gather agents!", base_api); @@ -154,6 +159,7 @@ pub fn agents() -> [&'static str; 9] {{ let chrome_agent_list: Vec = match get(&format!("{base_api}?chrome=true&list=true")) .set("apikey", &token) + .set("user-agent", "spider-rs") .call() { Ok(req) => { @@ -181,9 +187,9 @@ pub const STATIC_CHROME_AGENTS: &'static [&'static str; {}] = &[ chrome_devices.push_str("];"); fs::write(dest_path, chrome_devices).unwrap(); - + bump_version_in_cargo_toml()?; println!("cargo:rerun-if-changed=build.rs"); } - bump_version_in_cargo_toml()?; + Ok(()) } diff --git a/ua_generator/src/chrome_ua_list.rs b/ua_generator/src/chrome_ua_list.rs index 7e55487..90ee160 100644 --- a/ua_generator/src/chrome_ua_list.rs +++ b/ua_generator/src/chrome_ua_list.rs @@ -1,23 +1,23 @@ /// List of real Chrome User-Agents. pub const STATIC_CHROME_AGENTS: &'static [&'static str; 20] = &[ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", - "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/129.0.6668.69 Mobile/15E148 Safari/604.1", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/130.0.6723.37 Mobile/15E148 Safari/604.1", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36", - "Mozilla/5.0 (iPhone; CPU iPhone OS 18_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/129.0.6668.69 Mobile/15E148 Safari/604.1", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.95 Safari/537.36", - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/129.0.6668.46 Mobile/15E148 Safari/604.1", -]; + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", +]; \ No newline at end of file diff --git a/ua_generator/src/ua_list.rs b/ua_generator/src/ua_list.rs index cbf7cdc..518072e 100644 --- a/ua_generator/src/ua_list.rs +++ b/ua_generator/src/ua_list.rs @@ -1,15 +1,15 @@ /// static list of agents pre-built pub const STATIC_AGENTS: &'static [&'static str; 9] = &[ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.95 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:125.0) Gecko/20100101 Firefox/125.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" -]; + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", + "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/26.0 Chrome/122.0.0.0 Mobile Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:131.0) Gecko/20100101 Firefox/131.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36" +]; /// user agent list pub fn agents() -> [&'static str; 9] {