Skip to content

Commit

Permalink
chore(ci): fix sk passing
Browse files Browse the repository at this point in the history
  • Loading branch information
@j-mendez
j-mendez committed Oct 25, 2024
1 parent cce0460 commit e6d611f
Show file tree
Hide file tree
Showing 6 changed files with 60 additions and 46 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/cron.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
clean: false
- uses: actions/cache@v3
- uses: actions/cache@v4
id: cache
with:
path: |
Expand Down
7 changes: 7 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion ua_generator/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ fastrand = "2"
ureq = { version = "2", features = ["json", "charset", "brotli"] }
serde = { version = "1", features = ["derive"] }
serde_json = { version = "1" }
toml = { version = "0" }
toml = { version = "0" }
dotenv = "0.15.0"
42 changes: 24 additions & 18 deletions ua_generator/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,25 +30,29 @@ fn bump_version_in_cargo_toml() -> Result<(), Box<dyn std::error::Error>> {
let cargo_toml_path = Path::new("Cargo.toml");

let mut cargo_toml_content = String::new();

crate::fs::OpenOptions::new()
.read(true)
.open(&cargo_toml_path)?
.read_to_string(&mut cargo_toml_content)?;

let mut parsed_toml: Value = cargo_toml_content.parse()?;
if let Some(version) = parsed_toml
.get_mut("package")
.and_then(|pkg| pkg.get_mut("version"))
{
if let Some(version_str) = version.as_str() {
let new_version = increment_version(version_str);
*version = Value::String(new_version.clone());
println!("Bumped version to: {}", new_version);
if !cargo_toml_content.is_empty() {
let mut parsed_toml: Value = cargo_toml_content.parse()?;

if let Some(version) = parsed_toml
.get_mut("package")
.and_then(|pkg| pkg.get_mut("version"))
{
if let Some(version_str) = version.as_str() {
let new_version = increment_version(version_str);
*version = Value::String(new_version.clone());
println!("Bumped version to: {}", new_version);
}
}
}

let new_cargo_toml_content = toml::to_string(&parsed_toml)?;
fs::write(cargo_toml_path, new_cargo_toml_content)?;
let new_cargo_toml_content = toml::to_string(&parsed_toml)?;
fs::write(cargo_toml_path, new_cargo_toml_content)?;
}

Ok(())
}
Expand All @@ -71,14 +75,15 @@ pub fn get_agent(url: &str, token: &String) -> String {

/// build entry for setting required agents
fn main() -> Result<(), Box<dyn std::error::Error>> {
let build_enabled = env::var("BUILD_ENABLED").map(|v| v == "1").unwrap_or(false);
let build_enabled = dotenv::var("BUILD_ENABLED")
.map(|v| v == "1" || v == "true")
.unwrap_or(false);

if build_enabled {
let base_api =
env::var("API_URL").unwrap_or("https://api.spider.cloud/data/user_agents".into());

dotenv::var("API_URL").unwrap_or("https://api.spider.cloud/data/user_agents".into());
// fetch the latest ua and parse to files.
let token: String = match env::var("APILAYER_KEY") {
let token: String = match dotenv::var("APILAYER_KEY") {
Ok(key) => key,
Err(_) => {
println!("You need a valid {} API key to gather agents!", base_api);
Expand Down Expand Up @@ -154,6 +159,7 @@ pub fn agents() -> [&'static str; 9] {{
let chrome_agent_list: Vec<ApiResult> =
match get(&format!("{base_api}?chrome=true&list=true"))
.set("apikey", &token)
.set("user-agent", "spider-rs")
.call()
{
Ok(req) => {
Expand Down Expand Up @@ -181,9 +187,9 @@ pub const STATIC_CHROME_AGENTS: &'static [&'static str; {}] = &[
chrome_devices.push_str("];");

fs::write(dest_path, chrome_devices).unwrap();

bump_version_in_cargo_toml()?;
println!("cargo:rerun-if-changed=build.rs");
}
bump_version_in_cargo_toml()?;

Ok(())
}
32 changes: 16 additions & 16 deletions ua_generator/src/chrome_ua_list.rs
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
/// List of real Chrome User-Agents.
pub const STATIC_CHROME_AGENTS: &'static [&'static str; 20] = &[
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/129.0.6668.69 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/130.0.6723.37 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/129.0.6668.69 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.95 Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/129.0.6668.46 Mobile/15E148 Safari/604.1",
];
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
];
18 changes: 9 additions & 9 deletions ua_generator/src/ua_list.rs
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
/// static list of agents pre-built
pub const STATIC_AGENTS: &'static [&'static str; 9] = &[
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.95 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:125.0) Gecko/20100101 Firefox/125.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
];
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/26.0 Chrome/122.0.0.0 Mobile Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:131.0) Gecko/20100101 Firefox/131.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"
];

/// user agent list
pub fn agents() -> [&'static str; 9] {
Expand Down

0 comments on commit e6d611f

Please sign in to comment.