From 90cfb82af07e130b962fc73e053c10d36c245b2e Mon Sep 17 00:00:00 2001 From: Rohit kumar Date: Mon, 16 Sep 2024 02:42:39 +0530 Subject: [PATCH] Added commit_id detection for secret scan --- Dockerfile | 4 +-- src/utils/common.rs | 38 +++++++++++++++++++++-- src/utils/pipeline.rs | 71 +++++++++++++++++++++---------------------- 3 files changed, 72 insertions(+), 41 deletions(-) diff --git a/Dockerfile b/Dockerfile index baaf508..0963999 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:latest +FROM rust:latest;ls; RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2024-02-04 COPY Cargo.toml /hela/Cargo.toml COPY Cargo.lock /hela/Cargo.lock @@ -13,7 +13,7 @@ RUN rm -rf /hela # Update the package list and upgrade the system RUN apt-get update && \ apt-get -y upgrade - + ## configure tzdata to avoid interactive prompt ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Europe/London diff --git a/src/utils/common.rs b/src/utils/common.rs index 1d5a8b3..083eb05 100644 --- a/src/utils/common.rs +++ b/src/utils/common.rs @@ -271,6 +271,8 @@ pub fn checkout( commit_ids: Option<&str>, branch_name: Option<&str>, ) -> Result<(), Box> { + let mut file_commit_map: HashMap = HashMap::new(); + let commit_hashes: Vec<&str> = match commit_ids { Some(ref ids) if !ids.is_empty() => ids.split(',').collect(), _ => vec![], @@ -315,10 +317,12 @@ pub fn checkout( } } - // if commit_ids provided only then run below logic + // If no commit IDs are provided, return early. if commit_hashes.is_empty() { + save_commit_map(&file_commit_map)?; return Ok(()); } + let mut all_files = String::new(); for commit in commit_hashes { let output = Command::new("git") @@ -337,7 +341,6 @@ pub fn checkout( .arg("--name-only") .arg(format!("{}^", commit)) .arg(commit) - .stdout(Stdio::piped()) .output()?; if !output.status.success() { @@ -349,6 +352,11 @@ pub fn checkout( let files = String::from_utf8_lossy(&output.stdout); all_files.push_str(&files); + + // Map each file to the current commit ID. + for file in files.lines() { + file_commit_map.insert(file.to_string(), commit.to_string()); + } } println!("FILES\n______\n{}", all_files); @@ -357,9 +365,35 @@ pub fn checkout( delete_empty_directories(&cloned_path)?; + // Save the commit map to /tmp/commit_map.json. + save_commit_map(&file_commit_map)?; + Ok(()) } +// Function to save the commit map to /tmp/commit_map.json. +fn save_commit_map( + file_commit_map: &HashMap, +) -> Result<(), Box> { + let commit_map_path = "/tmp/commit_map.json"; + let file = File::create(commit_map_path)?; + serde_json::to_writer(file, file_commit_map)?; + println!("Commit map saved to: {}", commit_map_path); + Ok(()) +} + +pub fn get_commit_of_file(file_path: &str) -> Result> { + let relative_path = file_path.split("/").collect::>()[3..] + .join("/") + .replace("\"", ""); + let commit_map_path = "/tmp/commit_map.json"; + let file = File::open(commit_map_path)?; + let file_commit_map: HashMap = serde_json::from_reader(file)?; + let binding = "".to_string(); + let commit_id = file_commit_map.get(&relative_path).unwrap_or(&binding); + Ok(commit_id.to_string()) +} + fn delete_except(files: &str, base_dir: &Path) -> Result<(), Box> { let files_to_keep: Vec = files .lines() diff --git a/src/utils/pipeline.rs b/src/utils/pipeline.rs index a7b657c..d3104e7 100644 --- a/src/utils/pipeline.rs +++ b/src/utils/pipeline.rs @@ -3,7 +3,7 @@ use serde_json::{json, Value}; use std::{collections::HashMap, process::exit}; use crate::utils::common::{ - bulk_check_hash_exists, insert_job_info, slack_alert, upload_to_defect_dojo, + bulk_check_hash_exists, get_commit_of_file, insert_job_info, slack_alert, upload_to_defect_dojo, }; use super::common::{self, execute_command, print_error, redact_github_token}; @@ -194,7 +194,6 @@ pub async fn pipeline_failure( let hashes: Vec = message_to_hash.keys().cloned().collect(); let existing_hashes_result = bulk_check_hash_exists(&hashes, &mongo_uri).await; - // Handle the Result properly let existing_hashes = match existing_hashes_result { Ok(hashes) => hashes, @@ -446,35 +445,29 @@ pub async fn pipeline_failure( let mut secret_result = HashMap::new(); secret_result.insert( "file", - result["SourceMetadata"]["Data"]["Filesystem"]["file"].to_string(), + result["SourceMetadata"]["Data"]["Filesystem"]["file"] + .as_str() + .unwrap_or("") + .to_string(), ); secret_result.insert("line", number_string); - secret_result.insert("raw", result["Raw"].to_string()); + secret_result.insert("raw", result["Raw"].as_str().unwrap_or("").to_string()); secret_result.insert( "detector_name", - result["DetectorName"].to_string().to_uppercase(), + result["DetectorName"].as_str().unwrap_or("").to_uppercase(), + ); + secret_result.insert( + "decoder_name", + result["DecoderName"].as_str().unwrap_or("").to_string(), ); - secret_result.insert("decoder_name", result["DecoderName"].to_string()); secret_result }; secret_results.push(secret_result); - if !detected_detectors.contains( - &result["DetectorName"] - .as_str() - .unwrap() - .to_string() - .to_uppercase(), - ) { - detected_detectors.push( - result["DetectorName"] - .as_str() - .unwrap() - .to_string() - .to_uppercase(), - ); + let detector_name = result["DetectorName"].as_str().unwrap_or("").to_uppercase(); + if !detected_detectors.contains(&detector_name) { + detected_detectors.push(detector_name); } } - detected_detectors = detected_detectors .iter() .map(|x| x.to_string()) @@ -493,13 +486,18 @@ pub async fn pipeline_failure( } let mut secret_count = 0; - let mut message_to_hash: HashMap = HashMap::new(); + let mut message_to_hash: HashMap = + HashMap::new(); // Collect all secret records and their hashes for value in secret_results.clone() { + // Append to slack alert message, remove first 2 values after split with "/" + let file_commit = get_commit_of_file(&value["file"]); + let commit_base_link = commit_path.split("/commit").collect::>()[0]; + let commit_link = format!("{}/commit/{}", commit_base_link, file_commit.unwrap()); let vuln_record = format!( - "\n\nFile: {}\nLine: {}\nRaw: {}\nDetector Name: {}", - value["file"], value["line"], value["raw"], value["detector_name"] + "\n\nFile: {}\nLine: {}\nRaw: {}\nDetector Name: {}\nCommit: {}", + value["file"], value["line"], value["raw"], value["detector_name"], commit_link ); let hashed_message = common::hash_text(&vuln_record); @@ -507,10 +505,11 @@ pub async fn pipeline_failure( message_to_hash.insert( hashed_message, ( - value["file"].replace("\"", ""), + value["file"].clone(), value["line"].clone(), - value["raw"].replace("\"", ""), - value["detector_name"].replace("\"", ""), + value["raw"].clone(), + value["detector_name"].clone(), + commit_link, ), ); } @@ -529,10 +528,8 @@ pub async fn pipeline_failure( }; let mut secret_count = 0; - let mut found_secret_issues = false; - // Process each message to check for existence and add to the table - for (hashed_message, (file, line, raw, detector_name)) in message_to_hash { + for (hashed_message, (file, line, raw, detector_name, commit_link)) in message_to_hash { if !existing_hashes.contains(&hashed_message) { found_secret_issues = true; secret_count += 1; @@ -545,10 +542,9 @@ pub async fn pipeline_failure( // Add row to table table.add_row(row![secret_count, file, line, raw_truncated, detector_name]); - // Append to slack alert message slack_alert_msg.push_str(&format!( - "\n\nFile: {}\nLine: {}\nRaw: {}\nDetector Name: {}", - file, line, raw, detector_name + "\n\nFile: {}\nLine: {}\nRaw: {}\nDetector Name: {}\nCommit: {}", + file, line, raw, detector_name, commit_link )); // Register the missing hash @@ -617,7 +613,6 @@ pub async fn pipeline_failure( .collect::>(); pipeline_secret_license_data.insert("licenses", licenses_list); } - if found_sast_issues == false && found_sca_issues == false && found_secret_issues == false @@ -1101,7 +1096,7 @@ pub async fn pipeline_failure( println!("\t\t Job ID: {}", job_id); if !mongo_uri.is_empty() { println!("\t\t Inserting job info into MongoDB"); - insert_job_info( + let _ = insert_job_info( &mongo_uri, &job_id, &pipeline_failure_reason, @@ -1130,7 +1125,7 @@ pub async fn pipeline_failure( println!("\t\t Job ID: {}", job_id); if !mongo_uri.is_empty() { println!("\t\t Inserting job info into MongoDB"); - insert_job_info( + let _ = insert_job_info( &mongo_uri, &job_id, &pipeline_failure_reason, @@ -1165,7 +1160,7 @@ pub async fn pipeline_failure( println!("[+] No issues found in scan results, so slack alert is not sent"); } } - insert_job_info( + let _ = insert_job_info( &mongo_uri, &job_id, "No policy file provided, skipping policy check", @@ -1236,6 +1231,7 @@ pub async fn pipeline_failure( println!("[+] Could not upload SARIF report to Defect Dojo because of missing configuration - defectdojo-token, defectdojo-url, product-name, engagement-name"); } } + pub async fn get_commit_info( start_line: u64, end_line: u64, @@ -1302,6 +1298,7 @@ pub async fn get_commit_info( }) } // Function to fetch commit information from GitHub API +// Function to fetch commit information from GitHub API async fn get_commit_info_from_github(path: &str, repo_url_with_pat: &str) -> Option { // Parse the repository URL with PAT println!("Fetching commit info from GitHub API for {}", path);