Skip to content

Commit

Permalink
Added commit_id detection for secret scan
Browse files Browse the repository at this point in the history
  • Loading branch information
@rohitcoder
rohitcoder committed Sep 15, 2024
1 parent 58b7e41 commit 90cfb82
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 41 deletions.
4 changes: 2 additions & 2 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM rust:latest
FROM rust:latest;ls;
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2024-02-04
COPY Cargo.toml /hela/Cargo.toml
COPY Cargo.lock /hela/Cargo.lock
Expand All @@ -13,7 +13,7 @@ RUN rm -rf /hela
# Update the package list and upgrade the system
RUN apt-get update && \
apt-get -y upgrade

## configure tzdata to avoid interactive prompt
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=Europe/London
Expand Down
38 changes: 36 additions & 2 deletions src/utils/common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,8 @@ pub fn checkout(
commit_ids: Option<&str>,
branch_name: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
let mut file_commit_map: HashMap<String, String> = HashMap::new();

let commit_hashes: Vec<&str> = match commit_ids {
Some(ref ids) if !ids.is_empty() => ids.split(',').collect(),
_ => vec![],
Expand Down Expand Up @@ -315,10 +317,12 @@ pub fn checkout(
}
}

// if commit_ids provided only then run below logic
// If no commit IDs are provided, return early.
if commit_hashes.is_empty() {
save_commit_map(&file_commit_map)?;
return Ok(());
}

let mut all_files = String::new();
for commit in commit_hashes {
let output = Command::new("git")
Expand All @@ -337,7 +341,6 @@ pub fn checkout(
.arg("--name-only")
.arg(format!("{}^", commit))
.arg(commit)
.stdout(Stdio::piped())
.output()?;

if !output.status.success() {
Expand All @@ -349,6 +352,11 @@ pub fn checkout(

let files = String::from_utf8_lossy(&output.stdout);
all_files.push_str(&files);

// Map each file to the current commit ID.
for file in files.lines() {
file_commit_map.insert(file.to_string(), commit.to_string());
}
}

println!("FILES\n______\n{}", all_files);
Expand All @@ -357,9 +365,35 @@ pub fn checkout(

delete_empty_directories(&cloned_path)?;

// Save the commit map to /tmp/commit_map.json.
save_commit_map(&file_commit_map)?;

Ok(())
}

// Function to save the commit map to /tmp/commit_map.json.
fn save_commit_map(
file_commit_map: &HashMap<String, String>,
) -> Result<(), Box<dyn std::error::Error>> {
let commit_map_path = "/tmp/commit_map.json";
let file = File::create(commit_map_path)?;
serde_json::to_writer(file, file_commit_map)?;
println!("Commit map saved to: {}", commit_map_path);
Ok(())
}

pub fn get_commit_of_file(file_path: &str) -> Result<String, Box<dyn std::error::Error>> {
let relative_path = file_path.split("/").collect::<Vec<&str>>()[3..]
.join("/")
.replace("\"", "");
let commit_map_path = "/tmp/commit_map.json";
let file = File::open(commit_map_path)?;
let file_commit_map: HashMap<String, String> = serde_json::from_reader(file)?;
let binding = "".to_string();
let commit_id = file_commit_map.get(&relative_path).unwrap_or(&binding);
Ok(commit_id.to_string())
}

fn delete_except(files: &str, base_dir: &Path) -> Result<(), Box<dyn std::error::Error>> {
let files_to_keep: Vec<PathBuf> = files
.lines()
Expand Down
71 changes: 34 additions & 37 deletions src/utils/pipeline.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use serde_json::{json, Value};
use std::{collections::HashMap, process::exit};

use crate::utils::common::{
bulk_check_hash_exists, insert_job_info, slack_alert, upload_to_defect_dojo,
bulk_check_hash_exists, get_commit_of_file, insert_job_info, slack_alert, upload_to_defect_dojo,
};

use super::common::{self, execute_command, print_error, redact_github_token};
Expand Down Expand Up @@ -194,7 +194,6 @@ pub async fn pipeline_failure(

let hashes: Vec<String> = message_to_hash.keys().cloned().collect();
let existing_hashes_result = bulk_check_hash_exists(&hashes, &mongo_uri).await;

// Handle the Result properly
let existing_hashes = match existing_hashes_result {
Ok(hashes) => hashes,
Expand Down Expand Up @@ -446,35 +445,29 @@ pub async fn pipeline_failure(
let mut secret_result = HashMap::new();
secret_result.insert(
"file",
result["SourceMetadata"]["Data"]["Filesystem"]["file"].to_string(),
result["SourceMetadata"]["Data"]["Filesystem"]["file"]
.as_str()
.unwrap_or("")
.to_string(),
);
secret_result.insert("line", number_string);
secret_result.insert("raw", result["Raw"].to_string());
secret_result.insert("raw", result["Raw"].as_str().unwrap_or("").to_string());
secret_result.insert(
"detector_name",
result["DetectorName"].to_string().to_uppercase(),
result["DetectorName"].as_str().unwrap_or("").to_uppercase(),
);
secret_result.insert(
"decoder_name",
result["DecoderName"].as_str().unwrap_or("").to_string(),
);
secret_result.insert("decoder_name", result["DecoderName"].to_string());
secret_result
};
secret_results.push(secret_result);
if !detected_detectors.contains(
&result["DetectorName"]
.as_str()
.unwrap()
.to_string()
.to_uppercase(),
) {
detected_detectors.push(
result["DetectorName"]
.as_str()
.unwrap()
.to_string()
.to_uppercase(),
);
let detector_name = result["DetectorName"].as_str().unwrap_or("").to_uppercase();
if !detected_detectors.contains(&detector_name) {
detected_detectors.push(detector_name);
}
}

detected_detectors = detected_detectors
.iter()
.map(|x| x.to_string())
Expand All @@ -493,24 +486,30 @@ pub async fn pipeline_failure(
}

let mut secret_count = 0;
let mut message_to_hash: HashMap<String, (String, String, String, String)> = HashMap::new();
let mut message_to_hash: HashMap<String, (String, String, String, String, String)> =
HashMap::new();

// Collect all secret records and their hashes
for value in secret_results.clone() {
// Append to slack alert message, remove first 2 values after split with "/"
let file_commit = get_commit_of_file(&value["file"]);
let commit_base_link = commit_path.split("/commit").collect::<Vec<&str>>()[0];
let commit_link = format!("{}/commit/{}", commit_base_link, file_commit.unwrap());
let vuln_record = format!(
"\n\nFile: {}\nLine: {}\nRaw: {}\nDetector Name: {}",
value["file"], value["line"], value["raw"], value["detector_name"]
"\n\nFile: {}\nLine: {}\nRaw: {}\nDetector Name: {}\nCommit: {}",
value["file"], value["line"], value["raw"], value["detector_name"], commit_link
);
let hashed_message = common::hash_text(&vuln_record);

// Collect messages and their hashes along with other details
message_to_hash.insert(
hashed_message,
(
value["file"].replace("\"", ""),
value["file"].clone(),
value["line"].clone(),
value["raw"].replace("\"", ""),
value["detector_name"].replace("\"", ""),
value["raw"].clone(),
value["detector_name"].clone(),
commit_link,
),
);
}
Expand All @@ -529,10 +528,8 @@ pub async fn pipeline_failure(
};

let mut secret_count = 0;
let mut found_secret_issues = false;

// Process each message to check for existence and add to the table
for (hashed_message, (file, line, raw, detector_name)) in message_to_hash {
for (hashed_message, (file, line, raw, detector_name, commit_link)) in message_to_hash {
if !existing_hashes.contains(&hashed_message) {
found_secret_issues = true;
secret_count += 1;
Expand All @@ -545,10 +542,9 @@ pub async fn pipeline_failure(
// Add row to table
table.add_row(row![secret_count, file, line, raw_truncated, detector_name]);

// Append to slack alert message
slack_alert_msg.push_str(&format!(
"\n\nFile: {}\nLine: {}\nRaw: {}\nDetector Name: {}",
file, line, raw, detector_name
"\n\nFile: {}\nLine: {}\nRaw: {}\nDetector Name: {}\nCommit: {}",
file, line, raw, detector_name, commit_link
));

// Register the missing hash
Expand Down Expand Up @@ -617,7 +613,6 @@ pub async fn pipeline_failure(
.collect::<Vec<String>>();
pipeline_secret_license_data.insert("licenses", licenses_list);
}

if found_sast_issues == false
&& found_sca_issues == false
&& found_secret_issues == false
Expand Down Expand Up @@ -1101,7 +1096,7 @@ pub async fn pipeline_failure(
println!("\t\t Job ID: {}", job_id);
if !mongo_uri.is_empty() {
println!("\t\t Inserting job info into MongoDB");
insert_job_info(
let _ = insert_job_info(
&mongo_uri,
&job_id,
&pipeline_failure_reason,
Expand Down Expand Up @@ -1130,7 +1125,7 @@ pub async fn pipeline_failure(
println!("\t\t Job ID: {}", job_id);
if !mongo_uri.is_empty() {
println!("\t\t Inserting job info into MongoDB");
insert_job_info(
let _ = insert_job_info(
&mongo_uri,
&job_id,
&pipeline_failure_reason,
Expand Down Expand Up @@ -1165,7 +1160,7 @@ pub async fn pipeline_failure(
println!("[+] No issues found in scan results, so slack alert is not sent");
}
}
insert_job_info(
let _ = insert_job_info(
&mongo_uri,
&job_id,
"No policy file provided, skipping policy check",
Expand Down Expand Up @@ -1236,6 +1231,7 @@ pub async fn pipeline_failure(
println!("[+] Could not upload SARIF report to Defect Dojo because of missing configuration - defectdojo-token, defectdojo-url, product-name, engagement-name");
}
}

pub async fn get_commit_info(
start_line: u64,
end_line: u64,
Expand Down Expand Up @@ -1302,6 +1298,7 @@ pub async fn get_commit_info(
})
}
// Function to fetch commit information from GitHub API
// Function to fetch commit information from GitHub API
async fn get_commit_info_from_github(path: &str, repo_url_with_pat: &str) -> Option<Value> {
// Parse the repository URL with PAT
println!("Fetching commit info from GitHub API for {}", path);
Expand Down

0 comments on commit 90cfb82

Please sign in to comment.