From 829af611108e43e37c7fe434e8af0f06ca65477a Mon Sep 17 00:00:00 2001 From: Andrew Lilley Brinker Date: Fri, 6 Dec 2024 13:35:24 -0800 Subject: [PATCH] fix: Fix plugin downloading and prep bump for fuzz This fixes plugin downloading by ensuring plugins are downloaded and unarchived to the correct directory, and also fixes an error in the plugin manifests for `fuzz@0.1.0`, bumping to `0.1.1`. Signed-off-by: Andrew Lilley Brinker --- config/Hipcheck.kdl | 18 +++++----- hipcheck/src/plugin/download_manifest.rs | 5 ++- hipcheck/src/plugin/plugin_manifest.rs | 1 + hipcheck/src/plugin/retrieval.rs | 42 +++++++++++++++++++++--- plugins/fuzz/plugin.kdl | 4 +-- 5 files changed, 52 insertions(+), 18 deletions(-) diff --git a/config/Hipcheck.kdl b/config/Hipcheck.kdl index 126d5851..d1c831b3 100644 --- a/config/Hipcheck.kdl +++ b/config/Hipcheck.kdl @@ -1,18 +1,20 @@ plugins { - plugin "mitre/activity" version="0.1.0" manifest="./plugins/activity/plugin.kdl" - plugin "mitre/binary" version="0.1.0" manifest="./plugins/binary/plugin.kdl" - plugin "mitre/fuzz" version="0.1.0" manifest="./plugins/fuzz/plugin.kdl" - plugin "mitre/review" version="0.1.0" manifest="./plugins/review/plugin.kdl" - plugin "mitre/typo" version="0.1.0" manifest="./plugins/typo/plugin.kdl" - plugin "mitre/affiliation" version="0.1.0" manifest="./plugins/affiliation/plugin.kdl" - plugin "mitre/entropy" version="0.1.0" manifest="./plugins/entropy/plugin.kdl" - plugin "mitre/churn" version="0.1.0" manifest="./plugins/churn/plugin.kdl" + plugin "mitre/activity" version="0.1.0" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/activity.kdl" + plugin "mitre/affiliation" version="0.1.0" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/affiliation.kdl" + plugin "mitre/binary" version="0.1.0" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/binary.kdl" + plugin "mitre/churn" version="0.1.0" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/churn.kdl" + plugin "mitre/entropy" version="0.1.0" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/entropy.kdl" + plugin "mitre/fuzz" version="0.1.1" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/fuzz.kdl" + plugin "mitre/review" version="0.1.0" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/review.kdl" + plugin "mitre/typo" version="0.1.0" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/typo.kdl" } + patch { plugin "mitre/github" { api-token-var "HC_GITHUB_TOKEN" } } + analyze { investigate policy="(gt 0.5 $)" investigate-if-fail "mitre/typo" "mitre/binary" diff --git a/hipcheck/src/plugin/download_manifest.rs b/hipcheck/src/plugin/download_manifest.rs index c790be03..aaa27c34 100644 --- a/hipcheck/src/plugin/download_manifest.rs +++ b/hipcheck/src/plugin/download_manifest.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +#[cfg(test)] +use crate::plugin::arch::KnownArch; use crate::{ hc_error, plugin::{arch::Arch, PluginVersion}, @@ -8,9 +10,6 @@ use crate::{ use kdl::{KdlDocument, KdlNode, KdlValue}; use std::{fmt::Display, str::FromStr}; -#[cfg(test)] -use crate::plugin::arch::KnownArch; - // NOTE: the implementation in this crate was largely derived from RFD #0004 impl ParseKdlNode for url::Url { diff --git a/hipcheck/src/plugin/plugin_manifest.rs b/hipcheck/src/plugin/plugin_manifest.rs index 1326ec4a..bae3d360 100644 --- a/hipcheck/src/plugin/plugin_manifest.rs +++ b/hipcheck/src/plugin/plugin_manifest.rs @@ -94,6 +94,7 @@ impl ParseKdlNode for Entrypoints { .value() .as_string()? .to_string(); + if let Err(_e) = entrypoints.insert(arch.clone(), entrypoint) { log::error!("Duplicate entrypoint detected for [{}]", arch); return None; diff --git a/hipcheck/src/plugin/retrieval.rs b/hipcheck/src/plugin/retrieval.rs index 609a72a9..42eb42a1 100644 --- a/hipcheck/src/plugin/retrieval.rs +++ b/hipcheck/src/plugin/retrieval.rs @@ -5,17 +5,18 @@ use crate::{ error::Error, hc_error, plugin::{ - download_manifest::DownloadManifestEntry, try_get_bin_for_entrypoint, ArchiveFormat, - DownloadManifest, HashAlgorithm, HashWithDigest, PluginId, PluginManifest, + download_manifest::DownloadManifestEntry, get_current_arch, try_get_bin_for_entrypoint, + ArchiveFormat, DownloadManifest, HashAlgorithm, HashWithDigest, PluginId, PluginManifest, }, policy::policy_file::{ManifestLocation, PolicyPlugin}, util::{fs::file_sha256, http::agent::agent}, }; use flate2::read::GzDecoder; use fs_extra::{dir::remove, file::write_all}; +use pathbuf::pathbuf; use std::{ collections::HashSet, - fs::File, + fs::{read_dir, rename, DirEntry, File}, io::{Read, Write}, path::{Path, PathBuf}, str::FromStr, @@ -24,8 +25,6 @@ use tar::Archive; use url::Url; use xz2::read::XzDecoder; -use super::get_current_arch; - /// determine all of the plugins that need to be run and locate download them, if they do not exist pub fn retrieve_plugins( policy_plugins: &[PolicyPlugin], @@ -56,6 +55,7 @@ fn retrieve_plugin( if required_plugins.contains(&plugin_id) { return Ok(()); } + // TODO: if the plugin.kdl file for the plugin already exists, then should we skip the retrieval process? // if plugin_cache.plugin_kdl(&plugin_id).exists() @@ -201,6 +201,9 @@ fn download_and_unpack_plugin( ) })?; + println!("output_path: {}", output_path.display()); + println!("download_dir: {}", download_dir.display()); + extract_plugin( output_path.as_path(), download_dir.as_path(), @@ -350,6 +353,35 @@ fn extract_plugin( } }; + for child in read_dir(extract_dir)? { + let child = child?; + + if child.file_type()?.is_file() { + continue; + } + + for extracted_content in read_dir(child.path())? { + let extracted_content = extracted_content?; + move_to_extract_dir(extract_dir, &extracted_content)?; + } + } + + Ok(()) +} + +fn move_to_extract_dir(extract_dir: &Path, entry: &DirEntry) -> Result<(), Error> { + let remaining_path = entry + .path() + .components() + .last() + .ok_or_else(|| hc_error!("no last component: {}", entry.path().display())) + .map(|component| { + let path: &Path = component.as_ref(); + path.to_path_buf() + })?; + + let new_path = pathbuf![extract_dir, &remaining_path]; + rename(entry.path(), new_path)?; Ok(()) } diff --git a/plugins/fuzz/plugin.kdl b/plugins/fuzz/plugin.kdl index 3d170c62..5597711e 100644 --- a/plugins/fuzz/plugin.kdl +++ b/plugins/fuzz/plugin.kdl @@ -1,6 +1,6 @@ publisher "mitre" name "fuzz" -version "0.1.0" +version "0.1.1" license "Apache-2.0" entrypoint { @@ -11,5 +11,5 @@ entrypoint { } dependencies { - plugin "mitre/github" version="0.1.0" manifest="./plugins/github/plugin/github.kdl" + plugin "mitre/github" version="0.1.0" manifest="https://hipcheck.mitre.org/dl/plugin/mitre/github.kdl" }