diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 59b035b..04a6dce 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,5 +41,5 @@ jobs: run: cargo test if: github.ref != 'refs/heads/master' - name: 'Run all test' - run: cargo test --features "ci" + run: cargo test --features "ci-gcp" if: github.ref == 'refs/heads/master' diff --git a/Cargo.toml b/Cargo.toml index f3683e6..747e531 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ mime = "0.3" mime_guess = "2" zip = "2" globset = "0.4" -tempdir = "0.3" +tempfile = "3" csv-async = { version = "1", default-features = false, features = ["tokio", "tokio-stream"] } aws-config = { version = "1", features = ["behavior-version-latest"] } aws-sdk-s3 = { version = "1" } diff --git a/README.md b/README.md new file mode 100644 index 0000000..08984b4 --- /dev/null +++ b/README.md @@ -0,0 +1,48 @@ +[![Cargo](https://img.shields.io/crates/v/redacter.svg)](https://crates.io/crates/redacter) +![tests and formatting](https://github.com/abdolence/redacter-rs/workflows/tests%20&%20formatting/badge.svg) +![security audit](https://github.com/abdolence/redacter-rs/workflows/security%20audit/badge.svg) + +# Redacter + +Copy & Redact cli tool to securely copy and redact files across various sources and destinations, +utilizing Data Loss Prevention (DLP) capabilities. + +## Features + +* **Copy & Redact:** copy files while applying DLP redaction to protect sensitive information. +* **Multiple Sources & Destinations:** interact with: + * Local filesystem + * Google Cloud Storage (GCS) + * Amazon Simple Storage Service (S3) + * Zip files +* **GCP DLP Integration:** Leverage the power of GCP's DLP API for accurate and customizable redaction. +* **CLI:** Easy-to-use command-line interface for streamlined workflows. +* Built with Rust to ensure speed, safety, and reliability. + +## Installation + +**Cargo:** + +```sh +cargo install redacter +``` + +## Command line options + +TBD + +## Google authentication + +Looks for credentials in the following places, preferring the first location found: + +- A JSON file whose path is specified by the GOOGLE_APPLICATION_CREDENTIALS environment variable. +- A JSON file in a location known to the gcloud command-line tool using `gcloud auth application-default login`. +- On Google Compute Engine, it fetches credentials from the metadata server. + +## Licence + +Apache Software License (ASL) + +## Author + +Abdulla Abdurakhmanov diff --git a/src/filesystems/aws_s3.rs b/src/filesystems/aws_s3.rs index d81e710..711b0c5 100644 --- a/src/filesystems/aws_s3.rs +++ b/src/filesystems/aws_s3.rs @@ -75,8 +75,8 @@ impl<'a> AwsS3FileSystem<'a> { name.trim_start_matches(&self.object_name).into(); let media_type = mime_guess::from_path(&name).first(); FileSystemRef { - relative_path: relative_path, - media_type: media_type, + relative_path, + media_type, file_size: item.size.map(|v| v as u64), } }) @@ -246,6 +246,7 @@ impl<'a> FileSystemConnection<'a> for AwsS3FileSystem<'a> { } } +#[allow(unused_imports)] mod tests { use super::*; use crate::reporter::AppReporter; diff --git a/src/filesystems/file_matcher.rs b/src/filesystems/file_matcher.rs index 9314d6e..b79bce1 100644 --- a/src/filesystems/file_matcher.rs +++ b/src/filesystems/file_matcher.rs @@ -44,6 +44,7 @@ impl FileMatcher { } } +#[allow(unused_imports)] mod tests { use super::*; use crate::filesystems::*; diff --git a/src/filesystems/gcs.rs b/src/filesystems/gcs.rs index fd1d0b0..6cf69ea 100644 --- a/src/filesystems/gcs.rs +++ b/src/filesystems/gcs.rs @@ -238,6 +238,7 @@ impl<'a> FileSystemConnection<'a> for GoogleCloudStorageFileSystem<'a> { } } +#[allow(unused_imports)] mod tests { use super::*; use crate::reporter::AppReporter; diff --git a/src/filesystems/local.rs b/src/filesystems/local.rs index 694191c..9573b4f 100644 --- a/src/filesystems/local.rs +++ b/src/filesystems/local.rs @@ -169,6 +169,7 @@ impl<'a> FileSystemConnection<'a> for LocalFileSystem<'a> { } } +#[allow(unused_imports)] mod tests { use super::*; use crate::filesystems::DetectFileSystem; @@ -178,7 +179,7 @@ mod tests { async fn download_test() -> Result<(), Box> { let term = Term::stdout(); let reporter: AppReporter = AppReporter::from(&term); - let temp_dir = tempdir::TempDir::new("local_file_system_tests_download")?; + let temp_dir = tempfile::TempDir::with_prefix("local_file_system_tests_download")?; let temp_dir_path = temp_dir.path(); let fs = DetectFileSystem::open( @@ -217,7 +218,7 @@ mod tests { async fn upload_test() -> Result<(), Box> { let term = Term::stdout(); let reporter: AppReporter = AppReporter::from(&term); - let temp_dir = tempdir::TempDir::new("local_file_system_tests_upload")?; + let temp_dir = tempfile::TempDir::with_prefix("local_file_system_tests_upload")?; let temp_dir_path = temp_dir.path(); let fs = DetectFileSystem::open( @@ -252,7 +253,7 @@ mod tests { async fn list_test() -> Result<(), Box> { let term = Term::stdout(); let reporter: AppReporter = AppReporter::from(&term); - let temp_dir = tempdir::TempDir::new("local_file_system_tests_list")?; + let temp_dir = tempfile::TempDir::with_prefix("local_file_system_tests_list")?; let temp_dir_path = temp_dir.path(); let fs = DetectFileSystem::open( diff --git a/src/filesystems/zip.rs b/src/filesystems/zip.rs index 6e6fe55..99e0f65 100644 --- a/src/filesystems/zip.rs +++ b/src/filesystems/zip.rs @@ -10,7 +10,7 @@ use gcloud_sdk::prost::bytes::Bytes; use rvstruct::ValueStruct; use std::io::Write; use std::path::{Path, PathBuf}; -use tempdir::TempDir; +use tempfile::TempDir; use zip::*; pub struct ZipFileSystem<'a> { @@ -51,7 +51,7 @@ impl<'a> ZipFileSystem<'a> { if self.mode.is_none() { let file = std::fs::File::open(&self.zip_file_path)?; let mut archive = ZipArchive::new(file)?; - let temp_dir = TempDir::new("redacter")?; + let temp_dir = tempfile::TempDir::with_prefix("redacter")?; archive.extract(temp_dir.path())?; let temp_dir_str = temp_dir.path().to_string_lossy(); self.reporter @@ -177,17 +177,18 @@ impl<'a> FileSystemConnection<'a> for ZipFileSystem<'a> { } } +#[allow(unused_imports)] mod tests { use super::*; use gcloud_sdk::prost::bytes; use std::io::Read; - use tempdir::TempDir; + use tempfile::TempDir; #[tokio::test] async fn download_test() -> Result<(), Box> { let term = console::Term::stdout(); let reporter: AppReporter = AppReporter::from(&term); - let temp_dir = TempDir::new("zip_file_system_tests_download")?; + let temp_dir = TempDir::with_prefix("zip_file_system_tests_download")?; let temp_dir_path = temp_dir.path(); let zip_file_path = temp_dir_path.join("test.zip"); let mut zip = ZipWriter::new(std::fs::File::create(&zip_file_path)?); @@ -225,7 +226,7 @@ mod tests { async fn upload_test() -> Result<(), Box> { let term = console::Term::stdout(); let reporter: AppReporter = AppReporter::from(&term); - let temp_dir = TempDir::new("zip_file_system_tests_upload")?; + let temp_dir = TempDir::with_prefix("zip_file_system_tests_upload")?; let temp_dir_path = temp_dir.path(); let zip_file_path = temp_dir_path.join("test.zip"); @@ -262,7 +263,7 @@ mod tests { async fn list_files_test() -> Result<(), Box> { let term = console::Term::stdout(); let reporter: AppReporter = AppReporter::from(&term); - let temp_dir = TempDir::new("zip_file_system_tests_list_files")?; + let temp_dir = TempDir::with_prefix("zip_file_system_tests_list_files")?; let temp_dir_path = temp_dir.path(); let zip_file_path = temp_dir_path.join("test.zip"); let mut zip = ZipWriter::new(std::fs::File::create(&zip_file_path)?); diff --git a/src/redacters/gcp_dlp.rs b/src/redacters/gcp_dlp.rs index 6d6e1ba..16ec547 100644 --- a/src/redacters/gcp_dlp.rs +++ b/src/redacters/gcp_dlp.rs @@ -359,6 +359,7 @@ impl TryInto for Redacter } } +#[allow(unused_imports)] mod tests { use super::*; use crate::redacters::RedacterProviderOptions;