Skip to content

Commit

Permalink
Max files limit command line argument support (#26)
Browse files Browse the repository at this point in the history
  • Loading branch information
@abdolence
abdolence authored Aug 19, 2024
1 parent d5d405e commit 15a74f8
Show file tree
Hide file tree
Showing 19 changed files with 160 additions and 76 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "redacter"
version = "0.9.0"
version = "0.10.0"
edition = "2021"
authors = ["Abdulla Abdurakhmanov <[email protected]>"]
license = "Apache-2.0"
Expand Down
13 changes: 11 additions & 2 deletions src/args.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,17 @@ pub enum CliCommand {
help = "Destination directory or file such as /tmp, /tmp/file.txt or gs://bucket/file.txt and others supported providers"
)]
destination: String,

#[arg(short = 'm', long, help = "Maximum size of files to copy in bytes")]
max_size_limit: Option<u64>,
max_size_limit: Option<usize>,

#[arg(
short = 'n',
long,
help = "Maximum number of files to copy. Sort order is not guaranteed and depends on the provider"
)]
max_files_limit: Option<usize>,

#[arg(
short = 'f',
long,
Expand All @@ -50,7 +59,7 @@ pub enum CliCommand {
)]
source: String,
#[arg(short = 'm', long, help = "Maximum size of files to copy in bytes")]
max_size_limit: Option<u64>,
max_size_limit: Option<usize>,
#[arg(
short = 'f',
long,
Expand Down
15 changes: 10 additions & 5 deletions src/commands/copy_command.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ pub struct CopyCommandResult {
pub struct CopyCommandOptions {
pub file_matcher: FileMatcher,
pub file_mime_override: FileMimeOverride,
pub max_files_limit: Option<usize>,
}

impl CopyCommandOptions {
pub fn new(
filename_filter: Option<globset::Glob>,
max_size_limit: Option<u64>,
max_size_limit: Option<usize>,
max_files_limit: Option<usize>,
mime_override: Vec<(mime::Mime, globset::Glob)>,
) -> Self {
let filename_matcher = filename_filter
Expand All @@ -35,6 +37,7 @@ impl CopyCommandOptions {
CopyCommandOptions {
file_matcher: FileMatcher::new(filename_matcher, max_size_limit),
file_mime_override: FileMimeOverride::new(mime_override),
max_files_limit,
}
}
}
Expand Down Expand Up @@ -90,10 +93,12 @@ pub async fn command_copy(
});
}
bar.println("Copying directory and listing source files...");
let source_files_result = source_fs.list_files(Some(&options.file_matcher)).await?;
let source_files_result = source_fs
.list_files(Some(&options.file_matcher), options.max_files_limit)
.await?;
let source_files: Vec<FileSystemRef> = source_files_result.files;
let files_found = source_files.len();
let files_total_size: u64 = source_files
let files_total_size: usize = source_files
.iter()
.map(|file| file.file_size.unwrap_or(0))
.sum();
Expand All @@ -102,7 +107,7 @@ pub async fn command_copy(
format!(
"Found {} files. Total size: {}",
bold_style.apply_to(files_found),
bold_style.apply_to(HumanBytes(files_total_size))
bold_style.apply_to(HumanBytes(files_total_size as u64))
)
.as_str(),
);
Expand Down Expand Up @@ -292,7 +297,7 @@ async fn transfer_and_redact_file<
None
),
bold_style.apply_to(pad_str(
HumanBytes(file_ref.file_size.unwrap_or(0))
HumanBytes(file_ref.file_size.map(|sz| sz as u64).unwrap_or(0_u64))
.to_string()
.as_str(),
16,
Expand Down
16 changes: 11 additions & 5 deletions src/commands/ls_command.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ pub struct LsCommandOptions {
}

impl LsCommandOptions {
pub fn new(filename_filter: Option<globset::Glob>, max_size_limit: Option<u64>) -> Self {
pub fn new(filename_filter: Option<globset::Glob>, max_size_limit: Option<usize>) -> Self {
let filename_matcher = filename_filter
.as_ref()
.map(|filter| filter.compile_matcher());
Expand All @@ -29,8 +29,10 @@ pub async fn command_ls(term: &Term, source: &str, options: LsCommandOptions) ->
term.write_line(format!("Listing files in {}.", bold_style.apply_to(source)).as_str())?;
let app_reporter = crate::reporter::AppReporter::from(term);
let mut source_fs = DetectFileSystem::open(source, &app_reporter).await?;
let list_files_result = source_fs.list_files(Some(&options.file_matcher)).await?;
let total_size: u64 = list_files_result
let list_files_result = source_fs
.list_files(Some(&options.file_matcher), None)
.await?;
let total_size: usize = list_files_result
.files
.iter()
.map(|f| f.file_size.unwrap_or(0))
Expand Down Expand Up @@ -83,7 +85,11 @@ pub async fn command_ls(term: &Term, source: &str, options: LsCommandOptions) ->
None
),
highlighted.apply_to(pad_str(
format!("{}", HumanBytes(file.file_size.unwrap_or(0))).as_str(),
format!(
"{}",
HumanBytes(file.file_size.map(|sz| sz as u64).unwrap_or(0))
)
.as_str(),
16,
Alignment::Left,
None
Expand All @@ -98,7 +104,7 @@ pub async fn command_ls(term: &Term, source: &str, options: LsCommandOptions) ->
format!(
"{} files found. Total size: {}",
highlighted.apply_to(list_files_result.files.len()),
highlighted.apply_to(HumanBytes(total_size))
highlighted.apply_to(HumanBytes(total_size as u64))
)
.as_str(),
)?;
Expand Down
50 changes: 31 additions & 19 deletions src/file_systems/aws_s3.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,12 @@ impl<'a> AwsS3FileSystem<'a> {
prefix: Option<String>,
continuation_token: Option<String>,
file_matcher: &Option<&FileMatcher>,
max_files_limit: Option<usize>,
) -> AppResult<ListFilesResult> {
if max_files_limit.iter().any(|v| *v == 0) {
return Ok(ListFilesResult::EMPTY);
}

let list_req = self
.client
.list_objects_v2()
Expand All @@ -76,12 +81,27 @@ impl<'a> AwsS3FileSystem<'a> {
FileSystemRef {
relative_path,
media_type,
file_size: item.size.map(|v| v as u64),
file_size: item.size.map(|v| v as usize),
}
})
})
.take(max_files_limit.unwrap_or(usize::MAX))
.collect();

let all_found_len = all_found.len();
let filtered_files: Vec<FileSystemRef> = all_found
.into_iter()
.filter(|file_ref| {
file_matcher.iter().all(|matcher| {
matches!(matcher.matches(file_ref), FileMatcherResult::Matched)
})
})
.collect();
let skipped = all_found_len - filtered_files.len();

let new_max_files_limit =
max_files_limit.map(|v| v.saturating_sub(filtered_files.len()));

let next_list_result = if list_resp
.next_continuation_token
.as_ref()
Expand All @@ -92,23 +112,13 @@ impl<'a> AwsS3FileSystem<'a> {
None,
list_resp.next_continuation_token,
file_matcher,
new_max_files_limit,
)
.await?
} else {
ListFilesResult::EMPTY
};

let all_found_len = all_found.len();
let filtered_files: Vec<FileSystemRef> = all_found
.into_iter()
.filter(|file_ref| {
file_matcher.iter().all(|matcher| {
matches!(matcher.matches(file_ref), FileMatcherResult::Matched)
})
})
.collect();
let skipped = all_found_len - filtered_files.len();

Ok(ListFilesResult {
files: [filtered_files, next_list_result.files].concat(),
skipped: next_list_result.skipped + skipped,
Expand Down Expand Up @@ -157,7 +167,7 @@ impl<'a> FileSystemConnection<'a> for AwsS3FileSystem<'a> {
.map(|v| v.parse())
.transpose()?
.or_else(|| mime_guess::from_path(relative_path.value()).first()),
file_size: object.content_length.map(|v| v as u64),
file_size: object.content_length.map(|v| v as usize),
};

let reader = object.body.into_async_read();
Expand Down Expand Up @@ -194,6 +204,7 @@ impl<'a> FileSystemConnection<'a> for AwsS3FileSystem<'a> {
async fn list_files(
&mut self,
file_matcher: Option<&FileMatcher>,
max_files_limit: Option<usize>,
) -> AppResult<ListFilesResult> {
self.reporter.report(format!(
"Listing files in bucket: {} with prefix: {}",
Expand All @@ -208,6 +219,7 @@ impl<'a> FileSystemConnection<'a> for AwsS3FileSystem<'a> {
},
None,
&file_matcher,
max_files_limit,
)
.await
} else {
Expand Down Expand Up @@ -277,7 +289,7 @@ mod tests {
Some(&FileSystemRef {
relative_path: "test-upload.txt".into(),
media_type: Some(mime::TEXT_PLAIN),
file_size: Some(test_data.len() as u64),
file_size: Some(test_data.len()),
}),
)
.await?;
Expand All @@ -286,7 +298,7 @@ mod tests {
.download(Some(&FileSystemRef {
relative_path: "test-upload.txt".into(),
media_type: Some(mime::TEXT_PLAIN),
file_size: Some(test_data.len() as u64),
file_size: Some(test_data.len()),
}))
.await?;

Expand All @@ -297,7 +309,7 @@ mod tests {

assert_eq!(file_ref.relative_path.value(), "test-upload.txt");
assert_eq!(file_ref.media_type, Some(mime::TEXT_PLAIN));
assert_eq!(file_ref.file_size, Some(test_data.len() as u64));
assert_eq!(file_ref.file_size, Some(test_data.len()));

fs.close().await?;

Expand Down Expand Up @@ -325,17 +337,17 @@ mod tests {
Some(&FileSystemRef {
relative_path: "test-upload.txt".into(),
media_type: Some(mime::TEXT_PLAIN),
file_size: Some(test_data.len() as u64),
file_size: Some(test_data.len()),
}),
)
.await?;

let list_result = fs.list_files(None).await?;
let list_result = fs.list_files(None, None).await?;
assert_eq!(list_result.files.len(), 1);
let file_ref = &list_result.files[0];
assert_eq!(file_ref.relative_path.value(), "test-upload.txt");
assert_eq!(file_ref.media_type, Some(mime::TEXT_PLAIN));
assert_eq!(file_ref.file_size, Some(test_data.len() as u64));
assert_eq!(file_ref.file_size, Some(test_data.len()));

fs.close().await?;

Expand Down
9 changes: 5 additions & 4 deletions src/file_systems/clipboard.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ impl<'a> FileSystemConnection<'a> for ClipboardFileSystem<'a> {
FileSystemRef {
relative_path: format!("{}.png", filename).into(),
media_type: Some(mime::IMAGE_PNG),
file_size: Some(png_image_bytes.len() as u64),
file_size: Some(png_image_bytes.len()),
},
Box::new(futures::stream::iter(vec![Ok(bytes::Bytes::from(
png_image_bytes,
Expand All @@ -77,7 +77,7 @@ impl<'a> FileSystemConnection<'a> for ClipboardFileSystem<'a> {
FileSystemRef {
relative_path: format!("{}.txt", filename).into(),
media_type: Some(mime::TEXT_PLAIN),
file_size: Some(text.len() as u64),
file_size: Some(text.len()),
},
Box::new(futures::stream::iter(vec![Ok(bytes::Bytes::from(text))])),
))
Expand Down Expand Up @@ -137,6 +137,7 @@ impl<'a> FileSystemConnection<'a> for ClipboardFileSystem<'a> {
async fn list_files(
&mut self,
_file_matcher: Option<&FileMatcher>,
_max_files_limit: Option<usize>,
) -> AppResult<ListFilesResult> {
self.reporter
.report("Listing in clipboard is not supported")?;
Expand Down Expand Up @@ -201,7 +202,7 @@ mod tests {
let downloaded_content = std::str::from_utf8(&flattened_bytes)?;
assert_eq!(downloaded_content, test_content);
assert_eq!(file_ref.media_type, Some(mime::TEXT_PLAIN));
assert_eq!(file_ref.file_size, Some(test_content.len() as u64));
assert_eq!(file_ref.file_size, Some(test_content.len()));

fs.close().await?;

Expand All @@ -220,7 +221,7 @@ mod tests {
let mut writer = std::io::Cursor::new(Vec::new());
test_content.write_to(&mut writer, ImageFormat::Png)?;
let png_image_bytes = writer.into_inner();
let png_images_bytes_len = png_image_bytes.len() as u64;
let png_images_bytes_len = png_image_bytes.len();

fs.upload(
futures::stream::iter(vec![Ok(bytes::Bytes::from(png_image_bytes))]),
Expand Down
Loading

0 comments on commit 15a74f8

Please sign in to comment.