Skip to content

Commit

Permalink
display files as counts if over threshold
Browse files Browse the repository at this point in the history
  • Loading branch information
JR-1991 committed Feb 13, 2024
1 parent d113439 commit 6e0431b
Showing 1 changed file with 14 additions and 0 deletions.
14 changes: 14 additions & 0 deletions dvuploader/dvuploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def upload(
print("\n")
info = "\n".join(
[
f"Server: [bold]{dataverse_url}[/bold]", # type: ignore
f"PID: [bold]{persistent_id}[/bold]", # type: ignore
f"Files: {len(self.files)}",
]
Expand Down Expand Up @@ -207,18 +208,23 @@ def _check_duplicates(
table.add_column("Action")

to_remove = []
over_threshold = len(self.files) > 50
n_new_files = 0
n_skip_files = 0

for file in self.files:
has_same_hash = any(
map(lambda dsFile: self._check_hashes(file, dsFile), ds_files)
)

if has_same_hash and file.checksum:
n_skip_files += 1
table.add_row(
file.fileName, "[bright_black]Same hash", "[bright_black]Skip"
)
to_remove.append(file)
else:
n_new_files += 1
table.add_row(
file.fileName, "[spring_green3]New", "[spring_green3]Upload"
)
Expand All @@ -231,6 +237,14 @@ def _check_duplicates(
self.files.remove(file)

console = Console()

if over_threshold:
table = Table(title="[bold white]🔎 Checking dataset files")

table.add_column("New", style="spring_green3", no_wrap=True)
table.add_column("Skipped", style="bright_black", no_wrap=True)
table.add_row(str(n_new_files), str(n_skip_files))

console.print(table)

@staticmethod
Expand Down

0 comments on commit 6e0431b

Please sign in to comment.