From 84965d35af651c985a98119149b911d717329dec Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 29 Oct 2024 11:15:16 -0400 Subject: [PATCH] add workflow file --- download.py | 169 +++------------------------------------------------ download2.py | 16 ----- 2 files changed, 10 insertions(+), 175 deletions(-) delete mode 100644 download2.py diff --git a/download.py b/download.py index d7acaf0..03d967f 100644 --- a/download.py +++ b/download.py @@ -1,165 +1,16 @@ import os -import subprocess -import re -import sys -from typing import Optional -from datetime import datetime -import time -import threading +from huggingface_hub import snapshot_download -class ProgressMonitor: - def __init__(self): - self.last_update = datetime.now() - self.current_file = None - self.spinner_active = False - self.spinner_thread = None - self._lock = threading.Lock() - - def start_spinner(self): - """Start the spinner in a separate thread.""" - self.spinner_active = True - def spin(): - spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] - idx = 0 - while self.spinner_active: - with self._lock: - sys.stdout.write('\r {} Working...'.format(spinner[idx])) - sys.stdout.flush() - idx = (idx + 1) % len(spinner) - time.sleep(0.1) - - self.spinner_thread = threading.Thread(target=spin, daemon=False) - self.spinner_thread.start() - - def stop_spinner(self): - """Stop the spinner thread.""" - self.spinner_active = False - if self.spinner_thread: - self.spinner_thread.join() - sys.stdout.write('\r' + ' ' * 50 + '\r') - sys.stdout.flush() - - def update_progress(self, line: str): - with self._lock: - if not line.strip(): - return - - # Clear the current line in case spinner was active - sys.stdout.write('\r' + ' ' * 50 + '\r') - - # Handle different types of progress messages - if line.startswith("Downloading"): - print(f" → {line}") - elif "of" in line and "git lfs fetch" not in line: - print(f" → {line}") - elif "Git LFS:" in line: - print(f" → {line}") - elif line.startswith("fetch:"): - print(f" → {line}") - else: - print(f" {line}") - - sys.stdout.flush() +# Set the repository name +repo_id = "microsoft/OmniParser" -def run_with_progress(cmd: list, cwd: Optional[str] = None, env: Optional[dict] = None) -> None: - """Run a command with real-time progress monitoring.""" - monitor = ProgressMonitor() - - try: - process = subprocess.Popen( - cmd, - cwd=cwd, - env=env or os.environ.copy(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=1, - universal_newlines=True - ) - - # Start spinner - monitor.start_spinner() - - def process_output(pipe): - while True: - line = pipe.readline() - if not line: - break - monitor.update_progress(line.strip()) - - # Process stdout and stderr - stdout_thread = threading.Thread(target=process_output, args=(process.stdout,)) - stderr_thread = threading.Thread(target=process_output, args=(process.stderr,)) - - stdout_thread.start() - stderr_thread.start() - - # Wait for completion - stdout_thread.join() - stderr_thread.join() - process.wait() - - # Stop spinner - monitor.stop_spinner() - - if process.returncode != 0: - raise subprocess.CalledProcessError(process.returncode, cmd) - - except Exception as e: - monitor.stop_spinner() - raise e +# Set the local directory where you want to save the files +local_dir = "weights" -def check_git_lfs(): - """Check if git and git-lfs are installed with version information.""" - try: - git_version = subprocess.check_output(["git", "--version"]).decode().strip() - lfs_version = subprocess.check_output(["git-lfs", "--version"]).decode().strip() - print(f"✓ {git_version}") - print(f"✓ {lfs_version}") - except subprocess.CalledProcessError: - print("❌ Error: git and git-lfs must be installed and available in PATH.") - sys.exit(1) +# Create the local directory if it doesn't exist +os.makedirs(local_dir, exist_ok=True) -def shallow_clone_no_smudge(repo_url: str, target_dir: str): - """Perform shallow clone with detailed progress monitoring.""" - if os.path.exists(target_dir): - print(f"\n[1/3] 🔄 Updating existing repository in '{target_dir}'...") - run_with_progress(["git", "pull"], cwd=target_dir) - else: - print(f"\n[1/3] 📥 Cloning '{repo_url}' into '{target_dir}' (shallow clone)...") - run_with_progress( - ["git", "clone", "--depth", "1", repo_url, target_dir], - env=dict(os.environ, GIT_LFS_SKIP_SMUDGE="1") - ) +# Download the entire repository +snapshot_download(repo_id, local_dir=local_dir, ignore_patterns=["*.md"]) - print(f"\n[2/3] 📦 Fetching LFS files...") - run_with_progress( - ["git", "lfs", "fetch", "--all"], - cwd=target_dir, - env=dict(os.environ, GIT_TRANSFER_PROGRESS="1") - ) - - print(f"\n[3/3] ✨ Checking out LFS files...") - run_with_progress( - ["git", "lfs", "checkout"], - cwd=target_dir - ) - - print("\n✅ Repository clone completed successfully!") - -if __name__ == "__main__": - # Define the repository and target directory - repo_url = "https://huggingface.co/microsoft/OmniParser" - target_dir = "weights" - - print("🚀 Starting Git LFS download process...") - check_git_lfs() - shallow_clone_no_smudge(repo_url, target_dir) - - - # Print confirmation - downloaded_file = "./weights/icon_detect/model.safetensors" - if os.path.exists(downloaded_file): - print(f"Downloaded file at: {downloaded_file}") - else: - print(f"Download failed or file is missing: {downloaded_file}") +print(f"All files and folders have been downloaded to {local_dir}") diff --git a/download2.py b/download2.py deleted file mode 100644 index 03d967f..0000000 --- a/download2.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -from huggingface_hub import snapshot_download - -# Set the repository name -repo_id = "microsoft/OmniParser" - -# Set the local directory where you want to save the files -local_dir = "weights" - -# Create the local directory if it doesn't exist -os.makedirs(local_dir, exist_ok=True) - -# Download the entire repository -snapshot_download(repo_id, local_dir=local_dir, ignore_patterns=["*.md"]) - -print(f"All files and folders have been downloaded to {local_dir}")