-
Notifications
You must be signed in to change notification settings - Fork 419
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
2 changed files
with
10 additions
and
175 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,165 +1,16 @@ | ||
import os | ||
import subprocess | ||
import re | ||
import sys | ||
from typing import Optional | ||
from datetime import datetime | ||
import time | ||
import threading | ||
from huggingface_hub import snapshot_download | ||
|
||
class ProgressMonitor: | ||
def __init__(self): | ||
self.last_update = datetime.now() | ||
self.current_file = None | ||
self.spinner_active = False | ||
self.spinner_thread = None | ||
self._lock = threading.Lock() | ||
|
||
def start_spinner(self): | ||
"""Start the spinner in a separate thread.""" | ||
self.spinner_active = True | ||
def spin(): | ||
spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] | ||
idx = 0 | ||
while self.spinner_active: | ||
with self._lock: | ||
sys.stdout.write('\r {} Working...'.format(spinner[idx])) | ||
sys.stdout.flush() | ||
idx = (idx + 1) % len(spinner) | ||
time.sleep(0.1) | ||
|
||
self.spinner_thread = threading.Thread(target=spin, daemon=False) | ||
self.spinner_thread.start() | ||
|
||
def stop_spinner(self): | ||
"""Stop the spinner thread.""" | ||
self.spinner_active = False | ||
if self.spinner_thread: | ||
self.spinner_thread.join() | ||
sys.stdout.write('\r' + ' ' * 50 + '\r') | ||
sys.stdout.flush() | ||
|
||
def update_progress(self, line: str): | ||
with self._lock: | ||
if not line.strip(): | ||
return | ||
|
||
# Clear the current line in case spinner was active | ||
sys.stdout.write('\r' + ' ' * 50 + '\r') | ||
|
||
# Handle different types of progress messages | ||
if line.startswith("Downloading"): | ||
print(f" → {line}") | ||
elif "of" in line and "git lfs fetch" not in line: | ||
print(f" → {line}") | ||
elif "Git LFS:" in line: | ||
print(f" → {line}") | ||
elif line.startswith("fetch:"): | ||
print(f" → {line}") | ||
else: | ||
print(f" {line}") | ||
|
||
sys.stdout.flush() | ||
# Set the repository name | ||
repo_id = "microsoft/OmniParser" | ||
|
||
def run_with_progress(cmd: list, cwd: Optional[str] = None, env: Optional[dict] = None) -> None: | ||
"""Run a command with real-time progress monitoring.""" | ||
monitor = ProgressMonitor() | ||
|
||
try: | ||
process = subprocess.Popen( | ||
cmd, | ||
cwd=cwd, | ||
env=env or os.environ.copy(), | ||
stdout=subprocess.PIPE, | ||
stderr=subprocess.PIPE, | ||
text=True, | ||
bufsize=1, | ||
universal_newlines=True | ||
) | ||
|
||
# Start spinner | ||
monitor.start_spinner() | ||
|
||
def process_output(pipe): | ||
while True: | ||
line = pipe.readline() | ||
if not line: | ||
break | ||
monitor.update_progress(line.strip()) | ||
|
||
# Process stdout and stderr | ||
stdout_thread = threading.Thread(target=process_output, args=(process.stdout,)) | ||
stderr_thread = threading.Thread(target=process_output, args=(process.stderr,)) | ||
|
||
stdout_thread.start() | ||
stderr_thread.start() | ||
|
||
# Wait for completion | ||
stdout_thread.join() | ||
stderr_thread.join() | ||
process.wait() | ||
|
||
# Stop spinner | ||
monitor.stop_spinner() | ||
|
||
if process.returncode != 0: | ||
raise subprocess.CalledProcessError(process.returncode, cmd) | ||
|
||
except Exception as e: | ||
monitor.stop_spinner() | ||
raise e | ||
# Set the local directory where you want to save the files | ||
local_dir = "weights" | ||
|
||
def check_git_lfs(): | ||
"""Check if git and git-lfs are installed with version information.""" | ||
try: | ||
git_version = subprocess.check_output(["git", "--version"]).decode().strip() | ||
lfs_version = subprocess.check_output(["git-lfs", "--version"]).decode().strip() | ||
print(f"✓ {git_version}") | ||
print(f"✓ {lfs_version}") | ||
except subprocess.CalledProcessError: | ||
print("❌ Error: git and git-lfs must be installed and available in PATH.") | ||
sys.exit(1) | ||
# Create the local directory if it doesn't exist | ||
os.makedirs(local_dir, exist_ok=True) | ||
|
||
def shallow_clone_no_smudge(repo_url: str, target_dir: str): | ||
"""Perform shallow clone with detailed progress monitoring.""" | ||
if os.path.exists(target_dir): | ||
print(f"\n[1/3] 🔄 Updating existing repository in '{target_dir}'...") | ||
run_with_progress(["git", "pull"], cwd=target_dir) | ||
else: | ||
print(f"\n[1/3] 📥 Cloning '{repo_url}' into '{target_dir}' (shallow clone)...") | ||
run_with_progress( | ||
["git", "clone", "--depth", "1", repo_url, target_dir], | ||
env=dict(os.environ, GIT_LFS_SKIP_SMUDGE="1") | ||
) | ||
# Download the entire repository | ||
snapshot_download(repo_id, local_dir=local_dir, ignore_patterns=["*.md"]) | ||
|
||
print(f"\n[2/3] 📦 Fetching LFS files...") | ||
run_with_progress( | ||
["git", "lfs", "fetch", "--all"], | ||
cwd=target_dir, | ||
env=dict(os.environ, GIT_TRANSFER_PROGRESS="1") | ||
) | ||
|
||
print(f"\n[3/3] ✨ Checking out LFS files...") | ||
run_with_progress( | ||
["git", "lfs", "checkout"], | ||
cwd=target_dir | ||
) | ||
|
||
print("\n✅ Repository clone completed successfully!") | ||
|
||
if __name__ == "__main__": | ||
# Define the repository and target directory | ||
repo_url = "https://huggingface.co/microsoft/OmniParser" | ||
target_dir = "weights" | ||
|
||
print("🚀 Starting Git LFS download process...") | ||
check_git_lfs() | ||
shallow_clone_no_smudge(repo_url, target_dir) | ||
|
||
|
||
# Print confirmation | ||
downloaded_file = "./weights/icon_detect/model.safetensors" | ||
if os.path.exists(downloaded_file): | ||
print(f"Downloaded file at: {downloaded_file}") | ||
else: | ||
print(f"Download failed or file is missing: {downloaded_file}") | ||
print(f"All files and folders have been downloaded to {local_dir}") |
This file was deleted.
Oops, something went wrong.