Skip to content

Commit

Permalink
add workflow file
Browse files Browse the repository at this point in the history
  • Loading branch information
abrichr committed Oct 29, 2024
1 parent 7dc8286 commit 84965d3
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 175 deletions.
169 changes: 10 additions & 159 deletions download.py
Original file line number Diff line number Diff line change
@@ -1,165 +1,16 @@
import os
import subprocess
import re
import sys
from typing import Optional
from datetime import datetime
import time
import threading
from huggingface_hub import snapshot_download

class ProgressMonitor:
def __init__(self):
self.last_update = datetime.now()
self.current_file = None
self.spinner_active = False
self.spinner_thread = None
self._lock = threading.Lock()

def start_spinner(self):
"""Start the spinner in a separate thread."""
self.spinner_active = True
def spin():
spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
idx = 0
while self.spinner_active:
with self._lock:
sys.stdout.write('\r {} Working...'.format(spinner[idx]))
sys.stdout.flush()
idx = (idx + 1) % len(spinner)
time.sleep(0.1)

self.spinner_thread = threading.Thread(target=spin, daemon=False)
self.spinner_thread.start()

def stop_spinner(self):
"""Stop the spinner thread."""
self.spinner_active = False
if self.spinner_thread:
self.spinner_thread.join()
sys.stdout.write('\r' + ' ' * 50 + '\r')
sys.stdout.flush()

def update_progress(self, line: str):
with self._lock:
if not line.strip():
return

# Clear the current line in case spinner was active
sys.stdout.write('\r' + ' ' * 50 + '\r')

# Handle different types of progress messages
if line.startswith("Downloading"):
print(f" → {line}")
elif "of" in line and "git lfs fetch" not in line:
print(f" → {line}")
elif "Git LFS:" in line:
print(f" → {line}")
elif line.startswith("fetch:"):
print(f" → {line}")
else:
print(f" {line}")

sys.stdout.flush()
# Set the repository name
repo_id = "microsoft/OmniParser"

def run_with_progress(cmd: list, cwd: Optional[str] = None, env: Optional[dict] = None) -> None:
"""Run a command with real-time progress monitoring."""
monitor = ProgressMonitor()

try:
process = subprocess.Popen(
cmd,
cwd=cwd,
env=env or os.environ.copy(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
universal_newlines=True
)

# Start spinner
monitor.start_spinner()

def process_output(pipe):
while True:
line = pipe.readline()
if not line:
break
monitor.update_progress(line.strip())

# Process stdout and stderr
stdout_thread = threading.Thread(target=process_output, args=(process.stdout,))
stderr_thread = threading.Thread(target=process_output, args=(process.stderr,))

stdout_thread.start()
stderr_thread.start()

# Wait for completion
stdout_thread.join()
stderr_thread.join()
process.wait()

# Stop spinner
monitor.stop_spinner()

if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode, cmd)

except Exception as e:
monitor.stop_spinner()
raise e
# Set the local directory where you want to save the files
local_dir = "weights"

def check_git_lfs():
"""Check if git and git-lfs are installed with version information."""
try:
git_version = subprocess.check_output(["git", "--version"]).decode().strip()
lfs_version = subprocess.check_output(["git-lfs", "--version"]).decode().strip()
print(f"✓ {git_version}")
print(f"✓ {lfs_version}")
except subprocess.CalledProcessError:
print("❌ Error: git and git-lfs must be installed and available in PATH.")
sys.exit(1)
# Create the local directory if it doesn't exist
os.makedirs(local_dir, exist_ok=True)

def shallow_clone_no_smudge(repo_url: str, target_dir: str):
"""Perform shallow clone with detailed progress monitoring."""
if os.path.exists(target_dir):
print(f"\n[1/3] 🔄 Updating existing repository in '{target_dir}'...")
run_with_progress(["git", "pull"], cwd=target_dir)
else:
print(f"\n[1/3] 📥 Cloning '{repo_url}' into '{target_dir}' (shallow clone)...")
run_with_progress(
["git", "clone", "--depth", "1", repo_url, target_dir],
env=dict(os.environ, GIT_LFS_SKIP_SMUDGE="1")
)
# Download the entire repository
snapshot_download(repo_id, local_dir=local_dir, ignore_patterns=["*.md"])

print(f"\n[2/3] 📦 Fetching LFS files...")
run_with_progress(
["git", "lfs", "fetch", "--all"],
cwd=target_dir,
env=dict(os.environ, GIT_TRANSFER_PROGRESS="1")
)

print(f"\n[3/3] ✨ Checking out LFS files...")
run_with_progress(
["git", "lfs", "checkout"],
cwd=target_dir
)

print("\n✅ Repository clone completed successfully!")

if __name__ == "__main__":
# Define the repository and target directory
repo_url = "https://huggingface.co/microsoft/OmniParser"
target_dir = "weights"

print("🚀 Starting Git LFS download process...")
check_git_lfs()
shallow_clone_no_smudge(repo_url, target_dir)


# Print confirmation
downloaded_file = "./weights/icon_detect/model.safetensors"
if os.path.exists(downloaded_file):
print(f"Downloaded file at: {downloaded_file}")
else:
print(f"Download failed or file is missing: {downloaded_file}")
print(f"All files and folders have been downloaded to {local_dir}")
16 changes: 0 additions & 16 deletions download2.py

This file was deleted.

0 comments on commit 84965d3

Please sign in to comment.