Project improvements (jahwag#55)

* Truncate command to clean out Claude Projects (do this if you find
that loading Projects is becoming slow even after clearing browser
history).
* Only create new submodule(s) when running submodule creat
* Possible improvement to rate limiting / intermittent 403 from
Cloudflare especially when performing large syncs

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "claudesync"
-version = "0.5.2"
+version = "0.5.3"
 authors = [
     {name = "Jahziah Wagner", email = "[email protected]"},
 ]

src/claudesync/cli/project.py

@@ -1,9 +1,11 @@
 import os
 
 import click
+from tqdm import tqdm
+
 from claudesync.exceptions import ProviderError
 from .submodule import submodule
-from ..syncmanager import SyncManager
+from ..syncmanager import SyncManager, retry_on_403
 from ..utils import (
     handle_errors,
     validate_and_get_provider,
@@ -235,4 +237,59 @@ def sync(config, category):
     click.echo("Project sync completed successfully, including available submodules.")
 
 
+@project.command()
+@click.option(
+    "-a", "--include-archived", is_flag=True, help="Include archived projects"
+)
+@click.option("-y", "--yes", is_flag=True, help="Skip confirmation prompt")
+@click.pass_obj
+@handle_errors
+def truncate(config, include_archived, yes):
+    """Truncate all projects."""
+    provider = validate_and_get_provider(config)
+    active_organization_id = config.get("active_organization_id")
+
+    projects = provider.get_projects(
+        active_organization_id, include_archived=include_archived
+    )
+
+    if not projects:
+        click.echo("No projects found.")
+        return
+
+    if not yes:
+        click.echo("This will delete ALL files from the following projects:")
+        for project in projects:
+            status = " (Archived)" if project.get("archived_at") else ""
+            click.echo(f"  - {project['name']} (ID: {project['id']}){status}")
+        if not click.confirm(
+            "Are you sure you want to continue? This may take some time."
+        ):
+            click.echo("Operation cancelled.")
+            return
+
+    with tqdm(total=len(projects), desc="Deleting files from projects") as pbar:
+        for project in projects:
+            delete_files_from_project(
+                provider, active_organization_id, project["id"], project["name"]
+            )
+            pbar.update(1)
+
+    click.echo("All files have been deleted from all projects.")
+
+
+@retry_on_403()
+def delete_files_from_project(provider, organization_id, project_id, project_name):
+    try:
+        files = provider.list_files(organization_id, project_id)
+        with tqdm(
+            total=len(files), desc=f"Deleting files from {project_name}", leave=False
+        ) as file_pbar:
+            for file in files:
+                provider.delete_file(organization_id, project_id, file["uuid"])
+                file_pbar.update(1)
+    except ProviderError as e:
+        click.echo(f"Error deleting files from project {project_name}: {str(e)}")
+
+
 project.add_command(submodule)

src/claudesync/cli/submodule.py

@@ -43,7 +43,7 @@ def ls(config):
 @click.pass_obj
 @handle_errors
 def create(config):
-    """Create new projects for each detected submodule."""
+    """Create new projects for each detected submodule that doesn't already exist remotely."""
     provider = validate_and_get_provider(config, require_project=True)
     active_organization_id = config.get("active_organization_id")
     active_project_id = config.get("active_project_id")
@@ -67,25 +67,41 @@ def create(config):
         click.echo("No submodules detected in the current project.")
         return
 
-    click.echo(f"Detected {len(submodules)} submodule(s). Creating projects for each:")
+    # Fetch all remote projects
+    all_remote_projects = provider.get_projects(
+        active_organization_id, include_archived=False
+    )
+
+    click.echo(
+        f"Detected {len(submodules)} submodule(s). Checking for existing remote projects:"
+    )
 
     for i, submodule in enumerate(submodules, 1):
         submodule_name = os.path.basename(submodule)
         new_project_name = f"{active_project_name}-SubModule-{submodule_name}"
-        description = f"Submodule '{submodule_name}' for project '{active_project_name}' (ID: {active_project_id})"
 
-        try:
-            new_project = provider.create_project(
-                active_organization_id, new_project_name, description
-            )
-            click.echo(
-                f"{i}. Created project '{new_project_name}' (ID: {new_project['uuid']}) for submodule '{submodule_name}'"
-            )
-        except ProviderError as e:
+        # Check if the submodule project already exists
+        existing_project = next(
+            (p for p in all_remote_projects if p["name"] == new_project_name), None
+        )
+
+        if existing_project:
             click.echo(
-                f"Failed to create project for submodule '{submodule_name}': {str(e)}"
+                f"{i}. Submodule '{submodule_name}' already exists as project "
+                f"'{new_project_name}' (ID: {existing_project['id']}). Skipping."
             )
-
-    click.echo(
-        "\nSubmodule projects created successfully. You can now select and sync these projects individually."
-    )
+        else:
+            description = f"Submodule '{submodule_name}' for project '{active_project_name}' (ID: {active_project_id})"
+            try:
+                new_project = provider.create_project(
+                    active_organization_id, new_project_name, description
+                )
+                click.echo(
+                    f"{i}. Created project '{new_project_name}' (ID: {new_project['uuid']}) for submodule '{submodule_name}'"
+                )
+            except ProviderError as e:
+                click.echo(
+                    f"Failed to create project for submodule '{submodule_name}': {str(e)}"
+                )
+
+    click.echo("\nSubmodule project creation process completed.")

src/claudesync/providers/claude_ai.py

@@ -76,8 +76,22 @@ def handle_http_error(self, e):
         self.logger.debug(f"Request failed: {str(e)}")
         self.logger.debug(f"Response status code: {e.code}")
         self.logger.debug(f"Response headers: {e.headers}")
-        content = e.read().decode("utf-8")
-        self.logger.debug(f"Response content: {content}")
+
+        try:
+            # Check if the content is gzip-encoded
+            if e.headers.get("Content-Encoding") == "gzip":
+                content = gzip.decompress(e.read())
+            else:
+                content = e.read()
+
+            # Try to decode the content as UTF-8
+            content_str = content.decode("utf-8")
+        except UnicodeDecodeError:
+            # If UTF-8 decoding fails, try to decode as ISO-8859-1
+            content_str = content.decode("iso-8859-1")
+
+        self.logger.debug(f"Response content: {content_str}")
+
         if e.code == 403:
             error_msg = (
                 "Received a 403 Forbidden error. Your session key might be invalid. "
@@ -88,19 +102,23 @@ def handle_http_error(self, e):
             )
             self.logger.error(error_msg)
             raise ProviderError(error_msg)
-        if e.code == 429:
+        elif e.code == 429:
             try:
-                error_data = json.loads(content)
+                error_data = json.loads(content_str)
                 resets_at_unix = json.loads(error_data["error"]["message"])["resetsAt"]
                 resets_at_local = datetime.fromtimestamp(
                     resets_at_unix, tz=timezone.utc
                 ).astimezone()
                 formatted_time = resets_at_local.strftime("%a %b %d %Y %H:%M:%S %Z%z")
-                print(f"Message limit exceeded. Try again after {formatted_time}")
+                error_msg = f"Message limit exceeded. Try again after {formatted_time}"
             except (KeyError, json.JSONDecodeError) as parse_error:
-                print(f"Failed to parse error response: {parse_error}")
-            raise ProviderError("HTTP 429: Too Many Requests")
-        raise ProviderError(f"API request failed: {str(e)}")
+                error_msg = f"HTTP 429: Too Many Requests. Failed to parse error response: {parse_error}"
+            self.logger.error(error_msg)
+            raise ProviderError(error_msg)
+        else:
+            error_msg = f"API request failed with status code {e.code}: {content_str}"
+            self.logger.error(error_msg)
+            raise ProviderError(error_msg)
 
     def _make_request_stream(self, method, endpoint, data=None):
         url = f"{self.BASE_URL}{endpoint}"

src/claudesync/syncmanager.py

@@ -1,3 +1,4 @@
+import functools
 import os
 import time
 import logging
@@ -12,6 +13,33 @@
 logger = logging.getLogger(__name__)
 
 
+def retry_on_403(max_retries=3, delay=1):
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            self = args[0] if len(args) > 0 else None
+            for attempt in range(max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except ProviderError as e:
+                    if "403 Forbidden" in str(e) and attempt < max_retries - 1:
+                        if self and hasattr(self, "logger"):
+                            self.logger.warning(
+                                f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})"
+                            )
+                        else:
+                            print(
+                                f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})"
+                            )
+                        time.sleep(delay)
+                    else:
+                        raise
+
+        return wrapper
+
+    return decorator
+
+
 class SyncManager:
     """
     Manages the synchronization process between local and remote files.
@@ -35,29 +63,6 @@ def __init__(self, provider, config):
         self.max_retries = 3  # Maximum number of retries for 403 errors
         self.retry_delay = 1  # Delay between retries in seconds
 
-    def retry_on_403(func):
-        """
-        Decorator to retry a function on 403 Forbidden error.
-
-        This decorator will retry the wrapped function up to max_retries times
-        if a ProviderError with a 403 Forbidden message is encountered.
-        """
-
-        def wrapper(self, *args, **kwargs):
-            for attempt in range(self.max_retries):
-                try:
-                    return func(self, *args, **kwargs)
-                except ProviderError as e:
-                    if "403 Forbidden" in str(e) and attempt < self.max_retries - 1:
-                        logger.warning(
-                            f"Received 403 error. Retrying in {self.retry_delay} seconds..."
-                        )
-                        time.sleep(self.retry_delay)
-                    else:
-                        raise
-
-        return wrapper
-
     def sync(self, local_files, remote_files):
         """
         Main synchronization method that orchestrates the sync process.
@@ -98,7 +103,7 @@ def sync(self, local_files, remote_files):
 
         self.prune_remote_files(remote_files, remote_files_to_delete)
 
-    @retry_on_403
+    @retry_on_403()
     def update_existing_file(
         self,
         local_file,
@@ -142,7 +147,7 @@ def update_existing_file(
             synced_files.add(local_file)
         remote_files_to_delete.remove(local_file)
 
-    @retry_on_403
+    @retry_on_403()
     def upload_new_file(self, local_file, synced_files):
         """
         Upload a new file to the remote project.
@@ -271,7 +276,7 @@ def prune_remote_files(self, remote_files, remote_files_to_delete):
         for file_to_delete in list(remote_files_to_delete):
             self.delete_remote_files(file_to_delete, remote_files)
 
-    @retry_on_403
+    @retry_on_403()
     def delete_remote_files(self, file_to_delete, remote_files):
         """
         Delete a file from the remote project that no longer exists locally.