From 0fe20c9b746edd73166e7ae9bdf664ea83f45571 Mon Sep 17 00:00:00 2001 From: jahwag <540380+jahwag@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:31:56 +0200 Subject: [PATCH 1/5] Only create submodule if it does not already exist on remote --- pyproject.toml | 2 +- src/claudesync/cli/submodule.py | 47 ++++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9eab00c..423986b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "claudesync" -version = "0.5.2" +version = "0.5.3" authors = [ {name = "Jahziah Wagner", email = "jahziah.wagner+pypi@gmail.com"}, ] diff --git a/src/claudesync/cli/submodule.py b/src/claudesync/cli/submodule.py index f89cfc8..efb6cba 100644 --- a/src/claudesync/cli/submodule.py +++ b/src/claudesync/cli/submodule.py @@ -43,7 +43,7 @@ def ls(config): @click.pass_obj @handle_errors def create(config): - """Create new projects for each detected submodule.""" + """Create new projects for each detected submodule that doesn't already exist remotely.""" provider = validate_and_get_provider(config, require_project=True) active_organization_id = config.get("active_organization_id") active_project_id = config.get("active_project_id") @@ -67,25 +67,40 @@ def create(config): click.echo("No submodules detected in the current project.") return - click.echo(f"Detected {len(submodules)} submodule(s). Creating projects for each:") + # Fetch all remote projects + all_remote_projects = provider.get_projects( + active_organization_id, include_archived=False + ) + + click.echo( + f"Detected {len(submodules)} submodule(s). Checking for existing remote projects:" + ) for i, submodule in enumerate(submodules, 1): submodule_name = os.path.basename(submodule) new_project_name = f"{active_project_name}-SubModule-{submodule_name}" - description = f"Submodule '{submodule_name}' for project '{active_project_name}' (ID: {active_project_id})" - try: - new_project = provider.create_project( - active_organization_id, new_project_name, description - ) - click.echo( - f"{i}. Created project '{new_project_name}' (ID: {new_project['uuid']}) for submodule '{submodule_name}'" - ) - except ProviderError as e: + # Check if the submodule project already exists + existing_project = next( + (p for p in all_remote_projects if p["name"] == new_project_name), None + ) + + if existing_project: click.echo( - f"Failed to create project for submodule '{submodule_name}': {str(e)}" + f"{i}. Submodule '{submodule_name}' already exists as project '{new_project_name}' (ID: {existing_project['id']}). Skipping." ) - - click.echo( - "\nSubmodule projects created successfully. You can now select and sync these projects individually." - ) + else: + description = f"Submodule '{submodule_name}' for project '{active_project_name}' (ID: {active_project_id})" + try: + new_project = provider.create_project( + active_organization_id, new_project_name, description + ) + click.echo( + f"{i}. Created project '{new_project_name}' (ID: {new_project['uuid']}) for submodule '{submodule_name}'" + ) + except ProviderError as e: + click.echo( + f"Failed to create project for submodule '{submodule_name}': {str(e)}" + ) + + click.echo("\nSubmodule project creation process completed.") From ea435807fe2c04521ad606355338ad7ec07af192 Mon Sep 17 00:00:00 2001 From: jahwag <540380+jahwag@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:36:40 +0200 Subject: [PATCH 2/5] Handle intermittent 403 --- src/claudesync/providers/claude_ai.py | 38 +++++++++++++++++++-------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/claudesync/providers/claude_ai.py b/src/claudesync/providers/claude_ai.py index d32c8a7..f6a84fe 100644 --- a/src/claudesync/providers/claude_ai.py +++ b/src/claudesync/providers/claude_ai.py @@ -76,8 +76,22 @@ def handle_http_error(self, e): self.logger.debug(f"Request failed: {str(e)}") self.logger.debug(f"Response status code: {e.code}") self.logger.debug(f"Response headers: {e.headers}") - content = e.read().decode("utf-8") - self.logger.debug(f"Response content: {content}") + + try: + # Check if the content is gzip-encoded + if e.headers.get('Content-Encoding') == 'gzip': + content = gzip.decompress(e.read()) + else: + content = e.read() + + # Try to decode the content as UTF-8 + content_str = content.decode('utf-8') + except UnicodeDecodeError: + # If UTF-8 decoding fails, try to decode as ISO-8859-1 + content_str = content.decode('iso-8859-1') + + self.logger.debug(f"Response content: {content_str}") + if e.code == 403: error_msg = ( "Received a 403 Forbidden error. Your session key might be invalid. " @@ -88,19 +102,21 @@ def handle_http_error(self, e): ) self.logger.error(error_msg) raise ProviderError(error_msg) - if e.code == 429: + elif e.code == 429: try: - error_data = json.loads(content) + error_data = json.loads(content_str) resets_at_unix = json.loads(error_data["error"]["message"])["resetsAt"] - resets_at_local = datetime.fromtimestamp( - resets_at_unix, tz=timezone.utc - ).astimezone() + resets_at_local = datetime.fromtimestamp(resets_at_unix, tz=timezone.utc).astimezone() formatted_time = resets_at_local.strftime("%a %b %d %Y %H:%M:%S %Z%z") - print(f"Message limit exceeded. Try again after {formatted_time}") + error_msg = f"Message limit exceeded. Try again after {formatted_time}" except (KeyError, json.JSONDecodeError) as parse_error: - print(f"Failed to parse error response: {parse_error}") - raise ProviderError("HTTP 429: Too Many Requests") - raise ProviderError(f"API request failed: {str(e)}") + error_msg = f"HTTP 429: Too Many Requests. Failed to parse error response: {parse_error}" + self.logger.error(error_msg) + raise ProviderError(error_msg) + else: + error_msg = f"API request failed with status code {e.code}: {content_str}" + self.logger.error(error_msg) + raise ProviderError(error_msg) def _make_request_stream(self, method, endpoint, data=None): url = f"{self.BASE_URL}{endpoint}" From 55e04ffaa4a03e7cb0cad476ae3e8bb243139ef1 Mon Sep 17 00:00:00 2001 From: jahwag <540380+jahwag@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:39:48 +0200 Subject: [PATCH 3/5] Retry on 403 --- src/claudesync/syncmanager.py | 66 +++++++++++++++++------------------ 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/src/claudesync/syncmanager.py b/src/claudesync/syncmanager.py index fb883c8..e598b73 100644 --- a/src/claudesync/syncmanager.py +++ b/src/claudesync/syncmanager.py @@ -1,3 +1,4 @@ +import functools import os import time import logging @@ -12,6 +13,26 @@ logger = logging.getLogger(__name__) +def retry_on_403(max_retries=3, delay=1): + def decorator(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + for attempt in range(max_retries): + try: + return func(self, *args, **kwargs) + except ProviderError as e: + if "403 Forbidden" in str(e) and attempt < max_retries - 1: + self.logger.warning( + f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})") + time.sleep(delay) + else: + raise + + return wrapper + + return decorator + + class SyncManager: """ Manages the synchronization process between local and remote files. @@ -35,29 +56,6 @@ def __init__(self, provider, config): self.max_retries = 3 # Maximum number of retries for 403 errors self.retry_delay = 1 # Delay between retries in seconds - def retry_on_403(func): - """ - Decorator to retry a function on 403 Forbidden error. - - This decorator will retry the wrapped function up to max_retries times - if a ProviderError with a 403 Forbidden message is encountered. - """ - - def wrapper(self, *args, **kwargs): - for attempt in range(self.max_retries): - try: - return func(self, *args, **kwargs) - except ProviderError as e: - if "403 Forbidden" in str(e) and attempt < self.max_retries - 1: - logger.warning( - f"Received 403 error. Retrying in {self.retry_delay} seconds..." - ) - time.sleep(self.retry_delay) - else: - raise - - return wrapper - def sync(self, local_files, remote_files): """ Main synchronization method that orchestrates the sync process. @@ -100,12 +98,12 @@ def sync(self, local_files, remote_files): @retry_on_403 def update_existing_file( - self, - local_file, - local_checksum, - remote_file, - remote_files_to_delete, - synced_files, + self, + local_file, + local_checksum, + remote_file, + remote_files_to_delete, + synced_files, ): """ Update an existing file on the remote if it has changed locally. @@ -128,7 +126,7 @@ def update_existing_file( ) pbar.update(1) with open( - os.path.join(self.local_path, local_file), "r", encoding="utf-8" + os.path.join(self.local_path, local_file), "r", encoding="utf-8" ) as file: content = file.read() self.provider.upload_file( @@ -153,7 +151,7 @@ def upload_new_file(self, local_file, synced_files): """ logger.debug(f"Uploading new file {local_file} to remote...") with open( - os.path.join(self.local_path, local_file), "r", encoding="utf-8" + os.path.join(self.local_path, local_file), "r", encoding="utf-8" ) as file: content = file.read() with tqdm(total=1, desc=f"Uploading {local_file}", leave=False) as pbar: @@ -204,7 +202,7 @@ def sync_remote_to_local(self, remote_file, remote_files_to_delete, synced_files ) def update_existing_local_file( - self, local_file_path, remote_file, remote_files_to_delete, synced_files + self, local_file_path, remote_file, remote_files_to_delete, synced_files ): """ Update an existing local file if the remote version is newer. @@ -232,7 +230,7 @@ def update_existing_local_file( remote_files_to_delete.remove(remote_file["file_name"]) def create_new_local_file( - self, local_file_path, remote_file, remote_files_to_delete, synced_files + self, local_file_path, remote_file, remote_files_to_delete, synced_files ): """ Create a new local file from a remote file. @@ -247,7 +245,7 @@ def create_new_local_file( f"Creating new local file {remote_file['file_name']} from remote..." ) with tqdm( - total=1, desc=f"Creating {remote_file['file_name']}", leave=False + total=1, desc=f"Creating {remote_file['file_name']}", leave=False ) as pbar: with open(local_file_path, "w", encoding="utf-8") as file: file.write(remote_file["content"]) From cf2ef89b2011ab0fd1ae44b2b78b97ac2f277cca Mon Sep 17 00:00:00 2001 From: jahwag <540380+jahwag@users.noreply.github.com> Date: Sat, 24 Aug 2024 11:45:52 +0200 Subject: [PATCH 4/5] Added truncate --- src/claudesync/cli/project.py | 47 ++++++++++++++++++++++++++++++++++- src/claudesync/syncmanager.py | 19 +++++++------- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/src/claudesync/cli/project.py b/src/claudesync/cli/project.py index af9e871..7f34f94 100644 --- a/src/claudesync/cli/project.py +++ b/src/claudesync/cli/project.py @@ -1,9 +1,11 @@ import os import click +from tqdm import tqdm + from claudesync.exceptions import ProviderError from .submodule import submodule -from ..syncmanager import SyncManager +from ..syncmanager import SyncManager, retry_on_403 from ..utils import ( handle_errors, validate_and_get_provider, @@ -235,4 +237,47 @@ def sync(config, category): click.echo("Project sync completed successfully, including available submodules.") +@project.command() +@click.option('-a', '--include-archived', is_flag=True, help='Include archived projects') +@click.option('-y', '--yes', is_flag=True, help='Skip confirmation prompt') +@click.pass_obj +@handle_errors +def truncate(config, include_archived, yes): + """Truncate all projects.""" + provider = validate_and_get_provider(config) + active_organization_id = config.get("active_organization_id") + + projects = provider.get_projects(active_organization_id, include_archived=include_archived) + + if not projects: + click.echo("No projects found.") + return + + if not yes: + click.echo("This will delete ALL files from the following projects:") + for project in projects: + status = " (Archived)" if project.get("archived_at") else "" + click.echo(f" - {project['name']} (ID: {project['id']}){status}") + if not click.confirm("Are you sure you want to continue? This may take some time."): + click.echo("Operation cancelled.") + return + + with tqdm(total=len(projects), desc="Deleting files from projects") as pbar: + for project in projects: + delete_files_from_project(provider, active_organization_id, project['id'], project['name']) + pbar.update(1) + + click.echo("All files have been deleted from all projects.") + +@retry_on_403() +def delete_files_from_project(provider, organization_id, project_id, project_name): + try: + files = provider.list_files(organization_id, project_id) + with tqdm(total=len(files), desc=f"Deleting files from {project_name}", leave=False) as file_pbar: + for file in files: + provider.delete_file(organization_id, project_id, file['uuid']) + file_pbar.update(1) + except ProviderError as e: + click.echo(f"Error deleting files from project {project_name}: {str(e)}") + project.add_command(submodule) diff --git a/src/claudesync/syncmanager.py b/src/claudesync/syncmanager.py index e598b73..347e09d 100644 --- a/src/claudesync/syncmanager.py +++ b/src/claudesync/syncmanager.py @@ -16,20 +16,21 @@ def retry_on_403(max_retries=3, delay=1): def decorator(func): @functools.wraps(func) - def wrapper(self, *args, **kwargs): + def wrapper(*args, **kwargs): + self = args[0] if len(args) > 0 else None for attempt in range(max_retries): try: - return func(self, *args, **kwargs) + return func(*args, **kwargs) except ProviderError as e: if "403 Forbidden" in str(e) and attempt < max_retries - 1: - self.logger.warning( - f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})") + if self and hasattr(self, 'logger'): + self.logger.warning(f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})") + else: + print(f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})") time.sleep(delay) else: raise - return wrapper - return decorator @@ -96,7 +97,7 @@ def sync(self, local_files, remote_files): self.prune_remote_files(remote_files, remote_files_to_delete) - @retry_on_403 + @retry_on_403() def update_existing_file( self, local_file, @@ -140,7 +141,7 @@ def update_existing_file( synced_files.add(local_file) remote_files_to_delete.remove(local_file) - @retry_on_403 + @retry_on_403() def upload_new_file(self, local_file, synced_files): """ Upload a new file to the remote project. @@ -269,7 +270,7 @@ def prune_remote_files(self, remote_files, remote_files_to_delete): for file_to_delete in list(remote_files_to_delete): self.delete_remote_files(file_to_delete, remote_files) - @retry_on_403 + @retry_on_403() def delete_remote_files(self, file_to_delete, remote_files): """ Delete a file from the remote project that no longer exists locally. From 85a3188f73c5c4270ff3e865247687613ef9d17b Mon Sep 17 00:00:00 2001 From: jahwag <540380+jahwag@users.noreply.github.com> Date: Sun, 25 Aug 2024 18:58:59 +0200 Subject: [PATCH 5/5] Formatting --- src/claudesync/cli/project.py | 26 ++++++++++++++------ src/claudesync/cli/submodule.py | 3 ++- src/claudesync/providers/claude_ai.py | 10 ++++---- src/claudesync/syncmanager.py | 34 ++++++++++++++++----------- 4 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/claudesync/cli/project.py b/src/claudesync/cli/project.py index 7f34f94..14a2a9b 100644 --- a/src/claudesync/cli/project.py +++ b/src/claudesync/cli/project.py @@ -238,8 +238,10 @@ def sync(config, category): @project.command() -@click.option('-a', '--include-archived', is_flag=True, help='Include archived projects') -@click.option('-y', '--yes', is_flag=True, help='Skip confirmation prompt') +@click.option( + "-a", "--include-archived", is_flag=True, help="Include archived projects" +) +@click.option("-y", "--yes", is_flag=True, help="Skip confirmation prompt") @click.pass_obj @handle_errors def truncate(config, include_archived, yes): @@ -247,7 +249,9 @@ def truncate(config, include_archived, yes): provider = validate_and_get_provider(config) active_organization_id = config.get("active_organization_id") - projects = provider.get_projects(active_organization_id, include_archived=include_archived) + projects = provider.get_projects( + active_organization_id, include_archived=include_archived + ) if not projects: click.echo("No projects found.") @@ -258,26 +262,34 @@ def truncate(config, include_archived, yes): for project in projects: status = " (Archived)" if project.get("archived_at") else "" click.echo(f" - {project['name']} (ID: {project['id']}){status}") - if not click.confirm("Are you sure you want to continue? This may take some time."): + if not click.confirm( + "Are you sure you want to continue? This may take some time." + ): click.echo("Operation cancelled.") return with tqdm(total=len(projects), desc="Deleting files from projects") as pbar: for project in projects: - delete_files_from_project(provider, active_organization_id, project['id'], project['name']) + delete_files_from_project( + provider, active_organization_id, project["id"], project["name"] + ) pbar.update(1) click.echo("All files have been deleted from all projects.") + @retry_on_403() def delete_files_from_project(provider, organization_id, project_id, project_name): try: files = provider.list_files(organization_id, project_id) - with tqdm(total=len(files), desc=f"Deleting files from {project_name}", leave=False) as file_pbar: + with tqdm( + total=len(files), desc=f"Deleting files from {project_name}", leave=False + ) as file_pbar: for file in files: - provider.delete_file(organization_id, project_id, file['uuid']) + provider.delete_file(organization_id, project_id, file["uuid"]) file_pbar.update(1) except ProviderError as e: click.echo(f"Error deleting files from project {project_name}: {str(e)}") + project.add_command(submodule) diff --git a/src/claudesync/cli/submodule.py b/src/claudesync/cli/submodule.py index efb6cba..b2386f6 100644 --- a/src/claudesync/cli/submodule.py +++ b/src/claudesync/cli/submodule.py @@ -87,7 +87,8 @@ def create(config): if existing_project: click.echo( - f"{i}. Submodule '{submodule_name}' already exists as project '{new_project_name}' (ID: {existing_project['id']}). Skipping." + f"{i}. Submodule '{submodule_name}' already exists as project " + f"'{new_project_name}' (ID: {existing_project['id']}). Skipping." ) else: description = f"Submodule '{submodule_name}' for project '{active_project_name}' (ID: {active_project_id})" diff --git a/src/claudesync/providers/claude_ai.py b/src/claudesync/providers/claude_ai.py index f6a84fe..104179d 100644 --- a/src/claudesync/providers/claude_ai.py +++ b/src/claudesync/providers/claude_ai.py @@ -79,16 +79,16 @@ def handle_http_error(self, e): try: # Check if the content is gzip-encoded - if e.headers.get('Content-Encoding') == 'gzip': + if e.headers.get("Content-Encoding") == "gzip": content = gzip.decompress(e.read()) else: content = e.read() # Try to decode the content as UTF-8 - content_str = content.decode('utf-8') + content_str = content.decode("utf-8") except UnicodeDecodeError: # If UTF-8 decoding fails, try to decode as ISO-8859-1 - content_str = content.decode('iso-8859-1') + content_str = content.decode("iso-8859-1") self.logger.debug(f"Response content: {content_str}") @@ -106,7 +106,9 @@ def handle_http_error(self, e): try: error_data = json.loads(content_str) resets_at_unix = json.loads(error_data["error"]["message"])["resetsAt"] - resets_at_local = datetime.fromtimestamp(resets_at_unix, tz=timezone.utc).astimezone() + resets_at_local = datetime.fromtimestamp( + resets_at_unix, tz=timezone.utc + ).astimezone() formatted_time = resets_at_local.strftime("%a %b %d %Y %H:%M:%S %Z%z") error_msg = f"Message limit exceeded. Try again after {formatted_time}" except (KeyError, json.JSONDecodeError) as parse_error: diff --git a/src/claudesync/syncmanager.py b/src/claudesync/syncmanager.py index 347e09d..a760bc3 100644 --- a/src/claudesync/syncmanager.py +++ b/src/claudesync/syncmanager.py @@ -23,14 +23,20 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) except ProviderError as e: if "403 Forbidden" in str(e) and attempt < max_retries - 1: - if self and hasattr(self, 'logger'): - self.logger.warning(f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})") + if self and hasattr(self, "logger"): + self.logger.warning( + f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})" + ) else: - print(f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})") + print( + f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})" + ) time.sleep(delay) else: raise + return wrapper + return decorator @@ -99,12 +105,12 @@ def sync(self, local_files, remote_files): @retry_on_403() def update_existing_file( - self, - local_file, - local_checksum, - remote_file, - remote_files_to_delete, - synced_files, + self, + local_file, + local_checksum, + remote_file, + remote_files_to_delete, + synced_files, ): """ Update an existing file on the remote if it has changed locally. @@ -127,7 +133,7 @@ def update_existing_file( ) pbar.update(1) with open( - os.path.join(self.local_path, local_file), "r", encoding="utf-8" + os.path.join(self.local_path, local_file), "r", encoding="utf-8" ) as file: content = file.read() self.provider.upload_file( @@ -152,7 +158,7 @@ def upload_new_file(self, local_file, synced_files): """ logger.debug(f"Uploading new file {local_file} to remote...") with open( - os.path.join(self.local_path, local_file), "r", encoding="utf-8" + os.path.join(self.local_path, local_file), "r", encoding="utf-8" ) as file: content = file.read() with tqdm(total=1, desc=f"Uploading {local_file}", leave=False) as pbar: @@ -203,7 +209,7 @@ def sync_remote_to_local(self, remote_file, remote_files_to_delete, synced_files ) def update_existing_local_file( - self, local_file_path, remote_file, remote_files_to_delete, synced_files + self, local_file_path, remote_file, remote_files_to_delete, synced_files ): """ Update an existing local file if the remote version is newer. @@ -231,7 +237,7 @@ def update_existing_local_file( remote_files_to_delete.remove(remote_file["file_name"]) def create_new_local_file( - self, local_file_path, remote_file, remote_files_to_delete, synced_files + self, local_file_path, remote_file, remote_files_to_delete, synced_files ): """ Create a new local file from a remote file. @@ -246,7 +252,7 @@ def create_new_local_file( f"Creating new local file {remote_file['file_name']} from remote..." ) with tqdm( - total=1, desc=f"Creating {remote_file['file_name']}", leave=False + total=1, desc=f"Creating {remote_file['file_name']}", leave=False ) as pbar: with open(local_file_path, "w", encoding="utf-8") as file: file.write(remote_file["content"])