Skip to content

Commit

Permalink
Support correct destination for two way sync
Browse files Browse the repository at this point in the history
  • Loading branch information
jahwag committed Aug 6, 2024
1 parent e62c5e6 commit abadd33
Show file tree
Hide file tree
Showing 12 changed files with 879 additions and 81 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -172,4 +172,5 @@ claudesync.log
claude_chats
some_value

ROADMAP.md
ROADMAP.md
.claudesync
9 changes: 9 additions & 0 deletions ClaudeSync.iml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "claudesync"
version = "0.4.9"
version = "0.5.0"
authors = [
{name = "Jahziah Wagner", email = "[email protected]"},
]
Expand Down
240 changes: 240 additions & 0 deletions src/claudesync/base_syncmanager.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
import os
import json
import logging
import time
from datetime import datetime
from tqdm import tqdm
from claudesync.utils import compute_md5_hash

logger = logging.getLogger(__name__)

CLAUDESYNC_PATH_COMMENT = "// CLAUDESYNC_PATH: {}\n"


class BaseSyncManager:
"""
Base class for managing synchronization between local files and remote Claude.ai projects.
"""

def __init__(self, provider, config):
"""
Initialize the BaseSyncManager with the given provider and configuration.
Args:
provider (Provider): The provider instance to interact with the remote storage.
config (dict): Configuration dictionary containing sync settings.
"""
self.provider = provider
self.config = config
self.active_organization_id = config.get("active_organization_id")
self.active_project_id = config.get("active_project_id")
self.local_path = config.get("local_path")
self.upload_delay = config.get("upload_delay", 0.5)
self.autocrlf = config.get("autocrlf", "true")
self.last_known_times_file = os.path.join(
self.local_path, ".claudesync", "last_known_times.json"
)
self.prune_remote_files = config.get("prune_remote_files", False)

def load_last_known_times(self):
"""
Load the last known modification times of files from a JSON file.
Returns:
dict: A dictionary of file names to their last known modification times.
"""
os.makedirs(os.path.dirname(self.last_known_times_file), exist_ok=True)
if os.path.exists(self.last_known_times_file):
with open(self.last_known_times_file, "r") as f:
return {k: datetime.fromisoformat(v) for k, v in json.load(f).items()}
return {}

def save_last_known_times(self, last_known_times):
"""
Save the last known modification times of files to a JSON file.
Args:
last_known_times (dict): A dictionary of file names to their last known modification times.
"""
os.makedirs(os.path.dirname(self.last_known_times_file), exist_ok=True)
with open(self.last_known_times_file, "w") as f:
json.dump({k: v.isoformat() for k, v in last_known_times.items()}, f)

def normalize_line_endings(self, content, for_local=True):
"""
Normalize line endings based on the autocrlf setting.
Args:
content (str): The content to normalize.
for_local (bool): True if normalizing for local file, False for remote.
Returns:
str: The content with normalized line endings.
"""
# First, standardize to LF
content = content.replace("\r\n", "\n").replace("\r", "\n")

if for_local:
if self.autocrlf == "true" and os.name == "nt":
# Convert to CRLF for Windows when autocrlf is true
content = content.replace("\n", "\r\n")
else: # for remote
if self.autocrlf == "input":
# Keep LF for remote when autocrlf is input
pass
elif self.autocrlf == "true":
# Convert to LF for remote when autocrlf is true
content = content.replace("\r\n", "\n")

return content

def _add_path_comment(self, content, file_path):
"""
Add a path comment to the content if it doesn't already exist.
Args:
content (str): The file content.
file_path (str): The full path of the file.
Returns:
str: The content with the path comment added.
"""
relative_path = os.path.relpath(file_path, self.local_path)
if not content.startswith("// CLAUDESYNC_PATH:"):
return CLAUDESYNC_PATH_COMMENT.format(relative_path) + content
return content

def _remove_path_comment(self, content):
"""
Remove the path comment from the content if it exists.
Args:
content (str): The file content.
Returns:
str: The content with the path comment removed.
"""
lines = content.split("\n", 1)
if lines and lines[0].startswith("// CLAUDESYNC_PATH:"):
return lines[1] if len(lines) > 1 else ""
return content

def _extract_path_from_comment(self, content):
"""
Extract the file path from the path comment if it exists.
Args:
content (str): The file content.
Returns:
str or None: The extracted file path, or None if no path comment is found.
"""
lines = content.split("\n", 1)
if lines and lines[0].startswith("// CLAUDESYNC_PATH:"):
return lines[0].split(": ", 1)[1].strip()
return None

def sync(self, local_files, remote_files):
"""
Main synchronization method that orchestrates the sync process.
This method should be implemented by derived classes.
Args:
local_files (dict): Dictionary of local file names and their corresponding checksums.
remote_files (list): List of dictionaries representing remote files.
"""
raise NotImplementedError("Sync method must be implemented by derived classes.")

def get_all_local_files(self):
"""
Get a set of all files in the local directory.
Returns:
set: A set of all file paths relative to the local_path.
"""
all_files = set()
for root, _, files in os.walk(self.local_path):
for file in files:
relative_path = os.path.relpath(
os.path.join(root, file), self.local_path
)
all_files.add(relative_path)
return all_files

def update_existing_file(
self,
local_file,
local_checksum,
remote_file,
remote_files_to_delete,
synced_files,
):
"""
Update an existing file on the remote if it has changed locally or if the path comment needs to be added.
Args:
local_file (str): Name of the local file.
local_checksum (str): MD5 checksum of the local file content.
remote_file (dict): Dictionary representing the remote file.
remote_files_to_delete (set): Set of remote file names to be considered for deletion.
synced_files (set): Set of file names that have been synchronized.
"""
file_path = os.path.join(self.local_path, local_file)
with open(file_path, "r", encoding="utf-8") as file:
local_content = file.read()

local_content_with_comment = self._add_path_comment(local_content, file_path)
local_content_normalized = self.normalize_line_endings(
local_content_with_comment, for_local=False
)
local_checksum_with_comment = compute_md5_hash(local_content_normalized)

remote_content = remote_file["content"]
remote_checksum = compute_md5_hash(remote_content)

if local_checksum_with_comment != remote_checksum:
logger.debug(f"Updating {local_file} on remote...")
with tqdm(total=2, desc=f"Updating {local_file}", leave=False) as pbar:
self.provider.delete_file(
self.active_organization_id,
self.active_project_id,
remote_file["uuid"],
)
pbar.update(1)
self.provider.upload_file(
self.active_organization_id,
self.active_project_id,
local_file,
local_content_normalized,
)
pbar.update(1)
time.sleep(self.upload_delay)
synced_files.add(local_file)
remote_files_to_delete.remove(local_file)

def upload_new_file(self, local_file, synced_files):
"""
Upload a new file to the remote project.
Args:
local_file (str): Name of the local file to be uploaded.
synced_files (set): Set of file names that have been synchronized.
"""
logger.debug(f"Uploading new file {local_file} to remote...")
file_path = os.path.join(self.local_path, local_file)
with open(file_path, "r", encoding="utf-8") as file:
content = file.read()
content_with_comment = self._add_path_comment(content, file_path)
normalized_content = self.normalize_line_endings(
content_with_comment, for_local=False
)
with tqdm(total=1, desc=f"Uploading {local_file}", leave=False) as pbar:
self.provider.upload_file(
self.active_organization_id,
self.active_project_id,
local_file,
normalized_content,
)
pbar.update(1)
time.sleep(self.upload_delay)
synced_files.add(local_file)
37 changes: 32 additions & 5 deletions src/claudesync/cli/project.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
import os

import click

from claudesync.exceptions import ProviderError
from ..syncmanager import SyncManager
from ..one_way_syncmanager import OneWaySyncManager
from ..two_way_syncmanager import TwoWaySyncManager
from ..utils import (
get_local_files,
)
from ..utils import (
handle_errors,
validate_and_get_provider,
validate_and_store_local_path,
get_local_files,
)


Expand Down Expand Up @@ -46,6 +47,27 @@ def create(config):

validate_and_store_local_path(config)

# Prompt for custom instruction / system prompt
if click.confirm(
"Would you like to configure our recommended custom instruction / system prompt?",
default=True,
):
prompt_template = (
'When processing files, prepend "// CLAUDESYNC_PATH: {relative_path}\\n" to the start of each file\'s '
"content, where {relative_path} is the file's path relative to the project root."
)
try:
provider.set_project_prompt_template(
active_organization_id, new_project["uuid"], prompt_template
)
click.echo(
"Custom instruction / system prompt has been set successfully."
)
except Exception as e:
click.echo(
f"Failed to set custom instruction / system prompt: {str(e)}"
)

except ProviderError as e:
click.echo(f"Failed to create project: {str(e)}")

Expand Down Expand Up @@ -136,11 +158,16 @@ def sync(config):
"""Synchronize only the project files."""
provider = validate_and_get_provider(config, require_project=True)

sync_manager = SyncManager(provider, config)
if config.get("two_way_sync", False):
sync_manager = TwoWaySyncManager(provider, config)
else:
sync_manager = OneWaySyncManager(provider, config)

remote_files = provider.list_files(
sync_manager.active_organization_id, sync_manager.active_project_id
)
local_files = get_local_files(config.get("local_path"))

sync_manager.sync(local_files, remote_files)

click.echo("Project sync completed successfully.")
9 changes: 7 additions & 2 deletions src/claudesync/cli/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@
from crontab import CronTab

from claudesync.utils import get_local_files
from ..one_way_syncmanager import OneWaySyncManager
from ..two_way_syncmanager import TwoWaySyncManager
from ..utils import handle_errors, validate_and_get_provider
from ..syncmanager import SyncManager
from ..chat_sync import sync_chats


Expand Down Expand Up @@ -39,7 +40,11 @@ def sync(config):
provider = validate_and_get_provider(config, require_project=True)

# Sync projects
sync_manager = SyncManager(provider, config)
if config.get("two_way_sync", False):
sync_manager = TwoWaySyncManager(provider, config)
else:
sync_manager = OneWaySyncManager(provider, config)

remote_files = provider.list_files(
sync_manager.active_organization_id, sync_manager.active_project_id
)
Expand Down
20 changes: 20 additions & 0 deletions src/claudesync/config_manager.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import datetime
import json
import os
import subprocess
from pathlib import Path


Expand Down Expand Up @@ -42,8 +44,26 @@ def _get_default_config(self):
"max_file_size": 32 * 1024, # Default 32 KB
"two_way_sync": False, # Default to False
"curl_use_file_input": False,
"autocrlf": self._get_git_autocrlf(),
"prune_remote_files": False,
}

def _get_git_autocrlf(self):
try:
result = subprocess.run(
["git", "config", "--get", "core.autocrlf"],
capture_output=True,
text=True,
check=True,
)
value = result.stdout.strip().lower()
if value in ["true", "false", "input"]:
return value
except subprocess.CalledProcessError:
pass
# Default to 'true' on Windows, 'input' on other systems if git config is not available
return "true" if os.name == "nt" else "input"

def _load_config(self):
"""
Loads the configuration from the JSON file, applying default values for missing keys.
Expand Down
Loading

0 comments on commit abadd33

Please sign in to comment.