diff --git a/press/agent.py b/press/agent.py index 2e815631e1..8417754a0c 100644 --- a/press/agent.py +++ b/press/agent.py @@ -450,6 +450,7 @@ def backup_site(self, site, with_files=False, offsite=False): "ACCESS_KEY": settings.offsite_backups_access_key_id, "SECRET_KEY": settings.get_password("offsite_backups_secret_access_key"), "REGION": backup_bucket.get("region") if isinstance(backup_bucket, dict) else "", + "ENDPOINT_URL": backup_bucket.get("endpoint_url") if isinstance(backup_bucket, dict) else "", } data.update({"offsite": {"bucket": bucket_name, "auth": auth, "path": backups_path}}) diff --git a/press/api/site.py b/press/api/site.py index bc47e40a36..b7895ea82c 100644 --- a/press/api/site.py +++ b/press/api/site.py @@ -15,7 +15,6 @@ from frappe.core.utils import find from frappe.desk.doctype.tag.tag import add_tag from frappe.utils import flt, sbool, time_diff_in_hours -from frappe.utils.password import get_decrypted_password from frappe.utils.user import is_system_user from press.exceptions import AAAARecordExists, ConflictingCAARecord @@ -47,6 +46,7 @@ from press.press.doctype.deploy_candidate_app.deploy_candidate_app import ( DeployCandidateApp, ) + from press.press.doctype.press_settings.press_settings import PressSettings NAMESERVERS = ["1.1.1.1", "1.0.0.1", "8.8.8.8", "8.8.4.4"] @@ -1779,18 +1779,18 @@ def get_trial_plan(): @frappe.whitelist() def get_upload_link(file, parts=1): - bucket_name = frappe.db.get_single_value("Press Settings", "remote_uploads_bucket") - expiration = frappe.db.get_single_value("Press Settings", "remote_link_expiry") or 3600 + press_settings: "PressSettings" = frappe.get_single("Press Settings") # type: ignore + bucket_name = press_settings.remote_uploads_bucket + expiration = press_settings.remote_link_expiry or 3600 object_name = get_remote_key(file) parts = int(parts) s3_client = client( "s3", - aws_access_key_id=frappe.db.get_single_value("Press Settings", "remote_access_key_id"), - aws_secret_access_key=get_decrypted_password( - "Press Settings", "Press Settings", "remote_secret_access_key" - ), - region_name="ap-south-1", + aws_access_key_id=press_settings.remote_access_key_id, + aws_secret_access_key=press_settings.get_password("remote_secret_access_key"), + region_name=press_settings.get("remote_uploads_region") or "ap-south-1", + endpoint_url=press_settings.get("remote_uploads_endpoint") or None, ) try: # The response contains the presigned URL and required fields @@ -1822,24 +1822,24 @@ def get_upload_link(file, parts=1): @frappe.whitelist() def multipart_exit(file, id, action, parts=None): + press_settings: "PressSettings" = frappe.get_single("Press Settings") # type: ignore + bucket_name = "uploads.frappe.cloud" # TODO: Use press_settings.remote_uploads_bucket + s3_client = client( "s3", - aws_access_key_id=frappe.db.get_single_value("Press Settings", "remote_access_key_id"), - aws_secret_access_key=get_decrypted_password( - "Press Settings", - "Press Settings", - "remote_secret_access_key", - raise_exception=False, - ), - region_name="ap-south-1", + aws_access_key_id=press_settings.remote_access_key_id, + aws_secret_access_key=press_settings.get_password("remote_secret_access_key", raise_exception=False), + region_name=press_settings.get("remote_uploads_region") or "ap-south-1", + endpoint_url=press_settings.get("remote_uploads_endpoint") or None, ) + if action == "abort": - response = s3_client.abort_multipart_upload(Bucket="uploads.frappe.cloud", Key=file, UploadId=id) + response = s3_client.abort_multipart_upload(Bucket=bucket_name, Key=file, UploadId=id) elif action == "complete": parts = json.loads(parts) # After completing for all parts, you will use complete_multipart_upload api which requires that parts list response = s3_client.complete_multipart_upload( - Bucket="uploads.frappe.cloud", + Bucket=bucket_name, Key=file, UploadId=id, MultipartUpload={"Parts": parts}, diff --git a/press/press/doctype/press_settings/press_settings.json b/press/press/doctype/press_settings/press_settings.json index 5ac010d495..bdc523b1ca 100644 --- a/press/press/doctype/press_settings/press_settings.json +++ b/press/press/doctype/press_settings/press_settings.json @@ -55,9 +55,8 @@ "backups_tab", "offsite_backups_section", "backup_region", - "offsite_backups_provider", "aws_s3_bucket", - "data_40", + "offsite_backups_endpoint", "backup_rotation_scheme", "column_break_35", "offsite_backups_access_key_id", @@ -90,10 +89,12 @@ "auto_update_queue_size", "remote_files_section", "remote_uploads_bucket", - "remote_link_expiry", + "remote_uploads_region", + "remote_uploads_endpoint", "column_break_51", "remote_access_key_id", "remote_secret_access_key", + "remote_link_expiry", "product_documentation_section", "publish_docs", "storage_and_disk_limits_section", @@ -405,22 +406,11 @@ "fieldtype": "Data", "label": "Backup Region" }, - { - "default": "AWS S3", - "fieldname": "offsite_backups_provider", - "fieldtype": "Select", - "label": "Backup Provider", - "options": "AWS S3" - }, { "fieldname": "aws_s3_bucket", "fieldtype": "Data", "label": "Bucket Name" }, - { - "fieldname": "data_40", - "fieldtype": "Data" - }, { "fieldname": "backup_rotation_scheme", "fieldtype": "Select", @@ -1246,11 +1236,26 @@ "fieldtype": "Link", "label": "Press Trial Plan", "options": "Site Plan" + }, + { + "fieldname": "remote_uploads_region", + "fieldtype": "Data", + "label": "Uploads Bucket Region" + }, + { + "fieldname": "remote_uploads_endpoint", + "fieldtype": "Data", + "label": "Uploads Bucket Endpoint" + }, + { + "fieldname": "offsite_backups_endpoint", + "fieldtype": "Data", + "label": "Bucket Endpoint" } ], "issingle": 1, "links": [], - "modified": "2024-10-14 21:02:12.948747", + "modified": "2024-11-14 14:21:14.359536", "modified_by": "Administrator", "module": "Press", "name": "Press Settings", diff --git a/press/press/doctype/press_settings/press_settings.py b/press/press/doctype/press_settings/press_settings.py index bd99a7db4f..1e27348c51 100644 --- a/press/press/doctype/press_settings/press_settings.py +++ b/press/press/doctype/press_settings/press_settings.py @@ -50,7 +50,6 @@ class PressSettings(Document): code_server_password: DF.Data | None commission: DF.Float compress_app_cache: DF.Check - data_40: DF.Data | None default_outgoing_id: DF.Data | None default_outgoing_pass: DF.Data | None disable_agent_job_deduplication: DF.Check @@ -98,7 +97,7 @@ class PressSettings(Document): ngrok_auth_token: DF.Data | None offsite_backups_access_key_id: DF.Data | None offsite_backups_count: DF.Int - offsite_backups_provider: DF.Literal["AWS S3"] + offsite_backups_endpoint: DF.Data | None offsite_backups_secret_access_key: DF.Password | None plausible_api_key: DF.Password | None plausible_site_id: DF.Data | None @@ -115,6 +114,8 @@ class PressSettings(Document): remote_link_expiry: DF.Int remote_secret_access_key: DF.Password | None remote_uploads_bucket: DF.Data | None + remote_uploads_endpoint: DF.Data | None + remote_uploads_region: DF.Data | None root_domain: DF.Data | None rsa_key_size: DF.Literal["2048", "3072", "4096"] spaces_domain: DF.Link | None diff --git a/press/press/doctype/remote_file/remote_file.py b/press/press/doctype/remote_file/remote_file.py index fd17d32b13..f9f9862546 100644 --- a/press/press/doctype/remote_file/remote_file.py +++ b/press/press/doctype/remote_file/remote_file.py @@ -5,12 +5,16 @@ import json import pprint +from typing import TYPE_CHECKING import frappe import requests from boto3 import client, resource from frappe.model.document import Document -from frappe.utils.password import get_decrypted_password + + +if TYPE_CHECKING: + from press.press.doctype.press_settings.press_settings import PressSettings def get_remote_key(file): @@ -27,34 +31,30 @@ def get_remote_key(file): def poll_file_statuses(): - aws_access_key = frappe.db.get_single_value( - "Press Settings", "offsite_backups_access_key_id" - ) - aws_secret_key = get_decrypted_password( - "Press Settings", "Press Settings", "offsite_backups_secret_access_key" - ) - default_region = frappe.db.get_single_value("Press Settings", "backup_region") + press_settings: "PressSettings" = frappe.get_single("Press Settings") # type: ignore + + aws_access_key = press_settings.offsite_backups_access_key_id + aws_secret_key = press_settings.offsite_backups_secret_access_key + default_region = press_settings.backup_region + buckets = [ { - "name": frappe.db.get_single_value("Press Settings", "aws_s3_bucket"), + "name": press_settings.aws_s3_bucket, "region": default_region, "access_key_id": aws_access_key, "secret_access_key": aws_secret_key, + "endpoint_url": press_settings.offsite_backups_endpoint, }, { - "name": frappe.db.get_single_value("Press Settings", "remote_uploads_bucket"), - "region": default_region, - "access_key_id": frappe.db.get_single_value( - "Press Settings", "remote_access_key_id" - ), - "secret_access_key": get_decrypted_password( - "Press Settings", "Press Settings", "remote_secret_access_key" - ), + "name": press_settings.remote_uploads_bucket, + "region": press_settings.get("remote_uploads_region") or "ap-south-1", + "access_key_id": press_settings.remote_access_key_id, + "secret_access_key": press_settings.get_password("remote_secret_access_key"), + "endpoint_url": press_settings.get("remote_uploads_endpoint") or None, }, ] for b in frappe.get_all("Backup Bucket", ["bucket_name", "cluster", "region"]): - buckets.append( { "name": b["bucket_name"], @@ -83,6 +83,7 @@ def poll_file_statuses_from_bucket(bucket): aws_access_key_id=bucket["access_key_id"], aws_secret_access_key=bucket["secret_access_key"], region_name=bucket["region"], + endpoint_url=bucket.get("endpoint_url") or None, ) available_files = set() @@ -174,31 +175,32 @@ def s3_client(self): if not self.bucket: return None - elif self.bucket == frappe.db.get_single_value( - "Press Settings", "remote_uploads_bucket" - ): - access_key_id = frappe.db.get_single_value("Press Settings", "remote_access_key_id") - secret_access_key = get_decrypted_password( - "Press Settings", "Press Settings", "remote_secret_access_key" - ) + press_settings: "PressSettings" = frappe.get_single("Press Settings") # type: ignore - elif self.bucket: - access_key_id = frappe.db.get_single_value( - "Press Settings", "offsite_backups_access_key_id" + if self.bucket == press_settings.remote_uploads_bucket: + access_key_id = press_settings.remote_access_key_id + secret_access_key = press_settings.get_password("remote_secret_access_key") + region_name = press_settings.get("remote_uploads_region") or "ap-south-1" + endpoint_url = press_settings.get("remote_uploads_endpoint") or None + else: + access_key_id = press_settings.offsite_backups_access_key_id + secret_access_key = press_settings.get_password("offsite_backups_secret_access_key") + region_name = ( + frappe.db.get_value("Backup Bucket", self.bucket, "region") + or press_settings.backup_region ) - secret_access_key = get_decrypted_password( - "Press Settings", "Press Settings", "offsite_backups_secret_access_key" + endpoint_url = ( + frappe.db.get_value("Backup Bucket", self.bucket, "endpoint_url") + or press_settings.get("offsite_backups_endpoint") + or None ) - else: - return None - return client( "s3", aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, - region_name=frappe.db.get_value("Backup Bucket", self.bucket, "region") - or frappe.db.get_single_value("Press Settings", "backup_region"), + region_name=region_name, + endpoint_url=endpoint_url or None, ) @property @@ -270,16 +272,34 @@ def delete_s3_files(buckets): from press.utils import chunk - press_settings = frappe.get_single("Press Settings") + press_settings: "PressSettings" = frappe.get_single("Press Settings") # type: ignore for bucket_name in buckets.keys(): + if bucket_name == press_settings.aws_s3_bucket: + endpoint_url = press_settings.offsite_backups_endpoint + region_name = press_settings.backup_region + aws_access_key_id = press_settings.aws_access_key_id + aws_secret_access_key = press_settings.aws_secret_access_key + elif bucket_name == press_settings.remote_uploads_bucket: + endpoint_url = press_settings.remote_uploads_endpoint + region_name = press_settings.remote_uploads_region + aws_access_key_id = press_settings.offsite_backups_access_key_id + aws_secret_access_key = press_settings.get_password( + "offsite_backups_secret_access_key", raise_exception=False + ) + else: + endpoint_url = frappe.db.get_value("Backup Bucket", bucket_name, "endpoint_url") + region_name = frappe.db.get_value("Backup Bucket", bucket_name, "region") + aws_access_key_id = press_settings.offsite_backups_access_key_id + aws_secret_access_key = press_settings.get_password( + "offsite_backups_secret_access_key", raise_exception=False + ) + s3 = resource( "s3", - aws_access_key_id=press_settings.offsite_backups_access_key_id, - aws_secret_access_key=press_settings.get_password( - "offsite_backups_secret_access_key", raise_exception=False - ), - endpoint_url=frappe.db.get_value("Backup Bucket", bucket_name, "endpoint_url") - or "https://s3.amazonaws.com", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=region_name or "ap-south-1", + endpoint_url=endpoint_url or "https://s3.amazonaws.com", ) bucket = s3.Bucket(bucket_name) for objects in chunk([{"Key": x} for x in buckets[bucket_name]], 1000): diff --git a/press/press/doctype/site_backup/site_backup.py b/press/press/doctype/site_backup/site_backup.py index 2ad63c277c..4966563dda 100644 --- a/press/press/doctype/site_backup/site_backup.py +++ b/press/press/doctype/site_backup/site_backup.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from datetime import datetime + from press.press.doctype.press_settings.press_settings import PressSettings class SiteBackup(Document): @@ -208,12 +209,23 @@ def process_backup_site_job_update(job): def get_backup_bucket(cluster, region=False): - bucket_for_cluster = frappe.get_all("Backup Bucket", {"cluster": cluster}, ["name", "region"], limit=1) - default_bucket = frappe.db.get_single_value("Press Settings", "aws_s3_bucket") + bucket_for_cluster = frappe.get_all( + "Backup Bucket", {"cluster": cluster}, ["name", "region", "endpoint_url"], limit=1 + ) + + if not bucket_for_cluster: + press_settings: "PressSettings" = frappe.get_single("Press Settings") # type: ignore + bucket_for_cluster = [ + { + "name": press_settings.aws_s3_bucket, + "region": press_settings.backup_region, + "endpoint_url": press_settings.offsite_backups_endpoint, + } + ] if region: - return bucket_for_cluster[0] if bucket_for_cluster else default_bucket - return bucket_for_cluster[0]["name"] if bucket_for_cluster else default_bucket + return bucket_for_cluster[0] + return bucket_for_cluster[0]["name"] def on_doctype_update():