refactor(s3): Don't hardcode endpoint and region

press/agent.py

@@ -450,6 +450,7 @@ def backup_site(self, site, with_files=False, offsite=False):
 					"ACCESS_KEY": settings.offsite_backups_access_key_id,
 					"SECRET_KEY": settings.get_password("offsite_backups_secret_access_key"),
 					"REGION": backup_bucket.get("region") if isinstance(backup_bucket, dict) else "",
+					"ENDPOINT_URL": backup_bucket.get("endpoint_url") if isinstance(backup_bucket, dict) else "",
 				}
 				data.update({"offsite": {"bucket": bucket_name, "auth": auth, "path": backups_path}})
 

press/api/site.py

@@ -15,7 +15,6 @@
 from frappe.core.utils import find
 from frappe.desk.doctype.tag.tag import add_tag
 from frappe.utils import flt, sbool, time_diff_in_hours
-from frappe.utils.password import get_decrypted_password
 from frappe.utils.user import is_system_user
 
 from press.exceptions import AAAARecordExists, ConflictingCAARecord
@@ -47,6 +46,7 @@
 	from press.press.doctype.deploy_candidate_app.deploy_candidate_app import (
 		DeployCandidateApp,
 	)
+	from press.press.doctype.press_settings.press_settings import PressSettings
 
 
 NAMESERVERS = ["1.1.1.1", "1.0.0.1", "8.8.8.8", "8.8.4.4"]
@@ -1779,18 +1779,18 @@
 
 @frappe.whitelist()
 def get_upload_link(file, parts=1):
-	bucket_name = frappe.db.get_single_value("Press Settings", "remote_uploads_bucket")
-	expiration = frappe.db.get_single_value("Press Settings", "remote_link_expiry") or 3600
+	press_settings: "PressSettings" = frappe.get_single("Press Settings")  # type: ignore
+	bucket_name = press_settings.remote_uploads_bucket
+	expiration = press_settings.remote_link_expiry or 3600
 	object_name = get_remote_key(file)
 	parts = int(parts)
 
 	s3_client = client(
 		"s3",
-		aws_access_key_id=frappe.db.get_single_value("Press Settings", "remote_access_key_id"),
-		aws_secret_access_key=get_decrypted_password(
-			"Press Settings", "Press Settings", "remote_secret_access_key"
-		),
-		region_name="ap-south-1",
+		aws_access_key_id=press_settings.remote_access_key_id,
+		aws_secret_access_key=press_settings.get_password("remote_secret_access_key"),
+		region_name=press_settings.get("remote_uploads_region") or "ap-south-1",
+		endpoint_url=press_settings.get("remote_uploads_endpoint") or None,
 	)
 	try:
 		# The response contains the presigned URL and required fields
@@ -1822,24 +1822,24 @@
 
 @frappe.whitelist()
 def multipart_exit(file, id, action, parts=None):
+	press_settings: "PressSettings" = frappe.get_single("Press Settings")  # type: ignore
+	bucket_name = "uploads.frappe.cloud"  # TODO: Use press_settings.remote_uploads_bucket
+
 	s3_client = client(
 		"s3",
-		aws_access_key_id=frappe.db.get_single_value("Press Settings", "remote_access_key_id"),
-		aws_secret_access_key=get_decrypted_password(
-			"Press Settings",
-			"Press Settings",
-			"remote_secret_access_key",
-			raise_exception=False,
-		),
-		region_name="ap-south-1",
+		aws_access_key_id=press_settings.remote_access_key_id,
+		aws_secret_access_key=press_settings.get_password("remote_secret_access_key", raise_exception=False),
+		region_name=press_settings.get("remote_uploads_region") or "ap-south-1",
+		endpoint_url=press_settings.get("remote_uploads_endpoint") or None,
 	)
+
 	if action == "abort":
-		response = s3_client.abort_multipart_upload(Bucket="uploads.frappe.cloud", Key=file, UploadId=id)
+		response = s3_client.abort_multipart_upload(Bucket=bucket_name, Key=file, UploadId=id)
 	elif action == "complete":
 		parts = json.loads(parts)
 		# After completing for all parts, you will use complete_multipart_upload api which requires that parts list
 		response = s3_client.complete_multipart_upload(
-			Bucket="uploads.frappe.cloud",
+			Bucket=bucket_name,
 			Key=file,
 			UploadId=id,
 			MultipartUpload={"Parts": parts},

press/press/doctype/press_settings/press_settings.json

@@ -55,9 +55,8 @@
   "backups_tab",
   "offsite_backups_section",
   "backup_region",
-  "offsite_backups_provider",
   "aws_s3_bucket",
-  "data_40",
+  "offsite_backups_endpoint",
   "backup_rotation_scheme",
   "column_break_35",
   "offsite_backups_access_key_id",
@@ -90,10 +89,12 @@
   "auto_update_queue_size",
   "remote_files_section",
   "remote_uploads_bucket",
-  "remote_link_expiry",
+  "remote_uploads_region",
+  "remote_uploads_endpoint",
   "column_break_51",
   "remote_access_key_id",
   "remote_secret_access_key",
+  "remote_link_expiry",
   "product_documentation_section",
   "publish_docs",
   "storage_and_disk_limits_section",
@@ -405,22 +406,11 @@
    "fieldtype": "Data",
    "label": "Backup Region"
   },
-  {
-   "default": "AWS S3",
-   "fieldname": "offsite_backups_provider",
-   "fieldtype": "Select",
-   "label": "Backup Provider",
-   "options": "AWS S3"
-  },
   {
    "fieldname": "aws_s3_bucket",
    "fieldtype": "Data",
    "label": "Bucket Name"
   },
-  {
-   "fieldname": "data_40",
-   "fieldtype": "Data"
-  },
   {
    "fieldname": "backup_rotation_scheme",
    "fieldtype": "Select",
@@ -1246,11 +1236,26 @@
    "fieldtype": "Link",
    "label": "Press Trial Plan",
    "options": "Site Plan"
+  },
+  {
+   "fieldname": "remote_uploads_region",
+   "fieldtype": "Data",
+   "label": "Uploads Bucket Region"
+  },
+  {
+   "fieldname": "remote_uploads_endpoint",
+   "fieldtype": "Data",
+   "label": "Uploads Bucket Endpoint"
+  },
+  {
+   "fieldname": "offsite_backups_endpoint",
+   "fieldtype": "Data",
+   "label": "Bucket Endpoint"
   }
  ],
  "issingle": 1,
  "links": [],
- "modified": "2024-10-14 21:02:12.948747",
+ "modified": "2024-11-14 14:21:14.359536",
  "modified_by": "Administrator",
  "module": "Press",
  "name": "Press Settings",

press/press/doctype/press_settings/press_settings.py

@@ -50,7 +50,6 @@ class PressSettings(Document):
 		code_server_password: DF.Data | None
 		commission: DF.Float
 		compress_app_cache: DF.Check
-		data_40: DF.Data | None
 		default_outgoing_id: DF.Data | None
 		default_outgoing_pass: DF.Data | None
 		disable_agent_job_deduplication: DF.Check
@@ -98,7 +97,7 @@ class PressSettings(Document):
 		ngrok_auth_token: DF.Data | None
 		offsite_backups_access_key_id: DF.Data | None
 		offsite_backups_count: DF.Int
-		offsite_backups_provider: DF.Literal["AWS S3"]
+		offsite_backups_endpoint: DF.Data | None
 		offsite_backups_secret_access_key: DF.Password | None
 		plausible_api_key: DF.Password | None
 		plausible_site_id: DF.Data | None
@@ -115,6 +114,8 @@ class PressSettings(Document):
 		remote_link_expiry: DF.Int
 		remote_secret_access_key: DF.Password | None
 		remote_uploads_bucket: DF.Data | None
+		remote_uploads_endpoint: DF.Data | None
+		remote_uploads_region: DF.Data | None
 		root_domain: DF.Data | None
 		rsa_key_size: DF.Literal["2048", "3072", "4096"]
 		spaces_domain: DF.Link | None

press/press/doctype/remote_file/remote_file.py

@@ -1,16 +1,20 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe and contributors
 # For license information, please see license.txt
 
 
 import json
 import pprint
+from typing import TYPE_CHECKING
 
 import frappe
 import requests
 from boto3 import client, resource
 from frappe.model.document import Document
-from frappe.utils.password import get_decrypted_password
+
+
+if TYPE_CHECKING:
+	from press.press.doctype.press_settings.press_settings import PressSettings
 
 
 def get_remote_key(file):
@@ -27,34 +31,30 @@
 
 
 def poll_file_statuses():
-	aws_access_key = frappe.db.get_single_value(
-		"Press Settings", "offsite_backups_access_key_id"
-	)
-	aws_secret_key = get_decrypted_password(
-		"Press Settings", "Press Settings", "offsite_backups_secret_access_key"
-	)
-	default_region = frappe.db.get_single_value("Press Settings", "backup_region")
+	press_settings: "PressSettings" = frappe.get_single("Press Settings")  # type: ignore
+
+	aws_access_key = press_settings.offsite_backups_access_key_id
+	aws_secret_key = press_settings.offsite_backups_secret_access_key
+	default_region = press_settings.backup_region
+
 	buckets = [
 		{
-			"name": frappe.db.get_single_value("Press Settings", "aws_s3_bucket"),
+			"name": press_settings.aws_s3_bucket,
 			"region": default_region,
 			"access_key_id": aws_access_key,
 			"secret_access_key": aws_secret_key,
+			"endpoint_url": press_settings.offsite_backups_endpoint,
 		},
 		{
-			"name": frappe.db.get_single_value("Press Settings", "remote_uploads_bucket"),
-			"region": default_region,
-			"access_key_id": frappe.db.get_single_value(
-				"Press Settings", "remote_access_key_id"
-			),
-			"secret_access_key": get_decrypted_password(
-				"Press Settings", "Press Settings", "remote_secret_access_key"
-			),
+			"name": press_settings.remote_uploads_bucket,
+			"region": press_settings.get("remote_uploads_region") or "ap-south-1",
+			"access_key_id": press_settings.remote_access_key_id,
+			"secret_access_key": press_settings.get_password("remote_secret_access_key"),
+			"endpoint_url": press_settings.get("remote_uploads_endpoint") or None,
 		},
 	]
 
 	for b in frappe.get_all("Backup Bucket", ["bucket_name", "cluster", "region"]):
-
 		buckets.append(
 			{
 				"name": b["bucket_name"],
@@ -83,6 +83,7 @@
 		aws_access_key_id=bucket["access_key_id"],
 		aws_secret_access_key=bucket["secret_access_key"],
 		region_name=bucket["region"],
+		endpoint_url=bucket.get("endpoint_url") or None,
 	)
 
 	available_files = set()
@@ -128,7 +129,7 @@
 	"""Delete specified objects identified by keys in the backups bucket."""
 	remote_files = list(set([x for x in remote_files if x]))
 	if not remote_files:
 		return
 
 	buckets = {bucket: [] for bucket in frappe.get_all("Backup Bucket", pluck="name")}
 	buckets.update({frappe.db.get_single_value("Press Settings", "aws_s3_bucket"): []})
@@ -159,14 +160,14 @@
 	if TYPE_CHECKING:
 		from frappe.types import DF
 
 		bucket: DF.Data | None
 		file_name: DF.Data | None
 		file_path: DF.Text | None
 		file_size: DF.Data | None
 		file_type: DF.Data | None
 		site: DF.Link | None
 		status: DF.Literal["Available", "Unavailable"]
 		url: DF.Code | None
 	# end: auto-generated types
 
 	@property
@@ -174,31 +175,32 @@
 		if not self.bucket:
 			return None
 
-		elif self.bucket == frappe.db.get_single_value(
-			"Press Settings", "remote_uploads_bucket"
-		):
-			access_key_id = frappe.db.get_single_value("Press Settings", "remote_access_key_id")
-			secret_access_key = get_decrypted_password(
-				"Press Settings", "Press Settings", "remote_secret_access_key"
-			)
+		press_settings: "PressSettings" = frappe.get_single("Press Settings")  # type: ignore
 
-		elif self.bucket:
-			access_key_id = frappe.db.get_single_value(
-				"Press Settings", "offsite_backups_access_key_id"
+		if self.bucket == press_settings.remote_uploads_bucket:
+			access_key_id = press_settings.remote_access_key_id
+			secret_access_key = press_settings.get_password("remote_secret_access_key")
+			region_name = press_settings.get("remote_uploads_region") or "ap-south-1"
+			endpoint_url = press_settings.get("remote_uploads_endpoint") or None
+		else:
+			access_key_id = press_settings.offsite_backups_access_key_id
+			secret_access_key = press_settings.get_password("offsite_backups_secret_access_key")
+			region_name = (
+				frappe.db.get_value("Backup Bucket", self.bucket, "region")
+				or press_settings.backup_region
 			)
-			secret_access_key = get_decrypted_password(
-				"Press Settings", "Press Settings", "offsite_backups_secret_access_key"
+			endpoint_url = (
+				frappe.db.get_value("Backup Bucket", self.bucket, "endpoint_url")
+				or press_settings.get("offsite_backups_endpoint")
+				or None
 			)
 
-		else:
-			return None
-
 		return client(
 			"s3",
 			aws_access_key_id=access_key_id,
 			aws_secret_access_key=secret_access_key,
-			region_name=frappe.db.get_value("Backup Bucket", self.bucket, "region")
-			or frappe.db.get_single_value("Press Settings", "backup_region"),
+			region_name=region_name,
+			endpoint_url=endpoint_url or None,
 		)
 
 	@property
@@ -270,16 +272,34 @@
 
 	from press.utils import chunk
 
-	press_settings = frappe.get_single("Press Settings")
+	press_settings: "PressSettings" = frappe.get_single("Press Settings")  # type: ignore
 	for bucket_name in buckets.keys():
+		if bucket_name == press_settings.aws_s3_bucket:
+			endpoint_url = press_settings.offsite_backups_endpoint
+			region_name = press_settings.backup_region
+			aws_access_key_id = press_settings.aws_access_key_id
+			aws_secret_access_key = press_settings.aws_secret_access_key
+		elif bucket_name == press_settings.remote_uploads_bucket:
+			endpoint_url = press_settings.remote_uploads_endpoint
+			region_name = press_settings.remote_uploads_region
+			aws_access_key_id = press_settings.offsite_backups_access_key_id
+			aws_secret_access_key = press_settings.get_password(
+				"offsite_backups_secret_access_key", raise_exception=False
+			)
+		else:
+			endpoint_url = frappe.db.get_value("Backup Bucket", bucket_name, "endpoint_url")
+			region_name = frappe.db.get_value("Backup Bucket", bucket_name, "region")
+			aws_access_key_id = press_settings.offsite_backups_access_key_id
+			aws_secret_access_key = press_settings.get_password(
+				"offsite_backups_secret_access_key", raise_exception=False
+			)
+
 		s3 = resource(
 			"s3",
-			aws_access_key_id=press_settings.offsite_backups_access_key_id,
-			aws_secret_access_key=press_settings.get_password(
-				"offsite_backups_secret_access_key", raise_exception=False
-			),
-			endpoint_url=frappe.db.get_value("Backup Bucket", bucket_name, "endpoint_url")
-			or "https://s3.amazonaws.com",
+			aws_access_key_id=aws_access_key_id,
+			aws_secret_access_key=aws_secret_access_key,
+			region_name=region_name or "ap-south-1",
+			endpoint_url=endpoint_url or "https://s3.amazonaws.com",
 		)
 		bucket = s3.Bucket(bucket_name)
 		for objects in chunk([{"Key": x} for x in buckets[bucket_name]], 1000):

press/press/doctype/site_backup/site_backup.py

@@ -14,6 +14,7 @@
 
 if TYPE_CHECKING:
 	from datetime import datetime
+	from press.press.doctype.press_settings.press_settings import PressSettings
 
 
 class SiteBackup(Document):
@@ -208,12 +209,23 @@ def process_backup_site_job_update(job):
 
 
 def get_backup_bucket(cluster, region=False):
-	bucket_for_cluster = frappe.get_all("Backup Bucket", {"cluster": cluster}, ["name", "region"], limit=1)
-	default_bucket = frappe.db.get_single_value("Press Settings", "aws_s3_bucket")
+	bucket_for_cluster = frappe.get_all(
+		"Backup Bucket", {"cluster": cluster}, ["name", "region", "endpoint_url"], limit=1
+	)
+
+	if not bucket_for_cluster:
+		press_settings: "PressSettings" = frappe.get_single("Press Settings")  # type: ignore
+		bucket_for_cluster = [
+			{
+				"name": press_settings.aws_s3_bucket,
+				"region": press_settings.backup_region,
+				"endpoint_url": press_settings.offsite_backups_endpoint,
+			}
+		]
 
 	if region:
-		return bucket_for_cluster[0] if bucket_for_cluster else default_bucket
-	return bucket_for_cluster[0]["name"] if bucket_for_cluster else default_bucket
+		return bucket_for_cluster[0]
+	return bucket_for_cluster[0]["name"]
 
 
 def on_doctype_update():