diff --git a/dashboard/src2/components/server/ServerPlansDialog.vue b/dashboard/src2/components/server/ServerPlansDialog.vue index 76175a76ab..788e01322f 100644 --- a/dashboard/src2/components/server/ServerPlansDialog.vue +++ b/dashboard/src2/components/server/ServerPlansDialog.vue @@ -101,7 +101,8 @@ export default { url: 'press.api.server.plans', params: { name: this.serverType, - cluster: this.$server.doc.cluster + cluster: this.$server.doc.cluster, + platform: this.$server.doc.current_plan.platform }, auto: true, initialData: [] diff --git a/press/api/server.py b/press/api/server.py index 4a7efe38e3..df37dbd7ab 100644 --- a/press/api/server.py +++ b/press/api/server.py @@ -386,7 +386,7 @@ def options(): @frappe.whitelist() -def plans(name, cluster=None): +def plans(name, cluster=None, platform="x86_64"): return Plan.get_plans( doctype="Server Plan", fields=[ @@ -401,7 +401,12 @@ def plans(name, cluster=None): "instance_type", "premium", ], - filters={"server_type": name, "cluster": cluster} if cluster else {"server_type": name}, + filters={"server_type": name, "platform": platform, "cluster": cluster} + if cluster + else { + "server_type": name, + "platform": platform, + }, ) diff --git a/press/infrastructure/doctype/virtual_machine_migration/README.md b/press/infrastructure/doctype/virtual_machine_migration/README.md new file mode 100644 index 0000000000..6d9f961db2 --- /dev/null +++ b/press/infrastructure/doctype/virtual_machine_migration/README.md @@ -0,0 +1,96 @@ +# Explaining Choices + +Most commits/comments already explain the decisions. Just putting them here for sanity. + +## Mounts + +Going forward the data (mostly machine-independent directories) will be kept on a separate volume. + +For the migration we + +1. Shut down the machine +2. Start a new machine (with a new ARM image) +3. Attach the root volume from the old machine to the new machine +4. Do some mount magic so all services find data where they expect it to be + +### AWS Quirks + +1. We can't attach a volume at boot. The VM must be in the Running state. +2. You can't rely on device_name provided during run_instance. +3. The device will have an alias that looks something like "/dev/disk/by-id/......" + +### Bind Mounts + +Instead of directly mounting the volume to the target mount point, we + +1. Mount the volume to /opt/volumes// +2. Bind mount the relative location from this path. /opt/volumes/mariadb/a/b/c to /a/b/c + +This gives us the ability to + +1. Have two different mounts (/etc/mysql and /var/lib/mysql) +2. Use the same old volumes as-is without any custom mounting scheme. + +### Mount Dependency + +We don't want MariaDB / Docker to start unless the data volume is mounted correctly. +Add a systemd mount dependency (BindsTo) so the services start if and only if the data volume is mounted. + +Note: We define the dependency only on the bind mount. /opt/volumes... is left out as convenience. + +### Relabeling + +The base images are configured to mount partitions labeled UEFI and cloudimg-rootfs + +1. We change these labels so the new machine doesn't accidentally boot from these +2. We update fstab so the old machine can still boot with the modified labels + +Note: EFI partitions have a dirty bit set on them. fatlabel messes this up. We need to run fsck to fix this. + +### UUID + +When we spawn a new machine from the base image, all volumes get their own volume-id. If we rely on volume-id to determine the data volume then we'll have to do some extra work after the first boot. (To tell the machine about the volume) + +When we format the data volume we get a new UUID. This UUID remains the same (since it's part of the data itself) across boots (unless we reformat the volume). This is the easiest way to recognize a volume in fstab. + +During the migration, we need to do the extra step of updating fstab to use the old UUID (from the old root volume). + +We could have modified the UUID of the old root volume (so we don't need to do any work after the migration). But + +1. e2label needs a freshly checked disk (fsck) +2. fsck needs an unmounted partition. We can't unmount the root partition. + +## Misc + +### Hardcoded values + +This is only going to be used for app and db servers. + +- App servers will have /home/frappe/benches stored on the the data volume +- DB servers will have /var/lib/mysql and /etc/mysql stored on the data volume + +### Wait for ping + cloud init + +During the first boot we + +1. Delete old host keys (to avoid collisions between multiple hosts) +2. Update SSH config +3. Restart SSHD + +During this restart, for a short period, we can't start a new SSH session. (Sometimes we get lucky). +To avoid this. Explicitly wait for cloud-init to finish (and then check if sshd is running). + +--- + +## TODO + +#### Disk Usage Alerts + +We'll need to add alerts for the modified mount points (old alerts rely on /) + +#### Disk Resize + +Resize logic resizes the first volume listed in the volumes table. + +1. This ordering isn't guaranteed to be [root, data] +2. We need a way to specify exactly which volume we need to resize diff --git a/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.js b/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.js index aee30e7952..69b26932c3 100644 --- a/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.js +++ b/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.js @@ -4,6 +4,8 @@ frappe.ui.form.on('Virtual Machine Migration', { refresh(frm) { [ + [__('Start'), 'execute', frm.doc.status === 'Pending'], + [__('Force Continue'), 'force_continue', frm.doc.status === 'Failure'], [__('Force Continue'), 'force_continue', frm.doc.status === 'Failure'], [__('Force Fail'), 'force_fail', frm.doc.status === 'Running'], ].forEach(([label, method, condition]) => { diff --git a/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.json b/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.json index 2e4e01432e..08333f338c 100644 --- a/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.json +++ b/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.json @@ -7,6 +7,7 @@ "field_order": [ "virtual_machine", "status", + "new_plan", "column_break_pega", "virtual_machine_image", "machine_type", @@ -17,6 +18,9 @@ "duration", "section_break_pplo", "volumes", + "mounts", + "raw_devices", + "parsed_devices", "section_break_mjhg", "steps" ], @@ -27,6 +31,7 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Virtual Machine", + "link_filters": "[[\"Virtual Machine\",\"status\",\"not in\",[\"Draft\",\"Terminated\",null]]]", "options": "Virtual Machine", "reqd": 1, "set_only_once": 1 @@ -80,6 +85,7 @@ "fieldtype": "Link", "in_list_view": 1, "label": "Virtual Machine Image", + "link_filters": "[[\"Virtual Machine Image\",\"status\",\"=\",\"Available\"]]", "options": "Virtual Machine Image", "reqd": 1, "set_only_once": 1 @@ -113,11 +119,40 @@ "fieldtype": "Duration", "label": "Duration", "read_only": 1 + }, + { + "fieldname": "raw_devices", + "fieldtype": "Code", + "hidden": 1, + "label": "Raw Devices", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "parsed_devices", + "fieldtype": "Code", + "hidden": 1, + "label": "Parsed Devices", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "mounts", + "fieldtype": "Table", + "label": "Mounts", + "options": "Virtual Machine Migration Mount" + }, + { + "fieldname": "new_plan", + "fieldtype": "Link", + "label": "New Plan", + "options": "Server Plan", + "read_only": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2024-09-20 15:27:50.984335", + "modified": "2024-12-09 16:31:29.250443", "modified_by": "Administrator", "module": "Infrastructure", "name": "Virtual Machine Migration", diff --git a/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.py b/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.py index 5a79870b84..0313efbfed 100644 --- a/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.py +++ b/press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.py @@ -2,13 +2,19 @@ # For license information, please see license.txt from __future__ import annotations +import json +import shlex +import subprocess import time from enum import Enum from typing import TYPE_CHECKING import frappe +from frappe.core.utils import find from frappe.model.document import Document +from press.press.doctype.ansible_console.ansible_console import AnsibleAdHoc + if TYPE_CHECKING: from press.infrastructure.doctype.virtual_machine_migration_step.virtual_machine_migration_step import ( VirtualMachineMigrationStep, @@ -27,6 +33,9 @@ class VirtualMachineMigration(Document): if TYPE_CHECKING: from frappe.types import DF + from press.infrastructure.doctype.virtual_machine_migration_mount.virtual_machine_migration_mount import ( + VirtualMachineMigrationMount, + ) from press.infrastructure.doctype.virtual_machine_migration_step.virtual_machine_migration_step import ( VirtualMachineMigrationStep, ) @@ -38,7 +47,11 @@ class VirtualMachineMigration(Document): duration: DF.Duration | None end: DF.Datetime | None machine_type: DF.Data + mounts: DF.Table[VirtualMachineMigrationMount] name: DF.Int | None + new_plan: DF.Link | None + parsed_devices: DF.Code | None + raw_devices: DF.Code | None start: DF.Datetime | None status: DF.Literal["Pending", "Running", "Success", "Failure"] steps: DF.Table[VirtualMachineMigrationStep] @@ -53,9 +66,89 @@ def before_insert(self): self.add_steps() self.add_volumes() self.create_machine_copy() + self.set_new_plan() def after_insert(self): - self.execute() + self.add_devices() + self.set_default_mounts() + + def add_devices(self): + command = "lsblk --json --output name,type,uuid,mountpoint,size,label,fstype" + output = self.ansible_run(command)["output"] + + """Sample output of the command + { + "blockdevices": [ + {"name":"loop0", "type":"loop", "uuid":null, "mountpoint":"/snap/amazon-ssm-agent/9882", "size":"22.9M", "label":null, "fstype":null}, + {"name":"loop1", "type":"loop", "uuid":null, "mountpoint":"/snap/core20/2437", "size":"59.5M", "label":null, "fstype":null}, + {"name":"loop2", "type":"loop", "uuid":null, "mountpoint":"/snap/core22/1666", "size":"68.9M", "label":null, "fstype":null}, + {"name":"loop3", "type":"loop", "uuid":null, "mountpoint":"/snap/snapd/21761", "size":"33.7M", "label":null, "fstype":null}, + {"name":"loop4", "type":"loop", "uuid":null, "mountpoint":"/snap/lxd/29631", "size":"92M", "label":null, "fstype":null}, + {"name":"nvme0n1", "type":"disk", "uuid":null, "mountpoint":null, "size":"25G", "label":null, "fstype":null, + "children": [ + {"name":"nvme0n1p1", "type":"part", "uuid":"b8932e17-9ed7-47b7-8bf3-75ff6669e018", "mountpoint":"/", "size":"24.9G", "label":"cloudimg-rootfs", "fstype":"ext4"}, + {"name":"nvme0n1p15", "type":"part", "uuid":"7569-BCF0", "mountpoint":"/boot/efi", "size":"99M", "label":"UEFI", "fstype":"vfat"} + ] + }, + {"name":"nvme1n1", "type":"disk", "uuid":"41527fb0-f6e9-404e-9dba-0451dfa2195e", "mountpoint":"/opt/volumes/mariadb", "size":"10G", "label":null, "fstype":"ext4"} + ] + }""" + devices = json.loads(output)["blockdevices"] + self.raw_devices = json.dumps(devices, indent=2) + self.parsed_devices = json.dumps(self._parse_devices(devices), indent=2) + self.save() + + def _parse_devices(self, devices): + parsed = [] + for device in devices: + # We only care about disks and partitions + if device["type"] != "disk": + continue + + # Disk has partitions. e.g root volume + if "children" in device: + for partition in device["children"]: + if partition["type"] == "part": + parsed.append(partition) + else: + # Single partition. e.g data volume + parsed.append(device) + return parsed + + def set_default_mounts(self): + # Set root partition from old machine as the data partition in the new machine + + if self.mounts: + # We've already set the mounts + return + + parsed_devices = json.loads(self.parsed_devices) + device = find(parsed_devices, lambda x: x["mountpoint"] == "/") + if not device: + # No root volume found + return + + server_type = self.machine.get_server().doctype + if server_type == "Server": + target_mount_point = "/opt/volumes/benches" + service = "docker" + elif server_type == "Database Server": + target_mount_point = "/opt/volumes/mariadb" + service = "mariadb" + else: + # Data volumes are only supported for Server and Database Server + return + + self.append( + "mounts", + { + "uuid": device["uuid"], + "source_mount_point": device["mountpoint"], + "target_mount_point": target_mount_point, + "service": service, + }, + ) + self.save() def add_steps(self): for step in self.migration_steps: @@ -71,6 +164,8 @@ def add_volumes(self): { "status": "Unattached", "volume_id": volume.volume_id, + # This is the device name that will be used in the new machine + # Only needed for the attach_volumes call "device_name": f"/dev/sd{device_name_index}", }, ) @@ -86,6 +181,24 @@ def create_machine_copy(self): copied_machine = frappe.copy_doc(self.machine) copied_machine.insert(set_name=self.copied_virtual_machine) + def set_new_plan(self): + server = self.machine.get_server() + old_plan = frappe.get_doc("Server Plan", server.plan) + matching_plans = frappe.get_all( + "Server Plan", + { + "enabled": True, + "server_type": old_plan.server_type, + "cluster": old_plan.cluster, + "instance_type": self.machine_type, + "premium": old_plan.premium, + }, + pluck="name", + limit=1, + ) + if matching_plans: + self.new_plan = matching_plans[0] + def validate_aws_only(self): if self.machine.cloud_provider != "AWS EC2": frappe.throw("This feature is only available for AWS EC2") @@ -113,55 +226,68 @@ def copied_machine(self): @property def migration_steps(self): - return [ - { - "step": self.stop_machine.__doc__, - "method": self.stop_machine.__name__, - "wait_for_completion": True, - }, - { - "step": self.wait_for_machine_to_stop.__doc__, - "method": self.wait_for_machine_to_stop.__name__, - "wait_for_completion": True, - }, - { - "step": self.disable_delete_on_termination_for_all_volumes.__doc__, - "method": self.disable_delete_on_termination_for_all_volumes.__name__, - }, - { - "step": self.terminate_previous_machine.__doc__, - "method": self.terminate_previous_machine.__name__, - "wait_for_completion": True, - }, - { - "step": self.wait_for_previous_machine_to_terminate.__doc__, - "method": self.wait_for_previous_machine_to_terminate.__name__, - "wait_for_completion": True, - }, - { - "step": self.reset_virtual_machine_attributes.__doc__, - "method": self.reset_virtual_machine_attributes.__name__, - }, - { - "step": self.provision_new_machine.__doc__, - "method": self.provision_new_machine.__name__, - }, - { - "step": self.wait_for_machine_to_start.__doc__, - "method": self.wait_for_machine_to_start.__name__, - "wait_for_completion": True, - }, - { - "step": self.attach_volumes.__doc__, - "method": self.attach_volumes.__name__, - }, - { - "step": self.wait_for_machine_to_be_accessible.__doc__, - "method": self.wait_for_machine_to_be_accessible.__name__, - "wait_for_completion": True, - }, + Wait = True + NoWait = False + methods = [ + (self.update_partition_labels, NoWait), + (self.stop_machine, Wait), + (self.wait_for_machine_to_stop, Wait), + (self.disable_delete_on_termination_for_all_volumes, NoWait), + (self.terminate_previous_machine, Wait), + (self.wait_for_previous_machine_to_terminate, Wait), + (self.reset_virtual_machine_attributes, NoWait), + (self.provision_new_machine, NoWait), + (self.wait_for_machine_to_start, Wait), + (self.attach_volumes, NoWait), + (self.wait_for_machine_to_be_accessible, Wait), + (self.check_cloud_init_status, NoWait), + (self.wait_for_cloud_init, Wait), + (self.remove_old_host_key, NoWait), + (self.update_mounts, NoWait), + (self.update_plan, NoWait), ] + steps = [] + for method, wait_for_completion in methods: + steps.append( + { + "step": method.__doc__, + "method": method.__name__, + "wait_for_completion": wait_for_completion, + } + ) + return steps + + def update_partition_labels(self) -> StepStatus: + "Update partition labels" + # Ubuntu images have labels for root (cloudimg-rootfs) and efi (UEFI) partitions + # Remove these labels from the old volume + # So the new machine doesn't mount these as root or efi partitions + # Important: Update fstab so we can still boot the old machine + parsed_devices = json.loads(self.parsed_devices) + for device in parsed_devices: + old_label = device["label"] + if not old_label: + continue + + labeler = {"ext4": "e2label", "vfat": "fatlabel"}[device["fstype"]] + new_label = {"cloudimg-rootfs": "old-rootfs", "UEFI": "OLD-UEFI"}[old_label] + commands = [ + # Reference: https://wiki.archlinux.org/title/Persistent_block_device_naming#by-label + f"{labeler} /dev/{device['name']} {new_label}", + f"sed -i 's/LABEL\\={old_label}/LABEL\\={new_label}/g' /etc/fstab", # Ansible implementation quirk + ] + if old_label == "UEFI": + # efi mounts have dirty bit set. This resets it. + commands.append(f"fsck -a /dev/{device['name']}") + + for command in commands: + result = self.ansible_run(command) + if result["status"] != "Success": + self.add_comment(text=f"Error updating partition labels: {result}") + return StepStatus.Failure + return StepStatus.Success + def stop_machine(self) -> StepStatus: "Stop machine" machine = self.machine @@ -222,6 +348,7 @@ def reset_virtual_machine_attributes(self) -> StepStatus: # Set new machine image and machine type machine.virtual_machine_image = self.virtual_machine_image machine.machine_type = self.machine_type + machine.disk_size = 10 # Default disk size for new machines machine.save() return StepStatus.Success @@ -240,7 +367,7 @@ def wait_for_machine_to_start(self) -> StepStatus: return StepStatus.Success return StepStatus.Pending - def attach_volumes(self): + def attach_volumes(self) -> StepStatus: "Attach volumes" machine = self.machine for volume in self.volumes: @@ -263,7 +390,7 @@ def wait_for_machine_to_be_accessible(self): plays = frappe.get_all( "Ansible Play", - {"server": server.name, "play": "Ping Server"}, + {"server": server.name, "play": "Ping Server", "creation": (">", self.creation)}, ["status"], order_by="creation desc", limit=1, @@ -272,6 +399,68 @@ def wait_for_machine_to_be_accessible(self): return StepStatus.Success return StepStatus.Pending + def check_cloud_init_status(self) -> StepStatus: + "Check cloud-init status" + server = self.machine.get_server() + server.wait_for_cloud_init() + return StepStatus.Success + + def wait_for_cloud_init(self) -> StepStatus: + "Wait for cloud-init to finish" + server = self.machine.get_server() + plays = frappe.get_all( + "Ansible Play", + { + "server": server.name, + "play": "Wait for Cloud Init to finish", + "creation": (">", self.creation), + }, + ["status"], + order_by="creation desc", + limit=1, + ) + if plays and plays[0].status in ("Success", "Failure"): + return StepStatus.Success + return StepStatus.Pending + + def remove_old_host_key(self) -> StepStatus: + "Remove old host key" + command = f"ssh-keygen -R '{self.virtual_machine}'" + subprocess.check_call(shlex.split(command)) + return StepStatus.Success + + def update_mounts(self) -> StepStatus: + "Update mounts" + # Mount the volume using the old UUID + # Update fstab + # 1. Find mount matching the source mount point in fstab + # 2. Update UUID for this mountpoint + for mount in self.mounts: + escaped_mount_point = mount.target_mount_point.replace("/", "\\/") + # Reference: https://stackoverflow.com/questions/16637799/sed-error-invalid-reference-1-on-s-commands-rhs#comment88576787_16637847 + commands = [ + f"sed -Ei 's/^UUID\\=.*\\s({escaped_mount_point}\\s.*$)/UUID\\={mount.uuid} \\1/g' /etc/fstab", + "systemctl daemon-reload", + ] + if mount.service: + commands.append(f"systemctl start {mount.service}") + for command in commands: + result = self.ansible_run(command) + if result["status"] != "Success": + self.add_comment(text=f"Error updating mounts: {result}") + return StepStatus.Failure + + return StepStatus.Success + + def update_plan(self) -> StepStatus: + "Update plan" + if self.new_plan: + server = self.machine.get_server() + plan = frappe.get_doc("Server Plan", self.new_plan) + server._change_plan(plan) + return StepStatus.Success + + @frappe.whitelist() def execute(self): self.status = "Running" self.start = frappe.utils.now_datetime() @@ -294,9 +483,9 @@ def succeed(self) -> None: self.save() @frappe.whitelist() - def next(self, arguments=None) -> None: + def next(self, ignore_version=False) -> None: self.status = "Running" - self.save() + self.save(ignore_version=ignore_version) next_step = self.next_step if not next_step: @@ -342,6 +531,7 @@ def execute_step(self, step_name): if not step.start: step.start = frappe.utils.now_datetime() step.status = "Running" + ignore_version_while_saving = False try: result = getattr(self, step.method)() step.status = result.name @@ -349,6 +539,7 @@ def execute_step(self, step_name): step.attempts = step.attempts + 1 if result == StepStatus.Pending: # Wait some time before the next run + ignore_version_while_saving = True time.sleep(1) except Exception: step.status = "Failure" @@ -360,10 +551,21 @@ def execute_step(self, step_name): if step.status == "Failure": self.fail() else: - self.next() + self.next(ignore_version_while_saving) def get_step(self, step_name) -> VirtualMachineMigrationStep | None: for step in self.steps: if step.name == step_name: return step return None + + def ansible_run(self, command): + inventory = f"{self.virtual_machine}," + result = AnsibleAdHoc(sources=inventory).run(command, self.name)[0] + self.add_command(command, result) + return result + + def add_command(self, command, result): + pretty_result = json.dumps(result, indent=2, sort_keys=True, default=str) + comment = f"
{command}
{pretty_result}
" + self.add_comment(text=comment) diff --git a/press/infrastructure/doctype/virtual_machine_migration_mount/__init__.py b/press/infrastructure/doctype/virtual_machine_migration_mount/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/press/infrastructure/doctype/virtual_machine_migration_mount/virtual_machine_migration_mount.json b/press/infrastructure/doctype/virtual_machine_migration_mount/virtual_machine_migration_mount.json new file mode 100644 index 0000000000..50188bc1e4 --- /dev/null +++ b/press/infrastructure/doctype/virtual_machine_migration_mount/virtual_machine_migration_mount.json @@ -0,0 +1,56 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2024-11-26 15:14:54.328130", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "source_mount_point", + "target_mount_point", + "uuid", + "service" + ], + "fields": [ + { + "columns": 2, + "fieldname": "source_mount_point", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Source Mount Point" + }, + { + "columns": 3, + "fieldname": "target_mount_point", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Target Mount Point" + }, + { + "columns": 4, + "fieldname": "uuid", + "fieldtype": "Data", + "in_list_view": 1, + "label": "UUID" + }, + { + "columns": 1, + "fieldname": "service", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Service" + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2024-12-09 16:32:43.323509", + "modified_by": "Administrator", + "module": "Infrastructure", + "name": "Virtual Machine Migration Mount", + "owner": "Administrator", + "permissions": [], + "sort_field": "creation", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/press/infrastructure/doctype/virtual_machine_migration_mount/virtual_machine_migration_mount.py b/press/infrastructure/doctype/virtual_machine_migration_mount/virtual_machine_migration_mount.py new file mode 100644 index 0000000000..f27405797e --- /dev/null +++ b/press/infrastructure/doctype/virtual_machine_migration_mount/virtual_machine_migration_mount.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +# import frappe +from __future__ import annotations + +from frappe.model.document import Document + + +class VirtualMachineMigrationMount(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + service: DF.Data | None + source_mount_point: DF.Data | None + target_mount_point: DF.Data | None + uuid: DF.Data | None + # end: auto-generated types + + pass diff --git a/press/playbooks/database.yml b/press/playbooks/database.yml index 4ae1d20b41..3ea40380c0 100644 --- a/press/playbooks/database.yml +++ b/press/playbooks/database.yml @@ -7,7 +7,9 @@ roles: - role: essentials - role: user + - role: mount - role: mariadb + - role: mariadb_memory_allocator - role: nginx - role: agent - role: node_exporter diff --git a/press/playbooks/mount.yml b/press/playbooks/mount.yml new file mode 100644 index 0000000000..b2b17fb9ed --- /dev/null +++ b/press/playbooks/mount.yml @@ -0,0 +1,8 @@ +--- +- name: Mount Volumes + hosts: all + become: yes + become_user: root + gather_facts: no + roles: + - role: mount diff --git a/press/playbooks/roles/agent/tasks/main.yml b/press/playbooks/roles/agent/tasks/main.yml index 54af16600b..3a5dcdb3da 100644 --- a/press/playbooks/roles/agent/tasks/main.yml +++ b/press/playbooks/roles/agent/tasks/main.yml @@ -19,7 +19,7 @@ - name: Generate Agent Configuration File become: yes become_user: frappe - command: '/home/frappe/agent/env/bin/agent setup config --name {{ server }} --workers {{ workers }} {% if proxy_ip is defined %}--proxy-ip {{ proxy_ip }}{% endif %} {% if agent_sentry_dsn is defined and agent_sentry_dsn is truthy %}--sentry-dsn {{ agent_sentry_dsn }}{% endif %}' + command: '/home/frappe/agent/env/bin/agent setup config --name {{ server }} --workers {{ workers }} {% if proxy_ip is defined and proxy_ip is truthy %}--proxy-ip {{ proxy_ip }}{% endif %} {% if agent_sentry_dsn is defined and agent_sentry_dsn is truthy %}--sentry-dsn {{ agent_sentry_dsn }}{% endif %}' args: chdir: /home/frappe/agent diff --git a/press/playbooks/roles/docker/tasks/main.yml b/press/playbooks/roles/docker/tasks/main.yml index 7adb02d8ac..e9db7f65a5 100644 --- a/press/playbooks/roles/docker/tasks/main.yml +++ b/press/playbooks/roles/docker/tasks/main.yml @@ -54,8 +54,27 @@ src: daemon.json dest: /etc/docker/daemon.json +- name: Create Docker SystemD drop-in directory + file: + dest: /etc/systemd/system/docker.service.d + state: directory + owner: root + group: root + mode: 0644 + recurse: true + +- name: Set Docker to depend on Mounts + template: + src: mounts.conf + dest: /etc/systemd/system/docker.service.d/mounts.conf + owner: root + group: root + mode: 0644 + when: docker_depends_on_mounts | default(false) | bool + - name: Restart Docker Daemon systemd: + daemon_reload: true name: docker state: restarted diff --git a/press/playbooks/roles/docker/templates/mounts.conf b/press/playbooks/roles/docker/templates/mounts.conf new file mode 100644 index 0000000000..15b97e0e2d --- /dev/null +++ b/press/playbooks/roles/docker/templates/mounts.conf @@ -0,0 +1,14 @@ +[Unit] +# If Docker gets activated, then the mount will be activated as well. +# If the mount fails to activate, Docker will not be started. +# If the mount is explicitly stopped (or restarted), Docker will be stopped (or restarted). + +# BindsTo imposes a stronger condition than RequiresTo. +# If the mount is stopped, Docker will be stopped too. + +# When used in conjunction with After +# The mount strictly has to be in active state for Docker to also be in active state. +# Reference: https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#BindsTo= + +After=home-frappe-benches.mount +BindsTo=home-frappe-benches.mount \ No newline at end of file diff --git a/press/playbooks/roles/mariadb/tasks/main.yml b/press/playbooks/roles/mariadb/tasks/main.yml index da5774b29a..cb418901ea 100644 --- a/press/playbooks/roles/mariadb/tasks/main.yml +++ b/press/playbooks/roles/mariadb/tasks/main.yml @@ -65,6 +65,15 @@ insertafter: '\[Service\]' state: present +- name: Set MariaDB to depend on Mounts + template: + src: mounts.conf + dest: /etc/systemd/system/mariadb.service.d/mounts.conf + owner: root + group: root + mode: 0644 + when: mariadb_depends_on_mounts | default(false) | bool + - name: Restart MariaDB Service systemd: daemon_reload: true diff --git a/press/playbooks/roles/mariadb/templates/mounts.conf b/press/playbooks/roles/mariadb/templates/mounts.conf new file mode 100644 index 0000000000..195ecde230 --- /dev/null +++ b/press/playbooks/roles/mariadb/templates/mounts.conf @@ -0,0 +1,14 @@ +[Unit] +# If MariaDB gets activated, then mounts will be activated as well. +# If one of the mounts fails to activate, MariaDB will not be started. +# If one of the mounts is explicitly stopped (or restarted), MariaDB will be stopped (or restarted). + +# BindsTo imposes a stronger condition than RequiresTo. +# If one of the mounts are stopped, MariaDB will be stopped too. + +# When used in conjunction with After +# The mounts strictly have to be in active state for MariaDB to also be in active state. +# Reference: https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#BindsTo= + +After=etc-mysql.mount var-lib-mysql.mount +BindsTo=etc-mysql.mount var-lib-mysql.mount \ No newline at end of file diff --git a/press/playbooks/roles/mount/tasks/main.yml b/press/playbooks/roles/mount/tasks/main.yml new file mode 100644 index 0000000000..3ccf19620a --- /dev/null +++ b/press/playbooks/roles/mount/tasks/main.yml @@ -0,0 +1,55 @@ +--- +- name: Set JSON Variables + set_fact: + all_mounts: '{{ all_mounts_json | from_json }}' + volume_mounts: '{{ volume_mounts_json | from_json }}' + bind_mounts: '{{ bind_mounts_json | from_json }}' + +- name: Create Mount Points + file: + dest: "{{ item.mount_point }}" + state: directory + owner: "{{ item.mount_point_owner }}" + group: "{{ item.mount_point_group }}" + mode: "{{ item.mount_point_mode }}" + loop: "{{ all_mounts }}" + +- name: Format Volumes + filesystem: + fstype: "{{ item.filesystem }}" + dev: "{{ item.source }}" + force: false + loop: "{{ volume_mounts }}" + +- name: Show Block Device UUIDs + command: 'lsblk {{ item.source }} -no UUID' + loop: "{{ volume_mounts }}" + register: block_devices + +- name: Mount Volumes + mount: + src: "UUID={{ item.stdout.strip() }}" + path: "{{ item.item.mount_point }}" + fstype: "{{ item.item.filesystem }}" + opts: "{{ item.item.mount_options }}" + state: mounted + loop: "{{ block_devices.results }}" + +- name: Create Mount Source Directories + file: + dest: "{{ item.source }}" + state: directory + owner: "{{ item.mount_point_owner }}" + group: "{{ item.mount_point_group }}" + mode: "{{ item.mount_point_mode }}" + loop: "{{ bind_mounts }}" + +- name: Mount Bind Mounts + mount: + src: "{{ item.source }}" + path: "{{ item.mount_point }}" + fstype: none + opts: "{{ item.mount_options }}" + state: mounted + loop: "{{ bind_mounts }}" + diff --git a/press/playbooks/roles/wait_for_cloud_init/tasks/main.yml b/press/playbooks/roles/wait_for_cloud_init/tasks/main.yml new file mode 100644 index 0000000000..465c27b11e --- /dev/null +++ b/press/playbooks/roles/wait_for_cloud_init/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Wait for Cloud Init to finish + command: 'cloud-init status --wait' + +- name: Wait for SSH to be available + ansible.builtin.shell: systemctl is-active ssh + register: result + until: result.stdout.strip() == "active" + retries: 50 + delay: 2 diff --git a/press/playbooks/server.yml b/press/playbooks/server.yml index ed7991a64b..ee907a8ced 100644 --- a/press/playbooks/server.yml +++ b/press/playbooks/server.yml @@ -9,6 +9,7 @@ - role: user - role: nginx - role: agent + - role: mount - role: bench - role: docker - role: node_exporter @@ -24,3 +25,4 @@ - role: sshd_hardening - role: pam - role: user_ssh_certificate + - role: earlyoom_memory_limits diff --git a/press/playbooks/wait_for_cloud_init.yml b/press/playbooks/wait_for_cloud_init.yml index 931a9d6c1d..76d623d5dd 100644 --- a/press/playbooks/wait_for_cloud_init.yml +++ b/press/playbooks/wait_for_cloud_init.yml @@ -4,8 +4,5 @@ become: yes become_user: root gather_facts: no - - tasks: - - name: Wait for Cloud Init to finish - command: 'cloud-init status --wait' - + roles: + - role: wait_for_cloud_init diff --git a/press/press/doctype/ansible_play/ansible_play.js b/press/press/doctype/ansible_play/ansible_play.js index 2bc37ef293..f7d5fafc3f 100644 --- a/press/press/doctype/ansible_play/ansible_play.js +++ b/press/press/doctype/ansible_play/ansible_play.js @@ -2,6 +2,19 @@ // For license information, please see license.txt frappe.ui.form.on('Ansible Play', { - // refresh: function(frm) { - // } + refresh: function (frm) { + frappe.realtime.on('ansible_play_progress', (data) => { + if (data.progress && data.play === frm.doc.name) { + const progress_title = __('Ansible Play Progress'); + frm.dashboard.show_progress( + progress_title, + (data.progress / data.total) * 100, + `Ansible Play Progress (${data.progress} tasks completed out of ${data.total})`, + ); + if (data.progress === data.total) { + frm.dashboard.hide_progress(progress_title); + } + } + }); + }, }); diff --git a/press/press/doctype/cluster/cluster.py b/press/press/doctype/cluster/cluster.py index bd2cccd8a0..d6d54158a1 100644 --- a/press/press/doctype/cluster/cluster.py +++ b/press/press/doctype/cluster/cluster.py @@ -667,9 +667,9 @@ def provision_on_oci(self): self.save() - def get_available_vmi(self, series) -> str | None: + def get_available_vmi(self, series, platform=None) -> str | None: """Virtual Machine Image available in region for given series""" - return VirtualMachineImage.get_available_for_series(series, self.region) + return VirtualMachineImage.get_available_for_series(series, self.region, platform=platform) @property def server_doctypes(self): @@ -754,7 +754,7 @@ def create_servers(self): ) def create_vm( - self, machine_type: str, disk_size: int, domain: str, series: str, team: str + self, machine_type: str, platform: str, disk_size: int, domain: str, series: str, team: str ) -> "VirtualMachine": return frappe.get_doc( { @@ -764,7 +764,7 @@ def create_vm( "series": series, "disk_size": disk_size, "machine_type": machine_type, - "virtual_machine_image": self.get_available_vmi(series), + "virtual_machine_image": self.get_available_vmi(series, platform=platform), "team": team, }, ).insert() @@ -801,7 +801,9 @@ def create_server( server_series = {**self.base_servers, **self.private_servers} team = team or get_current_team() plan = plan or self.get_or_create_basic_plan(doctype) - vm = self.create_vm(plan.instance_type, plan.disk, domain, server_series[doctype], team) + vm = self.create_vm( + plan.instance_type, plan.platform, plan.disk, domain, server_series[doctype], team + ) server = None match doctype: case "Database Server": diff --git a/press/press/doctype/database_server/database_server.js b/press/press/doctype/database_server/database_server.js index 7e87b2027d..896979b0b9 100644 --- a/press/press/doctype/database_server/database_server.js +++ b/press/press/doctype/database_server/database_server.js @@ -154,6 +154,12 @@ frappe.ui.form.on('Database Server', { true, frm.doc.is_self_hosted, ], + [ + __('Mount Volumes'), + 'mount_volumes', + true, + frm.doc.virtual_machine && frm.doc.mounts, + ], ].forEach(([label, method, confirm, condition]) => { if (typeof condition === 'undefined' || condition) { frm.add_custom_button( diff --git a/press/press/doctype/database_server/database_server.json b/press/press/doctype/database_server/database_server.json index a6e9b86534..586171d14c 100644 --- a/press/press/doctype/database_server/database_server.json +++ b/press/press/doctype/database_server/database_server.json @@ -54,6 +54,8 @@ "column_break_apox", "tags_section", "tags", + "mounts_section", + "mounts", "mariadb_settings_tab", "memory_limits_section", "memory_high", @@ -488,7 +490,7 @@ "label": "Public" }, { - "default": "System", + "default": "TCMalloc", "fieldname": "memory_allocator", "fieldtype": "Select", "label": "Memory Allocator", @@ -517,11 +519,22 @@ "fieldtype": "Int", "label": "Auto Add Storage Max", "non_negative": 1 + }, + { + "fieldname": "mounts_section", + "fieldtype": "Section Break", + "label": "Mounts" + }, + { + "fieldname": "mounts", + "fieldtype": "Table", + "label": "Mounts", + "options": "Server Mount" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2024-08-13 11:02:07.399141", + "modified": "2024-12-09 19:35:23.683437", "modified_by": "Administrator", "module": "Press", "name": "Database Server", diff --git a/press/press/doctype/database_server/database_server.py b/press/press/doctype/database_server/database_server.py index b5bcd197c4..6c53c7c45c 100644 --- a/press/press/doctype/database_server/database_server.py +++ b/press/press/doctype/database_server/database_server.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe and contributors # For license information, please see license.txt - +from __future__ import annotations import json from typing import Any @@ -27,10 +26,12 @@ class DatabaseServer(BaseServer): if TYPE_CHECKING: from frappe.types import DF + from press.press.doctype.database_server_mariadb_variable.database_server_mariadb_variable import ( DatabaseServerMariaDBVariable, ) from press.press.doctype.resource_tag.resource_tag import ResourceTag + from press.press.doctype.server_mount.server_mount import ServerMount agent_password: DF.Password | None auto_add_storage_max: DF.Int @@ -57,6 +58,7 @@ class DatabaseServer(BaseServer): memory_high: DF.Float memory_max: DF.Float memory_swap_max: DF.Float + mounts: DF.Table[ServerMount] plan: DF.Link | None primary: DF.Link | None private_ip: DF.Data | None @@ -112,11 +114,7 @@ def on_update(self): ): self.update_memory_limits() - if ( - self.has_value_changed("team") - and self.subscription - and self.subscription.team != self.team - ): + if self.has_value_changed("team") and self.subscription and self.subscription.team != self.team: self.subscription.disable() # enable subscription if exists @@ -146,9 +144,7 @@ def on_update(self): frappe.log_error("Database Subscription Creation Error") def update_memory_limits(self): - frappe.enqueue_doc( - self.doctype, self.name, "_update_memory_limits", enqueue_after_commit=True - ) + frappe.enqueue_doc(self.doctype, self.name, "_update_memory_limits", enqueue_after_commit=True) def _update_memory_limits(self): self.memory_swap_max = self.memory_swap_max or 0.1 @@ -205,13 +201,13 @@ def get_variables_to_update(self) -> list[DatabaseServerMariaDBVariable]: if not old_doc: return self.mariadb_system_variables diff = get_diff(old_doc, self) or {} - return self.get_changed_variables( - diff.get("row_changed", {}) - ) + self.get_newly_added_variables(diff.get("added", [])) + return self.get_changed_variables(diff.get("row_changed", {})) + self.get_newly_added_variables( + diff.get("added", []) + ) - def _update_mariadb_system_variables( - self, variables: list[DatabaseServerMariaDBVariable] = [] - ): + def _update_mariadb_system_variables(self, variables: list[DatabaseServerMariaDBVariable] | None = None): + if variables is None: + variables = [] restart = False for variable in variables: variable.update_on_server() @@ -323,9 +319,7 @@ def add_mariadb_variable( persist: bool = True, ): """Add or update MariaDB variable on the server""" - existing = find( - self.mariadb_system_variables, lambda x: x.mariadb_variable == variable - ) + existing = find(self.mariadb_system_variables, lambda x: x.mariadb_variable == variable) if existing: existing.set(value_type, value) existing.set("skip", skip) @@ -344,9 +338,7 @@ def add_mariadb_variable( def validate_server_id(self): if self.is_new() and not self.server_id: - server_ids = frappe.get_all( - "Database Server", fields=["server_id"], pluck="server_id" - ) + server_ids = frappe.get_all("Database Server", fields=["server_id"], pluck="server_id") if server_ids: self.server_id = max(server_ids or []) + 1 else: @@ -356,9 +348,7 @@ def _setup_server(self): config = self._get_config() try: ansible = Ansible( - playbook="self_hosted_db.yml" - if getattr(self, "is_self_hosted", False) - else "database.yml", + playbook="self_hosted_db.yml" if getattr(self, "is_self_hosted", False) else "database.yml", server=self, user=self.ssh_user or "root", port=self.ssh_port or 22, @@ -372,14 +362,18 @@ def _setup_server(self): "kibana_password": config.kibana_password, "private_ip": self.private_ip, "server_id": self.server_id, + "allocator": self.memory_allocator.lower(), "mariadb_root_password": config.mariadb_root_password, "certificate_private_key": config.certificate.private_key, "certificate_full_chain": config.certificate.full_chain, "certificate_intermediate_chain": config.certificate.intermediate_chain, + "mariadb_depends_on_mounts": self.mariadb_depends_on_mounts, + **self.get_mount_variables(), }, ) play = ansible.run() self.reload() + self._set_mount_status(play) if play.status == "Success": self.status = "Active" self.is_server_setup = True @@ -400,9 +394,7 @@ def _get_config(self): log_server = frappe.db.get_single_value("Press Settings", "log_server") if log_server: - kibana_password = frappe.get_doc("Log Server", log_server).get_password( - "kibana_password" - ) + kibana_password = frappe.get_doc("Log Server", log_server).get_password("kibana_password") else: kibana_password = None @@ -457,9 +449,7 @@ def setup_essentials(self): def process_hybrid_server_setup(self): try: - hybird_server = frappe.db.get_value( - "Self Hosted Server", {"database_server": self.name}, "name" - ) + hybird_server = frappe.db.get_value("Self Hosted Server", {"database_server": self.name}, "name") if hybird_server: hybird_server = frappe.get_doc("Self Hosted Server", hybird_server) @@ -471,9 +461,7 @@ def process_hybrid_server_setup(self): def _setup_primary(self, secondary): mariadb_root_password = self.get_password("mariadb_root_password") - secondary_root_public_key = frappe.db.get_value( - "Database Server", secondary, "root_public_key" - ) + secondary_root_public_key = frappe.db.get_value("Database Server", secondary, "root_public_key") try: ansible = Ansible( playbook="primary.yml", @@ -532,9 +520,7 @@ def setup_replication(self): return self.status = "Installing" self.save() - frappe.enqueue_doc( - self.doctype, self.name, "_setup_replication", queue="long", timeout=18000 - ) + frappe.enqueue_doc(self.doctype, self.name, "_setup_replication", queue="long", timeout=18000) @frappe.whitelist() def perform_physical_backup(self, path): @@ -597,9 +583,7 @@ def trigger_failover(self): return self.status = "Installing" self.save() - frappe.enqueue_doc( - self.doctype, self.name, "_trigger_failover", queue="long", timeout=1200 - ) + frappe.enqueue_doc(self.doctype, self.name, "_trigger_failover", queue="long", timeout=1200) def _convert_from_frappe_server(self): mariadb_root_password = self.get_password("mariadb_root_password") @@ -633,15 +617,11 @@ def _convert_from_frappe_server(self): def convert_from_frappe_server(self): self.status = "Installing" self.save() - frappe.enqueue_doc( - self.doctype, self.name, "_convert_from_frappe_server", queue="long", timeout=1200 - ) + frappe.enqueue_doc(self.doctype, self.name, "_convert_from_frappe_server", queue="long", timeout=1200) def _install_exporters(self): mariadb_root_password = self.get_password("mariadb_root_password") - monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password( - "monitoring_password" - ) + monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password("monitoring_password") try: ansible = Ansible( playbook="database_exporters.yml", @@ -690,9 +670,7 @@ def enable_performance_schema(self): elif isinstance(value, str): type_key = "value_str" - existing_variable = find( - self.mariadb_system_variables, lambda x: x.mariadb_variable == key - ) + existing_variable = find(self.mariadb_system_variables, lambda x: x.mariadb_variable == key) if existing_variable: existing_variable.set(type_key, value) @@ -741,9 +719,7 @@ def reset_root_password_secondary(self): @frappe.whitelist() def setup_deadlock_logger(self): - frappe.enqueue_doc( - self.doctype, self.name, "_setup_deadlock_logger", queue="long", timeout=1200 - ) + frappe.enqueue_doc(self.doctype, self.name, "_setup_deadlock_logger", queue="long", timeout=1200) def _setup_deadlock_logger(self): try: @@ -761,9 +737,7 @@ def _setup_deadlock_logger(self): @frappe.whitelist() def setup_pt_stalk(self): - frappe.enqueue_doc( - self.doctype, self.name, "_setup_pt_stalk", queue="long", timeout=1200 - ) + frappe.enqueue_doc(self.doctype, self.name, "_setup_pt_stalk", queue="long", timeout=1200) def _setup_pt_stalk(self): extra_port_variable = find( @@ -845,14 +819,10 @@ def _rename_server(self): "TLS Certificate", {"wildcard": True, "domain": self.domain}, "name" ) certificate = frappe.get_doc("TLS Certificate", certificate_name) - monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password( - "monitoring_password" - ) + monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password("monitoring_password") log_server = frappe.db.get_single_value("Press Settings", "log_server") if log_server: - kibana_password = frappe.get_doc("Log Server", log_server).get_password( - "kibana_password" - ) + kibana_password = frappe.get_doc("Log Server", log_server).get_password("kibana_password") else: kibana_password = None @@ -926,9 +896,7 @@ def _reconfigure_mariadb_exporter(self): ) ansible.run() except Exception: - log_error( - "Database Server MariaDB Exporter Reconfigure Exception", server=self.as_dict() - ) + log_error("Database Server MariaDB Exporter Reconfigure Exception", server=self.as_dict()) @frappe.whitelist() def update_memory_allocator(self, memory_allocator): @@ -970,10 +938,14 @@ def _update_memory_allocator(self, memory_allocator): self.memory_allocator_version = query_result[0][0]["Value"] self.save() + @property + def mariadb_depends_on_mounts(self): + mount_points = set(mount.mount_point for mount in self.mounts) + mariadb_mount_points = set(["/var/lib/mysql", "/etc/mysql"]) + return mariadb_mount_points.issubset(mount_points) -get_permission_query_conditions = get_permission_query_conditions_for_doctype( - "Database Server" -) + +get_permission_query_conditions = get_permission_query_conditions_for_doctype("Database Server") PERFORMANCE_SCHEMA_VARIABLES = { "performance_schema": "1", diff --git a/press/press/doctype/server/server.js b/press/press/doctype/server/server.js index fd2a352dfc..6483165ce9 100644 --- a/press/press/doctype/server/server.js +++ b/press/press/doctype/server/server.js @@ -192,6 +192,12 @@ frappe.ui.form.on('Server', { false, frm.doc.is_server_setup, ], + [ + __('Mount Volumes'), + 'mount_volumes', + true, + frm.doc.virtual_machine && frm.doc.mounts, + ], ].forEach(([label, method, confirm, condition]) => { if (typeof condition === 'undefined' || condition) { frm.add_custom_button( diff --git a/press/press/doctype/server/server.json b/press/press/doctype/server/server.json index 36b2a46f1b..1a1a20100c 100644 --- a/press/press/doctype/server/server.json +++ b/press/press/doctype/server/server.json @@ -76,7 +76,9 @@ "column_break_edyf", "is_standalone_setup", "tags_section", - "tags" + "tags", + "mounts_section", + "mounts" ], "fields": [ { @@ -523,6 +525,17 @@ "fieldtype": "Int", "label": "Auto Add Storage Max", "non_negative": 1 + }, + { + "fieldname": "mounts_section", + "fieldtype": "Section Break", + "label": "Mounts" + }, + { + "fieldname": "mounts", + "fieldtype": "Table", + "label": "Mounts", + "options": "Server Mount" } ], "links": [], diff --git a/press/press/doctype/server/server.py b/press/press/doctype/server/server.py index 3705152dba..963a3542f6 100644 --- a/press/press/doctype/server/server.py +++ b/press/press/doctype/server/server.py @@ -12,7 +12,7 @@ import boto3 import frappe from frappe import _ -from frappe.core.utils import find +from frappe.core.utils import find, find_all from frappe.installer import subprocess from frappe.model.document import Document from frappe.utils.user import is_system_user @@ -205,6 +205,8 @@ def validate(self): if not self.hostname_abbreviation: self._set_hostname_abbreviation() + self.validate_mounts() + def _set_hostname_abbreviation(self): self.hostname_abbreviation = get_hostname_abbreviation(self.hostname) @@ -721,6 +723,10 @@ def can_change_plan(self, ignore_card_setup): def change_plan(self, plan, ignore_card_setup=False): self.can_change_plan(ignore_card_setup) plan = frappe.get_doc("Server Plan", plan) + self._change_plan(plan) + self.run_press_job("Resize Server", {"machine_type": plan.instance_type}) + + def _change_plan(self, plan): self.ram = plan.memory self.save() self.reload() @@ -733,7 +739,6 @@ def change_plan(self, plan, ignore_card_setup=False): "to_plan": plan.name, } ).insert() - self.run_press_job("Resize Server", {"machine_type": plan.instance_type}) @frappe.whitelist() def create_image(self): @@ -941,6 +946,148 @@ def rename(self, title): self.title = title self.save() + def validate_mounts(self): + if not self.virtual_machine: + return + machine = frappe.get_doc("Virtual Machine", self.virtual_machine) + if len(machine.volumes) > 1 and not self.mounts: + self.fetch_volumes_from_virtual_machine() + self.set_default_mount_points() + self.set_mount_properties() + + def fetch_volumes_from_virtual_machine(self): + machine = frappe.get_doc("Virtual Machine", self.virtual_machine) + for volume in machine.volumes: + if volume.device == "/dev/sda1": + # Skip root volume. This is for AWS other providers may have different root volume + continue + self.append("mounts", {"volume_id": volume.volume_id}) + + def set_default_mount_points(self): + first = self.mounts[0] + if self.doctype == "Server": + first.mount_point = "/opt/volumes/benches" + self.append( + "mounts", + { + "mount_type": "Bind", + "mount_point": "/home/frappe/benches", + "source": "/opt/volumes/benches/home/frappe/benches", + "mount_point_owner": "frappe", + "mount_point_group": "frappe", + }, + ) + elif self.doctype == "Database Server": + first.mount_point = "/opt/volumes/mariadb" + self.append( + "mounts", + { + "mount_type": "Bind", + "mount_point": "/var/lib/mysql", + "source": "/opt/volumes/mariadb/var/lib/mysql", + }, + ) + self.append( + "mounts", + { + "mount_type": "Bind", + "mount_point": "/etc/mysql", + "source": "/opt/volumes/mariadb/etc/mysql", + }, + ) + + def set_mount_properties(self): + for mount in self.mounts: + # set_defaults doesn't seem to work on children in a controller hook + default_fields = find_all(frappe.get_meta("Server Mount").fields, lambda x: x.default) + for field in default_fields: + fieldname = field.fieldname + if not mount.get(fieldname): + mount.set(fieldname, field.default) + + mount_options = "defaults,nofail" # Set default mount options + if mount.mount_options: + mount_options = f"{default_mount_options},{mount.mount_options}" + + mount.mount_options = mount_options + if mount.mount_type == "Bind": + mount.filesystem = "none" + mount.mount_options = f"{mount.mount_options},bind" + + if mount.volume_id: + # EBS volumes are named by their volume id + # There's likely a better way to do this + # https://docs.aws.amazon.com/ebs/latest/userguide/ebs-using-volumes.html + stripped_id = mount.volume_id.replace("-", "") + mount.source = f"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_{ stripped_id }" + if not mount.mount_point: + # If we don't know where to mount, mount it in /mnt/ + mount.mount_point = f"/mnt/{stripped_id}" + + def get_mount_variables(self): + return { + "all_mounts_json": json.dumps([mount.as_dict() for mount in self.mounts], indent=4, default=str), + "volume_mounts_json": json.dumps( + [mount.as_dict() for mount in self.mounts if mount.mount_type == "Volume"], + indent=4, + default=str, + ), + "bind_mounts_json": json.dumps( + [mount.as_dict() for mount in self.mounts if mount.mount_type == "Bind"], + indent=4, + default=str, + ), + } + + @frappe.whitelist() + def mount_volumes(self): + frappe.enqueue_doc(self.doctype, self.name, "_mount_volumes", queue="short", timeout=1200) + + def _mount_volumes(self): + try: + ansible = Ansible( + playbook="mount.yml", + server=self, + variables={**self.get_mount_variables()}, + ) + play = ansible.run() + self.reload() + if self._set_mount_status(play): + self.save() + except Exception: + log_error("Server Mount Exception", server=self.as_dict()) + + def _set_mount_status(self, play): + tasks = frappe.get_all( + "Ansible Task", + ["result", "task"], + { + "play": play.name, + "status": ("in", ("Success", "Failure")), + "task": ("in", ("Mount Volumes", "Mount Bind Mounts", "Show Block Device UUIDs")), + }, + ) + mounts_changed = False + for task in tasks: + result = json.loads(task.result) + for row in result.get("results", []): + mount = find(self.mounts, lambda x: x.name == row.get("item", {}).get("name")) + if not mount: + mount = find( + self.mounts, lambda x: x.name == row.get("item", {}).get("item", {}).get("name") + ) + if not mount: + continue + if task.task == "Show Block Device UUIDs": + mount.uuid = row.get("stdout", "").strip() + mounts_changed = True + else: + mount_status = {True: "Failure", False: "Success"}[row.get("failed", False)] + if mount.status != mount_status: + mount.status = mount_status + mounts_changed = True + return mounts_changed + def wait_for_cloud_init(self): frappe.enqueue_doc( self.doctype, @@ -1089,6 +1236,7 @@ class Server(BaseServer): from frappe.types import DF from press.press.doctype.resource_tag.resource_tag import ResourceTag + from press.press.doctype.server_mount.server_mount import ServerMount agent_password: DF.Password | None auto_add_storage_max: DF.Int @@ -1114,6 +1262,7 @@ class Server(BaseServer): is_standalone_setup: DF.Check is_upstream_setup: DF.Check managed_database_service: DF.Link | None + mounts: DF.Table[ServerMount] new_worker_allocation: DF.Check plan: DF.Link | None primary: DF.Link | None @@ -1301,10 +1450,13 @@ def _setup_server(self): "certificate_private_key": certificate.private_key, "certificate_full_chain": certificate.full_chain, "certificate_intermediate_chain": certificate.intermediate_chain, + "docker_depends_on_mounts": self.docker_depends_on_mounts, + **self.get_mount_variables(), }, ) play = ansible.run() self.reload() + self._set_mount_status(play) if play.status == "Success": self.status = "Active" self.is_server_setup = True @@ -1708,6 +1860,12 @@ def _install_earlyoom(self): except Exception: log_error("Earlyoom Install Exception", server=self.as_dict()) + @property + def docker_depends_on_mounts(self): + mount_points = set(mount.mount_point for mount in self.mounts) + bench_mount_points = set(["/home/frappe/benches"]) + return bench_mount_points.issubset(mount_points) + def scale_workers(now=False): servers = frappe.get_all("Server", {"status": "Active", "is_primary": True}) diff --git a/press/press/doctype/server_mount/__init__.py b/press/press/doctype/server_mount/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/press/press/doctype/server_mount/server_mount.json b/press/press/doctype/server_mount/server_mount.json new file mode 100644 index 0000000000..9a9bd2ce1e --- /dev/null +++ b/press/press/doctype/server_mount/server_mount.json @@ -0,0 +1,145 @@ +{ + "actions": [], + "autoname": "autoincrement", + "creation": "2024-10-28 17:06:07.172615", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "mount_type", + "volume_id", + "filesystem", + "column_break_ygbk", + "status", + "source", + "column_break_uvrc", + "uuid", + "mount_point", + "mount_options", + "permissions_section", + "mount_point_owner", + "mount_point_group", + "column_break_kwsz", + "mount_point_mode" + ], + "fields": [ + { + "columns": 2, + "fieldname": "volume_id", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Volume ID", + "mandatory_depends_on": "eval: doc.mount_type === \"Volume\"", + "read_only": 1 + }, + { + "fieldname": "column_break_ygbk", + "fieldtype": "Column Break" + }, + { + "default": "ext4", + "fieldname": "filesystem", + "fieldtype": "Select", + "label": "Filesystem", + "options": "ext4\nnone", + "reqd": 1 + }, + { + "fieldname": "mount_options", + "fieldtype": "Data", + "label": "Mount Options" + }, + { + "columns": 1, + "default": "Volume", + "fieldname": "mount_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Mount Type", + "options": "Volume\nBind", + "reqd": 1 + }, + { + "columns": 3, + "fieldname": "mount_point", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Mount Point", + "reqd": 1 + }, + { + "columns": 3, + "fieldname": "source", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Source", + "mandatory_depends_on": "eval: doc.mount_type === \"Bind\"", + "read_only_depends_on": "eval: doc.mount_type === \"Volume\"", + "reqd": 1 + }, + { + "columns": 1, + "default": "Pending", + "fieldname": "status", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Status", + "options": "Pending\nSuccess\nFailure", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "uuid", + "fieldtype": "Data", + "label": "UUID", + "read_only": 1 + }, + { + "fieldname": "permissions_section", + "fieldtype": "Section Break", + "label": "Permissions" + }, + { + "default": "root", + "fieldname": "mount_point_owner", + "fieldtype": "Data", + "label": "Mount Point Owner", + "reqd": 1 + }, + { + "fieldname": "column_break_kwsz", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_uvrc", + "fieldtype": "Column Break" + }, + { + "default": "0755", + "fieldname": "mount_point_mode", + "fieldtype": "Data", + "label": "Mount Point Mode", + "reqd": 1 + }, + { + "default": "root", + "fieldname": "mount_point_group", + "fieldtype": "Data", + "label": "Mount Point Group", + "reqd": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2024-11-15 17:24:16.761964", + "modified_by": "Administrator", + "module": "Press", + "name": "Server Mount", + "naming_rule": "Autoincrement", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/press/press/doctype/server_mount/server_mount.py b/press/press/doctype/server_mount/server_mount.py new file mode 100644 index 0000000000..aa6c0f3434 --- /dev/null +++ b/press/press/doctype/server_mount/server_mount.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +# import frappe +from __future__ import annotations + +from frappe.model.document import Document + + +class ServerMount(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + filesystem: DF.Literal["ext4", "none"] + mount_options: DF.Data | None + mount_point: DF.Data + mount_point_group: DF.Data + mount_point_mode: DF.Data + mount_point_owner: DF.Data + mount_type: DF.Literal["Volume", "Bind"] + name: DF.Int | None + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + source: DF.Data + status: DF.Literal["Pending", "Success", "Failure"] + uuid: DF.Data | None + volume_id: DF.Data | None + # end: auto-generated types + + pass diff --git a/press/press/doctype/server_plan/server_plan.json b/press/press/doctype/server_plan/server_plan.json index 555c0f4189..fd49828fd8 100644 --- a/press/press/doctype/server_plan/server_plan.json +++ b/press/press/doctype/server_plan/server_plan.json @@ -17,6 +17,7 @@ "server_type", "cluster", "instance_type", + "platform", "column_break_ypkt", "vcpu", "memory", @@ -118,11 +119,19 @@ "fieldname": "premium", "fieldtype": "Check", "label": "Premium" + }, + { + "default": "x86_64", + "fieldname": "platform", + "fieldtype": "Select", + "label": "Platform", + "options": "x86_64\narm64", + "reqd": 1 } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2024-03-29 12:19:06.709403", + "modified": "2024-11-21 13:49:02.682602", "modified_by": "Administrator", "module": "Press", "name": "Server Plan", diff --git a/press/press/doctype/server_plan/server_plan.py b/press/press/doctype/server_plan/server_plan.py index a07dc6dad3..fadac5726a 100644 --- a/press/press/doctype/server_plan/server_plan.py +++ b/press/press/doctype/server_plan/server_plan.py @@ -1,5 +1,6 @@ # Copyright (c) 2024, Frappe and contributors # For license information, please see license.txt +from __future__ import annotations from press.press.doctype.site_plan.plan import Plan @@ -19,25 +20,25 @@ class ServerPlan(Plan): enabled: DF.Check instance_type: DF.Data | None memory: DF.Int + platform: DF.Literal["x86_64", "arm64"] premium: DF.Check price_inr: DF.Currency price_usd: DF.Currency roles: DF.Table[HasRole] - server_type: DF.Literal[ - "Server", "Database Server", "Proxy Server", "Self Hosted Server" - ] + server_type: DF.Literal["Server", "Database Server", "Proxy Server", "Self Hosted Server"] title: DF.Data | None vcpu: DF.Int # end: auto-generated types - dashboard_fields = [ + dashboard_fields = ( "title", "price_inr", "price_usd", "vcpu", "memory", "disk", - ] + "platform", + ) def get_doc(self, doc): doc["price_per_day_inr"] = self.get_price_per_day("INR") diff --git a/press/press/doctype/virtual_machine/cloud-init.yml.jinja2 b/press/press/doctype/virtual_machine/cloud-init.yml.jinja2 index 1147e8f070..65993021f0 100644 --- a/press/press/doctype/virtual_machine/cloud-init.yml.jinja2 +++ b/press/press/doctype/virtual_machine/cloud-init.yml.jinja2 @@ -34,6 +34,7 @@ runcmd: - systemctl daemon-reload - systemctl restart mariadb - systemctl restart mysqld_exporter +- systemctl restart deadlock_logger {% endif %} {% if server.provider == 'OCI' %} - iptables -D INPUT -j REJECT --reject-with icmp-host-prohibited @@ -74,6 +75,14 @@ write_files: - path: /etc/systemd/system/mysqld_exporter.service content: | {{ mariadb_exporter_config | indent(4) }} + +- path: /root/.my.cnf + content: | + {{ mariadb_root_config | indent(4) }} + +- path: /etc/systemd/system/deadlock_logger.service + content: | + {{ deadlock_logger_config | indent(4) }} {% endif %} swap: diff --git a/press/press/doctype/virtual_machine/virtual_machine.js b/press/press/doctype/virtual_machine/virtual_machine.js index d6b0b20861..1bb458e8f8 100644 --- a/press/press/doctype/virtual_machine/virtual_machine.js +++ b/press/press/doctype/virtual_machine/virtual_machine.js @@ -270,6 +270,33 @@ frappe.ui.form.on('Virtual Machine', { ); } }); + if (frm.doc.status == 'Running') { + frm.add_custom_button( + 'Attach New Volume', + () => { + frappe.prompt( + [ + { + fieldtype: 'Int', + label: 'Size', + fieldname: 'size', + reqd: 1, + default: 10, + }, + ], + ({ size }) => { + frm + .call('attach_new_volume', { + size, + }) + .then((r) => frm.refresh()); + }, + __('Attach New Volume'), + ); + }, + __('Actions'), + ); + } if (frm.doc.instance_id) { if (frm.doc.cloud_provider === 'AWS EC2') { frm.add_web_link( @@ -285,3 +312,16 @@ frappe.ui.form.on('Virtual Machine', { } }, }); + +frappe.ui.form.on('Virtual Machine Volume', { + detach(frm, cdt, cdn) { + let row = frm.selected_doc; + frappe.confirm( + `Are you sure you want to detach volume ${row.volume_id}?`, + () => + frm + .call('detach', { volume_id: row.volume_id }) + .then((r) => frm.refresh()), + ); + }, +}); diff --git a/press/press/doctype/virtual_machine/virtual_machine.json b/press/press/doctype/virtual_machine/virtual_machine.json index 6dc0986146..db63004de4 100644 --- a/press/press/doctype/virtual_machine/virtual_machine.json +++ b/press/press/doctype/virtual_machine/virtual_machine.json @@ -198,6 +198,7 @@ "fieldname": "virtual_machine_image", "fieldtype": "Link", "label": "Virtual Machine Image", + "link_filters": "[[\"Virtual Machine Image\",\"status\",\"=\",\"Available\"]]", "options": "Virtual Machine Image", "read_only_depends_on": "eval: doc.virtual_machine_image" }, @@ -281,12 +282,10 @@ "read_only": 1 }, { - "default": "x86_64", "fieldname": "platform", "fieldtype": "Select", "label": "Platform", "options": "x86_64\narm64", - "read_only_depends_on": "eval: doc.platform", "reqd": 1 } ], @@ -323,7 +322,7 @@ "link_fieldname": "virtual_machine" } ], - "modified": "2024-10-09 10:54:27.750679", + "modified": "2024-11-18 18:55:50.846189", "modified_by": "Administrator", "module": "Press", "name": "Virtual Machine", diff --git a/press/press/doctype/virtual_machine/virtual_machine.py b/press/press/doctype/virtual_machine/virtual_machine.py index 1ace6b9d2d..5483cc95e4 100644 --- a/press/press/doctype/virtual_machine/virtual_machine.py +++ b/press/press/doctype/virtual_machine/virtual_machine.py @@ -4,6 +4,7 @@ import base64 import ipaddress +import time import boto3 import botocore @@ -284,6 +285,8 @@ def _provision_oci(self): def get_cloud_init(self): server = self.get_server() + if not server: + return "" log_server, kibana_password = server.get_log_server() cloud_init_template = "press/press/doctype/virtual_machine/cloud-init.yml.jinja2" context = { @@ -328,11 +331,21 @@ def get_cloud_init(self): mariadb_context, is_path=True, ), + "mariadb_root_config": frappe.render_template( + "press/playbooks/roles/mariadb/templates/my.cnf", + mariadb_context, + is_path=True, + ), "mariadb_exporter_config": frappe.render_template( "press/playbooks/roles/mysqld_exporter/templates/mysqld_exporter.service", mariadb_context, is_path=True, ), + "deadlock_logger_config": frappe.render_template( + "press/playbooks/roles/deadlock_logger/templates/deadlock_logger.service", + mariadb_context, + is_path=True, + ), } ) @@ -617,7 +630,7 @@ def _sync_aws(self, response=None): # noqa: C901 if not existing_volume: self.append("volumes", row) - self.disk_size = self.volumes[0].size if self.volumes else self.disk_size + self.disk_size = self._get_root_volume_size() for volume in list(self.volumes): if volume.volume_id not in attached_volumes: @@ -635,6 +648,14 @@ def _sync_aws(self, response=None): # noqa: C901 self.save() self.update_servers() + def _get_root_volume_size(self): + if self.volumes: + volume = find(self.volumes, lambda v: v.device == "/dev/sda1") + if volume: + return volume.size + return self.volumes[0].size + return self.disk_size + def update_servers(self): status_map = { "Pending": "Pending", @@ -1164,6 +1185,48 @@ def convert_to_arm(self, virtual_machine_image, machine_type): machine_type=machine_type, ).insert() + @frappe.whitelist() + def attach_new_volume(self, size): + if self.cloud_provider != "AWS EC2": + return + volume_id = self.client().create_volume( + AvailabilityZone=self.availability_zone, + Size=size, + VolumeType="gp3", + TagSpecifications=[ + { + "ResourceType": "volume", + "Tags": [{"Key": "Name", "Value": f"Frappe Cloud - {self.name}"}], + }, + ], + )["VolumeId"] + # Wait for the volume to be available + while ( + self.client().describe_volumes( + VolumeIds=[ + volume_id, + ], + )["Volumes"][0]["State"] + != "available" + ): + time.sleep(1) + # First volume starts from /dev/sdf + device_name_index = chr(ord("f") + len(self.volumes) - 1) + self.client().attach_volume( + Device=f"/dev/sd{device_name_index}", + InstanceId=self.instance_id, + VolumeId=volume_id, + ) + self.sync() + + @frappe.whitelist() + def detach(self, volume_id): + volume = find(self.volumes, lambda v: v.volume_id == volume_id) + self.client().detach_volume( + Device=volume.device, InstanceId=self.instance_id, VolumeId=volume.volume_id + ) + self.sync() + get_permission_query_conditions = get_permission_query_conditions_for_doctype("Virtual Machine") diff --git a/press/press/doctype/virtual_machine_image/virtual_machine_image.json b/press/press/doctype/virtual_machine_image/virtual_machine_image.json index ec6306767a..78ffee5ecf 100644 --- a/press/press/doctype/virtual_machine_image/virtual_machine_image.json +++ b/press/press/doctype/virtual_machine_image/virtual_machine_image.json @@ -146,7 +146,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2024-09-25 14:10:12.525448", + "modified": "2024-11-18 17:51:55.233635", "modified_by": "Administrator", "module": "Press", "name": "Virtual Machine Image", @@ -165,6 +165,7 @@ "write": 1 } ], + "show_title_field_in_link": 1, "sort_field": "modified", "sort_order": "DESC", "states": [], diff --git a/press/press/doctype/virtual_machine_image/virtual_machine_image.py b/press/press/doctype/virtual_machine_image/virtual_machine_image.py index 67473e01cc..66a341266a 100644 --- a/press/press/doctype/virtual_machine_image/virtual_machine_image.py +++ b/press/press/doctype/virtual_machine_image/virtual_machine_image.py @@ -213,7 +213,9 @@ def client(self): return None @classmethod - def get_available_for_series(cls, series: str, region: str | None = None) -> str | None: + def get_available_for_series( + cls, series: str, region: str | None = None, platform: str | None = None + ) -> str | None: images = frappe.qb.DocType(cls.DOCTYPE) get_available_images = ( frappe.qb.from_(images) @@ -223,9 +225,12 @@ def get_available_for_series(cls, series: str, region: str | None = None) -> str .where( images.series == series, ) + .orderby(images.creation, order=frappe.qb.desc) ) if region: get_available_images = get_available_images.where(images.region == region) + if platform: + get_available_images = get_available_images.where(images.platform == platform) available_images = get_available_images.run(as_dict=True) if not available_images: return None diff --git a/press/press/doctype/virtual_machine_volume/virtual_machine_volume.json b/press/press/doctype/virtual_machine_volume/virtual_machine_volume.json index 54597ccd16..1e527afe0d 100644 --- a/press/press/doctype/virtual_machine_volume/virtual_machine_volume.json +++ b/press/press/doctype/virtual_machine_volume/virtual_machine_volume.json @@ -14,10 +14,13 @@ "throughput", "last_updated_at", "section_break_frlu", - "device" + "device", + "column_break_buwy", + "detach" ], "fields": [ { + "columns": 1, "fieldname": "volume_type", "fieldtype": "Select", "in_list_view": 1, @@ -26,6 +29,7 @@ "read_only_depends_on": "eval: doc.volume_type" }, { + "columns": 1, "fieldname": "size", "fieldtype": "Int", "in_list_view": 1, @@ -33,6 +37,7 @@ "read_only_depends_on": "eval: doc.size" }, { + "columns": 1, "fieldname": "iops", "fieldtype": "Int", "in_list_view": 1, @@ -40,6 +45,7 @@ "read_only": 1 }, { + "columns": 1, "fieldname": "throughput", "fieldtype": "Int", "in_list_view": 1, @@ -47,6 +53,7 @@ "read_only": 1 }, { + "columns": 4, "fieldname": "volume_id", "fieldtype": "Data", "in_list_view": 1, @@ -68,16 +75,27 @@ "fieldtype": "Section Break" }, { + "columns": 2, "fieldname": "device", "fieldtype": "Data", + "in_list_view": 1, "label": "Device", "read_only": 1 + }, + { + "fieldname": "column_break_buwy", + "fieldtype": "Column Break" + }, + { + "fieldname": "detach", + "fieldtype": "Button", + "label": "Detach" } ], "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2024-10-09 10:45:41.294356", + "modified": "2024-11-21 16:20:36.656076", "modified_by": "Administrator", "module": "Press", "name": "Virtual Machine Volume", diff --git a/press/runner.py b/press/runner.py index 02620ed294..e8f3a60421 100644 --- a/press/runner.py +++ b/press/runner.py @@ -13,6 +13,7 @@ from ansible.plugins.callback import CallbackBase from ansible.utils.display import Display from ansible.vars.manager import VariableManager +from frappe.utils import cstr from frappe.utils import now_datetime as now from pymysql.err import InterfaceError @@ -31,15 +32,13 @@ def wrapper(wrapped, instance, args, kwargs): class AnsibleCallback(CallbackBase): def __init__(self, *args, **kwargs): - super(AnsibleCallback, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) @reconnect_on_failure() def process_task_success(self, result): result, action = frappe._dict(result._result), result._task.action if action == "user": - server_type, server = frappe.db.get_value( - "Ansible Play", self.play, ["server_type", "server"] - ) + server_type, server = frappe.db.get_value("Ansible Play", self.play, ["server_type", "server"]) server = frappe.get_doc(server_type, server) if result.name == "root": server.root_public_key = result.ssh_public_key @@ -74,7 +73,7 @@ def update_play(self, status=None, stats=None): play = frappe.get_doc("Ansible Play", self.play) if stats: # Assume we're running on one host - host = list(stats.processed.keys())[0] + host = next(iter(stats.processed.keys())) play.update(stats.summarize(host)) if play.failures or play.unreachable: play.status = "Failure" @@ -114,8 +113,18 @@ def update_task(self, status, result=None, task=None): else: task.start = now() task.save() + self.publish_play_progress(task.name) frappe.db.commit() + def publish_play_progress(self, task): + frappe.publish_realtime( + "ansible_play_progress", + {"progress": self.task_list.index(task), "total": len(self.task_list), "play": self.play}, + doctype="Ansible Play", + docname=self.play, + user=frappe.session.user, + ) + def parse_result(self, result): task = result._task.name role = result._task._role.get_name() @@ -132,9 +141,7 @@ def on_async_start(self, role, task, job_id): @reconnect_on_failure() def on_async_poll(self, result): job_id = result["ansible_job_id"] - task_name = frappe.get_value( - "Ansible Task", {"play": self.play, "job_id": job_id}, "name" - ) + task_name = frappe.get_value("Ansible Task", {"play": self.play, "job_id": job_id}, "name") task = frappe.get_doc("Ansible Task", task_name) task.result = json.dumps(result, indent=4) task.duration = now() - task.start @@ -157,7 +164,7 @@ def __init__(self, server, playbook, user="root", variables=None, port=22): check=False, connection="ssh", # This is the only way to pass variables that preserves newlines - extra_vars=[f"{key}='{value}'" for key, value in self.variables.items()], + extra_vars=[f"{cstr(key)}='{cstr(value)}'" for key, value in self.variables.items()], remote_user=user, start_at_task=None, syntax=False, @@ -212,6 +219,7 @@ def run(self): self.executor._tqm._stdout_callback = self.callback self.callback.play = self.play self.callback.tasks = self.tasks + self.callback.task_list = self.task_list self.executor.run() self.unpatch() return frappe.get_doc("Ansible Play", self.play) @@ -235,6 +243,7 @@ def create_ansible_play(self): ).insert() self.play = play_doc.name self.tasks = {} + self.task_list = [] for role in play.get_roles(): for block in role.get_task_blocks(): for task in block.block: @@ -247,3 +256,4 @@ def create_ansible_play(self): } ).insert() self.tasks.setdefault(role.get_name(), {})[task.name] = task_doc.name + self.task_list.append(task_doc.name)