diff --git a/Automation/psi_ops.py b/Automation/psi_ops.py index 4cc6ac01..0721d238 100644 --- a/Automation/psi_ops.py +++ b/Automation/psi_ops.py @@ -373,6 +373,14 @@ def copy_server_capabilities(caps): 'base_image_ssh_public_keys, base_image_rsa_private_key', default=None) +VultrAccount = psi_utils.recordtype( + 'VultrAccount', + 'api_key, regions, ' + + 'base_image_root_password, base_image_ssh_private_key, ' + + 'base_image_ssh_public_key, base_image_ssh_port, ' + + 'base_image_ssh_key_id', + default=None) + ElasticHostsAccount = psi_utils.recordtype( 'ElasticHostsAccount', 'zone, uuid, api_key, base_drive_id, cpu, mem, base_host_public_key, ' + @@ -456,6 +464,7 @@ def __init__(self, initialize_plugins=True): self.__scaleway_account = ScalewayAccount() self.__ramnode_account = RamnodeAccount() self.__oci_account = OracleAccount() + self.__vultr_account = VultrAccount() self.__elastichosts_accounts = [] self.__deploy_implementation_required_for_hosts = set() self.__deploy_data_required_for_all = False @@ -923,6 +932,9 @@ def upgrade(self): host.ipmi_password = "" host.ipmi_vpn_profile_location = None self.version = '0.71' + if cmp(parse_version(self.version), parse_version('0.72')) < 0: + self.__vultr_account = VultrAccount() + self.version = '0.72' def initialize_plugins(self): for plugin in plugins: diff --git a/Automation/psi_vultr.py b/Automation/psi_vultr.py new file mode 100644 index 00000000..9e417d8a --- /dev/null +++ b/Automation/psi_vultr.py @@ -0,0 +1,273 @@ +#!/usr/bin/python +# +# Copyright (c) 2023, Psiphon Inc. +# All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import os +import sys +import json +import random +import string +import time +import psi_ssh +import psi_utils + +# Import Vultr Python Library +# Requirement: vultr directory with library file +from vultr import vultr + +# VARIABLE +TCS_BASE_IMAGE_ID = '5eaa342e-8b40-4236-b7d1-2fb5a7a34635' +TCS_VULTR_DEFAULT_PLAN = 'vc2-2c-4gb' # default 2vCore 4G RAM 'vc2-2c-4gb', Sao Paulo 'vc2-2c-4gb-sc1' + +### +# +# Helper functions +# +### +def wait_while_condition(condition, max_wait_seconds, description): + total_wait_seconds = 0 + wait_seconds = 5 + while condition() == True: + if total_wait_seconds > max_wait_seconds: + raise Exception('Took more than %d seconds to %s' % (max_wait_seconds, description)) + time.sleep(wait_seconds) + total_wait_seconds = total_wait_seconds + wait_seconds + +#============================================================================== +### +# +# General API Interaction functions +# +### +class PsiVultr: + def __init__(self, vultr_account, debug=False): + self.api_key = vultr_account.api_key + self.regions = vultr_account.regions + self.plan = TCS_VULTR_DEFAULT_PLAN + self.base_image_id = TCS_BASE_IMAGE_ID + self.ssh_key_id = vultr_account.base_image_ssh_key_id + self.client = vultr.Vultr(api_key=self.api_key) + + def get_region(self, select_region=None): + # Load region from API + # region_id required for create_instance + all_regions = self.client.list_regions() + if select_region != None: + regions = [r for r in all_regions if r['id'] == select_region] + else: + regions = all_regions + + region = random.choice(regions) + + return region['country'], region['id'] + + def get_datacenter_names(self, select_datacenter): + datacenters = { + "ewr" : "VULT New Jersey, US", + "ord" : "VULT Chicago, US", + "dfw" : "VULT Dallas, US", + "sea" : "VULT Seattle, US", + "lax" : "VULT Los Angeles, US", + "atl" : "VULT Atlanta, US", + "ams" : "VULT Amsterdam, NL", + "lhr" : "VULT London, GB", + "fra" : "VULT Frankfurt, DE", + "sjc" : "VULT Silicon Valley, US", + "syd" : "VULT Sydney, AU", + "yto" : "VULT Toronto, CA", + "cdg" : "VULT Paris, FR", + "nrt" : "VULT Tokyo, JP", + "waw" : "VULT Warsaw, PL", + "mad" : "VULT Madrid, ES", + "icn" : "VULT Seoul, KR", + "mia" : "VULT Miami, US", + "sgp" : "VULT Singapore, SG", + "sto" : "VULT Stockholm, SE", + "mex" : "VULT Mexico City, MX", + "mel" : "VULT Melbourne, AU", + "bom" : "VULT Mumbai, IN", + "jnb" : "VULT Johannesburg, ZA", + "tlv" : "VULT Tel Aviv, IL", + "blr" : "VULT Bangalore, IN", + "del" : "VULT Delhi NCR, IN", + "scl" : "VULT Santiago, CL", + "itm" : "VULT Osaka, JP", + "man" : "VULT Manchester, GB", + "hnl" : "VULT Honolulu, US", #hnl and sao does not have vc2-2c-4gb available. + "sao" : "VULT Sao Paulo, BR" # this is a mistake/error as per their api: https://api.vultr.com/v2/regions; should be: Sao Paulo + } + + return datacenters.get(select_datacenter, '') + + def list_instances(self): + all_instances = self.client.list_instances() + return all_instances + + def get_instance(self, instance_id): + instance = self.client.get_instance(instance_id) + return instance + + def remove_instance(self, instance_id): + print("Deleting Instances: {}".format(instance_id)) + self.client.delete_instance(instance_id) + + def create_instance(self, host_id, datacenter_code): + # Launch Instnace + instance = self.client.create_instance( + region=datacenter_code, + plan=self.plan, + label=host_id, + hostname=host_id, + backups="disabled", + snapshot_id=self.base_image_id, + sshkey_id=[self.ssh_key_id], + tags=["psiphond"] + ) + + return instance, self.get_datacenter_names(instance['region']), self.get_region(instance['region']) + +### +# +# Server side SSH Interaction functions (Migrated from old code) +# +### +def refresh_credentials(vultr_account, ip_address, new_root_password, new_stats_password, stats_username): + ssh = psi_ssh.make_ssh_session(ip_address, vultr_account.base_image_ssh_port, + 'root', None, None, + host_auth_key=vultr_account.base_image_ssh_private_key) + try: + ssh.exec_command('echo "root:%s" | chpasswd' % (new_root_password,)) + ssh.exec_command('useradd -M -d /var/log -s /bin/sh -g adm %s' % (stats_username)) + ssh.exec_command('echo "%s:%s" | chpasswd' % (stats_username, new_stats_password)) + ssh.exec_command('rm /etc/ssh/ssh_host_*') + ssh.exec_command('rm -rf /root/.ssh') + ssh.exec_command('export DEBIAN_FRONTEND=noninteractive && dpkg-reconfigure openssh-server') + return ssh.exec_command('cat /etc/ssh/ssh_host_rsa_key.pub') + finally: + ssh.close() + +def set_allowed_users(vultr_account, ip_address, stats_username): + ssh = psi_ssh.make_ssh_session(ip_address, vultr_account.base_image_ssh_port, + 'root', None, None, + host_auth_key=vultr_account.base_image_ssh_private_key) + try: + user_exists = ssh.exec_command('grep %s /etc/ssh/sshd_config' % stats_username) + if not user_exists: + ssh.exec_command('sed -i "s/^AllowUsers.*/& %s/" /etc/ssh/sshd_config' % stats_username) + ssh.exec_command('service ssh restart') + finally: + ssh.close() + +def get_host_name(vultr_account, ip_address): + # Note: using base image credentials; call before changing credentials + ssh = psi_ssh.make_ssh_session(ip_address, vultr_account.base_image_ssh_port, + 'root',None, None, + host_auth_key=vultr_account.base_image_ssh_private_key) + try: + return ssh.exec_command('hostname').strip() + finally: + ssh.close() + +def set_host_name(vultr_account, ip_address, new_hostname): + # Note: hostnamectl is for systemd servers + ssh = psi_ssh.make_ssh_session(ip_address, vultr_account.base_image_ssh_port, + 'root', None, None, + host_auth_key=vultr_account.base_image_ssh_private_key) + try: + ssh.exec_command('hostnamectl set-hostname %s' % new_hostname) + finally: + ssh.close() + +def add_swap_file(vultr_account, ip_address): + ssh = psi_ssh.make_ssh_session(ip_address, vultr_account.base_image_ssh_port, 'root', None, None, host_auth_key=vultr_account.base_image_ssh_private_key) + try: + has_swap = ssh.exec_command('grep swap /etc/fstab') + + if not has_swap: + ssh.exec_command('dd if=/dev/zero of=/swapfile bs=1024 count=1048576 && mkswap /swapfile && chown root:root /swapfile && chmod 0600 /swapfile') + ssh.exec_command('echo "/swapfile swap swap defaults 0 0" >> /etc/fstab') + ssh.exec_command('swapon -a') + finally: + ssh.close() + +### +# +# Main function +# +### +def get_servers(vultr_account): + vultr_api = PsiVultr(vultr_account) + instances = vultr_api.list_instances() + #return [(v['region'] + '_' + v['id'], v['label']) for v in vultrs] + return instances + +def get_server(vultr_account, vultr_id): + vultr_api = PsiVultr(vultr_account) + return vultr_api.get_instance(vultr_id) + +def remove_server(vultr_account, vultr_id): + vultr_api = PsiVultr(vultr_account) + vultr_api.remove_instance(vultr_id) + +def launch_new_server(vultr_account, is_TCS, plugins, multi_ip=False): + + instance = None + vultr_api = PsiVultr(vultr_account) # Use API interface + + try: + #Create a new Vultr instance + region, datacenter_code = vultr_api.get_region() + host_id = "vt" + '-' + region.lower() + datacenter_code.lower() + ''.join(random.choice(string.ascii_lowercase) for x in range(8)) + instance_info, datacenter_name, region_info = vultr_api.create_instance(host_id, datacenter_code) + + # Wait for job completion + wait_while_condition(lambda: vultr_api.client.get_instance(instance_info['id'])['power_status'] != 'running', + 30, + 'Creating VULTR Instance') + # Wait for Restorying fron snapshot + time.sleep(30) + instance = vultr_api.client.get_instance(instance_info['id']) + + instance_ip_address = instance["main_ip"] + + new_stats_username = psi_utils.generate_stats_username() + set_host_name(vultr_account, instance_ip_address, host_id) + set_allowed_users(vultr_account, instance_ip_address, new_stats_username) + add_swap_file(vultr_account, instance_ip_address) + + # Change the new vultr instance's credentials + new_root_password = psi_utils.generate_password() + new_stats_password = psi_utils.generate_password() + new_host_public_key = refresh_credentials(vultr_account, instance_ip_address, + new_root_password, new_stats_password, + new_stats_username) + + except Exception as ex: + if instance: + vultr_api.remove_instance(instance['id']) + raise ex + + return (host_id, is_TCS, 'NATIVE' if is_TCS else None, None, + instance['id'], instance_ip_address, + vultr_account.base_image_ssh_port, 'root', new_root_password, + ' '.join(new_host_public_key.split(' ')[:2]), + new_stats_username, new_stats_password, + datacenter_name, region, egress_ip_address if multi_ip else None, None) + +if __name__ == '__main__': + print(launch_new_server) diff --git a/Automation/vultr/README.md b/Automation/vultr/README.md new file mode 100644 index 00000000..6d0c938b --- /dev/null +++ b/Automation/vultr/README.md @@ -0,0 +1 @@ +The code downloaded from [Github `cssnr/vultr-python`](https://github.com/cssnr/vultr-python/blob/master/vultr.py) with some modifications diff --git a/Automation/vultr/__init__.py b/Automation/vultr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Automation/vultr/vultr.py b/Automation/vultr/vultr.py new file mode 100644 index 00000000..dc75c385 --- /dev/null +++ b/Automation/vultr/vultr.py @@ -0,0 +1,180 @@ +import os +import typing +import requests + +class Vultr(object): + API_URL = 'https://api.vultr.com/v2' + + def __init__(self, api_key: typing.Union[str, None]): + """ + :param str api_key: Vultr API Key or VULTR_API_KEY environment variable + """ + self.api_key = api_key or os.getenv('VULTR_API_KEY') + self.s = requests.session() + if self.api_key: + self.s.headers.update({'Authorization': f'Bearer {self.api_key}'}) + + # Remove all extra functions + # Only leave necessary functions + def list_os(self): + url = f'{self.API_URL}/os' + return self._get(url)['os'] + + def list_plans(self): + url = f'{self.API_URL}/plans' + return self._get(url)['plans'] + + def list_regions(self): + url = f'{self.API_URL}/regions' + return self._get(url)['regions'] + + def list_instances(self): + url = f'{self.API_URL}/instances' + return self._get(url)['instances'] + + def get_instance(self, instance: typing.Union[str, dict]): + instance_id = self._get_obj_key(instance) + url = f'{self.API_URL}/instances/{instance_id}' + return self._get(url)['instance'] + + def create_instance(self, region: str, plan: str, **kwargs): + data = {'region': region, 'plan': plan} + data.update(kwargs) + url = f'{self.API_URL}/instances' + return self._post(url, data)['instance'] + + def update_instance(self, instance: typing.Union[str, dict], **kwargs): + instance_id = self._get_obj_key(instance) + url = f'{self.API_URL}/instances/{instance_id}' + return self._patch(url, kwargs)['instance'] + + def delete_instance(self, instance: typing.Union[str, dict]): + instance_id = self._get_obj_key(instance) + url = f'{self.API_URL}/instances/{instance_id}' + return self._delete(url) + + def list_keys(self): + url = f'{self.API_URL}/ssh-keys' + return self._get(url)['ssh_keys'] + + def get_key(self, key: typing.Union[str, dict]): + key_id = self._get_obj_key(key) + url = f'{self.API_URL}/ssh-keys/{key_id}' + return self._get(url)['ssh_key'] + + def create_key(self, name: str, key: str, **kwargs): + data = {'name': name, 'ssh_key': key} + data.update(kwargs) + url = f'{self.API_URL}/ssh-keys' + return self._post(url, data)['ssh_key'] + + def update_key(self, key: typing.Union[str, dict], **kwargs): + key_id = self._get_obj_key(key) + url = f'{self.API_URL}/ssh-keys/{key_id}' + return self._patch(url, kwargs)['ssh_key'] + + def delete_key(self, key: typing.Union[str, dict]): + key_id = self._get_obj_key(key) + url = f'{self.API_URL}/ssh-keys/{key_id}' + return self._delete(url) + + def list_scripts(self): + url = f'{self.API_URL}/startup-scripts' + return self._get(url)['startup_scripts'] + + def get_script(self, script: typing.Union[str, dict]): + script_id = self._get_obj_key(script) + url = f'{self.API_URL}/startup-scripts/{script_id}' + return self._get(url)['startup_script'] + + def create_script(self, name: str, script: str, **kwargs): + data = {'name': name, 'script': script} + data.update(kwargs) + url = f'{self.API_URL}/startup-scripts' + return self._post(url, data)['startup_script'] + + def update_script(self, script: typing.Union[str, dict], **kwargs): + script_id = self._get_obj_key(script) + url = f'{self.API_URL}/startup-scripts/{script_id}' + return self._patch(url, kwargs)['startup_script'] + + def delete_script(self, script: typing.Union[str, dict]): + script_id = self._get_obj_key(script) + url = f'{self.API_URL}/startup-scripts/{script_id}' + return self._delete(url) + + def list_ipv4(self, instance: typing.Union[str, dict]): + instance_id = self._get_obj_key(instance) + url = f'{self.API_URL}/instances/{instance_id}/ipv4' + return self._get(url)['ipv4s'] + + def create_ipv4(self, instance: typing.Union[str, dict], **kwargs): + instance_id = self._get_obj_key(instance) + url = f'{self.API_URL}/instances/{instance_id}/ipv4' + return self._post(url, kwargs)['ipv4'] + + def delete_ipv4(self, instance: typing.Union[str, dict]): + instance_id = self._get_obj_key(instance) + url = f'{self.API_URL}/instances/{instance_id}/ipv4' + return self._delete(url) + + @staticmethod + def filter_keys(keys: list, name: str) -> dict: + try: + return next(d for d in keys if d['name'].lower() == name.lower()) + except StopIteration: + return {} + + @staticmethod + def filter_os(os_list: list, name: str) -> dict: + try: + return next(d for d in os_list if d['name'].lower() == name.lower()) + except StopIteration: + return {} + + @staticmethod + def filter_scripts(scripts: list, name: str) -> dict: + try: + return next(d for d in scripts if d['name'].lower() == name.lower()) + except StopIteration: + return {} + + @staticmethod + def filter_regions(regions: list, locations: list) -> list: + return [d for d in regions if d['id'] in locations] + + def _get(self, url): + r = self.s.get(url, timeout=10) + if not r.ok: + r.raise_for_status() + return r.json() + + def _post(self, url, data): + r = self.s.post(url, json=data, timeout=10) + if not r.ok: + r.raise_for_status() + return r.json() + + def _patch(self, url, data): + r = self.s.patch(url, json=data, timeout=10) + if not r.ok: + r.raise_for_status() + return r.json() + + def _delete(self, url): + r = self.s.delete(url, timeout=10) + if not r.ok: + r.raise_for_status() + return None + + @staticmethod + def _get_obj_key(obj, key='id'): + if isinstance(obj, str): + return obj + elif isinstance(obj, int): + return str(obj) + elif isinstance(obj, dict): + if key in obj: + return obj[key] + else: + raise ValueError(f'Unable to parse object: {key}')