From 999b9d127fecfe45a20a8681f252e415b43e7b26 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Thu, 19 Jul 2018 20:40:05 -0400 Subject: [PATCH] Issue #151: Factor out application configuration for reuse Introduces a new abstraction, AppConfigurer, that configures an application on a host. Currently this is done by SSHing in and either running a script or running an Ansible playbook. The Ansible playbook execution logic was extracted out of the CloudMan2AppPlugin. --- .../backend_plugins/base_vm_app.py | 47 ++- .../backend_plugins/cloudman2_app.py | 204 ++----------- django-cloudlaunch/cloudlaunch/configurers.py | 279 ++++++++++++++++++ setup.py | 3 +- 4 files changed, 349 insertions(+), 184 deletions(-) create mode 100644 django-cloudlaunch/cloudlaunch/configurers.py diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/base_vm_app.py b/django-cloudlaunch/cloudlaunch/backend_plugins/base_vm_app.py index eca3d6ad..9e6b4f13 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/base_vm_app.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/base_vm_app.py @@ -9,6 +9,8 @@ from cloudbridge.cloud.interfaces import InstanceState from cloudbridge.cloud.interfaces.resources import TrafficDirection +from cloudlaunch import configurers + from .app_plugin import AppPlugin log = get_task_logger('cloudlaunch') @@ -353,12 +355,49 @@ def _provision_host(self, name, task, app_config, provider_config): return {"cloudLaunch": results} def _configure_host(self, name, task, app_config, provider_config): - host = provider_config.get('host_address') + try: + configurer = self._get_configurer(app_config) + except Exception as e: + task.update_state( + state='ERROR', + meta={'action': + "Unable to create app configurer: {}".format(e)} + ) + return {} + task.update_state( + state='PROGRESSING', + meta={'action': 'Validating provider connection info...'} + ) + try: + configurer.validate(app_config, provider_config) + except Exception as e: + task.update_state( + state='ERROR', + meta={'action': "Validation of provider connection info " + "failed: {}".format(e)} + ) + return {} task.update_state( state='PROGRESSING', - meta={"action": "Configuring host % s" % host}) - log.info("Configuring host %s", host) - return {} + meta={'action': 'Configuring application...'} + ) + try: + result = configurer.configure(app_config, provider_config) + task.update_state( + state='PROGRESSING', + meta={'action': 'Application configuration completed ' + 'successfully.'} + ) + return result + except Exception as e: + task.update_state( + state='ERROR', + meta={'action': "Configuration failed: {}".format(e)} + ) + return {} + + def _get_configurer(self, app_config): + return configurers.create_configurer(app_config) def _get_deployment_iid(self, deployment): """ diff --git a/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2_app.py b/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2_app.py index c32464c7..773c023d 100644 --- a/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2_app.py +++ b/django-cloudlaunch/cloudlaunch/backend_plugins/cloudman2_app.py @@ -1,31 +1,16 @@ """CloudMan 2.0 application plugin implementation.""" import json -import os -import paramiko import random -import shutil -import socket import string -import subprocess import base64 -from io import StringIO -from paramiko.ssh_exception import AuthenticationException -from paramiko.ssh_exception import BadHostKeyException -from paramiko.ssh_exception import SSHException -import requests -from retrying import retry, RetryError -from string import Template from urllib.parse import urljoin -from django.conf import settings +from cloudlaunch.configurers import AnsibleAppConfigurer + from djcloudbridge.view_helpers import get_credentials_from_dict -from git import Repo from .simple_web_app import SimpleWebAppPlugin -from celery.utils.log import get_task_logger -log = get_task_logger(__name__) - class CloudMan2AppPlugin(SimpleWebAppPlugin): """CloudLaunch appliance implementation for CloudMan 2.0.""" @@ -43,159 +28,32 @@ def sanitise_app_config(app_config): return super(CloudMan2AppPlugin, CloudMan2AppPlugin).sanitise_app_config(app_config) - def _remove_known_host(self, host): - """ - Remove a host from ~/.ssh/known_hosts. - - :type host: ``str`` - :param host: Hostname or IP address of the host to remove from the - known hosts file. - - :rtype: ``bool`` - :return: True if the host was successfully removed. - """ - cmd = "ssh-keygen -R {0}".format(host) - p = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (out, err) = p.communicate() - if p.wait() == 0: - return True - return False - - @retry(retry_on_result=lambda result: result is False, wait_fixed=5000, - stop_max_delay=180000) - def _check_ssh(self, host, pk=None, user='ubuntu'): - """ - Check for ssh availability on a host. - - :type host: ``str`` - :param host: Hostname or IP address of the host to check. - - :type pk: ``str`` - :param pk: Private portion of an ssh key. - - :type user: ``str`` - :param user: Username to use when trying to login. - - :rtype: ``bool`` - :return: True if ssh connection was successful. - """ - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - pkey = None - if pk: - if 'RSA' not in pk: - # AWS at least does not specify key type yet paramiko requires - pk = pk.replace(' PRIVATE', ' RSA PRIVATE') - key_file_object = StringIO(pk) - pkey = paramiko.RSAKey.from_private_key(key_file_object) - key_file_object.close() - try: - log.info("Trying to ssh {0}@{1}".format(user, host)) - ssh.connect(host, username=user, pkey=pkey) - self._remove_known_host(host) - return True - except (BadHostKeyException, AuthenticationException, - SSHException, socket.error) as e: - log.warn("ssh connection exception for {0}: {1}".format(host, e)) - self._remove_known_host(host) - return False - - def _run_playbook(self, playbook, inventory, host, pk, user='ubuntu', - playbook_vars=None): - """ - Run an Ansible playbook to configure a host. - - First clone a playbook from the supplied repo if not already - available, configure the Ansible inventory, and run the playbook. - - The method assumes ``ansible-playbook`` system command is available. - - :type playbook: ``str`` - :param playbook: A URL of a git repository where the playbook resides. - - :type inventory: ``str`` - :param inventory: A URL pointing to a string ``Template``-like file - that will be used for running the playbook. The - file should have defined variables for ``host`` and - ``user``. - - :type playbook_vars: ``list`` of tuples - :param playbook_vars: A list of key/value tuples with variables to pass - to the playbook via command line arguments - (i.e., --extra-vars key=value). + def _configure_host(self, name, task, app_config, provider_config): + result = super()._configure_host(name, task, app_config, provider_config) + host = provider_config.get('host_address') + pulsar_token = app_config['config_cloudman2'].get('pulsar_token') + result['cloudLaunch'] = {'applicationURL': + 'https://{0}/'.format(host), + 'pulsar_token': pulsar_token} + task.update_state( + state='PROGRESSING', + meta={'action': "Waiting for CloudMan to become available at %s" + % result['cloudLaunch']['applicationURL']}) + login_url = urljoin(result['cloudLaunch']['applicationURL'], + 'cloudman/openid/openid/KeyCloak') + self.wait_for_http(login_url, ok_status_codes=[302]) + return result - :type host: ``str`` - :param host: Hostname or IP of a machine as the playbook target. + def _get_configurer(self, app_config): + # CloudMan2 can only be configured with ansible + return CloudMan2AnsibleAppConfigurer() - :type pk: ``str`` - :param pk: Private portion of an ssh key. - :type user: ``str`` - :param user: Target host system username with which to login. - """ - # Clone the repo in its own dir if multiple tasks run simultaneously - # The path must be to a folder that doesn't already contain a git repo, - # including any parent folders - repo_path = '/tmp/cloudlaunch_plugin_runners/rancher_ansible_%s' % host - try: - log.info("Delete plugin runner folder %s if not empty", repo_path) - shutil.rmtree(repo_path) - except FileNotFoundError: - pass - # Ensure the playbook is available - log.info("Cloning Ansible playbook %s to %s", playbook, repo_path) - Repo.clone_from(playbook, to_path=repo_path) - # Create a private ssh key file - pkf = os.path.join(repo_path, 'pk') - with os.fdopen(os.open(pkf, os.O_WRONLY | os.O_CREAT, 0o600), - 'w') as f: - f.writelines(pk) - # Create an inventory file - r = requests.get(inventory) - inv = Template((r.content).decode('utf-8')) - inventory_path = os.path.join(repo_path, 'inventory') - with open(inventory_path, 'w') as f: - log.info("Creating inventory file %s", inventory_path) - f.writelines(inv.substitute({'host': host, 'user': user})) - # Run the playbook - cmd = ["ansible-playbook", "-i", "inventory", "playbook.yml"] - for pev in playbook_vars or []: - cmd += ["--extra-vars", "{0}={1}".format(pev[0], pev[1])] - log.info("Running Ansible with command %s", cmd) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, cwd=repo_path) - (out, _) = p.communicate() - p_status = p.wait() - log.info("Playbook stdout: %s\nstatus: %s", out, p_status) - if not settings.DEBUG: - log.info("Deleting ansible playbook %s", repo_path) - shutil.rmtree(repo_path) - return (p_status, out) +class CloudMan2AnsibleAppConfigurer(AnsibleAppConfigurer): + """Add CloudMan2 specific vars to playbook.""" - def _configure_host(self, name, task, app_config, provider_config): - log.debug("Running CloudMan2AppPlugin _configure_host for %s", name) + def configure(self, app_config, provider_config): host = provider_config.get('host_address') - user = provider_config.get('ssh_user') - ssh_private_key = provider_config.get('ssh_private_key') - if settings.DEBUG: - log.info("Using config ssh key:\n%s", ssh_private_key) - task.update_state( - state='PROGRESSING', - meta={'action': 'Waiting for ssh on host {0}...'.format(host)}) - try: - self._check_ssh(host, pk=ssh_private_key, user=user) - except RetryError as rte: - task.update_state( - state='ERROR', - meta={'action': "Error ssh to host {0}: {1}".format(host, rte)} - ) - return {'applicationURL': '{0}'.format(host)} - task.update_state( - state='PROGRESSING', - meta={'action': 'Booting CloudMan on host {0}...'.format(host)}) - playbook = app_config.get('config_appliance', {}).get('repository') - inventory = app_config.get( - 'config_appliance', {}).get('inventoryTemplate') cloud_info = get_credentials_from_dict( provider_config['cloud_provider'].config.copy()) # Create a random token for Pulsar if it's set to be used @@ -219,17 +77,5 @@ def _configure_host(self, name, task, app_config, provider_config): ('cm_bootstrap_data', base64.b64encode( json.dumps(cm_bd).encode('utf-8')).decode('utf-8')) ] - self._run_playbook(playbook, inventory, host, ssh_private_key, user, - playbook_vars) - result = {} - result['cloudLaunch'] = {'applicationURL': - 'https://{0}/'.format(host), - 'pulsar_token': pulsar_token} - task.update_state( - state='PROGRESSING', - meta={'action': "Waiting for CloudMan to become available at %s" - % result['cloudLaunch']['applicationURL']}) - login_url = urljoin(result['cloudLaunch']['applicationURL'], - 'cloudman/openid/openid/KeyCloak') - self.wait_for_http(login_url, ok_status_codes=[302]) - return result + return super().configure(app_config, provider_config, + playbook_vars=playbook_vars) diff --git a/django-cloudlaunch/cloudlaunch/configurers.py b/django-cloudlaunch/cloudlaunch/configurers.py new file mode 100644 index 00000000..3f76ae86 --- /dev/null +++ b/django-cloudlaunch/cloudlaunch/configurers.py @@ -0,0 +1,279 @@ +"""Application configurers.""" +import abc +import logging +import os +import shutil +import socket +import subprocess +from io import StringIO +from string import Template + +from django.conf import settings + +from git import Repo + +import paramiko +from paramiko.ssh_exception import AuthenticationException +from paramiko.ssh_exception import BadHostKeyException +from paramiko.ssh_exception import SSHException + +import requests + +from retrying import RetryError, retry + + +log = logging.getLogger(__name__) + +DEFAULT_INVENTORY_TEMPLATE = """ +${host} + +[all:vars] +ansible_ssh_port=22 +ansible_user='${user}' +ansible_ssh_private_key_file=pk +ansible_ssh_extra_args='-o StrictHostKeyChecking=no' +""".strip() + + +def create_configurer(app_config): + """Create a configurer based on the 'runner' in app_config.""" + # Default to ansible if no runner + runner = app_config.get('config_appliance', {}).get('runner', 'ansible') + if runner == "ansible": + return AnsibleAppConfigurer() + elif runner == "script": + return ScriptAppConfigurer() + else: + raise ValueError("Unsupported value of 'runner': {}".format(runner)) + + +class AppConfigurer(): + """Interface class for application configurer.""" + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def validate(self, app_config, provider_config): + """Throws exception if provider_config or app_config isn't valid.""" + pass + + @abc.abstractmethod + def configure(self, app_config, provider_config): + """ + Configure application on already provisioned host. + + See AppPlugin.deploy for additional documentation on arguments. + """ + pass + + +class SSHBasedConfigurer(AppConfigurer): + + def validate(self, app_config, provider_config): + # Validate SSH connection info in provider_config + host = provider_config.get('host_address') + user = provider_config.get('ssh_user') + ssh_private_key = provider_config.get('ssh_private_key') + log.debug("Using config ssh key:\n%s", ssh_private_key) + try: + self._check_ssh(host, pk=ssh_private_key, user=user) + except RetryError as rte: + raise Exception("Error trying to ssh to host {}: {}".format( + host, rte)) + + def _remove_known_host(self, host): + """ + Remove a host from ~/.ssh/known_hosts. + + :type host: ``str`` + :param host: Hostname or IP address of the host to remove from the + known hosts file. + + :rtype: ``bool`` + :return: True if the host was successfully removed. + """ + cmd = "ssh-keygen -R {0}".format(host) + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (out, err) = p.communicate() + if p.wait() == 0: + return True + return False + + @retry(retry_on_result=lambda result: result is False, wait_fixed=5000, + stop_max_delay=180000) + def _check_ssh(self, host, pk=None, user='ubuntu'): + """ + Check for ssh availability on a host. + + :type host: ``str`` + :param host: Hostname or IP address of the host to check. + + :type pk: ``str`` + :param pk: Private portion of an ssh key. + + :type user: ``str`` + :param user: Username to use when trying to login. + + :rtype: ``bool`` + :return: True if ssh connection was successful. + """ + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + pkey = self._get_private_key_from_string(pk) + try: + log.info("Trying to ssh {0}@{1}".format(user, host)) + ssh.connect(host, username=user, pkey=pkey) + self._remove_known_host(host) + return True + except (BadHostKeyException, AuthenticationException, + SSHException, socket.error) as e: + log.warn("ssh connection exception for {0}: {1}".format(host, e)) + self._remove_known_host(host) + return False + + def _get_private_key_from_string(self, private_key): + pkey = None + if private_key: + if 'RSA' not in private_key: + # AWS at least does not specify key type yet paramiko requires + private_key = private_key.replace(' PRIVATE', ' RSA PRIVATE') + key_file_object = StringIO(private_key) + pkey = paramiko.RSAKey.from_private_key(key_file_object) + key_file_object.close() + return pkey + + +class ScriptAppConfigurer(SSHBasedConfigurer): + + def validate(self, app_config, provider_config): + super().validate(app_config, provider_config) + config_script = app_config.get('config_appliance', {}).get( + 'config_script') + if not config_script: + raise Exception("config_appliance missing required parameter: " + "config_script") + + def configure(self, app_config, provider_config): + host = provider_config.get('host_address') + user = provider_config.get('ssh_user') + ssh_private_key = provider_config.get('ssh_private_key') + # TODO: maybe add support for running multiple commands, but how to + # distinguish from run_cmd? + config_script = app_config.get('config_appliance', {}).get( + 'config_script') + + try: + pkey = self._get_private_key_from_string(ssh_private_key) + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + log.info("Trying to ssh {0}@{1}".format(user, host)) + ssh.connect(host, username=user, pkey=pkey) + stdin, stdout, stderr = ssh.exec_command(config_script) + self._remove_known_host(host) + return { + 'stdout': stdout.read(), + 'stderr': stderr.read() + } + except SSHException as sshe: + raise Exception("Failed to execute '{}' on {}".format( + config_script, host)) from sshe + + +class AnsibleAppConfigurer(SSHBasedConfigurer): + + def validate(self, app_config, provider_config): + super().validate(app_config, provider_config) + + # validate required app_config values + playbook = app_config.get('config_appliance', {}).get('repository') + if not playbook: + raise Exception("config_applicance missing required parameter: " + "repository") + + def configure(self, app_config, provider_config, playbook_vars=None): + host = provider_config.get('host_address') + user = provider_config.get('ssh_user') + ssh_private_key = provider_config.get('ssh_private_key') + playbook = app_config.get('config_appliance', {}).get('repository') + if 'inventoryTemplate' in app_config.get('config_appliance', {}): + inventory = app_config.get( + 'config_appliance', {}).get('inventoryTemplate') + else: + inventory = DEFAULT_INVENTORY_TEMPLATE + self._run_playbook(playbook, inventory, host, ssh_private_key, user, + playbook_vars) + return {} + + def _run_playbook(self, playbook, inventory, host, pk, user='ubuntu', + playbook_vars=None): + """ + Run an Ansible playbook to configure a host. + + First clone a playbook from the supplied repo if not already + available, configure the Ansible inventory, and run the playbook. + + The method assumes ``ansible-playbook`` system command is available. + + :type playbook: ``str`` + :param playbook: A URL of a git repository where the playbook resides. + + :type inventory: ``str`` + :param inventory: A URL pointing to a string ``Template``-like file + that will be used for running the playbook. The + file should have defined variables for ``host`` and + ``user``. + + :type playbook_vars: ``list`` of tuples + :param playbook_vars: A list of key/value tuples with variables to pass + to the playbook via command line arguments + (i.e., --extra-vars key=value). + + :type host: ``str`` + :param host: Hostname or IP of a machine as the playbook target. + + :type pk: ``str`` + :param pk: Private portion of an ssh key. + + :type user: ``str`` + :param user: Target host system username with which to login. + """ + # Clone the repo in its own dir if multiple tasks run simultaneously + # The path must be to a folder that doesn't already contain a git repo, + # including any parent folders + # TODO: generalize this temporary directory + repo_path = '/tmp/cloudlaunch_plugin_runners/rancher_ansible_%s' % host + try: + log.info("Delete plugin runner folder %s if not empty", repo_path) + shutil.rmtree(repo_path) + except FileNotFoundError: + pass + # Ensure the playbook is available + log.info("Cloning Ansible playbook %s to %s", playbook, repo_path) + Repo.clone_from(playbook, to_path=repo_path) + # Create a private ssh key file + pkf = os.path.join(repo_path, 'pk') + with os.fdopen(os.open(pkf, os.O_WRONLY | os.O_CREAT, 0o600), + 'w') as f: + f.writelines(pk) + # Create an inventory file + r = requests.get(inventory) + inv = Template((r.content).decode('utf-8')) + inventory_path = os.path.join(repo_path, 'inventory') + with open(inventory_path, 'w') as f: + log.info("Creating inventory file %s", inventory_path) + f.writelines(inv.substitute({'host': host, 'user': user})) + # Run the playbook + cmd = ["ansible-playbook", "-i", "inventory", "playbook.yml"] + for pev in playbook_vars or []: + cmd += ["--extra-vars", "{0}={1}".format(pev[0], pev[1])] + log.info("Running Ansible with command %s", cmd) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + universal_newlines=True, cwd=repo_path) + (out, _) = p.communicate() + p_status = p.wait() + log.info("Playbook stdout: %s\nstatus: %s", out, p_status) + if not settings.DEBUG: + log.info("Deleting ansible playbook %s", repo_path) + shutil.rmtree(repo_path) + return (p_status, out) diff --git a/setup.py b/setup.py index ae2749d7..cd5367b2 100755 --- a/setup.py +++ b/setup.py @@ -89,7 +89,8 @@ def get_version(*file_paths): # Integration with Sentry 'raven', # For CloudMan2 plugin - 'gitpython' + 'gitpython', + 'ansible' ] REQS_PROD = ([