diff --git a/IM/__init__.py b/IM/__init__.py index 6f8ee2046..4f459e52b 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -19,5 +19,5 @@ 'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry', 'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPConnection', 'uriparse', 'VirtualMachine', 'VMRC', 'xmlobject'] -__version__ = '1.5.4' +__version__ = '1.5.5' __author__ = 'Miguel Caballer' diff --git a/IM/connectors/Azure.py b/IM/connectors/Azure.py index 7d3b0bbb2..85e7c3dcb 100644 --- a/IM/connectors/Azure.py +++ b/IM/connectors/Azure.py @@ -21,12 +21,14 @@ from IM.VirtualMachine import VirtualMachine from .CloudConnector import CloudConnector from radl.radl import Feature +from IM.config import Config try: from azure.mgmt.resource import ResourceManagementClient from azure.mgmt.storage import StorageManagementClient from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.network import NetworkManagementClient + from azure.mgmt.dns import DnsManagementClient from azure.common.credentials import UserPassCredentials except Exception as ex: print("WARN: Python Azure SDK not correctly installed. AzureCloudConnector will not work!.") @@ -142,28 +144,25 @@ def get_instance_type(self, system, credentials, subscription_id): disk_free_op = system.getFeature('memory.size').getLogOperator() compute_client = ComputeManagementClient(credentials, subscription_id) - instace_types = compute_client.virtual_machine_sizes.list(location) + instace_types = list(compute_client.virtual_machine_sizes.list(location)) + instace_types.sort(key=lambda x: (x.number_of_cores, x.memory_in_mb, x.resource_disk_size_in_mb)) res = None default = None - for instace_type in list(instace_types): + for instace_type in instace_types: if instace_type.name == self.INSTANCE_TYPE: default = instace_type # get the instance type with the lowest Memory - if res is None or (instace_type.memory_in_mb <= res.memory_in_mb): + if res is None: str_compare = "instace_type.number_of_cores " + cpu_op + " cpu " str_compare += " and instace_type.memory_in_mb " + memory_op + " memory " - str_compare += " and instace_type.resource_disk_size_in_mb " + \ - disk_free_op + " disk_free" + str_compare += " and instace_type.resource_disk_size_in_mb " + disk_free_op + " disk_free" if eval(str_compare): if not instance_type_name or instace_type.name == instance_type_name: - res = instace_type + return instace_type - if res is None: - return default - else: - return res + return default def update_system_info_from_instance(self, system, instance_type): """ @@ -283,7 +282,21 @@ def create_nics(self, inf, radl, credentials, subscription_id, group_name, subne if radl.systems[0].getValue('availability_zone'): location = radl.systems[0].getValue('availability_zone') - hasPublicIP = radl.hasPublicNet(system.name) + i = 0 + hasPublicIP = False + hasPrivateIP = False + while system.getValue("net_interface." + str(i) + ".connection"): + network_name = system.getValue("net_interface." + str(i) + ".connection") + # TODO: check how to do that + # fixed_ip = system.getValue("net_interface." + str(i) + ".ip") + network = radl.get_network_by_id(network_name) + + if network.isPublic(): + hasPublicIP = True + else: + hasPrivateIP = True + + i += 1 i = 0 res = [] @@ -294,7 +307,7 @@ def create_nics(self, inf, radl, credentials, subscription_id, group_name, subne # fixed_ip = system.getValue("net_interface." + str(i) + ".ip") network = radl.get_network_by_id(network_name) - if network.isPublic(): + if network.isPublic() and hasPrivateIP: # Public nets are not added as nics i += 1 continue @@ -442,7 +455,7 @@ def create_nets(self, inf, radl, credentials, subscription_id, group_name): if not vnet: # Create VNet in the RG of the Inf - async_vnet_creation = network_client.virtual_networks.create_or_update( + network_client.virtual_networks.create_or_update( group_name, "privates", { @@ -452,7 +465,6 @@ def create_nets(self, inf, radl, credentials, subscription_id, group_name): } } ) - async_vnet_creation.wait() subnets = {} for i, net in enumerate(radl.networks): @@ -497,6 +509,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res = [] i = 0 + all_ok = True while i < num_vm: group_name = None try: @@ -523,9 +536,11 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): credentials, subscription_id, location) if not storage_account: + all_ok = False res.append((False, error_msg)) + # delete VM group resource_client.resource_groups.delete(group_name) - break + continue nics = self.create_nics(inf, radl, credentials, subscription_id, group_name, subnets) @@ -534,7 +549,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): compute_client = ComputeManagementClient(credentials, subscription_id) async_vm_creation = compute_client.virtual_machines.create_or_update(group_name, vm_name, vm_parameters) - azure_vm = async_vm_creation.result() + # azure_vm = async_vm_creation.result() vm = VirtualMachine(inf, group_name + '/' + vm_name, self.cloud, radl, requested_radl, self) vm.info.systems[0].setValue('instance_id', group_name + '/' + vm_name) @@ -543,6 +558,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res.append((True, vm)) except Exception as ex: + all_ok = False self.log_exception("Error creating the VM") res.append((False, "Error creating the VM: " + str(ex))) @@ -553,6 +569,10 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): i += 1 + if not all_ok: + # Remove the general group + resource_client.resource_groups.delete("rg-%s" % inf.id) + return res def attach_data_disks(self, vm, storage_account_name, credentials, subscription_id, location): @@ -571,7 +591,7 @@ def attach_data_disks(self, vm, storage_account_name, credentials, subscription_ try: # Attach data disk - async_vm_update = compute_client.virtual_machines.create_or_update( + compute_client.virtual_machines.create_or_update( group_name, vm_name, { @@ -590,7 +610,6 @@ def attach_data_disks(self, vm, storage_account_name, credentials, subscription_ } } ) - async_vm_update.wait() except Exception as ex: self.log_exception("Error attaching disk %d to VM %s" % (cont, vm_name)) return False, "Error attaching disk %d to VM %s: %s" % (cont, vm_name, str(ex)) @@ -626,8 +645,60 @@ def updateVMInfo(self, vm, auth_data): # Update IP info self.setIPs(vm, virtual_machine.network_profile, credentials, subscription_id) + self.add_dns_entries(vm, credentials, subscription_id) return (True, vm) + def add_dns_entries(self, vm, credentials, subscription_id): + """ + Add the required entries in the Azure DNS service + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - credentials, subscription_id: Authentication data to access cloud provider. + """ + try: + group_name = vm.id.split('/')[0] + dns_client = DnsManagementClient(credentials, subscription_id) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain" and ip: + zone = None + try: + zone = dns_client.zones.get(group_name, domain) + except Exception: + pass + if not zone: + self.log_debug("Creating DNS zone %s" % domain) + zone = dns_client.zones.create_or_update(group_name, domain, + {'location': 'global'}) + else: + self.log_debug("DNS zone %s exists. Do not create." % domain) + + if zone: + record = None + try: + record = dns_client.record_sets.get(group_name, domain, hostname, 'A') + except Exception: + pass + if not record: + self.log_debug("Creating DNS record %s." % hostname) + record_data = {"ttl": 300, "arecords": [{"ipv4_address": ip}]} + record_set = dns_client.record_sets.create_or_update(group_name, domain, + hostname, 'A', + record_data) + else: + self.log_debug("DNS record %s exists. Do not create." % hostname) + + return True + except Exception: + self.log_exception("Error creating DNS entries") + return False + def setIPs(self, vm, network_profile, credentials, subscription_id): """ Set the information about the IPs of the VM @@ -664,7 +735,7 @@ def finalize(self, vm, last, auth_data): # Delete Resource group and everything in it resource_client = ResourceManagementClient(credentials, subscription_id) self.log_debug("Removing RG: %s" % group_name) - resource_client.resource_groups.delete(group_name).wait() + resource_client.resource_groups.delete(group_name) # if it is the last VM delete the RG of the Inf if last: @@ -724,7 +795,7 @@ def alterVM(self, vm, radl, auth_data): # Start the VM async_vm_start = compute_client.virtual_machines.start(group_name, vm_name) - async_vm_start.wait() + # async_vm_start.wait() return self.updateVMInfo(vm, auth_data) except Exception as ex: diff --git a/IM/connectors/EC2.py b/IM/connectors/EC2.py index 8eea25b19..cc71805ab 100644 --- a/IM/connectors/EC2.py +++ b/IM/connectors/EC2.py @@ -22,6 +22,7 @@ try: import boto.ec2 import boto.vpc + import boto.route53 except Exception as ex: print("WARN: Boto library not correctly installed. EC2CloudConnector will not work!.") print(ex) @@ -30,6 +31,7 @@ from IM.VirtualMachine import VirtualMachine from .CloudConnector import CloudConnector from radl.radl import Feature +from IM.config import Config class InstanceTypeInfo: @@ -87,6 +89,7 @@ class EC2CloudConnector(CloudConnector): def __init__(self, cloud_info, inf): self.connection = None + self.route53_connection = None self.auth = None CloudConnector.__init__(self, cloud_info, inf) @@ -197,6 +200,45 @@ def get_connection(self, region_name, auth_data): self.connection = conn return conn + # Get the Route53 connection object + def get_route53_connection(self, region_name, auth_data): + """ + Get a :py:class:`boto.route53.connection` to interact with. + + Arguments: + - region_name(str): AWS region to connect. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + Returns: a :py:class:`boto.route53.connection` or None in case of error + """ + auths = auth_data.getAuthInfo(self.type) + if not auths: + raise Exception("No auth data has been specified to EC2.") + else: + auth = auths[0] + + if self.route53_connection and self.auth.compare(auth_data, self.type): + return self.route53_connection + else: + self.auth = auth_data + conn = None + try: + if 'username' in auth and 'password' in auth: + conn = boto.route53.connect_to_region(region_name, + aws_access_key_id=auth['username'], + aws_secret_access_key=auth['password']) + else: + self.log_error("No correct auth data has been specified to EC2: " + "username (Access Key) and password (Secret Key)") + raise Exception("No correct auth data has been specified to EC2: " + "username (Access Key) and password (Secret Key)") + + except Exception as ex: + self.log_exception("Error conneting Route53 in region " + region_name) + raise Exception("Error conneting Route53 in region" + region_name + ": " + str(ex)) + + self.route53_connection = conn + return conn + # el path sera algo asi: aws://eu-west-1/ami-00685b74 def getAMIData(self, path): """ @@ -420,7 +462,7 @@ def create_keypair(self, system, conn): system.setUserKeyCredentials( system.getCredentials().username, public, private) else: - self.log_debug("Creating the Keypair") + self.log_debug("Creating the Keypair name: %s" % keypair_name) keypair_file = self.KEYPAIR_DIR + '/' + keypair_name + '.pem' keypair = conn.create_key_pair(keypair_name) created = True @@ -1075,6 +1117,7 @@ def updateVMInfo(self, vm, auth_data): self.setIPsFromInstance(vm, instance) self.attach_volumes(instance, vm) + self.add_dns_entries(vm, auth_data) try: vm.info.systems[0].setValue('launch_time', int(time.mktime( @@ -1088,6 +1131,91 @@ def updateVMInfo(self, vm, auth_data): return (True, vm) + def add_dns_entries(self, vm, auth_data): + """ + Add the required entries in the AWS Route53 service + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + """ + try: + region = vm.id.split(";")[0] + conn = self.get_route53_connection(region, auth_data) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain" and ip: + if not domain.endswith("."): + domain += "." + zone = conn.get_zone(domain) + if not zone: + self.log_debug("Creating DNS zone %s" % domain) + zone = conn.create_zone(domain) + else: + self.log_debug("DNS zone %s exists. Do not create." % domain) + + if zone: + fqdn = hostname + "." + domain + record = zone.get_a(fqdn) + if not record: + self.log_debug("Creating DNS record %s." % fqdn) + changes = boto.route53.record.ResourceRecordSets(conn, zone.id) + change = changes.add_change("CREATE", fqdn, "A") + change.add_value(ip) + result = changes.commit() + else: + self.log_debug("DNS record %s exists. Do not create." % fqdn) + + return True + except Exception: + self.log_exception("Error creating DNS entries") + return False + + def del_dns_entries(self, vm, auth_data): + """ + Delete the added entries in the AWS Route53 service + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + """ + region = vm.id.split(";")[0] + conn = self.get_route53_connection(region, auth_data) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain" and ip: + if not domain.endswith("."): + domain += "." + zone = conn.get_zone(domain) + if not zone: + self.log_debug("The DNS zone %s does not exists. Do not delete records." % domain) + else: + fqdn = hostname + "." + domain + record = zone.get_a(fqdn) + if not record: + self.log_debug("DNS record %s does not exists. Do not delete." % fqdn) + else: + self.log_debug("Deleting DNS record %s." % fqdn) + changes = boto.route53.record.ResourceRecordSets(conn, zone.id) + change = changes.add_change("DELETE", fqdn, "A") + change.add_value(ip) + result = changes.commit() + + # if there are no A records + all_a_records = [r for r in conn.get_all_rrsets(zone.id) if r.type == "A"] + if not all_a_records: + conn.delete_hosted_zone(zone.id) + def cancel_spot_requests(self, conn, vm): """ Cancel the spot requests of a VM @@ -1143,6 +1271,12 @@ def finalize(self, vm, last, auth_data): except: self.log_exception("Error deleting keypair.") + # Delete the DNS entries + try: + self.del_dns_entries(vm, auth_data) + except: + self.log_exception("Error deleting DNS entries") + # Delete the elastic IPs try: self.delete_elastic_ips(conn, vm) @@ -1159,7 +1293,7 @@ def finalize(self, vm, last, auth_data): try: self.delete_volumes(conn, volumes, instance_id) except: - self.log_exception("Error deleting EBS volumess") + self.log_exception("Error deleting EBS volumes") return (True, "") diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 30182ceff..ca1e8e72f 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -19,10 +19,13 @@ import os try: - from libcloud.compute.base import Node, NodeSize + from libcloud.compute.base import NodeSize from libcloud.compute.types import NodeState, Provider from libcloud.compute.providers import get_driver from libcloud.common.google import ResourceNotFoundError + from libcloud.dns.types import Provider as DNSProvider + from libcloud.dns.types import RecordType + from libcloud.dns.providers import get_driver as get_dns_driver except Exception as ex: print("WARN: libcloud library not correctly installed. GCECloudConnector will not work!.") print(ex) @@ -31,6 +34,7 @@ from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from radl.radl import Feature +from IM.config import Config class GCECloudConnector(CloudConnector): @@ -46,11 +50,12 @@ def __init__(self, cloud_info, inf): self.auth = None self.datacenter = None self.driver = None + self.dns_driver = None CloudConnector.__init__(self, cloud_info, inf) def get_driver(self, auth_data, datacenter=None): """ - Get the driver from the auth data + Get the compute driver from the auth data Arguments: - auth(Authentication): parsed authentication tokens. @@ -91,6 +96,46 @@ def get_driver(self, auth_data, datacenter=None): raise Exception( "No correct auth data has been specified to GCE: username, password and project") + def get_dns_driver(self, auth_data): + """ + Get the DNS driver from the auth data + + Arguments: + - auth(Authentication): parsed authentication tokens. + + Returns: a :py:class:`libcloud.dns.base.DNSDriver` or None in case of error + """ + auths = auth_data.getAuthInfo(self.type) + if not auths: + raise Exception("No auth data has been specified to GCE.") + else: + auth = auths[0] + + if self.dns_driver and self.auth.compare(auth_data, self.type): + return self.dns_driver + else: + self.auth = auth_data + + if 'username' in auth and 'password' in auth and 'project' in auth: + cls = get_dns_driver(DNSProvider.GOOGLE) + # Patch to solve some client problems with \\n + auth['password'] = auth['password'].replace('\\n', '\n') + lines = len(auth['password'].replace(" ", "").split()) + if lines < 2: + raise Exception("The certificate provided to the GCE plugin has an incorrect format." + " Check that it has more than one line.") + + driver = cls(auth['username'], auth['password'], project=auth['project']) + + self.dns_driver = driver + return driver + else: + self.log_error( + "No correct auth data has been specified to GCE: username, password and project") + self.log_debug(auth) + raise Exception( + "No correct auth data has been specified to GCE: username, password and project") + def concreteSystem(self, radl_system, auth_data): image_urls = radl_system.getValue("disk.0.image.url") if not image_urls: @@ -199,6 +244,12 @@ def get_instance_type(self, sizes, radl): """ instance_type_name = radl.getValue('instance_type') + cpu = 1 + cpu_op = ">=" + if radl.getFeature('cpu.count'): + cpu = radl.getValue('cpu.count') + cpu_op = radl.getFeature('cpu.count').getLogOperator() + memory = 1 memory_op = ">1" if radl.getFeature('memory.size'): @@ -212,7 +263,11 @@ def get_instance_type(self, sizes, radl): if size.price is None: size.price = 0 if res is None or (size.price <= res.price and size.ram <= res.ram): - str_compare = "size.ram " + memory_op + " memory" + str_compare = "" + if 'guestCpus' in size.extra and size.extra['guestCpus']: + str_compare = "size.extra['guestCpus'] " + cpu_op + " cpu and " + str_compare += "size.ram " + memory_op + " memory" + if eval(str_compare): if not instance_type_name or size.name == instance_type_name: res = size @@ -440,6 +495,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm.info.systems[0].setValue('instance_id', str(vm.id)) vm.info.systems[0].setValue('instance_name', str(vm.id)) self.log_debug("Node successfully created.") + res.append((True, vm)) for _ in range(len(nodes), num_vm): @@ -457,6 +513,7 @@ def finalize(self, vm, last, auth_data): if node: success = node.destroy() self.delete_disks(node) + self.del_dns_entries(vm, auth_data) if last: self.delete_firewall(vm, node.driver) @@ -663,11 +720,104 @@ def updateVMInfo(self, vm, auth_data): vm.setIps(node.public_ips, node.private_ips) self.attach_volumes(vm, node) + self.add_dns_entries(vm, auth_data) else: vm.state = VirtualMachine.OFF return (True, vm) + def add_dns_entries(self, vm, auth_data): + """ + Add the required entries in the Google DNS system + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + """ + try: + driver = self.get_dns_driver(auth_data) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain" and ip: + if not domain.endswith("."): + domain += "." + zone = [z for z in driver.iterate_zones() if z.domain == domain] + if not zone: + self.log_debug("Creating DNS zone %s" % domain) + zone = driver.create_zone(domain) + else: + zone = zone[0] + self.log_debug("DNS zone %s exists. Do not create." % domain) + + if zone: + fqdn = hostname + "." + domain + record = [r for r in driver.iterate_records(zone) if r.name == fqdn] + if not record: + self.log_debug("Creating DNS record %s." % fqdn) + driver.create_record(fqdn, zone, RecordType.A, dict(ttl=300, rrdatas=[ip])) + else: + self.log_debug("DNS record %s exists. Do not create." % fqdn) + + return True + except Exception: + self.log_exception("Error creating DNS entries") + return False + + def del_dns_entries(self, vm, auth_data): + """ + Delete the added entries in the Google DNS system + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + """ + try: + driver = self.get_dns_driver(auth_data) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain" and ip: + if not domain.endswith("."): + domain += "." + zone = [z for z in driver.iterate_zones() if z.domain == domain] + if not zone: + self.log_debug("The DNS zone %s does not exists. Do not delete records." % domain) + else: + zone = zone[0] + fqdn = hostname + "." + domain + record = [r for r in driver.iterate_records(zone) if r.name == fqdn] + if not record: + self.log_debug("DNS record %s does not exists. Do not delete." % fqdn) + else: + record = record[0] + if record.data['rrdatas'] != [ip]: + self.log_debug("DNS record %s mapped to unexpected IP: %s != %s." + "Do not delete." % (fqdn, record.data['rrdatas'], ip)) + else: + self.log_debug("Deleting DNS record %s." % fqdn) + if not driver.delete_record(record): + self.log_error("Error deleting DNS record %s." % fqdn) + + # if there are no records (except the NS and SOA auto added ones), delete the zone + all_records = [r for r in driver.iterate_records(zone) + if r.type not in [RecordType.NS, RecordType.SOA]] + if not all_records: + driver.delete_zone(zone) + + return True + except Exception: + self.log_exception("Error deleting DNS entries") + return False + def start(self, vm, auth_data): driver = self.get_driver(auth_data) diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index b88ce7f78..fe1271a41 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -284,7 +284,7 @@ def get_property_from_category(self, occi_res, category, prop_name): def get_floating_pool(self, auth_data): """ - Get the first floating pool available (For OpenStack sites with Neutron) + Get a random floating pool available (For OpenStack sites with Neutron) """ _, occi_data = self.query_occi(auth_data) lines = occi_data.split("\n") @@ -989,26 +989,25 @@ def alterVM(self, vm, radl, auth_data): self.log_debug("Error waiting volume %s. Deleting it." % volume_id) self.delete_volume(volume_id, auth_data) return (False, "Error waiting volume %s. Deleting it." % volume_id) + else: + self.log_debug("Attaching to the instance") + attached = self.attach_volume(vm, volume_id, disk_device, mount_path, auth_data) + if attached: + orig_system.setValue("disk." + str(cont) + ".size", disk_size, "G") + orig_system.setValue("disk." + str(cont) + ".provider_id", volume_id) + if disk_device: + orig_system.setValue("disk." + str(cont) + ".device", disk_device) + if mount_path: + orig_system.setValue("disk." + str(cont) + ".mount_path", mount_path) + else: + self.log_error("Error attaching a %d GB volume for the disk %d." + " Deleting it." % (int(disk_size), cont)) + self.delete_volume(volume_id, auth_data) + return (False, "Error attaching the new volume") else: self.log_error("Error creating volume: %s" % volume_id) + return (False, "Error creating volume: %s" % volume_id) - if wait_ok: - self.log_debug("Attaching to the instance") - attached = self.attach_volume(vm, volume_id, disk_device, mount_path, auth_data) - if attached: - orig_system.setValue("disk." + str(cont) + ".size", disk_size, "G") - orig_system.setValue("disk." + str(cont) + ".provider_id", volume_id) - if disk_device: - orig_system.setValue("disk." + str(cont) + ".device", disk_device) - if mount_path: - orig_system.setValue("disk." + str(cont) + ".mount_path", mount_path) - else: - self.log_error("Error attaching a %d GB volume for the disk %d." - " Deleting it." % (int(disk_size), cont)) - self.delete_volume(volume_id, auth_data) - return (False, "Error attaching the new volume") - else: - return (False, "Error creating the new volume: " + volume_id) cont += 1 except Exception as ex: self.log_exception("Error connecting with OCCI server") diff --git a/IM/connectors/OpenNebula.py b/IM/connectors/OpenNebula.py index 5e8072bc5..387ebd4b3 100644 --- a/IM/connectors/OpenNebula.py +++ b/IM/connectors/OpenNebula.py @@ -533,7 +533,8 @@ def finalize(self, vm, last, auth_data): else: return (False, "Error in the one.vm.action return value") - self.delete_security_groups(vm.inf, auth_data) + if success: + self.delete_security_groups(vm.inf, auth_data) return (success, err) diff --git a/INSTALL b/INSTALL index cad57c446..42f9f238e 100644 --- a/INSTALL +++ b/INSTALL @@ -72,7 +72,7 @@ In case of using the SSL secured version of the REST API pyOpenSSL must be insta 1.3.1 Using installer (Recommended option) ------------------------------------------ -The IM provides a script to install the IM in one single step. +The IM provides a script to install the IM in one single step (using pip). You only need to execute the following command: $ wget -qO- https://raw.githubusercontent.com/grycap/im/master/install.sh | bash diff --git a/changelog b/changelog index a09f01834..196062a03 100644 --- a/changelog +++ b/changelog @@ -301,3 +301,10 @@ IM 1.5.4 * Fix error in Azure connector creating a VM with two nics. * Fix error in Azure connector creating Storage Account with more than 24 chars. +IM 1.5.5 + * Fix error getting IP info in OCCI conn. + * Enable to reset the add_public_ip_count in the OCCI/OST conns. + * Improve Azure instance_type selection. + * Improve GCE instance type selection. + * Manage DNS records in EC2, Azure and GCE connectors. + * Fix error in Azure conn creating a VM with only a public net attached. diff --git a/doc/gitbook/docker-image.md b/doc/gitbook/docker-image.md index 29904d3dd..33a8bacfe 100644 --- a/doc/gitbook/docker-image.md +++ b/doc/gitbook/docker-image.md @@ -9,6 +9,13 @@ How to launch the IM service using docker:: ```sh $ sudo docker run -d -p 8899:8899 -p 8800:8800 --name im indigodatacloud/im ``` + +To make the IM data persistent you also have to specify a persistent location for the IM database using the IM_DATA_DB environment variable and adding a volume:: + +```sh +$ sudo docker run -d -p 8899:8899 -p 8800:8800 -v "/some_local_path/db:/db" -e IM_DATA_DB=/db/inf.dat --name im indigodatacloud/im +``` + You can also specify an external MySQL server to store IM data using the IM_DATA_DB environment variable:: ```sh diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 88c027bcf..b08dcce98 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -79,7 +79,7 @@ Installation Using installer (Recommended option) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The IM provides a script to install the IM in one single step. +The IM provides a script to install the IM in one single step (using pip). You only need to execute the following command:: $ wget -qO- https://raw.githubusercontent.com/grycap/im/master/install.sh | bash @@ -136,7 +136,7 @@ Then install the downloaded RPMs:: Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip:: - $ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource + $ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns From Deb package (Tested with Ubuntu 14.04 and 16.04) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -547,6 +547,11 @@ How to launch the IM service using docker:: $ sudo docker run -d -p 8899:8899 --name im grycap/im +To make the IM data persistent you also have to specify a persistent location for the IM database using +the IM_DATA_DB environment variable and adding a volume:: + + $ sudo docker run -d -p 8899:8899 -p 8800:8800 -v "/some_local_path/db:/db" -e IM_DATA_DB=/db/inf.dat --name im grycap/im + You can also specify an external MySQL server to store IM data using the IM_DATA_DB environment variable:: $ sudo docker run -d -p 8899:8899 -e IM_DATA_DB=mysql://username:password@server/db_name --name im grycap/im diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index a4cb833a2..0a5874966 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -10,7 +10,7 @@ RUN pip3 install setuptools --upgrade -I RUN pip3 install cheroot RUN pip3 install pyOpenSSL --upgrade -I # Install pip optional libraries -RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource +RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns RUN apt install -y git RUN cd tmp \ diff --git a/test/unit/connectors/Azure.py b/test/unit/connectors/Azure.py index b414592d1..dfee2cadf 100755 --- a/test/unit/connectors/Azure.py +++ b/test/unit/connectors/Azure.py @@ -34,7 +34,7 @@ from IM.VirtualMachine import VirtualMachine from IM.InfrastructureInfo import InfrastructureInfo from IM.connectors.Azure import AzureCloudConnector -from mock import patch, MagicMock +from mock import patch, MagicMock, call def read_file_as_string(file_name): @@ -179,8 +179,9 @@ def test_20_launch(self, credentials, network_client, compute_client, storage_cl @patch('IM.connectors.Azure.NetworkManagementClient') @patch('IM.connectors.Azure.ComputeManagementClient') + @patch('IM.connectors.Azure.DnsManagementClient') @patch('IM.connectors.Azure.UserPassCredentials') - def test_30_updateVMInfo(self, credentials, compute_client, network_client): + def test_30_updateVMInfo(self, credentials, dns_client, compute_client, network_client): radl_data = """ network net (outbound = 'yes') system test ( @@ -188,7 +189,7 @@ def test_30_updateVMInfo(self, credentials, compute_client, network_client): cpu.count=1 and memory.size=512m and net_interface.0.connection = 'net' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'test.domain.com' and disk.0.os.name = 'linux' and disk.0.image.url = 'azr://Canonical/UbuntuServer/16.04.0-LTS/latest' and disk.0.os.credentials.username = 'user' and @@ -214,14 +215,14 @@ def test_30_updateVMInfo(self, credentials, compute_client, network_client): compute_client.return_value = cclient cclient.virtual_machine_sizes.list.return_value = instace_types - vm = MagicMock() - vm.provisioning_state = "Succeeded" - vm.hardware_profile.vm_size = "instance_type1" - vm.location = "northeurope" + avm = MagicMock() + avm.provisioning_state = "Succeeded" + avm.hardware_profile.vm_size = "instance_type1" + avm.location = "northeurope" ni = MagicMock() ni.id = "/subscriptions/subscription-id/resourceGroups/rg0/providers/Microsoft.Network/networkInterfaces/ni-0" - vm.network_profile.network_interfaces = [ni] - cclient.virtual_machines.get.return_value = vm + avm.network_profile.network_interfaces = [ni] + cclient.virtual_machines.get.return_value = avm nclient = MagicMock() network_client.return_value = nclient @@ -237,9 +238,19 @@ def test_30_updateVMInfo(self, credentials, compute_client, network_client): pub_ip_res.ip_address = "13.0.0.1" nclient.public_ip_addresses.get.return_value = pub_ip_res + dclient = MagicMock() + dns_client.return_value = dclient + dclient.zones.get.return_value = None + dclient.record_sets.get.return_value = None + success, vm = azure_cloud.updateVMInfo(vm, auth) self.assertTrue(success, msg="ERROR: updating VM info.") + self.assertEquals(dclient.zones.create_or_update.call_args_list, + [call('rg0', 'domain.com', {'location': 'global'})]) + self.assertEquals(dclient.record_sets.create_or_update.call_args_list, + [call('rg0', 'domain.com', 'test', 'A', + {'arecords': [{'ipv4_address': '13.0.0.1'}], 'ttl': 300})]) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.Azure.ComputeManagementClient') @@ -351,9 +362,19 @@ def test_60_finalize(self, credentials, resource_client): auth = Authentication([{'id': 'azure', 'type': 'Azure', 'subscription_id': 'subscription_id', 'username': 'user', 'password': 'password'}]) azure_cloud = self.get_azure_cloud() + radl_data = """ + network net (outbound = 'yes') + system test ( + cpu.count=1 and + memory.size=512m and + net_interface.0.connection = 'net' and + net_interface.0.ip = '158.42.1.1' and + net_interface.0.dns_name = 'test.domain.com' + )""" + radl = radl_parse.parse_radl(radl_data) inf = MagicMock() - vm = VirtualMachine(inf, "rg0/vm0", azure_cloud.cloud, "", "", azure_cloud, 1) + vm = VirtualMachine(inf, "rg0/vm0", azure_cloud.cloud, radl, radl, azure_cloud, 1) success, _ = azure_cloud.finalize(vm, True, auth) diff --git a/test/unit/connectors/EC2.py b/test/unit/connectors/EC2.py index d614b3c17..22912b2dd 100755 --- a/test/unit/connectors/EC2.py +++ b/test/unit/connectors/EC2.py @@ -280,7 +280,9 @@ def test_25_launch_spot(self, blockdevicemapping, VPCConnection, get_region): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.EC2.EC2CloudConnector.get_connection') - def test_30_updateVMInfo(self, get_connection): + @patch('boto.route53.connect_to_region') + @patch('boto.route53.record.ResourceRecordSets') + def test_30_updateVMInfo(self, record_sets, connect_to_region, get_connection): radl_data = """ network net (outbound = 'yes') system test ( @@ -289,7 +291,7 @@ def test_30_updateVMInfo(self, get_connection): memory.size=512m and net_interface.0.connection = 'net' and net_interface.0.ip = '158.42.1.1' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'test.domain.com' and disk.0.os.name = 'linux' and disk.0.image.url = 'one://server.com/1' and disk.0.os.credentials.username = 'user' and @@ -335,9 +337,25 @@ def test_30_updateVMInfo(self, get_connection): conn.create_volume.return_value = volume conn.attach_volume.return_value = True + dns_conn = MagicMock() + connect_to_region.return_value = dns_conn + + dns_conn.get_zone.return_value = None + zone = MagicMock() + zone.get_a.return_value = None + dns_conn.create_zone.return_value = zone + changes = MagicMock() + record_sets.return_value = changes + change = MagicMock() + changes.add_change.return_value = change + success, vm = ec2_cloud.updateVMInfo(vm, auth) self.assertTrue(success, msg="ERROR: updating VM info.") + self.assertEquals(dns_conn.create_zone.call_count, 1) + self.assertEquals(dns_conn.create_zone.call_args_list[0][0][0], "domain.com.") + self.assertEquals(changes.add_change.call_args_list, [call('CREATE', 'test.domain.com.', 'A')]) + self.assertEquals(change.add_value.call_args_list, [call('158.42.1.1')]) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.EC2.EC2CloudConnector.get_connection') @@ -497,7 +515,9 @@ def test_55_alter(self, get_connection): @patch('IM.connectors.EC2.EC2CloudConnector.get_connection') @patch('time.sleep') - def test_60_finalize(self, sleep, get_connection): + @patch('boto.route53.connect_to_region') + @patch('boto.route53.record.ResourceRecordSets') + def test_60_finalize(self, record_sets, connect_to_region, sleep, get_connection): radl_data = """ network net (outbound = 'yes') system test ( @@ -506,7 +526,7 @@ def test_60_finalize(self, sleep, get_connection): memory.size=512m and net_interface.0.connection = 'net' and net_interface.0.ip = '158.42.1.1' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'test.domain.com' and disk.0.os.name = 'linux' and disk.0.image.url = 'one://server.com/1' and disk.0.os.credentials.username = 'user' and @@ -561,9 +581,27 @@ def test_60_finalize(self, sleep, get_connection): sg.delete.return_value = True conn.get_all_security_groups.return_value = [sg] + dns_conn = MagicMock() + connect_to_region.return_value = dns_conn + + zone = MagicMock() + record = MagicMock() + zone.id = "zid" + zone.get_a.return_value = record + dns_conn.get_all_rrsets.return_value = [] + dns_conn.get_zone.return_value = zone + changes = MagicMock() + record_sets.return_value = changes + change = MagicMock() + changes.add_change.return_value = change + success, _ = ec2_cloud.finalize(vm, True, auth) self.assertTrue(success, msg="ERROR: finalizing VM info.") + self.assertEquals(dns_conn.delete_hosted_zone.call_count, 1) + self.assertEquals(dns_conn.delete_hosted_zone.call_args_list[0][0][0], zone.id) + self.assertEquals(changes.add_change.call_args_list, [call('DELETE', 'test.domain.com.', 'A')]) + self.assertEquals(change.add_value.call_args_list, [call('158.42.1.1')]) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.EC2.EC2CloudConnector.get_connection') diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index 0cba0a73b..7c9f88d3a 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -34,7 +34,7 @@ from IM.VirtualMachine import VirtualMachine from IM.InfrastructureInfo import InfrastructureInfo from IM.connectors.GCE import GCECloudConnector -from mock import patch, MagicMock +from mock import patch, MagicMock, call from libcloud.compute.base import NodeSize @@ -109,11 +109,13 @@ def test_10_concrete(self, get_driver): node_size.price = 1.0 node_size.disk = 1 node_size.name = "small" + node_size.extra = {'guestCpus': 1} node_size2 = MagicMock() node_size2.ram = 1024 node_size2.price = None node_size2.disk = 2 node_size2.name = "medium" + node_size2.extra = {'guestCpus': 2} driver.list_sizes.return_value = [node_size, node_size2] gce_cloud = self.get_gce_cloud() @@ -156,6 +158,7 @@ def test_20_launch(self, get_driver): node_size.disk = 1 node_size.vcpus = 1 node_size.name = "small" + node_size.extra = {'guestCpus': 1} driver.list_sizes.return_value = [node_size] driver.ex_get_image.return_value = "image" @@ -188,7 +191,8 @@ def test_20_launch(self, get_driver): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.gce.GCENodeDriver') - def test_30_updateVMInfo(self, get_driver): + @patch('libcloud.dns.drivers.google.GoogleDNSDriver') + def test_30_updateVMInfo(self, get_dns_driver, get_driver): radl_data = """ network net (outbound = 'yes') system test ( @@ -196,7 +200,7 @@ def test_30_updateVMInfo(self, get_driver): cpu.count=1 and memory.size=512m and net_interface.0.connection = 'net' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'test.domain.com' and disk.0.os.name = 'linux' and disk.0.image.url = 'gce://us-central1-a/centos-6' and disk.0.os.credentials.username = 'user' and @@ -216,13 +220,14 @@ def test_30_updateVMInfo(self, get_driver): driver = MagicMock() get_driver.return_value = driver + dns_driver = MagicMock() + get_dns_driver.return_value = dns_driver node = MagicMock() zone = MagicMock() node.id = "1" node.state = "running" node.extra = {'flavorId': 'small'} - node.public_ips = [] node.public_ips = ['158.42.1.1'] node.private_ips = ['10.0.0.1'] node.driver = driver @@ -237,9 +242,20 @@ def test_30_updateVMInfo(self, get_driver): volume.extra = {'status': 'READY'} driver.create_volume.return_value = volume + dns_driver.iterate_zones.return_value = [] + dns_driver.iterate_records.return_value = [] + success, vm = gce_cloud.updateVMInfo(vm, auth) self.assertTrue(success, msg="ERROR: updating VM info.") + + self.assertEquals(dns_driver.create_zone.call_count, 1) + self.assertEquals(dns_driver.create_record.call_count, 1) + self.assertEquals(dns_driver.create_zone.call_args_list[0], call('domain.com.')) + self.assertEquals(dns_driver.create_record.call_args_list[0][0][0], 'test.domain.com.') + self.assertEquals(dns_driver.create_record.call_args_list[0][0][2], 'A') + self.assertEquals(dns_driver.create_record.call_args_list[0][0][3], {'rrdatas': ['158.42.1.1'], 'ttl': 300}) + self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.gce.GCENodeDriver') @@ -283,16 +299,21 @@ def test_50_start(self, get_driver): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.gce.GCENodeDriver') + @patch('libcloud.dns.drivers.google.GoogleDNSDriver') @patch('time.sleep') - def test_60_finalize(self, sleep, get_driver): + def test_60_finalize(self, sleep, get_dns_driver, get_driver): auth = Authentication([{'id': 'one', 'type': 'GCE', 'username': 'user', 'password': 'pass\npass', 'project': 'proj'}]) gce_cloud = self.get_gce_cloud() radl_data = """ + network net (outbound = 'yes') system test ( - cpu.count>=2 and - memory.size>=2048m + cpu.count=2 and + memory.size=2048m and + net_interface.0.connection = 'net' and + net_interface.0.dns_name = 'test.domain.com' and + net_interface.0.ip = '158.42.1.1' )""" radl = radl_parse.parse_radl(radl_data) @@ -300,8 +321,9 @@ def test_60_finalize(self, sleep, get_driver): vm = VirtualMachine(inf, "1", gce_cloud.cloud, radl, radl, gce_cloud, 1) driver = MagicMock() - driver.name = "OpenStack" get_driver.return_value = driver + dns_driver = MagicMock() + get_dns_driver.return_value = dns_driver node = MagicMock() node.destroy.return_value = True @@ -314,9 +336,19 @@ def test_60_finalize(self, sleep, get_driver): volume.destroy.return_value = True driver.ex_get_volume.return_value = volume + zone = MagicMock() + zone.domain = "domain.com." + dns_driver.iterate_zones.return_value = [zone] + record = MagicMock() + record.name = 'test.domain.com.' + record.data = {'rrdatas': ['158.42.1.1'], 'ttl': 300} + dns_driver.iterate_records.return_value = [record] + success, _ = gce_cloud.finalize(vm, True, auth) self.assertTrue(success, msg="ERROR: finalizing VM info.") + self.assertEquals(dns_driver.delete_record.call_count, 1) + self.assertEquals(dns_driver.delete_record.call_args_list[0][0][0].name, 'test.domain.com.') self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())