Merge pull request #230 from indigo-dc/devel

Devel

IM/InfrastructureManager.py

@@ -1221,10 +1221,12 @@ def check_auth_data(auth):
         # First check if an OIDC token is included
         if "token" in im_auth[0]:
             InfrastructureManager.check_oidc_token(im_auth[0])
-        else:
+        elif "username" in im_auth[0]:
             if im_auth[0]['username'].startswith(IM.InfrastructureInfo.InfrastructureInfo.OPENID_USER_PREFIX):
                 # This is a OpenID user do not enable to get data using user/pass creds
                 raise IncorrectVMCrecentialsException("Invalid username used for the InfrastructureManager.")
+        else:
+            raise IncorrectVMCrecentialsException("No username nor token for the InfrastructureManager.")
 
         # Now check if the user is in authorized
         if not InfrastructureManager.check_im_user(im_auth):

IM/VirtualMachine.py

@@ -581,32 +581,36 @@ def setIps(self, public_ips, private_ips, remove_old=False):
             if real_public_ips:
                 vm_system.setValue('net_interface.%s.connection' % num_net, public_net.id)
                 if len(real_public_ips) > 1:
-                    self.logger.warn("Node with more that one public IP!")
-                    self.logger.debug(real_public_ips)
+                    self.log_warn("Node with more that one public IP!")
+                    self.log_debug(real_public_ips)
                     if len(real_public_ips) == 2:
                         ip1 = IPAddress(real_public_ips[0])
                         ip2 = IPAddress(real_public_ips[1])
                         if ip1.version != ip2.version:
-                            self.logger.info("It seems that there are one IPv4 and other IPv6. Get the IPv4 one.")
+                            self.log_info("It seems that there are one IPv4 and other IPv6. Get the IPv4 one.")
                             if ip1.version == 4:
                                 vm_system.setValue('net_interface.%s.ip' % num_net, str(real_public_ips[0]))
                                 vm_system.setValue('net_interface.%s.ipv6' % num_net, str(real_public_ips[1]))
                             else:
                                 vm_system.setValue('net_interface.%s.ip' % num_net, str(real_public_ips[1]))
                                 vm_system.setValue('net_interface.%s.ipv6' % num_net, str(real_public_ips[0]))
                         else:
-                            self.logger.info("It seems that both are from the same version first one will be used")
+                            self.log_info("It seems that both are from the same version first one will be used")
                             vm_system.setValue('net_interface.%s.ip' % num_net, str(real_public_ips[0]))
                     else:
-                        self.logger.info("It seems that there are more that 2 last ones will be used")
+                        self.log_info("It seems that there are more that 2 last ones will be used")
                         for ip in real_public_ips:
                             if IPAddress(ip).version == 4:
                                 vm_system.setValue('net_interface.%s.ip' % num_net, str(ip))
                             else:
                                 vm_system.setValue('net_interface.%s.ipv6' % num_net, str(ip))
                 else:
                     # The usual case
-                    vm_system.setValue('net_interface.%s.ip' % num_net, str(real_public_ips[0]))
+                    if IPAddress(real_public_ips[0]).version == 6:
+                        self.log_warn("Node only with one IPv6!!")
+                        vm_system.setValue('net_interface.%s.ipv6' % num_net, str(real_public_ips[0]))
+                    else:
+                        vm_system.setValue('net_interface.%s.ip' % num_net, str(real_public_ips[0]))
 
         if private_ips:
             private_net_map = {}
@@ -664,7 +668,10 @@ def setIps(self, public_ips, private_ips, remove_old=False):
                         # this VM
                         num_net = self.getNumNetworkIfaces()
 
-                vm_system.setValue('net_interface.%s.ip' % num_net, str(private_ip))
+                if IPAddress(private_ip).version == 6:
+                    vm_system.setValue('net_interface.%s.ipv6' % num_net, str(private_ip))
+                else:
+                    vm_system.setValue('net_interface.%s.ip' % num_net, str(private_ip))
                 vm_system.setValue('net_interface.%s.connection' % num_net, private_net.id)
 
     def get_ssh(self, retry=False):

IM/__init__.py

@@ -19,5 +19,5 @@
            'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry',
            'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPAdapter', 'uriparse',
            'VirtualMachine', 'VMRC', 'xmlobject']
-__version__ = '1.7.0'
+__version__ = '1.7.1'
 __author__ = 'Miguel Caballer'

IM/config.py

@@ -103,6 +103,7 @@ class Config:
     VM_NUM_USE_CTXT_DIST = 30
     DELAY_BETWEEN_VM_RETRIES = 5
 
+
 config = ConfigParser()
 config.read([Config.IM_PATH + '/../im.cfg', Config.IM_PATH +
              '/../etc/im.cfg', '/etc/im/im.cfg'])
@@ -128,6 +129,7 @@ class ConfigOpenNebula:
     IMAGE_UNAME = ''
     TTS_URL = 'https://localhost:8443'
 
+
 if config.has_section("OpenNebula"):
     parse_options(config, 'OpenNebula', ConfigOpenNebula)
 

IM/connectors/Azure.py

@@ -66,12 +66,12 @@ class AzureCloudConnector(CloudConnector):
     }
 
     POWER_STATE_MAP = {
-        'Deallocated': VirtualMachine.OFF,
-        'Deallocating': VirtualMachine.OFF,
-        'Running': VirtualMachine.RUNNING,
-        'Starting': VirtualMachine.PENDING,
-        'Started': VirtualMachine.PENDING,
-        'Stopped': VirtualMachine.STOPPED
+        'PowerState/deallocated': VirtualMachine.OFF,
+        'PowerState/deallocating': VirtualMachine.OFF,
+        'PowerState/running': VirtualMachine.RUNNING,
+        'PowerState/starting': VirtualMachine.PENDING,
+        'PowerState/started': VirtualMachine.PENDING,
+        'PowerState/stopped': VirtualMachine.STOPPED
     }
 
     def __init__(self, cloud_info, inf):
@@ -645,14 +645,26 @@ def updateVMInfo(self, vm, auth_data):
             credentials, subscription_id = self.get_credentials(auth_data)
             compute_client = ComputeManagementClient(credentials, subscription_id)
             # Get one the virtual machine by name
-            virtual_machine = compute_client.virtual_machines.get(group_name, vm_name)
+            virtual_machine = compute_client.virtual_machines.get(group_name, vm_name, expand='instanceView')
         except Exception as ex:
+            self.log_warn("The VM does not exists.")
+            # check if the RG still exists
+            if self.get_rg(group_name, credentials, subscription_id):
+                self.log_info("But the RG %s does exits. Return OFF." % group_name)
+                vm.state = VirtualMachine.OFF
+                return (True, vm)
+
             self.log_exception("Error getting the VM info: " + vm.id)
             return (False, "Error getting the VM info: " + vm.id + ". " + str(ex))
 
         self.log_info("VM info: " + vm.id + " obtained.")
         vm.state = self.PROVISION_STATE_MAP.get(virtual_machine.provisioning_state, VirtualMachine.UNKNOWN)
-        self.log_info("The VM state is: " + vm.state)
+
+        if (vm.state == VirtualMachine.RUNNING and virtual_machine.instance_view and
+                len(virtual_machine.instance_view.statuses) > 1):
+            vm.state = self.POWER_STATE_MAP.get(virtual_machine.instance_view.statuses[1].code, VirtualMachine.UNKNOWN)
+
+        self.log_debug("The VM state is: " + vm.state)
 
         instance_type = self.get_instance_type_by_name(virtual_machine.hardware_profile.vm_size,
                                                        virtual_machine.location, credentials, subscription_id)

IM/connectors/EC2.py

@@ -1009,19 +1009,16 @@ def setIPsFromInstance(self, vm, instance):
                 # It will be used if it is different to the public IP of the
                 # instance
                 if str(address.public_ip) != instance.ip_address:
-                    vm_system.setValue(
-                        'net_interface.' + str(num_nets) + '.ip', str(instance.ip_address))
-                    vm_system.setValue(
-                        'net_interface.' + str(num_nets) + '.connection', public_net.id)
+                    vm_system.setValue('net_interface.' + str(num_nets) + '.ip', str(instance.ip_address))
+                    vm_system.setValue('net_interface.' + str(num_nets) + '.connection', public_net.id)
 
                     num_pub_nets += 1
                     num_nets += 1
 
         n = 0
         requested_ips = []
         while vm.getRequestedSystem().getValue("net_interface." + str(n) + ".connection"):
-            net_conn = vm.getRequestedSystem().getValue(
-                'net_interface.' + str(n) + '.connection')
+            net_conn = vm.getRequestedSystem().getValue('net_interface.' + str(n) + '.connection')
             if vm.info.get_network_by_id(net_conn).isPublic():
                 fixed_ip = vm.getRequestedSystem().getValue("net_interface." + str(n) + ".ip")
                 requested_ips.append(fixed_ip)

IM/connectors/OpenNebula.py

@@ -286,7 +286,11 @@ def setIPsFromTemplate(self, vm, template):
                 net = vm.info.get_network_by_id(net_name)
                 provider_id = net.getValue("provider_id")
                 if provider_id == nic.NETWORK:
-                    system.setValue("net_interface." + str(i) + ".ip", str(nic.IP))
+                    ip = str(nic.IP)
+                    if IPAddress(ip).version == 6:
+                        system.setValue("net_interface." + str(i) + ".ipv6", ip)
+                    else:
+                        system.setValue("net_interface." + str(i) + ".ip", ip)
                     break
                 i += 1
 

IM/connectors/OpenStack.py

@@ -289,26 +289,41 @@ def map_radl_ost_networks(self, radl_nets, ost_nets):
             if net_name:
                 for radl_net in radl_nets:
                     net_provider_id = radl_net.getValue('provider_id')
+
                     if net_provider_id:
+                        parts = net_provider_id.split(".")
+                        if len(parts) > 1:
+                            net_provider_id = parts[0]
                         if net_name == net_provider_id:
-                            res[radl_net.id] = ip
-                            break
+                            if radl_net.isPublic() == is_public:
+                                res[radl_net.id] = ip
+                                break
+                            else:
+                                # the ip not matches the is_public value
+                                res["#UNMAPPED#"].append(ip)
                     else:
                         if radl_net.id not in res:
                             if radl_net.isPublic() == is_public:
                                 res[radl_net.id] = ip
                                 radl_net.setValue('provider_id', net_name)
+                                if ip in res["#UNMAPPED#"]:
+                                    res["#UNMAPPED#"].remove(ip)
                                 break
                             else:
                                 # the ip not matches the is_public value
                                 res["#UNMAPPED#"].append(ip)
             else:
                 # It seems to be a floating IP
+                added = False
                 for radl_net in radl_nets:
                     if radl_net.id not in res and radl_net.isPublic() == is_public:
                         res[radl_net.id] = ip
+                        added = True
                         break
 
+                if not added:
+                    res["#UNMAPPED#"].append(ip)
+
         return res
 
     def get_node_floating_ips(self, node):
@@ -366,7 +381,10 @@ def setIPsFromInstance(self, vm, node):
                 net_name = system.getValue("net_interface." + str(i) + ".connection")
                 if net_name in map_nets:
                     ip = map_nets[net_name]
-                    system.setValue("net_interface." + str(i) + ".ip", ip)
+                    if IPAddress(ip).version == 6:
+                        system.setValue("net_interface." + str(i) + ".ipv6", ip)
+                    else:
+                        system.setValue("net_interface." + str(i) + ".ip", ip)
                     ips_assigned.append(ip)
                 i += 1
 
@@ -377,7 +395,10 @@ def setIPsFromInstance(self, vm, node):
                 if net_name != '#UNMAPPED#':
                     if ip not in ips_assigned:
                         num_net = system.getNumNetworkIfaces()
-                        system.setValue('net_interface.' + str(num_net) + '.ip', ip)
+                        if IPAddress(ip).version == 6:
+                            system.setValue('net_interface.' + str(num_net) + '.ipv6', ip)
+                        else:
+                            system.setValue('net_interface.' + str(num_net) + '.ip', ip)
                         system.setValue('net_interface.' + str(num_net) + '.connection', net_name)
                 else:
                     pub_ips = []
@@ -445,6 +466,10 @@ def get_networks(self, driver, radl):
             else:
                 # First check if the user has specified a provider ID
                 if net_provider_id:
+                    parts = net_provider_id.split(".")
+                    if len(parts) > 1:
+                        net_provider_id = parts[0]
+
                     for net in ost_nets:
                         if net.name == net_provider_id:
                             if net.name not in used_nets:
@@ -624,6 +649,14 @@ def manage_elastic_ips(self, vm, node, public_ips):
             if net.isPublic():
                 fixed_ip = vm.getRequestedSystem().getValue("net_interface." + str(n) + ".ip")
                 pool_name = net.getValue("pool_name")
+
+                if not pool_name:
+                    net_provider_id = net.getValue("provider_id")
+                    if net_provider_id:
+                        parts = net_provider_id.split(".")
+                        if len(parts) > 1:
+                            pool_name = parts[1]
+
                 requested_ips.append((fixed_ip, pool_name))
             n += 1
 
@@ -860,7 +893,7 @@ def finalize(self, vm, last, auth_data):
 
         return (True, "")
 
-    def delete_security_groups(self, driver, inf, timeout=90, delay=10):
+    def delete_security_groups(self, driver, inf, timeout=180, delay=10):
         """
         Delete the SG of this inf
         """

IM/db.py

@@ -227,6 +227,7 @@ def table_exists(self, table_name):
         else:
             return True
 
+
 try:
     class IntegrityError(sqlite.IntegrityError):
         """ Class to return IntegrityError independently of the DB used"""

IM/request.py

@@ -280,6 +280,7 @@ def serve_forever_in_thread(self):
         self.__thread.daemon = True
         self.__thread.start()
 
+
 if Config.XMLRCP_SSL:
     class AsyncSSLXMLRPCServer(ThreadingMixIn, SSLSimpleXMLRPCServer):
 

changelog

@@ -407,3 +407,8 @@ IM 1.7.0
     * Change in the DB schema in the case of MySQL DB. This command must be made
       in the current DB:
 			ALTER TABLE `inf_list` ADD COLUMN `rowid` INT AUTO_INCREMENT UNIQUE FIRST;
+
+IM 1.7.1:
+    * Fix problems with nodes with IPv6 in OpenStack conn.
+    * Fix Azure conn does not get the correct state (in case of suspended VMs).
+    * Enable to specify pool_name in the network provider_ip in OpenStack conn.

contextualization/conf-ansible.yml

@@ -158,11 +158,11 @@
       when: (ansible_os_family == "RedHat" and ansible_distribution_major_version|int < 6) or (ansible_os_family == "Suse" and ansible_distribution_major_version|int < 10)
 
     - name: Change ssh_args to set ControlPersist to 15 min in ansible.cfg
-      ini_file: dest=/etc/ansible/ansible.cfg section=ssh_connection option=ssh_args value="-o ControlMaster=auto -o ControlPersist=900s"
+      ini_file: dest=/etc/ansible/ansible.cfg section=ssh_connection option=ssh_args value="-o ControlMaster=auto -o ControlPersist=900s -o UserKnownHostsFile=/dev/null"
       when: ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 7) or (ansible_os_family == "Suse" and ansible_distribution_major_version|int >= 12)
 
     - name: Change ssh_args to remove ControlPersist in REL 6 and older in ansible.cfg
-      ini_file: dest=/etc/ansible/ansible.cfg section=ssh_connection option=ssh_args value=""
+      ini_file: dest=/etc/ansible/ansible.cfg section=ssh_connection option=ssh_args value="-o UserKnownHostsFile=/dev/null"
       when: (ansible_os_family == "RedHat" and ansible_distribution_major_version|int < 7) or (ansible_os_family == "Suse" and ansible_distribution_major_version|int < 12)
 
     - name: Activate SSH pipelining in ansible.cfg

contextualization/ctxt_agent.py

@@ -630,6 +630,7 @@ def run(general_conf_file, vm_conf_file):
 
         return res_data['OK']
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Contextualization Agent.')
     parser.add_argument('general', type=str, nargs=1)

contextualization/ctxt_agent_dist.py

@@ -960,6 +960,7 @@ def run(general_conf_file, vm_conf_file, local):
 
         return res_data['OK']
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Contextualization Agent.')
     parser.add_argument('general', type=str, nargs=1)

doc/source/client.rst

@@ -46,9 +46,11 @@ The :program:`im_client` is called like this::
    ``list``
       List the infrastructure IDs created by the user.
 
-   ``create radlfile``
+   ``create radlfile async_flag``
       Create an infrastructure using RADL specified in the file with path
-      ``radlfile``.
+      ``radlfile``. The ``async_flag`` parameter is optional
+      and is a flag to specify if the creation call will wait the resources
+      to be created or return immediately the id of the infrastructure.
 
    ``destroy infId``
       Destroy the infrastructure with ID ``infId``.