Merge pull request #228 from indigo-dc/devel

Devel

IM/__init__.py

@@ -19,5 +19,5 @@
            'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry',
            'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPAdapter', 'uriparse',
            'VirtualMachine', 'VMRC', 'xmlobject']
-__version__ = '1.6.7'
+__version__ = '1.6.8'
 __author__ = 'Miguel Caballer'

IM/connectors/Azure.py

@@ -745,14 +745,18 @@ def finalize(self, vm, last, auth_data):
 
             # Delete Resource group and everything in it
             if self.get_rg(group_name, credentials, subscription_id):
-                self.delete_resource_group(group_name, resource_client)
+                deleted, msg = self.delete_resource_group(group_name, resource_client)
+                if not deleted:
+                    return False, "Error terminating the VM: %s" % msg
             else:
                 self.log_info("RG: %s does not exist. Do not remove." % group_name)
 
             # if it is the last VM delete the RG of the Inf
             if last:
                 if self.get_rg("rg-%s" % vm.inf.id, credentials, subscription_id):
-                    self.delete_resource_group("rg-%s" % vm.inf.id, resource_client)
+                    deleted, msg = self.delete_resource_group("rg-%s" % vm.inf.id, resource_client)
+                    if not deleted:
+                        return False, "Error terminating the VM: %s" % msg
                 else:
                     self.log_info("RG: %s does not exist. Do not remove." % "rg-%s" % vm.inf.id)
 
@@ -823,6 +827,7 @@ def delete_resource_group(self, group_name, resource_client, max_retries=3):
         Delete a RG with retries
         """
         cont = 0
+        msg = ""
         deleted = False
 
         self.log_info("Delete RG %s." % group_name)
@@ -831,12 +836,13 @@ def delete_resource_group(self, group_name, resource_client, max_retries=3):
             try:
                 resource_client.resource_groups.delete(group_name).wait()
                 deleted = True
-            except:
+            except Exception as ex:
+                msg = str(ex)
                 self.log_exception("Error deleting Resource group %s (%d/%d)." % (group_name, cont, max_retries))
 
         if not deleted:
             self.log_error("Resource group %s cannot be deleted!!!" % group_name)
         else:
             self.log_info("Resource group %s successfully deleted." % group_name)
 
-        return deleted
+        return deleted, msg

IM/connectors/GCE.py

@@ -250,7 +250,7 @@ def get_instance_type(self, sizes, radl):
             cpu_op = radl.getFeature('cpu.count').getLogOperator()
 
         memory = 1
-        memory_op = ">1"
+        memory_op = ">="
         if radl.getFeature('memory.size'):
             memory = radl.getFeature('memory.size').getValue('M')
             memory_op = radl.getFeature('memory.size').getLogOperator()
@@ -284,15 +284,28 @@ def get_instance_type(self, sizes, radl):
         return res
 
     def request_external_ip(self, radl):
+        """
+        Check if the user has requested for a public ip
+        """
+        system = radl.systems[0]
+        n = 0
+        while system.getValue("net_interface." + str(n) + ".connection"):
+            net_conn = system.getValue('net_interface.' + str(n) + '.connection')
+            if radl.get_network_by_id(net_conn).isPublic():
+                return True
+            n += 1
+
+        return False
+
+    def request_fixed_ip(self, radl):
         """
         Check if the user has requested for a fixed ip
         """
         system = radl.systems[0]
         n = 0
         requested_ips = []
         while system.getValue("net_interface." + str(n) + ".connection"):
-            net_conn = system.getValue(
-                'net_interface.' + str(n) + '.connection')
+            net_conn = system.getValue('net_interface.' + str(n) + '.connection')
             if radl.get_network_by_id(net_conn).isPublic():
                 fixed_ip = system.getValue("net_interface." + str(n) + ".ip")
                 if fixed_ip:
@@ -302,8 +315,7 @@ def request_external_ip(self, radl):
         if requested_ips:
             self.log_info("The user requested for a fixed IP")
             if len(requested_ips) > 1:
-                self.log_warn(
-                    "The user has requested more than one fixed IP. Using only the first one")
+                self.log_warn("The user has requested more than one fixed IP. Using only the first one")
             return requested_ips[0]
         else:
             return None
@@ -413,14 +425,17 @@ def create_firewall(self, inf, net_name, radl, driver):
 
     def launch(self, inf, radl, requested_radl, num_vm, auth_data):
         system = radl.systems[0]
-        region, image_id = self.get_image_data(
-            system.getValue("disk.0.image.url"))
+        region, image_id = self.get_image_data(system.getValue("disk.0.image.url"))
 
         if system.getValue('availability_zone'):
             region = system.getValue('availability_zone')
 
         driver = self.get_driver(auth_data, region)
 
+        region = driver.ex_get_zone(region)
+        if not region:
+            return [(False, "Incorrect region specified.") for _ in range(num_vm)]
+
         image = driver.ex_get_image(image_id)
         if not image:
             return [(False, "Incorrect image name") for _ in range(num_vm)]
@@ -438,7 +453,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
 
         args = {'size': instance_type,
                 'image': image,
-                'external_ip': 'ephemeral',
+                'external_ip': None,
                 'location': region}
 
         # include the SSH_KEYS
@@ -479,10 +494,13 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
             self.create_firewall(inf, net_name, radl, driver)
 
         if self.request_external_ip(radl):
-            if num_vm > 1:
-                raise Exception("A fixed IP cannot be specified to a set of nodes (deploy is higher than 1)")
-            fixed_ip = self.request_external_ip(radl)
-            args['external_ip'] = driver.ex_create_address(name="im-" + fixed_ip, region=region, address=fixed_ip)
+            args['external_ip'] = 'ephemeral'
+            fixed_ip = self.request_fixed_ip(radl)
+            if fixed_ip:
+                if num_vm > 1:
+                    raise Exception("A fixed IP cannot be specified to a set of nodes (deploy is higher than 1)")
+
+                args['external_ip'] = driver.ex_create_address(name="im-" + fixed_ip, region=region, address=fixed_ip)
 
         res = []
         error_msg = "Error launching VM."

IM/tosca/Tosca.py

@@ -279,6 +279,12 @@ def _add_node_nets(self, node, radl, system, nodetemplates):
                         private_ip = self._final_function_result(cap_props["private_ip"].value, node)
                     if cap_props and "ports" in cap_props:
                         ports = self._final_function_result(cap_props["ports"].value, node)
+                    if cap_props and "port" in cap_props:
+                        port = self._final_function_result(cap_props["port"].value, node)
+                        protocol = "tcp"
+                        if "protocol" in cap_props:
+                            protocol = self._final_function_result(cap_props["protocol"].value, node)
+                        ports["im-%s-%s" % (protocol, port)] = {"protocol": protocol, "source": port}
 
             # The private net is always added
             if not public_ip or private_ip:

changelog

@@ -379,3 +379,8 @@ IM 1.6.7
     * Add --version option in im_service.py.
     * Fix error reconfiguring nodes in case of addition of nodes.
     * Avoid "Failed to lock apt for exclusive operation" in contextualization.
+
+IM 1.6.8
+    * Add support for the "endpoint" "port" field.
+    * Fix error that causes that some services left behind after deleting infrastructure on Azure.
+    * Fix error that causes that GCE conn assing public IPs to all VMs.

docker-devel/Dockerfile

@@ -1,7 +1,7 @@
 # Dockerfile to create a container with the IM service
 FROM grycap/jenkins:ubuntu16.04-im
 MAINTAINER Miguel Caballer <[email protected]>
-LABEL version="1.6.7"
+LABEL version="1.6.8"
 LABEL description="Container image to run the IM service with TOSCA support. (http://www.grycap.upv.es/im)"
 
 # Install tosca-parser

docker/Dockerfile

@@ -1,13 +1,13 @@
 # Dockerfile to create a container with the IM service and TOSCA support
 FROM ubuntu:16.04
 MAINTAINER Miguel Caballer <[email protected]>
-LABEL version="1.6.7"
+LABEL version="1.6.8"
 LABEL description="Container image to run the IM service with TOSCA support. (http://www.grycap.upv.es/im)"
 
 # Install Ansible
 RUN echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main" > /etc/apt/sources.list.d/ansible-ubuntu-ansible-xenial.list && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 && \ 
-    apt-get update && apt-get -y install wget ansible python-xmltodict && \
+    apt-get update && apt-get -y install wget ansible python-dbg python-xmltodict && \
     rm -rf /var/lib/apt/lists/*
 
 # Install Azure python SDK
@@ -23,7 +23,7 @@ RUN wget https://launchpad.net/ubuntu/+archive/primary/+files/python-msrest_0.4.
 
 # Install IM
 RUN wget https://github.com/grycap/RADL/releases/download/v1.1.0/python-radl_1.1.0-1_all.deb && \ 
-    wget https://github.com/indigo-dc/im/releases/download/v1.6.7/python-im_1.6.7-1_all.deb && \
+    wget https://github.com/indigo-dc/im/releases/download/v1.6.8/python-im_1.6.8-1_all.deb && \
     wget https://github.com/indigo-dc/tosca-parser/releases/download/0.9.1/python-tosca-parser_0.9.1-1_all.deb && \
     dpkg -i python-radl_*_all.deb ; \
     dpkg -i python-tosca-parser_*_all.deb ; \

test/files/tosca_long.yml

@@ -51,6 +51,7 @@ topology_template:
         endpoint:
           properties:
             network_name: PUBLIC
+            port: 8000
             ports:
               port_range:
                 protocol: tcp

test/unit/Tosca.py

@@ -52,12 +52,14 @@ def test_tosca_to_radl(self):
         net1 = radl.get_network_by_id('public_net_1')
         self.assertIn(net.getValue('provider_id'), ['vpc-XX.subnet-XX', None])
         if net.getValue('provider_id') is None:
-            self.assertEqual(net.getValue("outports"), '1:4/tcp')
+            self.assertIn('1:4/tcp', net.getValue("outports"))
+            self.assertIn('8000/tcp-8000/tcp', net.getValue("outports"))
             self.assertEqual(net1.getValue("outports"), '8080/tcp-8080/tcp')
         else:
             self.assertEqual(net.getValue('provider_id'), 'vpc-XX.subnet-XX')
             self.assertEqual(net.getValue("outports"), '8080/tcp-8080/tcp')
-            self.assertEqual(net1.getValue("outports"), '1:4/tcp')
+            self.assertIn('1:4/tcp', net1.getValue("outports"))
+            self.assertIn('8000/tcp-8000/tcp', net1.getValue("outports"))
         lrms_wn = radl.get_system_by_name('lrms_wn')
         self.assertEqual(lrms_wn.getValue('memory.size'), 2000000000)
         lrms_server = radl.get_system_by_name('lrms_server')