From 8bd9ae098d59b79a03b02d7cdab546e35d6d0429 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 12 Apr 2017 10:36:18 +0200 Subject: [PATCH 01/27] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 18e1ad978..9e05d9a3a 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ REST_SSL = True And then set the variables: XMLRCP_SSL_* or REST_SSL_* to your certificates paths. -2. DOCKER IMAGE +2 DOCKER IMAGE =============== A Docker image named `grycap/im` has been created to make easier the deployment of an IM service using the From f36280f9b3ee474b76db0a030785a5541288bb4e Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 24 May 2017 17:44:25 +0200 Subject: [PATCH 02/27] Implements #335 --- IM/connectors/Azure.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/IM/connectors/Azure.py b/IM/connectors/Azure.py index 7d3b0bbb2..8696accca 100644 --- a/IM/connectors/Azure.py +++ b/IM/connectors/Azure.py @@ -142,28 +142,25 @@ def get_instance_type(self, system, credentials, subscription_id): disk_free_op = system.getFeature('memory.size').getLogOperator() compute_client = ComputeManagementClient(credentials, subscription_id) - instace_types = compute_client.virtual_machine_sizes.list(location) + instace_types = list(compute_client.virtual_machine_sizes.list(location)) + instace_types.sort(key=lambda x: (x.number_of_cores, x.memory_in_mb, x.resource_disk_size_in_mb, x.name)) res = None default = None - for instace_type in list(instace_types): + for instace_type in instace_types: if instace_type.name == self.INSTANCE_TYPE: default = instace_type # get the instance type with the lowest Memory - if res is None or (instace_type.memory_in_mb <= res.memory_in_mb): + if res is None: str_compare = "instace_type.number_of_cores " + cpu_op + " cpu " str_compare += " and instace_type.memory_in_mb " + memory_op + " memory " - str_compare += " and instace_type.resource_disk_size_in_mb " + \ - disk_free_op + " disk_free" + str_compare += " and instace_type.resource_disk_size_in_mb " + disk_free_op + " disk_free" if eval(str_compare): if not instance_type_name or instace_type.name == instance_type_name: - res = instace_type + return instace_type - if res is None: - return default - else: - return res + return default def update_system_info_from_instance(self, system, instance_type): """ From 72d572283f180fa090c464354ad9fa764844cf26 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 24 May 2017 18:02:20 +0200 Subject: [PATCH 03/27] Implements #336 --- IM/connectors/GCE.py | 12 +++++++++++- test/unit/connectors/GCE.py | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 30182ceff..7a18060eb 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -199,6 +199,12 @@ def get_instance_type(self, sizes, radl): """ instance_type_name = radl.getValue('instance_type') + cpu = 1 + cpu_op = ">=" + if radl.getFeature('cpu.count'): + cpu = radl.getValue('cpu.count') + cpu_op = radl.getFeature('cpu.count').getLogOperator() + memory = 1 memory_op = ">1" if radl.getFeature('memory.size'): @@ -212,7 +218,11 @@ def get_instance_type(self, sizes, radl): if size.price is None: size.price = 0 if res is None or (size.price <= res.price and size.ram <= res.ram): - str_compare = "size.ram " + memory_op + " memory" + str_compare = "" + if 'guestCpus' in size.extra and size.extra['guestCpus']: + str_compare = "size.extra['guestCpus'] " + cpu_op + " cpu and " + str_compare += "size.ram " + memory_op + " memory" + if eval(str_compare): if not instance_type_name or size.name == instance_type_name: res = size diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index 0cba0a73b..f9fa4e418 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -109,11 +109,13 @@ def test_10_concrete(self, get_driver): node_size.price = 1.0 node_size.disk = 1 node_size.name = "small" + node_size.extra = {'guestCpus': 1} node_size2 = MagicMock() node_size2.ram = 1024 node_size2.price = None node_size2.disk = 2 node_size2.name = "medium" + node_size2.extra = {'guestCpus': 2} driver.list_sizes.return_value = [node_size, node_size2] gce_cloud = self.get_gce_cloud() @@ -156,6 +158,7 @@ def test_20_launch(self, get_driver): node_size.disk = 1 node_size.vcpus = 1 node_size.name = "small" + node_size.extra = {'guestCpus': 1} driver.list_sizes.return_value = [node_size] driver.ex_get_image.return_value = "image" From 144b82a791b2082f63a3337927850e1abe9ee322 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 09:51:58 +0200 Subject: [PATCH 04/27] Convert versions to int --- ansible_install.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible_install.yaml b/ansible_install.yaml index c285fdcbf..7b8a84ee0 100644 --- a/ansible_install.yaml +++ b/ansible_install.yaml @@ -19,19 +19,19 @@ - name: Set transport to ssh in ansible.cfg ini_file: dest=/etc/ansible/ansible.cfg section=defaults option=transport value=ssh - when: ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_major_version >= 6) or (ansible_os_family == "Suse" and ansible_distribution_major_version >= 10) + when: ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 6) or (ansible_os_family == "Suse" and ansible_distribution_major_version|int >= 10) - name: Set transport to smart in ansible.cfg ini_file: dest=/etc/ansible/ansible.cfg section=defaults option=transport value=smart - when: (ansible_os_family == "RedHat" and ansible_distribution_major_version < 6) or (ansible_os_family == "Suse" and ansible_distribution_major_version < 10) + when: (ansible_os_family == "RedHat" and ansible_distribution_major_version|int < 6) or (ansible_os_family == "Suse" and ansible_distribution_major_version|int < 10) - name: Change ssh_args to set ControlPersist to 15 min in ansible.cfg ini_file: dest=/etc/ansible/ansible.cfg section=ssh_connection option=ssh_args value="-o ControlMaster=auto -o ControlPersist=900s" - when: ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_major_version >= 7) or (ansible_os_family == "Suse" and ansible_distribution_major_version >= 12) + when: ansible_os_family == "Debian" or (ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 7) or (ansible_os_family == "Suse" and ansible_distribution_major_version|int >= 12) - name: Change ssh_args to remove ControlPersist in REL 6 and older in ansible.cfg ini_file: dest=/etc/ansible/ansible.cfg section=ssh_connection option=ssh_args value="" - when: (ansible_os_family == "RedHat" and ansible_distribution_major_version < 7) or (ansible_os_family == "Suse" and ansible_distribution_major_version < 12) + when: (ansible_os_family == "RedHat" and ansible_distribution_major_version|int < 7) or (ansible_os_family == "Suse" and ansible_distribution_major_version|int < 12) - name: Activate SSH pipelining in ansible.cfg ini_file: dest=/etc/ansible/ansible.cfg section=ssh_connection option=pipelining value=True From db343443119c101f444f50c906eaae47c2bca2fd Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 09:53:08 +0200 Subject: [PATCH 05/27] Update docs --- INSTALL | 2 +- README | 2 +- README.md | 2 +- doc/source/manual.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/INSTALL b/INSTALL index cad57c446..42f9f238e 100644 --- a/INSTALL +++ b/INSTALL @@ -72,7 +72,7 @@ In case of using the SSL secured version of the REST API pyOpenSSL must be insta 1.3.1 Using installer (Recommended option) ------------------------------------------ -The IM provides a script to install the IM in one single step. +The IM provides a script to install the IM in one single step (using pip). You only need to execute the following command: $ wget -qO- https://raw.githubusercontent.com/grycap/im/master/install.sh | bash diff --git a/README b/README index cf9286a3e..c17704af9 100644 --- a/README +++ b/README @@ -105,7 +105,7 @@ Microsoft Azure platform. 1.3.1 Using installer (Recommended option) ------------------------------------------ -The IM provides a script to install the IM in one single step. +The IM provides a script to install the IM in one single step (using pip). You only need to execute the following command: $ wget -qO- https://raw.githubusercontent.com/grycap/im/master/install.sh | bash diff --git a/README.md b/README.md index 6dddb3ec6..b96efc7cb 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Microsoft Azure platform. ### 1.3.1 Using installer (Recommended option) -The IM provides a script to install the IM in one single step. +The IM provides a script to install the IM in one single step (using pip). You only need to execute the following command: ```sh diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 88c027bcf..e84bb05b8 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -79,7 +79,7 @@ Installation Using installer (Recommended option) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The IM provides a script to install the IM in one single step. +The IM provides a script to install the IM in one single step (using pip). You only need to execute the following command:: $ wget -qO- https://raw.githubusercontent.com/grycap/im/master/install.sh | bash From ef0b62273762792437530119b96dee8ee2db7256 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 10:12:06 +0200 Subject: [PATCH 06/27] Improve ansible install --- ansible_install.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_install.yaml b/ansible_install.yaml index 7b8a84ee0..7aeac71d5 100644 --- a/ansible_install.yaml +++ b/ansible_install.yaml @@ -57,9 +57,9 @@ - name: Install last pip version easy_install: name=pip - # Step needed by ubuntu 14 - name: Install setuptools with pip pip: executable=pip name=setuptools,six extra_args="-I" + when: ansible_distribution == "Ubuntu" and ansible_distribution_major_version|int <= 14 - name: Install pip libraries pip: executable=pip name=pyOpenSSL,MySQL-python,msrest,msrestazure,azure-common,azure-mgmt-storage,azure-mgmt-compute,azure-mgmt-network,azure-mgmt-resource From a3b66047efd01053eca98fb53ed8cbc369bc5507 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 10:18:52 +0200 Subject: [PATCH 07/27] Minor changes --- docker/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2c0a53a7d..d07c37574 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,9 +6,7 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 RUN apt-get update && apt-get install -y gcc python-dbg python-dev python-pip libmysqld-dev python-pysqlite2 openssh-client sshpass libssl-dev libffi-dev python-requests # Install CherryPy to enable REST API -RUN pip install setuptools --upgrade -I RUN pip install cheroot -RUN pip install pyOpenSSL --upgrade -I # Install pip optional libraries RUN pip install MySQL-python msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource RUN pip install IM From 4d8587e9b71d07ce3785b3aaa11bed9e11faa5ad Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 11:51:05 +0200 Subject: [PATCH 08/27] Improve dockerfile --- docker/Dockerfile | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d07c37574..ce3c86040 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,11 +4,31 @@ MAINTAINER Miguel Caballer LABEL version="1.5.4" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 -RUN apt-get update && apt-get install -y gcc python-dbg python-dev python-pip libmysqld-dev python-pysqlite2 openssh-client sshpass libssl-dev libffi-dev python-requests -# Install CherryPy to enable REST API -RUN pip install cheroot -# Install pip optional libraries -RUN pip install MySQL-python msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource -RUN pip install IM +# Install Ansible +RUN echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main" > /etc/apt/sources.list.d/ansible-ubuntu-ansible-xenial.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 && \ + apt-get update && apt-get -y install wget ansible && \ + rm -rf /var/lib/apt/lists/* + +# Install Azure python SDK +RUN wget https://launchpad.net/ubuntu/+archive/primary/+files/python-msrest_0.4.4-1_all.deb && \ + wget https://launchpad.net/ubuntu/+archive/primary/+files/python-msrestazure_0.4.3-1_all.deb && \ + wget https://launchpad.net/ubuntu/+archive/primary/+files/python-azure_2.0.0~rc6+dfsg-2_all.deb && \ + dpkg -i python-msrest_*_all.deb ; \ + dpkg -i python-msrestazure_*_all.deb ; \ + dpkg -i python-azure_*_all.deb ; \ + rm *.deb && \ + apt update && apt install -f -y && \ + rm -rf /var/lib/apt/lists/* + +# Install IM +RUN wget https://github.com/grycap/RADL/releases/download/v1.1.0/python-radl_1.1.0-1_all.deb && \ + wget https://github.com/grycap/im/releases/download/v1.5.4/python-im_1.5.4-1_all.deb && \ + dpkg -i python-radl_*_all.deb ; \ + dpkg -i python-im_*_all.deb ; \ + rm *.deb && \ + apt update && apt install -f -y && \ + rm -rf /var/lib/apt/lists/* + COPY ansible.cfg /etc/ansible/ansible.cfg CMD im_service.py From 37117ee06aeae59265a12bfe2849f5e45bae427b Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 12:21:09 +0200 Subject: [PATCH 09/27] Add sshpass package --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ce3c86040..f8d8c0296 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ EXPOSE 8899 8800 # Install Ansible RUN echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main" > /etc/apt/sources.list.d/ansible-ubuntu-ansible-xenial.list && \ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 && \ - apt-get update && apt-get -y install wget ansible && \ + apt-get update && apt-get -y install wget ansible sshpass && \ rm -rf /var/lib/apt/lists/* # Install Azure python SDK From b33df7897538bebb0679c29b3c728bb79443bfbf Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 12:22:21 +0200 Subject: [PATCH 10/27] Remove sshpass package --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f8d8c0296..ce3c86040 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ EXPOSE 8899 8800 # Install Ansible RUN echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main" > /etc/apt/sources.list.d/ansible-ubuntu-ansible-xenial.list && \ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 && \ - apt-get update && apt-get -y install wget ansible sshpass && \ + apt-get update && apt-get -y install wget ansible && \ rm -rf /var/lib/apt/lists/* # Install Azure python SDK From ba147b824d89a0525f3da37a30c8275e908acbc6 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 12:42:49 +0200 Subject: [PATCH 11/27] Set version 1.5.5 --- IM/__init__.py | 2 +- changelog | 7 +++++++ docker-devel/Dockerfile | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/IM/__init__.py b/IM/__init__.py index 6f8ee2046..4f459e52b 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -19,5 +19,5 @@ 'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry', 'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPConnection', 'uriparse', 'VirtualMachine', 'VMRC', 'xmlobject'] -__version__ = '1.5.4' +__version__ = '1.5.5' __author__ = 'Miguel Caballer' diff --git a/changelog b/changelog index a09f01834..20cc5944d 100644 --- a/changelog +++ b/changelog @@ -301,3 +301,10 @@ IM 1.5.4 * Fix error in Azure connector creating a VM with two nics. * Fix error in Azure connector creating Storage Account with more than 24 chars. +IM 1.5.5 + * Fix error getting IP info in OCCI conn. + * Enable to reset the add_public_ip_count in the OCCI/OST conns. + * Improve Azure instance_type selection. + * Improve GCE instance type selection. + + diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 4a388dd79..52c1b9566 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM grycap/jenkins:ubuntu16.04-im MAINTAINER Miguel Caballer -LABEL version="1.5.4" +LABEL version="1.5.5" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 From 76237bce239c8b8c64f179a358e59184ba9339b6 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 25 May 2017 17:47:01 +0200 Subject: [PATCH 12/27] Do not delete SGs in case of VM deletion failuer --- IM/connectors/OpenNebula.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IM/connectors/OpenNebula.py b/IM/connectors/OpenNebula.py index d26e2b1b8..11ecf09d0 100644 --- a/IM/connectors/OpenNebula.py +++ b/IM/connectors/OpenNebula.py @@ -525,7 +525,8 @@ def finalize(self, vm, last, auth_data): else: return (False, "Error in the one.vm.action return value") - self.delete_security_groups(vm.inf, auth_data) + if success: + self.delete_security_groups(vm.inf, auth_data) return (success, err) From 111760f8a8ece0b9998b3859198b3b5f6e3118ba Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 29 May 2017 08:39:05 +0200 Subject: [PATCH 13/27] Minor changes --- IM/connectors/Azure.py | 2 +- IM/connectors/OCCI.py | 35 +++++++++++++++++------------------ 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/IM/connectors/Azure.py b/IM/connectors/Azure.py index 8696accca..1c2800424 100644 --- a/IM/connectors/Azure.py +++ b/IM/connectors/Azure.py @@ -143,7 +143,7 @@ def get_instance_type(self, system, credentials, subscription_id): compute_client = ComputeManagementClient(credentials, subscription_id) instace_types = list(compute_client.virtual_machine_sizes.list(location)) - instace_types.sort(key=lambda x: (x.number_of_cores, x.memory_in_mb, x.resource_disk_size_in_mb, x.name)) + instace_types.sort(key=lambda x: (x.number_of_cores, x.memory_in_mb, x.resource_disk_size_in_mb)) res = None default = None diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index b88ce7f78..fe1271a41 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -284,7 +284,7 @@ def get_property_from_category(self, occi_res, category, prop_name): def get_floating_pool(self, auth_data): """ - Get the first floating pool available (For OpenStack sites with Neutron) + Get a random floating pool available (For OpenStack sites with Neutron) """ _, occi_data = self.query_occi(auth_data) lines = occi_data.split("\n") @@ -989,26 +989,25 @@ def alterVM(self, vm, radl, auth_data): self.log_debug("Error waiting volume %s. Deleting it." % volume_id) self.delete_volume(volume_id, auth_data) return (False, "Error waiting volume %s. Deleting it." % volume_id) + else: + self.log_debug("Attaching to the instance") + attached = self.attach_volume(vm, volume_id, disk_device, mount_path, auth_data) + if attached: + orig_system.setValue("disk." + str(cont) + ".size", disk_size, "G") + orig_system.setValue("disk." + str(cont) + ".provider_id", volume_id) + if disk_device: + orig_system.setValue("disk." + str(cont) + ".device", disk_device) + if mount_path: + orig_system.setValue("disk." + str(cont) + ".mount_path", mount_path) + else: + self.log_error("Error attaching a %d GB volume for the disk %d." + " Deleting it." % (int(disk_size), cont)) + self.delete_volume(volume_id, auth_data) + return (False, "Error attaching the new volume") else: self.log_error("Error creating volume: %s" % volume_id) + return (False, "Error creating volume: %s" % volume_id) - if wait_ok: - self.log_debug("Attaching to the instance") - attached = self.attach_volume(vm, volume_id, disk_device, mount_path, auth_data) - if attached: - orig_system.setValue("disk." + str(cont) + ".size", disk_size, "G") - orig_system.setValue("disk." + str(cont) + ".provider_id", volume_id) - if disk_device: - orig_system.setValue("disk." + str(cont) + ".device", disk_device) - if mount_path: - orig_system.setValue("disk." + str(cont) + ".mount_path", mount_path) - else: - self.log_error("Error attaching a %d GB volume for the disk %d." - " Deleting it." % (int(disk_size), cont)) - self.delete_volume(volume_id, auth_data) - return (False, "Error attaching the new volume") - else: - return (False, "Error creating the new volume: " + volume_id) cont += 1 except Exception as ex: self.log_exception("Error connecting with OCCI server") From ed654bd4fdad2feffa933bc2c702ada216906173 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 31 May 2017 11:32:54 +0200 Subject: [PATCH 14/27] Manage DNS records in GCE conn: #340 --- IM/connectors/GCE.py | 143 +++++++++++++++++++++++++++++++++++- test/unit/connectors/GCE.py | 43 +++++++++-- 2 files changed, 176 insertions(+), 10 deletions(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 7a18060eb..38edc3f99 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -19,10 +19,13 @@ import os try: - from libcloud.compute.base import Node, NodeSize + from libcloud.compute.base import NodeSize from libcloud.compute.types import NodeState, Provider from libcloud.compute.providers import get_driver from libcloud.common.google import ResourceNotFoundError + from libcloud.dns.types import Provider as DNSProvider + from libcloud.dns.types import RecordType + from libcloud.dns.providers import get_driver as get_dns_driver except Exception as ex: print("WARN: libcloud library not correctly installed. GCECloudConnector will not work!.") print(ex) @@ -31,6 +34,7 @@ from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from radl.radl import Feature +from IM.config import Config class GCECloudConnector(CloudConnector): @@ -46,11 +50,12 @@ def __init__(self, cloud_info, inf): self.auth = None self.datacenter = None self.driver = None + self.dns_driver = None CloudConnector.__init__(self, cloud_info, inf) def get_driver(self, auth_data, datacenter=None): """ - Get the driver from the auth data + Get the compute driver from the auth data Arguments: - auth(Authentication): parsed authentication tokens. @@ -91,6 +96,46 @@ def get_driver(self, auth_data, datacenter=None): raise Exception( "No correct auth data has been specified to GCE: username, password and project") + def get_dns_driver(self, auth_data): + """ + Get the DNS driver from the auth data + + Arguments: + - auth(Authentication): parsed authentication tokens. + + Returns: a :py:class:`libcloud.dns.base.DNSDriver` or None in case of error + """ + auths = auth_data.getAuthInfo(self.type) + if not auths: + raise Exception("No auth data has been specified to GCE.") + else: + auth = auths[0] + + if self.dns_driver and self.auth.compare(auth_data, self.type): + return self.dns_driver + else: + self.auth = auth_data + + if 'username' in auth and 'password' in auth and 'project' in auth: + cls = get_dns_driver(DNSProvider.GOOGLE) + # Patch to solve some client problems with \\n + auth['password'] = auth['password'].replace('\\n', '\n') + lines = len(auth['password'].replace(" ", "").split()) + if lines < 2: + raise Exception("The certificate provided to the GCE plugin has an incorrect format." + " Check that it has more than one line.") + + driver = cls(auth['username'], auth['password'], project=auth['project']) + + self.dns_driver = driver + return driver + else: + self.log_error( + "No correct auth data has been specified to GCE: username, password and project") + self.log_debug(auth) + raise Exception( + "No correct auth data has been specified to GCE: username, password and project") + def concreteSystem(self, radl_system, auth_data): image_urls = radl_system.getValue("disk.0.image.url") if not image_urls: @@ -450,6 +495,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm.info.systems[0].setValue('instance_id', str(vm.id)) vm.info.systems[0].setValue('instance_name', str(vm.id)) self.log_debug("Node successfully created.") + res.append((True, vm)) for _ in range(len(nodes), num_vm): @@ -467,6 +513,7 @@ def finalize(self, vm, last, auth_data): if node: success = node.destroy() self.delete_disks(node) + self.del_dns_entries(vm, auth_data) if last: self.delete_firewall(vm, node.driver) @@ -673,11 +720,103 @@ def updateVMInfo(self, vm, auth_data): vm.setIps(node.public_ips, node.private_ips) self.attach_volumes(vm, node) + self.add_dns_entries(vm, auth_data) else: vm.state = VirtualMachine.OFF return (True, vm) + def add_dns_entries(self, vm, auth_data): + """ + Add the required entries in the Google DNS system + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + """ + try: + driver = self.get_dns_driver(auth_data) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain": + if not domain.endswith("."): + domain += "." + zone = [z for z in driver.iterate_zones() if z.domain == domain] + if not zone: + self.log_debug("Creating DNS zone %s" % domain) + zone = driver.create_zone(domain) + else: + zone = zone[0] + self.log_debug("DNS zone %s exists. Do not create." % domain) + + if zone: + fqdn = hostname + "." + domain + record = [r for r in driver.iterate_records(zone) if r.name == fqdn] + if not record: + self.log_debug("Creating DNS record %s." % fqdn) + driver.create_record(fqdn, zone, RecordType.A, ip) + else: + self.log_debug("DNS record %s exists. Do not create." % fqdn) + + return True + except Exception: + self.log_exception("Error creating DNS entries") + return False + + def del_dns_entries(self, vm, auth_data): + """ + Delete the added entries in the Google DNS system + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + """ + try: + driver = self.get_dns_driver(auth_data) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain": + if not domain.endswith("."): + domain += "." + zone = [z for z in driver.iterate_zones() if z.domain == domain] + if not zone: + self.log_debug("The DNS zone %s does not exists. Do not delete records." % domain) + else: + zone = zone[0] + fqdn = hostname + "." + domain + record = [r for r in driver.iterate_records(zone) if r.name == fqdn] + if not record: + self.log_debug("DNS record %s does not exists. Do not delete." % fqdn) + else: + record = record[0] + if record.data != ip: + self.log_debug("DNS record %s mapped to unexpected IP: %s != %s." + "Do not delete." % (fqdn, record.data, ip)) + else: + self.log_debug("Deleting DNS record %s." % fqdn) + if not driver.delete_record(record): + self.log_error("Error deleting DNS record %s." % fqdn) + + # if there are no records, delete the zone + all_records = [r for r in driver.iterate_records(zone)] + if not all_records: + driver.delete_zone(zone) + + return True + except Exception: + self.log_exception("Error deleting DNS entries") + return False + def start(self, vm, auth_data): driver = self.get_driver(auth_data) diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index f9fa4e418..1bb56fbb9 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -34,7 +34,7 @@ from IM.VirtualMachine import VirtualMachine from IM.InfrastructureInfo import InfrastructureInfo from IM.connectors.GCE import GCECloudConnector -from mock import patch, MagicMock +from mock import patch, MagicMock, call from libcloud.compute.base import NodeSize @@ -191,7 +191,8 @@ def test_20_launch(self, get_driver): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.gce.GCENodeDriver') - def test_30_updateVMInfo(self, get_driver): + @patch('libcloud.dns.drivers.google.GoogleDNSDriver') + def test_30_updateVMInfo(self, get_dns_driver, get_driver): radl_data = """ network net (outbound = 'yes') system test ( @@ -199,7 +200,7 @@ def test_30_updateVMInfo(self, get_driver): cpu.count=1 and memory.size=512m and net_interface.0.connection = 'net' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'test.domain.com' and disk.0.os.name = 'linux' and disk.0.image.url = 'gce://us-central1-a/centos-6' and disk.0.os.credentials.username = 'user' and @@ -219,13 +220,14 @@ def test_30_updateVMInfo(self, get_driver): driver = MagicMock() get_driver.return_value = driver + dns_driver = MagicMock() + get_dns_driver.return_value = dns_driver node = MagicMock() zone = MagicMock() node.id = "1" node.state = "running" node.extra = {'flavorId': 'small'} - node.public_ips = [] node.public_ips = ['158.42.1.1'] node.private_ips = ['10.0.0.1'] node.driver = driver @@ -240,9 +242,20 @@ def test_30_updateVMInfo(self, get_driver): volume.extra = {'status': 'READY'} driver.create_volume.return_value = volume + dns_driver.iterate_zones.return_value = [] + dns_driver.iterate_records.return_value = [] + success, vm = gce_cloud.updateVMInfo(vm, auth) self.assertTrue(success, msg="ERROR: updating VM info.") + + self.assertEquals(dns_driver.create_zone.call_count, 1) + self.assertEquals(dns_driver.create_record.call_count, 1) + self.assertEquals(dns_driver.create_zone.call_args_list[0], call('domain.com.')) + self.assertEquals(dns_driver.create_record.call_args_list[0][0][0], 'test.domain.com.') + self.assertEquals(dns_driver.create_record.call_args_list[0][0][2], 'A') + self.assertEquals(dns_driver.create_record.call_args_list[0][0][3], '158.42.1.1') + self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.gce.GCENodeDriver') @@ -286,16 +299,21 @@ def test_50_start(self, get_driver): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.gce.GCENodeDriver') + @patch('libcloud.dns.drivers.google.GoogleDNSDriver') @patch('time.sleep') - def test_60_finalize(self, sleep, get_driver): + def test_60_finalize(self, sleep, get_dns_driver, get_driver): auth = Authentication([{'id': 'one', 'type': 'GCE', 'username': 'user', 'password': 'pass\npass', 'project': 'proj'}]) gce_cloud = self.get_gce_cloud() radl_data = """ + network net (outbound = 'yes') system test ( - cpu.count>=2 and - memory.size>=2048m + cpu.count=2 and + memory.size=2048m and + net_interface.0.connection = 'net' and + net_interface.0.dns_name = 'test.domain.com' and + net_interface.0.ip = '158.42.1.1' )""" radl = radl_parse.parse_radl(radl_data) @@ -303,8 +321,9 @@ def test_60_finalize(self, sleep, get_driver): vm = VirtualMachine(inf, "1", gce_cloud.cloud, radl, radl, gce_cloud, 1) driver = MagicMock() - driver.name = "OpenStack" get_driver.return_value = driver + dns_driver = MagicMock() + get_dns_driver.return_value = dns_driver node = MagicMock() node.destroy.return_value = True @@ -317,6 +336,14 @@ def test_60_finalize(self, sleep, get_driver): volume.destroy.return_value = True driver.ex_get_volume.return_value = volume + zone = MagicMock() + zone.domain = "domain.com." + dns_driver.iterate_zones.return_value = [zone] + record = MagicMock() + record.name = 'test.domain.com.' + record.data = '158.42.1.1' + dns_driver.iterate_records.return_value = [record] + success, _ = gce_cloud.finalize(vm, True, auth) self.assertTrue(success, msg="ERROR: finalizing VM info.") From 2aa6703a4df4a099a4544ea95da9ea8c5d40324c Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 31 May 2017 11:42:37 +0200 Subject: [PATCH 15/27] Improve test --- test/unit/connectors/GCE.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index 1bb56fbb9..5f5c971f8 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -347,6 +347,8 @@ def test_60_finalize(self, sleep, get_dns_driver, get_driver): success, _ = gce_cloud.finalize(vm, True, auth) self.assertTrue(success, msg="ERROR: finalizing VM info.") + self.assertEquals(dns_driver.delete_record.call_count, 1) + self.assertEquals(dns_driver.delete_record.call_args_list[0][0][0].name, 'test.domain.com.') self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) From 6ac102daa012883d57c303cd7837831dcd0db3e0 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 31 May 2017 13:10:57 +0200 Subject: [PATCH 16/27] Fix errors in GCE conn: #340 --- IM/connectors/GCE.py | 6 +++--- test/unit/connectors/GCE.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 38edc3f99..dd72099f9 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -759,7 +759,7 @@ def add_dns_entries(self, vm, auth_data): record = [r for r in driver.iterate_records(zone) if r.name == fqdn] if not record: self.log_debug("Creating DNS record %s." % fqdn) - driver.create_record(fqdn, zone, RecordType.A, ip) + driver.create_record(fqdn, zone, RecordType.A, dict(ttl=300, rrdatas=[ip])) else: self.log_debug("DNS record %s exists. Do not create." % fqdn) @@ -799,9 +799,9 @@ def del_dns_entries(self, vm, auth_data): self.log_debug("DNS record %s does not exists. Do not delete." % fqdn) else: record = record[0] - if record.data != ip: + if record.data['rrdatas'] != [ip]: self.log_debug("DNS record %s mapped to unexpected IP: %s != %s." - "Do not delete." % (fqdn, record.data, ip)) + "Do not delete." % (fqdn, record.data['rrdatas'], ip)) else: self.log_debug("Deleting DNS record %s." % fqdn) if not driver.delete_record(record): diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index 5f5c971f8..7c9f88d3a 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -254,7 +254,7 @@ def test_30_updateVMInfo(self, get_dns_driver, get_driver): self.assertEquals(dns_driver.create_zone.call_args_list[0], call('domain.com.')) self.assertEquals(dns_driver.create_record.call_args_list[0][0][0], 'test.domain.com.') self.assertEquals(dns_driver.create_record.call_args_list[0][0][2], 'A') - self.assertEquals(dns_driver.create_record.call_args_list[0][0][3], '158.42.1.1') + self.assertEquals(dns_driver.create_record.call_args_list[0][0][3], {'rrdatas': ['158.42.1.1'], 'ttl': 300}) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @@ -341,7 +341,7 @@ def test_60_finalize(self, sleep, get_dns_driver, get_driver): dns_driver.iterate_zones.return_value = [zone] record = MagicMock() record.name = 'test.domain.com.' - record.data = '158.42.1.1' + record.data = {'rrdatas': ['158.42.1.1'], 'ttl': 300} dns_driver.iterate_records.return_value = [record] success, _ = gce_cloud.finalize(vm, True, auth) From 09c6399101c8493c74a4e4664bdde3e508d17b63 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 31 May 2017 13:21:23 +0200 Subject: [PATCH 17/27] Fix errors in GCE conn: #340 --- IM/connectors/GCE.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index dd72099f9..f8d44e47d 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -807,8 +807,8 @@ def del_dns_entries(self, vm, auth_data): if not driver.delete_record(record): self.log_error("Error deleting DNS record %s." % fqdn) - # if there are no records, delete the zone - all_records = [r for r in driver.iterate_records(zone)] + # if there are no records (except the NS and SOA auto added ones), delete the zone + all_records = [r for r in driver.iterate_records(zone) if r.type not in [RecordType.NS, RecordType.SOA]] if not all_records: driver.delete_zone(zone) From 9a1545a17303efd169fd6af554c3c173175fe430 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 31 May 2017 17:42:26 +0200 Subject: [PATCH 18/27] Style change --- IM/connectors/GCE.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index f8d44e47d..3cb667bbf 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -808,7 +808,8 @@ def del_dns_entries(self, vm, auth_data): self.log_error("Error deleting DNS record %s." % fqdn) # if there are no records (except the NS and SOA auto added ones), delete the zone - all_records = [r for r in driver.iterate_records(zone) if r.type not in [RecordType.NS, RecordType.SOA]] + all_records = [r for r in driver.iterate_records(zone) + if r.type not in [RecordType.NS, RecordType.SOA]] if not all_records: driver.delete_zone(zone) From f77e3ee779da10a31a2206fc19aed5fcd7d02fbe Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 31 May 2017 17:42:48 +0200 Subject: [PATCH 19/27] Manage DNS records in EC2 conn: #340 --- IM/connectors/EC2.py | 133 ++++++++++++++++++++++++++++++++++++ test/unit/connectors/EC2.py | 46 +++++++++++-- 2 files changed, 175 insertions(+), 4 deletions(-) diff --git a/IM/connectors/EC2.py b/IM/connectors/EC2.py index 8eea25b19..bea73d766 100644 --- a/IM/connectors/EC2.py +++ b/IM/connectors/EC2.py @@ -22,6 +22,7 @@ try: import boto.ec2 import boto.vpc + import boto.route53 except Exception as ex: print("WARN: Boto library not correctly installed. EC2CloudConnector will not work!.") print(ex) @@ -30,6 +31,7 @@ from IM.VirtualMachine import VirtualMachine from .CloudConnector import CloudConnector from radl.radl import Feature +from IM.config import Config class InstanceTypeInfo: @@ -87,6 +89,7 @@ class EC2CloudConnector(CloudConnector): def __init__(self, cloud_info, inf): self.connection = None + self.route53_connection = None self.auth = None CloudConnector.__init__(self, cloud_info, inf) @@ -197,6 +200,45 @@ def get_connection(self, region_name, auth_data): self.connection = conn return conn + # Get the Route53 connection object + def get_route53_connection(self, region_name, auth_data): + """ + Get a :py:class:`boto.route53.connection` to interact with. + + Arguments: + - region_name(str): AWS region to connect. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + Returns: a :py:class:`boto.route53.connection` or None in case of error + """ + auths = auth_data.getAuthInfo(self.type) + if not auths: + raise Exception("No auth data has been specified to EC2.") + else: + auth = auths[0] + + if self.route53_connection and self.auth.compare(auth_data, self.type): + return self.route53_connection + else: + self.auth = auth_data + conn = None + try: + if 'username' in auth and 'password' in auth: + conn = boto.route53.connect_to_region(region_name, + aws_access_key_id=auth['username'], + aws_secret_access_key=auth['password']) + else: + self.log_error("No correct auth data has been specified to EC2: " + "username (Access Key) and password (Secret Key)") + raise Exception("No correct auth data has been specified to EC2: " + "username (Access Key) and password (Secret Key)") + + except Exception as ex: + self.log_exception("Error conneting Route53 in region " + region_name) + raise Exception("Error conneting Route53 in region" + region_name + ": " + str(ex)) + + self.route53_connection = conn + return conn + # el path sera algo asi: aws://eu-west-1/ami-00685b74 def getAMIData(self, path): """ @@ -1075,6 +1117,7 @@ def updateVMInfo(self, vm, auth_data): self.setIPsFromInstance(vm, instance) self.attach_volumes(instance, vm) + self.add_dns_entries(vm, auth_data) try: vm.info.systems[0].setValue('launch_time', int(time.mktime( @@ -1088,6 +1131,90 @@ def updateVMInfo(self, vm, auth_data): return (True, vm) + def add_dns_entries(self, vm, auth_data): + """ + Add the required entries in the AWS Route53 service + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + """ + try: + region = vm.id.split(";")[0] + conn = self.get_route53_connection(region, auth_data) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain": + if not domain.endswith("."): + domain += "." + zone = conn.get_zone(domain) + if not zone: + self.log_debug("Creating DNS zone %s" % domain) + zone = conn.create_zone(domain) + else: + self.log_debug("DNS zone %s exists. Do not create." % domain) + + if zone: + fqdn = hostname + "." + domain + record = zone.get_a(fqdn) + if not record: + self.log_debug("Creating DNS record %s." % fqdn) + changes = boto.route53.record.ResourceRecordSets(conn, zone.id) + change = changes.add_change("CREATE", fqdn, "A") + change.add_value(ip) + result = changes.commit() + else: + self.log_debug("DNS record %s exists. Do not create." % fqdn) + + return True + except Exception: + self.log_exception("Error creating DNS entries") + return False + + def del_dns_entries(self, vm, auth_data): + """ + Delete the added entries in the AWS Route53 service + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. + """ + region = vm.id.split(";")[0] + conn = self.get_route53_connection(region, auth_data) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain": + if not domain.endswith("."): + domain += "." + zone = conn.get_zone(domain) + if not zone: + self.log_debug("The DNS zone %s does not exists. Do not delete records." % domain) + else: + fqdn = hostname + "." + domain + record = zone.get_a(fqdn) + if not record: + self.log_debug("DNS record %s does not exists. Do not delete." % fqdn) + else: + self.log_debug("Deleting DNS record %s." % fqdn) + changes = boto.route53.record.ResourceRecordSets(conn, zone.id) + change = changes.add_change("DELETE", fqdn, "A") + change.add_value(ip) + result = changes.commit() + + # if there are no A records + if not zone.find_records(type="A"): + conn.delete_hosted_zone(zone.id) + def cancel_spot_requests(self, conn, vm): """ Cancel the spot requests of a VM @@ -1161,6 +1288,12 @@ def finalize(self, vm, last, auth_data): except: self.log_exception("Error deleting EBS volumess") + # Delete the DNS entries + try: + self.del_dns_entries(vm, auth_data) + except: + self.log_exception("Error deleting EBS volumess") + return (True, "") def _get_security_groups(self, conn, vm): diff --git a/test/unit/connectors/EC2.py b/test/unit/connectors/EC2.py index d614b3c17..027039165 100755 --- a/test/unit/connectors/EC2.py +++ b/test/unit/connectors/EC2.py @@ -280,7 +280,9 @@ def test_25_launch_spot(self, blockdevicemapping, VPCConnection, get_region): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.EC2.EC2CloudConnector.get_connection') - def test_30_updateVMInfo(self, get_connection): + @patch('boto.route53.connect_to_region') + @patch('boto.route53.record.ResourceRecordSets') + def test_30_updateVMInfo(self, record_sets, connect_to_region, get_connection): radl_data = """ network net (outbound = 'yes') system test ( @@ -289,7 +291,7 @@ def test_30_updateVMInfo(self, get_connection): memory.size=512m and net_interface.0.connection = 'net' and net_interface.0.ip = '158.42.1.1' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'test.domain.com' and disk.0.os.name = 'linux' and disk.0.image.url = 'one://server.com/1' and disk.0.os.credentials.username = 'user' and @@ -335,9 +337,25 @@ def test_30_updateVMInfo(self, get_connection): conn.create_volume.return_value = volume conn.attach_volume.return_value = True + dns_conn = MagicMock() + connect_to_region.return_value = dns_conn + + dns_conn.get_zone.return_value = None + zone = MagicMock() + zone.get_a.return_value = None + dns_conn.create_zone.return_value = zone + changes = MagicMock() + record_sets.return_value = changes + change = MagicMock() + changes.add_change.return_value = change + success, vm = ec2_cloud.updateVMInfo(vm, auth) self.assertTrue(success, msg="ERROR: updating VM info.") + self.assertEquals(dns_conn.create_zone.call_count, 1) + self.assertEquals(dns_conn.create_zone.call_args_list[0][0][0], "domain.com.") + self.assertEquals(changes.add_change.call_args_list, [call('CREATE', 'test.domain.com.', 'A')]) + self.assertEquals(change.add_value.call_args_list, [call('158.42.1.1')]) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.EC2.EC2CloudConnector.get_connection') @@ -497,7 +515,9 @@ def test_55_alter(self, get_connection): @patch('IM.connectors.EC2.EC2CloudConnector.get_connection') @patch('time.sleep') - def test_60_finalize(self, sleep, get_connection): + @patch('boto.route53.connect_to_region') + @patch('boto.route53.record.ResourceRecordSets') + def test_60_finalize(self, record_sets, connect_to_region, sleep, get_connection): radl_data = """ network net (outbound = 'yes') system test ( @@ -506,7 +526,7 @@ def test_60_finalize(self, sleep, get_connection): memory.size=512m and net_interface.0.connection = 'net' and net_interface.0.ip = '158.42.1.1' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'test.domain.com' and disk.0.os.name = 'linux' and disk.0.image.url = 'one://server.com/1' and disk.0.os.credentials.username = 'user' and @@ -561,9 +581,27 @@ def test_60_finalize(self, sleep, get_connection): sg.delete.return_value = True conn.get_all_security_groups.return_value = [sg] + dns_conn = MagicMock() + connect_to_region.return_value = dns_conn + + zone = MagicMock() + record = MagicMock() + zone.id = "zid" + zone.get_a.return_value = record + zone.find_records.return_value = None + dns_conn.get_zone.return_value = zone + changes = MagicMock() + record_sets.return_value = changes + change = MagicMock() + changes.add_change.return_value = change + success, _ = ec2_cloud.finalize(vm, True, auth) self.assertTrue(success, msg="ERROR: finalizing VM info.") + self.assertEquals(dns_conn.delete_hosted_zone.call_count, 1) + self.assertEquals(dns_conn.delete_hosted_zone.call_args_list[0][0][0], zone.id) + self.assertEquals(changes.add_change.call_args_list, [call('DELETE', 'test.domain.com.', 'A')]) + self.assertEquals(change.add_value.call_args_list, [call('158.42.1.1')]) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.EC2.EC2CloudConnector.get_connection') From bddf4745a33684c44c88c55420096fea30930ae5 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 1 Jun 2017 08:45:36 +0200 Subject: [PATCH 20/27] Minor changes: #340 --- IM/connectors/EC2.py | 4 ++-- IM/connectors/GCE.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/IM/connectors/EC2.py b/IM/connectors/EC2.py index bea73d766..aa77f21a2 100644 --- a/IM/connectors/EC2.py +++ b/IM/connectors/EC2.py @@ -462,7 +462,7 @@ def create_keypair(self, system, conn): system.setUserKeyCredentials( system.getCredentials().username, public, private) else: - self.log_debug("Creating the Keypair") + self.log_debug("Creating the Keypair name: %s" % keypair_name) keypair_file = self.KEYPAIR_DIR + '/' + keypair_name + '.pem' keypair = conn.create_key_pair(keypair_name) created = True @@ -1149,7 +1149,7 @@ def add_dns_entries(self, vm, auth_data): (hostname, domain) = vm.getRequestedNameIface(num_conn, default_hostname=Config.DEFAULT_VM_NAME, default_domain=Config.DEFAULT_DOMAIN) - if domain != "localdomain": + if domain != "localdomain" and ip: if not domain.endswith("."): domain += "." zone = conn.get_zone(domain) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 3cb667bbf..98b4533e2 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -743,7 +743,7 @@ def add_dns_entries(self, vm, auth_data): (hostname, domain) = vm.getRequestedNameIface(num_conn, default_hostname=Config.DEFAULT_VM_NAME, default_domain=Config.DEFAULT_DOMAIN) - if domain != "localdomain": + if domain != "localdomain" and ip: if not domain.endswith("."): domain += "." zone = [z for z in driver.iterate_zones() if z.domain == domain] From 1ebed6480e8710c8d9c18250b67150a629c2c2fd Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 1 Jun 2017 08:47:17 +0200 Subject: [PATCH 21/27] Minor changes: #340 --- IM/connectors/EC2.py | 2 +- IM/connectors/GCE.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/IM/connectors/EC2.py b/IM/connectors/EC2.py index aa77f21a2..57b72d1c3 100644 --- a/IM/connectors/EC2.py +++ b/IM/connectors/EC2.py @@ -1193,7 +1193,7 @@ def del_dns_entries(self, vm, auth_data): (hostname, domain) = vm.getRequestedNameIface(num_conn, default_hostname=Config.DEFAULT_VM_NAME, default_domain=Config.DEFAULT_DOMAIN) - if domain != "localdomain": + if domain != "localdomain" and ip: if not domain.endswith("."): domain += "." zone = conn.get_zone(domain) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 98b4533e2..ca1e8e72f 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -785,7 +785,7 @@ def del_dns_entries(self, vm, auth_data): (hostname, domain) = vm.getRequestedNameIface(num_conn, default_hostname=Config.DEFAULT_VM_NAME, default_domain=Config.DEFAULT_DOMAIN) - if domain != "localdomain": + if domain != "localdomain" and ip: if not domain.endswith("."): domain += "." zone = [z for z in driver.iterate_zones() if z.domain == domain] From f6868ecf71f13a0501b79a304c619e18ed389b64 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 1 Jun 2017 09:54:46 +0200 Subject: [PATCH 22/27] Manage DNS records in Azure: #340 --- IM/connectors/Azure.py | 102 ++++++++++++++++++++++++++++++++++ test/unit/connectors/Azure.py | 50 +++++++++++++---- 2 files changed, 141 insertions(+), 11 deletions(-) diff --git a/IM/connectors/Azure.py b/IM/connectors/Azure.py index 1c2800424..fd1376006 100644 --- a/IM/connectors/Azure.py +++ b/IM/connectors/Azure.py @@ -21,12 +21,14 @@ from IM.VirtualMachine import VirtualMachine from .CloudConnector import CloudConnector from radl.radl import Feature +from IM.config import Config try: from azure.mgmt.resource import ResourceManagementClient from azure.mgmt.storage import StorageManagementClient from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.network import NetworkManagementClient + from azure.mgmt.dns import DnsManagementClient from azure.common.credentials import UserPassCredentials except Exception as ex: print("WARN: Python Azure SDK not correctly installed. AzureCloudConnector will not work!.") @@ -623,8 +625,106 @@ def updateVMInfo(self, vm, auth_data): # Update IP info self.setIPs(vm, virtual_machine.network_profile, credentials, subscription_id) + self.add_dns_entries(vm, credentials, subscription_id) return (True, vm) + def add_dns_entries(self, vm, credentials, subscription_id): + """ + Add the required entries in the Azure DNS service + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - credentials, subscription_id: Authentication data to access cloud provider. + """ + try: + group_name = vm.id.split('/')[0] + dns_client = DnsManagementClient(credentials, subscription_id) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain" and ip: + zone = None + try: + zone = dns_client.zones.get(group_name, domain) + except Exception: + pass + if not zone: + self.log_debug("Creating DNS zone %s" % domain) + zone = dns_client.zones.create_or_update(group_name, domain, + {'location': 'global'}) + else: + self.log_debug("DNS zone %s exists. Do not create." % domain) + + if zone: + record = None + try: + record = dns_client.record_sets.get(group_name, domain, hostname, 'A') + except Exception: + pass + if not record: + self.log_debug("Creating DNS record %s." % hostname) + record_data = {"ttl": 300, "arecords": [{"ipv4_address": ip}]} + record_set = dns_client.record_sets.create_or_update(group_name, domain, + hostname, 'A', + record_data) + else: + self.log_debug("DNS record %s exists. Do not create." % hostname) + + return True + except Exception: + self.log_exception("Error creating DNS entries") + return False + + def del_dns_entries(self, vm, credentials, subscription_id): + """ + Delete the added entries in the Azure DNS service + + Arguments: + - vm(:py:class:`IM.VirtualMachine`): VM information. + - credentials, subscription_id: Authentication data to access cloud provider. + """ + try: + group_name = vm.id.split('/')[0] + dns_client = DnsManagementClient(credentials, subscription_id) + system = vm.info.systems[0] + for net_name in system.getNetworkIDs(): + num_conn = system.getNumNetworkWithConnection(net_name) + ip = system.getIfaceIP(num_conn) + (hostname, domain) = vm.getRequestedNameIface(num_conn, + default_hostname=Config.DEFAULT_VM_NAME, + default_domain=Config.DEFAULT_DOMAIN) + if domain != "localdomain" and ip: + if not domain.endswith("."): + domain += "." + zone = None + try: + zone = dns_client.zones.get(group_name, domain) + except Exception: + pass + if not zone: + self.log_debug("The DNS zone %s does not exists. Do not delete records." % domain) + else: + record = None + try: + record = dns_client.record_sets.get(group_name, domain, hostname, 'A') + except Exception: + pass + if not record: + self.log_debug("DNS record %s does not exists. Do not delete." % hostname) + else: + self.log_debug("Deleting DNS record %s." % hostname) + dns_client.record_sets.delete(group_name, domain, hostname, 'A') + + # if there are no A records + if not dns_client.record_sets.list_by_type(group_name, domain, 'A'): + dns_client.zones.delete(group_name, domain) + except Exception as ex: + self.log_exception("Error removing DNS entries.") + def setIPs(self, vm, network_profile, credentials, subscription_id): """ Set the information about the IPs of the VM @@ -663,6 +763,8 @@ def finalize(self, vm, last, auth_data): self.log_debug("Removing RG: %s" % group_name) resource_client.resource_groups.delete(group_name).wait() + self.del_dns_entries(vm, credentials, subscription_id) + # if it is the last VM delete the RG of the Inf if last: self.log_debug("Removing RG: %s" % "rg-%s" % vm.inf.id) diff --git a/test/unit/connectors/Azure.py b/test/unit/connectors/Azure.py index b414592d1..f095453bf 100755 --- a/test/unit/connectors/Azure.py +++ b/test/unit/connectors/Azure.py @@ -34,7 +34,7 @@ from IM.VirtualMachine import VirtualMachine from IM.InfrastructureInfo import InfrastructureInfo from IM.connectors.Azure import AzureCloudConnector -from mock import patch, MagicMock +from mock import patch, MagicMock, call def read_file_as_string(file_name): @@ -179,8 +179,9 @@ def test_20_launch(self, credentials, network_client, compute_client, storage_cl @patch('IM.connectors.Azure.NetworkManagementClient') @patch('IM.connectors.Azure.ComputeManagementClient') + @patch('IM.connectors.Azure.DnsManagementClient') @patch('IM.connectors.Azure.UserPassCredentials') - def test_30_updateVMInfo(self, credentials, compute_client, network_client): + def test_30_updateVMInfo(self, credentials, dns_client, compute_client, network_client): radl_data = """ network net (outbound = 'yes') system test ( @@ -188,7 +189,7 @@ def test_30_updateVMInfo(self, credentials, compute_client, network_client): cpu.count=1 and memory.size=512m and net_interface.0.connection = 'net' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'test.domain.com' and disk.0.os.name = 'linux' and disk.0.image.url = 'azr://Canonical/UbuntuServer/16.04.0-LTS/latest' and disk.0.os.credentials.username = 'user' and @@ -214,14 +215,14 @@ def test_30_updateVMInfo(self, credentials, compute_client, network_client): compute_client.return_value = cclient cclient.virtual_machine_sizes.list.return_value = instace_types - vm = MagicMock() - vm.provisioning_state = "Succeeded" - vm.hardware_profile.vm_size = "instance_type1" - vm.location = "northeurope" + avm = MagicMock() + avm.provisioning_state = "Succeeded" + avm.hardware_profile.vm_size = "instance_type1" + avm.location = "northeurope" ni = MagicMock() ni.id = "/subscriptions/subscription-id/resourceGroups/rg0/providers/Microsoft.Network/networkInterfaces/ni-0" - vm.network_profile.network_interfaces = [ni] - cclient.virtual_machines.get.return_value = vm + avm.network_profile.network_interfaces = [ni] + cclient.virtual_machines.get.return_value = avm nclient = MagicMock() network_client.return_value = nclient @@ -237,9 +238,19 @@ def test_30_updateVMInfo(self, credentials, compute_client, network_client): pub_ip_res.ip_address = "13.0.0.1" nclient.public_ip_addresses.get.return_value = pub_ip_res + dclient = MagicMock() + dns_client.return_value = dclient + dclient.zones.get.return_value = None + dclient.record_sets.get.return_value = None + success, vm = azure_cloud.updateVMInfo(vm, auth) self.assertTrue(success, msg="ERROR: updating VM info.") + self.assertEquals(dclient.zones.create_or_update.call_args_list, + [call('rg0', 'domain.com', {'location': 'global'})]) + self.assertEquals(dclient.record_sets.create_or_update.call_args_list, + [call('rg0', 'domain.com', 'test', 'A', + {'arecords': [{'ipv4_address': '13.0.0.1'}], 'ttl': 300})]) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.Azure.ComputeManagementClient') @@ -346,18 +357,35 @@ def test_55_alter(self, credentials, network_client, compute_client, storage_cli self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.Azure.ResourceManagementClient') + @patch('IM.connectors.Azure.DnsManagementClient') @patch('IM.connectors.Azure.UserPassCredentials') - def test_60_finalize(self, credentials, resource_client): + def test_60_finalize(self, credentials, dns_client, resource_client): auth = Authentication([{'id': 'azure', 'type': 'Azure', 'subscription_id': 'subscription_id', 'username': 'user', 'password': 'password'}]) azure_cloud = self.get_azure_cloud() + radl_data = """ + network net (outbound = 'yes') + system test ( + cpu.count=1 and + memory.size=512m and + net_interface.0.connection = 'net' and + net_interface.0.ip = '158.42.1.1' and + net_interface.0.dns_name = 'test.domain.com' + )""" + radl = radl_parse.parse_radl(radl_data) inf = MagicMock() - vm = VirtualMachine(inf, "rg0/vm0", azure_cloud.cloud, "", "", azure_cloud, 1) + vm = VirtualMachine(inf, "rg0/vm0", azure_cloud.cloud, radl, radl, azure_cloud, 1) + + dclient = MagicMock() + dns_client.return_value = dclient + dclient.record_sets.list_by_type.return_value = [] success, _ = azure_cloud.finalize(vm, True, auth) self.assertTrue(success, msg="ERROR: finalizing VM info.") + self.assertEquals(dclient.record_sets.delete.call_args_list, [call('rg0', 'domain.com.', 'test', 'A')]) + self.assertEquals(dclient.zones.delete.call_args_list, [call('rg0', 'domain.com.')]) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) From 41947e246a5bcbc962cfa848f2abd998e19710e8 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 1 Jun 2017 10:50:26 +0200 Subject: [PATCH 23/27] Fix errors in EC2 conn: #340 --- IM/connectors/EC2.py | 17 +++++++++-------- test/unit/connectors/EC2.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/IM/connectors/EC2.py b/IM/connectors/EC2.py index 57b72d1c3..cc71805ab 100644 --- a/IM/connectors/EC2.py +++ b/IM/connectors/EC2.py @@ -1212,7 +1212,8 @@ def del_dns_entries(self, vm, auth_data): result = changes.commit() # if there are no A records - if not zone.find_records(type="A"): + all_a_records = [r for r in conn.get_all_rrsets(zone.id) if r.type == "A"] + if not all_a_records: conn.delete_hosted_zone(zone.id) def cancel_spot_requests(self, conn, vm): @@ -1270,6 +1271,12 @@ def finalize(self, vm, last, auth_data): except: self.log_exception("Error deleting keypair.") + # Delete the DNS entries + try: + self.del_dns_entries(vm, auth_data) + except: + self.log_exception("Error deleting DNS entries") + # Delete the elastic IPs try: self.delete_elastic_ips(conn, vm) @@ -1286,13 +1293,7 @@ def finalize(self, vm, last, auth_data): try: self.delete_volumes(conn, volumes, instance_id) except: - self.log_exception("Error deleting EBS volumess") - - # Delete the DNS entries - try: - self.del_dns_entries(vm, auth_data) - except: - self.log_exception("Error deleting EBS volumess") + self.log_exception("Error deleting EBS volumes") return (True, "") diff --git a/test/unit/connectors/EC2.py b/test/unit/connectors/EC2.py index 027039165..22912b2dd 100755 --- a/test/unit/connectors/EC2.py +++ b/test/unit/connectors/EC2.py @@ -588,7 +588,7 @@ def test_60_finalize(self, record_sets, connect_to_region, sleep, get_connection record = MagicMock() zone.id = "zid" zone.get_a.return_value = record - zone.find_records.return_value = None + dns_conn.get_all_rrsets.return_value = [] dns_conn.get_zone.return_value = zone changes = MagicMock() record_sets.return_value = changes From 9a9f4ca20f291f01846e9efec02653c12f693304 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 5 Jun 2017 17:00:08 +0200 Subject: [PATCH 24/27] Code improvements and Fix: #342 --- IM/connectors/Azure.py | 78 ++++++++++++----------------------- test/unit/connectors/Azure.py | 9 +--- 2 files changed, 27 insertions(+), 60 deletions(-) diff --git a/IM/connectors/Azure.py b/IM/connectors/Azure.py index fd1376006..9584da826 100644 --- a/IM/connectors/Azure.py +++ b/IM/connectors/Azure.py @@ -282,7 +282,21 @@ def create_nics(self, inf, radl, credentials, subscription_id, group_name, subne if radl.systems[0].getValue('availability_zone'): location = radl.systems[0].getValue('availability_zone') - hasPublicIP = radl.hasPublicNet(system.name) + i = 0 + hasPublicIP = False + hasPrivateIP = False + while system.getValue("net_interface." + str(i) + ".connection"): + network_name = system.getValue("net_interface." + str(i) + ".connection") + # TODO: check how to do that + # fixed_ip = system.getValue("net_interface." + str(i) + ".ip") + network = radl.get_network_by_id(network_name) + + if network.isPublic(): + hasPublicIP = True + else: + hasPrivateIP = True + + i += 1 i = 0 res = [] @@ -293,7 +307,7 @@ def create_nics(self, inf, radl, credentials, subscription_id, group_name, subne # fixed_ip = system.getValue("net_interface." + str(i) + ".ip") network = radl.get_network_by_id(network_name) - if network.isPublic(): + if network.isPublic() and hasPrivateIP: # Public nets are not added as nics i += 1 continue @@ -496,6 +510,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res = [] i = 0 + all_ok = True while i < num_vm: group_name = None try: @@ -522,9 +537,11 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): credentials, subscription_id, location) if not storage_account: + all_ok = False res.append((False, error_msg)) + # delete VM group resource_client.resource_groups.delete(group_name) - break + continue nics = self.create_nics(inf, radl, credentials, subscription_id, group_name, subnets) @@ -533,7 +550,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): compute_client = ComputeManagementClient(credentials, subscription_id) async_vm_creation = compute_client.virtual_machines.create_or_update(group_name, vm_name, vm_parameters) - azure_vm = async_vm_creation.result() + # azure_vm = async_vm_creation.result() vm = VirtualMachine(inf, group_name + '/' + vm_name, self.cloud, radl, requested_radl, self) vm.info.systems[0].setValue('instance_id', group_name + '/' + vm_name) @@ -542,6 +559,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res.append((True, vm)) except Exception as ex: + all_ok = False self.log_exception("Error creating the VM") res.append((False, "Error creating the VM: " + str(ex))) @@ -552,6 +570,10 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): i += 1 + if not all_ok: + # Remove the general group + resource_client.resource_groups.delete("rg-%s" % inf.id) + return res def attach_data_disks(self, vm, storage_account_name, credentials, subscription_id, location): @@ -679,52 +701,6 @@ def add_dns_entries(self, vm, credentials, subscription_id): self.log_exception("Error creating DNS entries") return False - def del_dns_entries(self, vm, credentials, subscription_id): - """ - Delete the added entries in the Azure DNS service - - Arguments: - - vm(:py:class:`IM.VirtualMachine`): VM information. - - credentials, subscription_id: Authentication data to access cloud provider. - """ - try: - group_name = vm.id.split('/')[0] - dns_client = DnsManagementClient(credentials, subscription_id) - system = vm.info.systems[0] - for net_name in system.getNetworkIDs(): - num_conn = system.getNumNetworkWithConnection(net_name) - ip = system.getIfaceIP(num_conn) - (hostname, domain) = vm.getRequestedNameIface(num_conn, - default_hostname=Config.DEFAULT_VM_NAME, - default_domain=Config.DEFAULT_DOMAIN) - if domain != "localdomain" and ip: - if not domain.endswith("."): - domain += "." - zone = None - try: - zone = dns_client.zones.get(group_name, domain) - except Exception: - pass - if not zone: - self.log_debug("The DNS zone %s does not exists. Do not delete records." % domain) - else: - record = None - try: - record = dns_client.record_sets.get(group_name, domain, hostname, 'A') - except Exception: - pass - if not record: - self.log_debug("DNS record %s does not exists. Do not delete." % hostname) - else: - self.log_debug("Deleting DNS record %s." % hostname) - dns_client.record_sets.delete(group_name, domain, hostname, 'A') - - # if there are no A records - if not dns_client.record_sets.list_by_type(group_name, domain, 'A'): - dns_client.zones.delete(group_name, domain) - except Exception as ex: - self.log_exception("Error removing DNS entries.") - def setIPs(self, vm, network_profile, credentials, subscription_id): """ Set the information about the IPs of the VM @@ -763,8 +739,6 @@ def finalize(self, vm, last, auth_data): self.log_debug("Removing RG: %s" % group_name) resource_client.resource_groups.delete(group_name).wait() - self.del_dns_entries(vm, credentials, subscription_id) - # if it is the last VM delete the RG of the Inf if last: self.log_debug("Removing RG: %s" % "rg-%s" % vm.inf.id) diff --git a/test/unit/connectors/Azure.py b/test/unit/connectors/Azure.py index f095453bf..dfee2cadf 100755 --- a/test/unit/connectors/Azure.py +++ b/test/unit/connectors/Azure.py @@ -357,9 +357,8 @@ def test_55_alter(self, credentials, network_client, compute_client, storage_cli self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('IM.connectors.Azure.ResourceManagementClient') - @patch('IM.connectors.Azure.DnsManagementClient') @patch('IM.connectors.Azure.UserPassCredentials') - def test_60_finalize(self, credentials, dns_client, resource_client): + def test_60_finalize(self, credentials, resource_client): auth = Authentication([{'id': 'azure', 'type': 'Azure', 'subscription_id': 'subscription_id', 'username': 'user', 'password': 'password'}]) azure_cloud = self.get_azure_cloud() @@ -377,15 +376,9 @@ def test_60_finalize(self, credentials, dns_client, resource_client): inf = MagicMock() vm = VirtualMachine(inf, "rg0/vm0", azure_cloud.cloud, radl, radl, azure_cloud, 1) - dclient = MagicMock() - dns_client.return_value = dclient - dclient.record_sets.list_by_type.return_value = [] - success, _ = azure_cloud.finalize(vm, True, auth) self.assertTrue(success, msg="ERROR: finalizing VM info.") - self.assertEquals(dclient.record_sets.delete.call_args_list, [call('rg0', 'domain.com.', 'test', 'A')]) - self.assertEquals(dclient.zones.delete.call_args_list, [call('rg0', 'domain.com.')]) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) From 6a41d6811ee8e105ddaf33b3fe5193552cb67709 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 6 Jun 2017 08:27:07 +0200 Subject: [PATCH 25/27] Code improvements. Remove unnecessary waits --- IM/connectors/Azure.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/IM/connectors/Azure.py b/IM/connectors/Azure.py index 9584da826..85e7c3dcb 100644 --- a/IM/connectors/Azure.py +++ b/IM/connectors/Azure.py @@ -455,7 +455,7 @@ def create_nets(self, inf, radl, credentials, subscription_id, group_name): if not vnet: # Create VNet in the RG of the Inf - async_vnet_creation = network_client.virtual_networks.create_or_update( + network_client.virtual_networks.create_or_update( group_name, "privates", { @@ -465,7 +465,6 @@ def create_nets(self, inf, radl, credentials, subscription_id, group_name): } } ) - async_vnet_creation.wait() subnets = {} for i, net in enumerate(radl.networks): @@ -592,7 +591,7 @@ def attach_data_disks(self, vm, storage_account_name, credentials, subscription_ try: # Attach data disk - async_vm_update = compute_client.virtual_machines.create_or_update( + compute_client.virtual_machines.create_or_update( group_name, vm_name, { @@ -611,7 +610,6 @@ def attach_data_disks(self, vm, storage_account_name, credentials, subscription_ } } ) - async_vm_update.wait() except Exception as ex: self.log_exception("Error attaching disk %d to VM %s" % (cont, vm_name)) return False, "Error attaching disk %d to VM %s: %s" % (cont, vm_name, str(ex)) @@ -737,7 +735,7 @@ def finalize(self, vm, last, auth_data): # Delete Resource group and everything in it resource_client = ResourceManagementClient(credentials, subscription_id) self.log_debug("Removing RG: %s" % group_name) - resource_client.resource_groups.delete(group_name).wait() + resource_client.resource_groups.delete(group_name) # if it is the last VM delete the RG of the Inf if last: @@ -797,7 +795,7 @@ def alterVM(self, vm, radl, auth_data): # Start the VM async_vm_start = compute_client.virtual_machines.start(group_name, vm_name) - async_vm_start.wait() + # async_vm_start.wait() return self.updateVMInfo(vm, auth_data) except Exception as ex: From 9391ca326fc617921b1b3a1843d4bb052737349c Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 6 Jun 2017 12:54:00 +0200 Subject: [PATCH 26/27] Update docs --- README | 2 +- README.md | 2 +- ansible_install.yaml | 2 +- changelog | 4 ++-- doc/source/manual.rst | 2 +- docker-devel/Dockerfile | 2 +- docker-py3/Dockerfile | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README b/README index c17704af9..811dc339d 100644 --- a/README +++ b/README @@ -165,7 +165,7 @@ Then install the downloaded RPMs:: Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip:: - $ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource + $ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns 1.3.4 From Deb package (Tested with Ubuntu 14.04 and 16.04) ----------------------------------------------------------- diff --git a/README.md b/README.md index b96efc7cb..9711ac471 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ $ yum localinstall IM-*.rpm RADL-*.rpm Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip: ```sh -$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource +$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns ``` ### 1.3.4 From Deb package (Tested with Ubuntu 14.04 and 16.04) diff --git a/ansible_install.yaml b/ansible_install.yaml index 7aeac71d5..05c4d73f8 100644 --- a/ansible_install.yaml +++ b/ansible_install.yaml @@ -62,7 +62,7 @@ when: ansible_distribution == "Ubuntu" and ansible_distribution_major_version|int <= 14 - name: Install pip libraries - pip: executable=pip name=pyOpenSSL,MySQL-python,msrest,msrestazure,azure-common,azure-mgmt-storage,azure-mgmt-compute,azure-mgmt-network,azure-mgmt-resource + pip: executable=pip name=pyOpenSSL,MySQL-python,msrest,msrestazure,azure-common,azure-mgmt-storage,azure-mgmt-compute,azure-mgmt-network,azure-mgmt-resource,azure-mgmt-dns - name: Install IM pip: executable=pip name=IM \ No newline at end of file diff --git a/changelog b/changelog index 20cc5944d..196062a03 100644 --- a/changelog +++ b/changelog @@ -306,5 +306,5 @@ IM 1.5.5 * Enable to reset the add_public_ip_count in the OCCI/OST conns. * Improve Azure instance_type selection. * Improve GCE instance type selection. - - + * Manage DNS records in EC2, Azure and GCE connectors. + * Fix error in Azure conn creating a VM with only a public net attached. diff --git a/doc/source/manual.rst b/doc/source/manual.rst index e84bb05b8..1807a86f5 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -136,7 +136,7 @@ Then install the downloaded RPMs:: Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip:: - $ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource + $ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns From Deb package (Tested with Ubuntu 14.04 and 16.04) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 52c1b9566..1ed48ce15 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -7,7 +7,7 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 # Install pip optional libraries -RUN pip install MySQL-python msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource +RUN pip install MySQL-python msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns # Install im - 'devel' branch RUN cd tmp \ && git clone -b devel https://github.com/grycap/im.git \ diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index a4cb833a2..0a5874966 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -10,7 +10,7 @@ RUN pip3 install setuptools --upgrade -I RUN pip3 install cheroot RUN pip3 install pyOpenSSL --upgrade -I # Install pip optional libraries -RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource +RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns RUN apt install -y git RUN cd tmp \ From 9d68e938f06715ae8d8239605fe2ecd2ec133069 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 6 Jun 2017 15:55:06 +0200 Subject: [PATCH 27/27] Update docs --- README | 7 ++++++- README.md | 7 +++++++ doc/source/manual.rst | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/README b/README index 811dc339d..6cd2f9819 100644 --- a/README +++ b/README @@ -276,7 +276,12 @@ default configuration. Information about this image can be found here: https://r How to launch the IM service using docker: - $ sudo docker run -d -p 8899:8899 -p 8800:8800 --name im grycap/im + $ sudo docker run -d -p 8899:8899 -p 8800:8800 --name im grycap/im + +To make the IM data persistent you also have to specify a persistent location for the IM database using the +IM_DATA_DB environment variable and adding a volume:: + + $ sudo docker run -d -p 8899:8899 -p 8800:8800 -v "/some_local_path/db:/db" -e IM_DATA_DB=/db/inf.dat --name im grycap/im You can also specify an external MySQL server to store IM data using the IM_DATA_DB environment variable:: diff --git a/README.md b/README.md index eebe2ce78..5786533bf 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,13 @@ How to launch the IM service using docker:: ```sh $ sudo docker run -d -p 8899:8899 -p 8800:8800 --name im grycap/im ``` + +To make the IM data persistent you also have to specify a persistent location for the IM database using the IM_DATA_DB environment variable and adding a volume:: + +```sh +$ sudo docker run -d -p 8899:8899 -p 8800:8800 -v "/some_local_path/db:/db" -e IM_DATA_DB=/db/inf.dat --name im grycap/im +``` + You can also specify an external MySQL server to store IM data using the IM_DATA_DB environment variable:: ```sh diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 1807a86f5..b08dcce98 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -547,6 +547,11 @@ How to launch the IM service using docker:: $ sudo docker run -d -p 8899:8899 --name im grycap/im +To make the IM data persistent you also have to specify a persistent location for the IM database using +the IM_DATA_DB environment variable and adding a volume:: + + $ sudo docker run -d -p 8899:8899 -p 8800:8800 -v "/some_local_path/db:/db" -e IM_DATA_DB=/db/inf.dat --name im grycap/im + You can also specify an external MySQL server to store IM data using the IM_DATA_DB environment variable:: $ sudo docker run -d -p 8899:8899 -e IM_DATA_DB=mysql://username:password@server/db_name --name im grycap/im