From 74e68cc66ed947a4da40113aec4d5fccab91b2fd Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Jun 2018 08:37:15 +0200 Subject: [PATCH 01/15] Update dockerfile --- docker/Dockerfile | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4b8661914..80b96d30b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,13 +12,13 @@ RUN pip install setuptools pip --upgrade -I RUN pip install pyOpenSSL --upgrade -I # Install IM -RUN apt-get update && apt-get install --no-install-recommends -y gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev libmysqlclient20 && \ +RUN apt-get update && apt-get install --no-install-recommends -y git gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev libmysqlclient20 && \ pip install pycrypto && \ pip install MySQL-python && \ # Install my version until PR #1215 is accepted cd /tmp && git clone https://github.com/micafer/libcloud && cd libcloud && pip install /tmp/libcloud && \ pip install IM==1.7.3 && \ - apt-get purge -y gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev python-dev python-pip && \ + apt-get purge -y git gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev python-dev python-pip && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install pip optional libraries diff --git a/setup.py b/setup.py index 606e5f27f..060172340 100644 --- a/setup.py +++ b/setup.py @@ -64,5 +64,5 @@ platforms=["any"], install_requires=["ansible >= 2.0", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot", "boto >= 2.29", "apache-libcloud >= 2.3.0", "RADL >= 1.1.1", "bottle", "netaddr", - "requests", "scp", "tosca-parser"] + "requests == 2.11.1", "scp", "tosca-parser"] ) From c1bc8cc040009bd45b99372ae504c3a380e94880 Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 8 Jun 2018 14:06:03 +0200 Subject: [PATCH 02/15] Fix #642 --- IM/VirtualMachine.py | 59 ++++++++++++++++++++++++----------------- IM/connectors/OCCI.py | 61 +++++++++++++++++++++++++++++-------------- 2 files changed, 76 insertions(+), 44 deletions(-) diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index f81625f05..8c87bb6b0 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -536,11 +536,43 @@ def update_status(self, auth, force=False): return updated + @staticmethod + def add_public_net(radl): + """ + Add a public net to the radl specified + """ + now = str(int(time.time() * 100)) + + public_nets = [] + for net in radl.networks: + if net.isPublic(): + public_nets.append(net) + + if public_nets: + public_net = None + for net in public_nets: + num_net = radl.systems[0].getNumNetworkWithConnection(net.id) + if num_net is not None: + public_net = net + break + + if not public_net: + # There are a public net but it has not been used in this + # VM + public_net = public_nets[0] + num_net = radl.systems[0].getNumNetworkIfaces() + else: + # There no public net, create one + public_net = network.createNetwork("public." + now, True) + radl.networks.append(public_net) + num_net = radl.systems[0].getNumNetworkIfaces() + + return public_net, num_net + def setIps(self, public_ips, private_ips, remove_old=False): """ Set the specified IPs in the VM RADL info """ - now = str(int(time.time() * 100)) vm_system = self.info.systems[0] # First remove old ip values @@ -548,34 +580,13 @@ def setIps(self, public_ips, private_ips, remove_old=False): if remove_old: cont = 0 while vm_system.getValue('net_interface.%d.connection' % cont): + vm_system.delValue('net_interface.%d.connection' % cont) if vm_system.getValue('net_interface.%d.ip' % cont): vm_system.delValue('net_interface.%d.ip' % cont) cont += 1 if public_ips and not set(public_ips).issubset(set(private_ips)): - public_nets = [] - for net in self.info.networks: - if net.isPublic(): - public_nets.append(net) - - if public_nets: - public_net = None - for net in public_nets: - num_net = self.getNumNetworkWithConnection(net.id) - if num_net is not None: - public_net = net - break - - if not public_net: - # There are a public net but it has not been used in this - # VM - public_net = public_nets[0] - num_net = self.getNumNetworkIfaces() - else: - # There no public net, create one - public_net = network.createNetwork("public." + now, True) - self.info.networks.append(public_net) - num_net = self.getNumNetworkIfaces() + public_net, num_net = self.add_public_net(self.info) real_public_ips = [public_ip for public_ip in public_ips if public_ip not in private_ips] if real_public_ips: diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index f004d0f85..6c4285835 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -30,7 +30,7 @@ from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from .CloudConnector import CloudConnector -from radl.radl import Feature +from radl.radl import Feature, network from IM.config import Config @@ -1169,9 +1169,30 @@ def manage_nics(self, vm, radl, auth_data): current_has_public_ip = vm.info.hasPublicNet(vm.info.systems[0].name) new_has_public_ip = radl.hasPublicNet(vm.info.systems[0].name) if new_has_public_ip and not current_has_public_ip: - return self.add_public_ip(vm, auth_data) + success = self.add_public_ip(vm, auth_data) + + if success: + # Add public net in the Requested RADL + public_net, num_net = VirtualMachine.add_public_net(vm.requested_radl) + vm.requested_radl.systems[0].setValue("net_interface.%d.connection" % num_net, public_net.id) + + return success if not new_has_public_ip and current_has_public_ip: - return self.remove_public_ip(vm, auth_data) + success = self.remove_public_ip(vm, auth_data) + + if success: + # Remove all public net connections in the Requested RADL + nets_id = [net.id for net in vm.requested_radl.networks if net.isPublic()] + system = vm.requested_radl.systems[0] + + i = 0 + while system.getValue('net_interface.%d.connection' % i): + f = system.getFeature("net_interface.%d.connection" % i) + if f.value in nets_id: + system.delValue('net_interface.%d.connection' % i) + i += 1 + + return success except Exception as ex: self.log_exception("Error adding new public IP") return (False, "Error adding new public IP: " + str(ex)) @@ -1303,12 +1324,12 @@ def get_keystone_uri(occi, auth_data): else: return None except SSLError as ex: - occi.logger.exception( + occi.log_exception( "Error with the credentials when contacting with the OCCI server.") raise Exception( "Error with the credentials when contacting with the OCCI server: %s. Check your proxy file." % str(ex)) except: - occi.logger.exception("Error contacting with the OCCI server.") + occi.log_exception("Error contacting with the OCCI server.") return None @staticmethod @@ -1330,10 +1351,10 @@ def check_keystone_token(occi, keystone_uri, version, auth): if resp.status_code == 200: return occi.keystone_token else: - occi.logger.warn("Old Keystone token invalid.") + occi.log_warn("Old Keystone token invalid.") return None except Exception: - occi.logger.exception("Error checking Keystone token") + occi.log_exception("Error checking Keystone token") return None else: return None @@ -1350,11 +1371,11 @@ def get_keystone_token(occi, keystone_uri, auth): return token if version == 2: - occi.logger.info("Getting Keystone v2 token") + occi.log_info("Getting Keystone v2 token") occi.keystone_token = KeyStoneAuth.get_keystone_token_v2(occi, keystone_uri, auth) return occi.keystone_token elif version == 3: - occi.logger.info("Getting Keystone v3 token") + occi.log_info("Getting Keystone v3 token") occi.keystone_token = KeyStoneAuth.get_keystone_token_v3(occi, keystone_uri, auth) return occi.keystone_token else: @@ -1380,7 +1401,7 @@ def get_keystone_version(occi, keystone_uri, auth): elif 'version' in json_data: versions = [json_data["version"]] else: - occi.logger.error("Error obtaining Keystone versions: versions or version expected.") + occi.log_error("Error obtaining Keystone versions: versions or version expected.") for elem in versions: if not token and elem["id"].startswith("v2"): @@ -1389,13 +1410,13 @@ def get_keystone_version(occi, keystone_uri, auth): # only use version 3 if 2 is not available version = 3 else: - occi.logger.error("Error obtaining Keystone versions: %s" % resp.text) + occi.log_error("Error obtaining Keystone versions: %s" % resp.text) except Exception: - occi.logger.exception("Error obtaining Keystone versions.") + occi.log_exception("Error obtaining Keystone versions.") if not version: # use version 2 as the default one in case of error - occi.logger.warn("Keystone Version not obtained, using default one v2.") + occi.log_warn("Keystone Version not obtained, using default one v2.") return 2 else: return version @@ -1425,7 +1446,7 @@ def get_keystone_token_v2(occi, keystone_uri, auth): if 'access' in output: token_id = output['access']['token']['id'] else: - occi.logger.exception("Error obtaining Keystone Token.") + occi.log_exception("Error obtaining Keystone Token.") raise Exception("Error obtaining Keystone Token: %s" % str(output)) if occi.keystone_tenant is None: @@ -1465,7 +1486,7 @@ def get_keystone_token_v2(occi, keystone_uri, auth): # \"metadata\": {\"is_admin\": 0, \"roles\": []}}}" output = resp.json() if 'access' in output: - occi.logger.info("Using tenant: %s" % tenant["name"]) + occi.log_info("Using tenant: %s" % tenant["name"]) occi.keystone_tenant = tenant tenant_token_id = str(output['access']['token']['id']) break @@ -1474,7 +1495,7 @@ def get_keystone_token_v2(occi, keystone_uri, auth): raise Exception("Error obtaining Keystone v2 Token: No tenant scoped token.") return tenant_token_id except Exception as ex: - occi.logger.exception("Error obtaining Keystone v2 Token.") + occi.log_exception("Error obtaining Keystone v2 Token.") raise Exception("Error obtaining Keystone v2 Token: %s" % str(ex)) @staticmethod @@ -1520,7 +1541,7 @@ def get_keystone_token_v3(occi, keystone_uri, auth): projects = [project_found] else: projects = output['projects'] - self.log_warn("Keystone 3 project %s not found." % auth["project"]) + occi.log_warn("Keystone 3 project %s not found." % auth["project"]) else: projects = [occi.keystone_project] @@ -1534,15 +1555,15 @@ def get_keystone_token_v3(occi, keystone_uri, auth): url = "%s/v3/auth/tokens" % keystone_uri resp = occi.create_request_static('POST', url, auth, headers, json.dumps(body)) if resp.status_code in [200, 201, 202]: - occi.logger.info("Using project: %s" % project["name"]) + occi.log_info("Using project: %s" % project["name"]) occi.keystone_project = project scoped_token = resp.headers['X-Subject-Token'] break if not scoped_token: - occi.logger.error("Not project accesible for the user.") + occi.log_error("Not project accesible for the user.") return scoped_token except Exception as ex: - occi.logger.exception("Error obtaining Keystone v3 Token.") + occi.log_exception("Error obtaining Keystone v3 Token.") raise Exception("Error obtaining Keystone v3 Token: %s" % str(ex)) From cf109de1ce7a0a2145c8e43b6bf19f7c0968afb6 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 11 Jun 2018 09:38:13 +0200 Subject: [PATCH 03/15] Fix #642 --- IM/connectors/OCCI.py | 15 ++++++++++----- test/unit/connectors/OCCI.py | 4 ++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index 6c4285835..20e684d1b 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -1169,16 +1169,17 @@ def manage_nics(self, vm, radl, auth_data): current_has_public_ip = vm.info.hasPublicNet(vm.info.systems[0].name) new_has_public_ip = radl.hasPublicNet(vm.info.systems[0].name) if new_has_public_ip and not current_has_public_ip: - success = self.add_public_ip(vm, auth_data) + success, msg = self.add_public_ip(vm, auth_data) if success: # Add public net in the Requested RADL public_net, num_net = VirtualMachine.add_public_net(vm.requested_radl) vm.requested_radl.systems[0].setValue("net_interface.%d.connection" % num_net, public_net.id) - - return success + return True, "" + else: + return False, msg if not new_has_public_ip and current_has_public_ip: - success = self.remove_public_ip(vm, auth_data) + success, msg = self.remove_public_ip(vm, auth_data) if success: # Remove all public net connections in the Requested RADL @@ -1190,9 +1191,13 @@ def manage_nics(self, vm, radl, auth_data): f = system.getFeature("net_interface.%d.connection" % i) if f.value in nets_id: system.delValue('net_interface.%d.connection' % i) + if system.getValue('net_interface.%d.ip' % i): + system.delValue('net_interface.%d.ip' % i) i += 1 - return success + return True, "" + else: + return False, msg except Exception as ex: self.log_exception("Error adding new public IP") return (False, "Error adding new public IP: " + str(ex)) diff --git a/test/unit/connectors/OCCI.py b/test/unit/connectors/OCCI.py index 98926b9c6..8c72f863e 100755 --- a/test/unit/connectors/OCCI.py +++ b/test/unit/connectors/OCCI.py @@ -412,6 +412,10 @@ def test_55_alter(self, get_keystone_uri, requests): self.assertTrue(success, msg="ERROR: modifying VM info.") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) + self.assertEqual(vm.requested_radl.systems[0].getValue("net_interface.1.connection"), None) + self.assertEqual(vm.requested_radl.systems[0].getValue("net_interface.1.ip"), None) + self.assertEqual(vm.requested_radl.systems[0].getValue("net_interface.0.connection"), "net") + @patch('requests.request') @patch('IM.connectors.OCCI.KeyStoneAuth.get_keystone_uri') def test_60_finalize(self, get_keystone_uri, requests): From e343f87028e8a26030a24cb9bb0c65b12d199891 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 11 Jun 2018 12:57:21 +0200 Subject: [PATCH 04/15] Add VERIFI_SSL variable --- IM/config.py | 1 + IM/connectors/AzureClassic.py | 2 +- IM/connectors/CloudConnector.py | 11 +++++++++++ IM/connectors/Docker.py | 4 ++-- IM/connectors/FogBow.py | 2 +- IM/connectors/Kubernetes.py | 2 +- IM/connectors/OCCI.py | 22 +++++++++++----------- IM/connectors/OpenStack.py | 23 ++++++++++++----------- doc/source/manual.rst | 6 ++++++ etc/im.cfg | 4 ++++ 10 files changed, 50 insertions(+), 27 deletions(-) diff --git a/IM/config.py b/IM/config.py index 72e723782..4fb094b4f 100644 --- a/IM/config.py +++ b/IM/config.py @@ -102,6 +102,7 @@ class Config: OIDC_SCOPES = [] VM_NUM_USE_CTXT_DIST = 30 DELAY_BETWEEN_VM_RETRIES = 5 + VERIFI_SSL = False config = ConfigParser() diff --git a/IM/connectors/AzureClassic.py b/IM/connectors/AzureClassic.py index d3f94f040..f41957d18 100644 --- a/IM/connectors/AzureClassic.py +++ b/IM/connectors/AzureClassic.py @@ -155,7 +155,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): subscription_id = self.get_subscription_id(auth_data) url = "https://%s:%d/%s%s" % (self.AZURE_SERVER, self.AZURE_PORT, subscription_id, url) cert = self.get_user_cert_data(auth) - resp = requests.request(method, url, verify=False, cert=cert, headers=headers, data=body) + resp = requests.request(method, url, verify=self.verify_ssl, cert=cert, headers=headers, data=body) return resp diff --git a/IM/connectors/CloudConnector.py b/IM/connectors/CloudConnector.py index f03d7f092..b69081c56 100644 --- a/IM/connectors/CloudConnector.py +++ b/IM/connectors/CloudConnector.py @@ -3,6 +3,7 @@ import shutil import tempfile import time +from IM.config import Config class CloudConnector: @@ -22,6 +23,16 @@ def __init__(self, cloud_info, inf): """Logger object.""" self.error_messages = "" """String with error messages to be shown to the user.""" + self.verify_ssl = Config.VERIFI_SSL + """Verify SSL connections """ + if not self.verify_ssl: + try: + # To avoid annoying InsecureRequestWarning messages in some Connectors + import requests.packages + from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + except: + pass def concreteSystem(self, radl_system, auth_data): """ diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index 8698f99c1..b2412b407 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -62,7 +62,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): url) session = requests.Session() session.mount('http+unix://', UnixHTTPAdapter.UnixHTTPAdapter()) - resp = session.request(method, url, verify=False, headers=headers, data=body) + resp = session.request(method, url, verify=self.verify_ssl, headers=headers, data=body) else: url = "%s://%s:%d%s%s" % (self.cloud.protocol, self.cloud.server, self.cloud.port, self.cloud.path, url) if 'public_key' in auth and 'private_key' in auth: @@ -71,7 +71,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): cert = None try: - resp = requests.request(method, url, verify=False, cert=cert, headers=headers, data=body) + resp = requests.request(method, url, verify=self.verify_ssl, cert=cert, headers=headers, data=body) finally: if cert: try: diff --git a/IM/connectors/FogBow.py b/IM/connectors/FogBow.py index 981fc025c..98430767a 100644 --- a/IM/connectors/FogBow.py +++ b/IM/connectors/FogBow.py @@ -65,7 +65,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): protocol = self.cloud.protocol url = "%s://%s:%d%s%s" % (protocol, self.cloud.server, self.cloud.port, self.cloud.path, url) - resp = requests.request(method, url, verify=False, headers=headers, data=body) + resp = requests.request(method, url, verify=self.verify_ssl, headers=headers, data=body) return resp diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index aeeed40c6..6784d97dc 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -56,7 +56,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): headers.update(auth_header) url = "%s://%s:%d%s%s" % (self.cloud.protocol, self.cloud.server, self.cloud.port, self.cloud.path, url) - resp = requests.request(method, url, verify=False, headers=headers, data=body) + resp = requests.request(method, url, verify=self.verify_ssl, headers=headers, data=body) return resp diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index 20e684d1b..94dfd27c8 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -68,7 +68,7 @@ def __init__(self, cloud_info, inf): CloudConnector.__init__(self, cloud_info, inf) @staticmethod - def create_request_static(method, url, auth, headers, body=None): + def create_request_static(method, url, auth, headers, verify=False, body=None): if auth and 'proxy' in auth: proxy = auth['proxy'] @@ -83,7 +83,7 @@ def create_request_static(method, url, auth, headers, body=None): headers.update({'Authorization': 'Bearer ' + auth["token"]}) try: - resp = requests.request(method, url, verify=False, cert=cert, headers=headers, data=body) + resp = requests.request(method, url, verify=verify, cert=cert, headers=headers, data=body) finally: if cert: try: @@ -106,7 +106,7 @@ def create_request(self, method, url, auth_data, headers, body=None): else: auth = auths[0] - return self.create_request_static(method, url, auth, headers, body) + return self.create_request_static(method, url, auth, headers, self.verify_ssl, body) def get_auth_header(self, auth_data): """ @@ -1352,7 +1352,7 @@ def check_keystone_token(occi, keystone_uri, version, auth): url = "%s/v3/auth/tokens" % keystone_uri else: return None - resp = occi.create_request_static('GET', url, auth, headers) + resp = occi.create_request_static('GET', url, auth, headers, occi.verify_ssl) if resp.status_code == 200: return occi.keystone_token else: @@ -1397,7 +1397,7 @@ def get_keystone_version(occi, keystone_uri, auth): try: headers = {"Accept": "application/json"} - resp = occi.create_request_static('GET', keystone_uri, None, headers) + resp = occi.create_request_static('GET', keystone_uri, None, headers, occi.verify_ssl) if resp.status_code in [200, 300]: versions = [] json_data = resp.json() @@ -1435,7 +1435,7 @@ def get_keystone_token_v2(occi, keystone_uri, auth): body = '{"auth":{"voms":true}}' headers = {'Accept': 'application/json', 'Connection': 'close', 'Content-Type': 'application/json'} url = "%s/v2.0/tokens" % keystone_uri - resp = occi.create_request_static('POST', url, auth, headers, body) + resp = occi.create_request_static('POST', url, auth, headers, occi.verify_ssl, body) resp.raise_for_status() # format: -> "{\"access\": {\"token\": {\"issued_at\": @@ -1458,7 +1458,7 @@ def get_keystone_token_v2(occi, keystone_uri, auth): headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'X-Auth-Token': token_id, 'Connection': 'close'} url = "%s/v2.0/tenants" % keystone_uri - resp = occi.create_request_static('GET', url, auth, headers) + resp = occi.create_request_static('GET', url, auth, headers, occi.verify_ssl) resp.raise_for_status() # format: -> "{\"tenants_links\": [], \"tenants\": @@ -1478,7 +1478,7 @@ def get_keystone_token_v2(occi, keystone_uri, auth): headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'X-Auth-Token': token_id, 'Connection': 'close'} url = "%s/v2.0/tokens" % keystone_uri - resp = occi.create_request_static('POST', url, auth, headers, body) + resp = occi.create_request_static('POST', url, auth, headers, occi.verify_ssl, body) if resp.status_code in [200, 202]: # format: -> "{\"access\": {\"token\": {\"issued_at\": # \"2014-12-29T17:10:49.609894\", \"expires\": @@ -1518,7 +1518,7 @@ def get_keystone_token_v3(occi, keystone_uri, auth): # Use VOMS proxy url = "%s/v3/OS-FEDERATION/identity_providers/egi.eu/protocols/mapped/auth" % keystone_uri - resp = occi.create_request_static('GET', url, auth, headers) + resp = occi.create_request_static('GET', url, auth, headers, occi.verify_ssl) resp.raise_for_status() token = resp.headers['X-Subject-Token'] @@ -1527,7 +1527,7 @@ def get_keystone_token_v3(occi, keystone_uri, auth): headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'X-Auth-Token': token, 'Connection': 'close'} url = "%s/v3/auth/projects" % keystone_uri - resp = occi.create_request_static('GET', url, auth, headers) + resp = occi.create_request_static('GET', url, auth, headers, occi.verify_ssl) resp.raise_for_status() output = resp.json() @@ -1558,7 +1558,7 @@ def get_keystone_token_v3(occi, keystone_uri, auth): body = {"auth": {"identity": {"methods": ["token"], "token": {"id": token}}, "scope": {"project": {"id": project["id"]}}}} url = "%s/v3/auth/tokens" % keystone_uri - resp = occi.create_request_static('POST', url, auth, headers, json.dumps(body)) + resp = occi.create_request_static('POST', url, auth, headers, occi.verify_ssl, json.dumps(body)) if resp.status_code in [200, 201, 202]: occi.log_info("Using project: %s" % project["name"]) occi.keystone_project = project diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index ea9bad86c..c46c3ba3f 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -128,17 +128,18 @@ def get_driver(self, auth_data): raise Exception( "No correct auth data has been specified to OpenStack: username, password and tenant or proxy") - # To avoid errors with host certificates - # if you want to do it in a more secure way check this: - # http://libcloud.readthedocs.org/en/latest/other/ssl-certificate-validation.html - import libcloud.security - libcloud.security.VERIFY_SSL_CERT = False - - try: - import ssl - ssl._create_default_https_context = ssl._create_unverified_context - except: - pass + if not self.verify_ssl: + # To avoid errors with host certificates + # if you want to do it in a more secure way check this: + # http://libcloud.readthedocs.org/en/latest/other/ssl-certificate-validation.html + import libcloud.security + libcloud.security.VERIFY_SSL_CERT = False + + try: + import ssl + ssl._create_default_https_context = ssl._create_unverified_context + except: + pass # Workaround to OTC to enable to set service_name as None service_name = parameters["service_name"] diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 6028c25db..ba940808b 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -337,6 +337,12 @@ Default Virtual Machine Options Default domain assigned to a virtual machine. The default value is ``localdomain``. +.. confval:: VERIFI_SSL + + Verify SSL hosts in CloudConnectors connections If you set it to True you must assure + the CA certificates are installed correctly + The default value is ``False``. + .. _options-ctxt: Contextualization diff --git a/etc/im.cfg b/etc/im.cfg index 41c5ece4f..ecdd075c6 100644 --- a/etc/im.cfg +++ b/etc/im.cfg @@ -130,6 +130,10 @@ OIDC_ISSUERS = # in memory. Only used in case of IM in HA mode. #INF_CACHE_TIME = 3600 +# Verify SSL hosts in CloudConnectors connections +# If you set it to True you must assure the CA certificates are installed correctly +VERIFI_SSL = False + [OpenNebula] # OpenNebula connector configuration values From 9399a10205b519f98484e7576328fdac58e165b5 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 11 Jun 2018 13:14:31 +0200 Subject: [PATCH 05/15] Fix #642 --- IM/VirtualMachine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index 8c87bb6b0..3f448e1cd 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -580,7 +580,6 @@ def setIps(self, public_ips, private_ips, remove_old=False): if remove_old: cont = 0 while vm_system.getValue('net_interface.%d.connection' % cont): - vm_system.delValue('net_interface.%d.connection' % cont) if vm_system.getValue('net_interface.%d.ip' % cont): vm_system.delValue('net_interface.%d.ip' % cont) cont += 1 From 207246c73e921bef8e8ebf00c650aab327e216da Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 11 Jun 2018 15:18:44 +0200 Subject: [PATCH 06/15] Fix #642 --- IM/connectors/OCCI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index 94dfd27c8..aef027bea 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -1166,7 +1166,7 @@ def manage_nics(self, vm, radl, auth_data): # update VM info try: vm.update_status(auth_data, force=True) - current_has_public_ip = vm.info.hasPublicNet(vm.info.systems[0].name) + current_has_public_ip = vm.hasPublicIP() new_has_public_ip = radl.hasPublicNet(vm.info.systems[0].name) if new_has_public_ip and not current_has_public_ip: success, msg = self.add_public_ip(vm, auth_data) From 50dc46b03da06d629b45c488dc191cd7423aa0af Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 12 Jun 2018 08:37:26 +0200 Subject: [PATCH 07/15] Improve error messages --- IM/InfrastructureManager.py | 43 +++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index de4725960..919732766 100644 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -457,24 +457,31 @@ def AddResource(inf_id, radl_data, auth, context=True): InfrastructureManager.logger.info("Adding resources to Inf ID: " + str(inf_id)) - if isinstance(radl_data, RADL): - radl = radl_data - else: - radl = radl_parse.parse_radl(radl_data) + sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) - InfrastructureManager.logger.debug("Inf ID: " + str(inf_id) + ": \n" + str(radl)) - radl.check() + try: + if isinstance(radl_data, RADL): + radl = radl_data + else: + radl = radl_parse.parse_radl(radl_data) - sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) + InfrastructureManager.logger.debug("Inf ID: " + str(inf_id) + ": \n" + str(radl)) + radl.check() - # Update infrastructure RADL with this new RADL - sel_inf.complete_radl(radl) + # Update infrastructure RADL with this new RADL + sel_inf.complete_radl(radl) - # If any deploy is defined, only update definitions. - if not radl.deploys: - sel_inf.update_radl(radl, []) - InfrastructureManager.logger.warn("Inf ID: " + sel_inf.id + ": without any deploy. Exiting.") - return [] + # If any deploy is defined, only update definitions. + if not radl.deploys: + sel_inf.update_radl(radl, []) + InfrastructureManager.logger.warn("Inf ID: " + sel_inf.id + ": without any deploy. Exiting.") + sel_inf.add_cont_msg("Infrastructure without any deploy. Exiting.") + return [] + except Exception as ex: + sel_inf.configured = False + sel_inf.add_cont_msg("Error parsing RADL: %s" % str(ex)) + InfrastructureManager.logger.exception("Inf ID: " + sel_inf.id + " error parsing RADL") + raise ex for system in radl.systems: # Add apps requirements to the RADL @@ -526,7 +533,8 @@ def AddResource(inf_id, radl_data, auth, context=True): for deploy_group in deploy_groups: if not deploy_group: InfrastructureManager.logger.warning("Inf ID: %s: No VMs to deploy!" % sel_inf.id) - return + sel_inf.add_cont_msg("No VMs to deploy. Exiting.") + return [] cloud_id = deploys_group_cloud[id(deploy_group)] cloud = cloud_list[cloud_id] @@ -896,7 +904,10 @@ def GetInfrastructureState(inf_id, auth): state = VirtualMachine.UNCONFIGURED if state is None: - state = VirtualMachine.UNKNOWN + if sel_inf.configured is False: + state = VirtualMachine.FAILED + else: + state = VirtualMachine.UNKNOWN InfrastructureManager.logger.info("Inf ID: " + str(inf_id) + " is in state: " + state) return {'state': state, 'vm_states': vm_states} From 36d3bed1d451b46cb94abd02ce55d76042a99d7c Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 12 Jun 2018 08:57:35 +0200 Subject: [PATCH 08/15] Improve error messages: #646 --- IM/InfrastructureManager.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index 919732766..f07409789 100644 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -586,6 +586,7 @@ def AddResource(inf_id, radl_data, auth, context=True): sel_inf.update_radl(radl, [(d, deployed_vm[d], concrete_systems[d.cloud_id][d.id][0]) for d in deployed_vm]) if all_failed: InfrastructureManager.logger.error("VMs failed when adding to Inf ID: %s" % sel_inf.id) + sel_inf.add_cont_msg("All VMs failed. No contextualize.") else: InfrastructureManager.logger.info("VMs %s successfully added to Inf ID: %s" % (new_vms, sel_inf.id)) @@ -1259,7 +1260,7 @@ def check_auth_data(auth): return auth @staticmethod - def CreateInfrastructure(radl, auth, async_call=False): + def CreateInfrastructure(radl_data, auth, async_call=False): """ Create a new infrastructure. @@ -1268,7 +1269,7 @@ def CreateInfrastructure(radl, auth, async_call=False): Args: - - radl(RADL): RADL description. + - radl_data(RADL): RADL description. - auth(Authentication): parsed authentication tokens. - async_call(bool): Create the inf in an async way. @@ -1278,6 +1279,14 @@ def CreateInfrastructure(radl, auth, async_call=False): # First check the auth data auth = InfrastructureManager.check_auth_data(auth) + # Then parse the RADL + if isinstance(radl_data, RADL): + radl = radl_data + else: + radl = radl_parse.parse_radl(radl_data) + + radl.check() + # Create a new infrastructure inf = IM.InfrastructureInfo.InfrastructureInfo() inf.auth = Authentication(auth.getAuthInfo("InfrastructureManager")) From e8e193edb5178c4b73a22702c6eee2f6dc4f0183 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 12 Jun 2018 12:30:25 +0200 Subject: [PATCH 09/15] Remove version in dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 060172340..606e5f27f 100644 --- a/setup.py +++ b/setup.py @@ -64,5 +64,5 @@ platforms=["any"], install_requires=["ansible >= 2.0", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot", "boto >= 2.29", "apache-libcloud >= 2.3.0", "RADL >= 1.1.1", "bottle", "netaddr", - "requests == 2.11.1", "scp", "tosca-parser"] + "requests", "scp", "tosca-parser"] ) From 3235d10d40d6240616743c6cce6e36343a2f7d16 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Jun 2018 12:33:07 +0200 Subject: [PATCH 10/15] Update doc --- doc/source/radl.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/source/radl.rst b/doc/source/radl.rst index 0f32b690f..1ce79ff01 100644 --- a/doc/source/radl.rst +++ b/doc/source/radl.rst @@ -253,6 +253,10 @@ machine. The supported features are: ``instance_type`` Set the instance type name of this VM. +``instance_tags`` + A set of keypair values to be set to the VMs. + With the following format: key=value,key2=value2 ... + ``disk..`` Features under this prefix refer to virtual storage devices attached to the virtual machine. ``disk.0`` refers to system boot device. From 84a92013f72c0532aa01c4c58092b88650ce7055 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Jun 2018 12:33:30 +0200 Subject: [PATCH 11/15] Set correct requests version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 606e5f27f..616f8bf0c 100644 --- a/setup.py +++ b/setup.py @@ -64,5 +64,5 @@ platforms=["any"], install_requires=["ansible >= 2.0", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot", "boto >= 2.29", "apache-libcloud >= 2.3.0", "RADL >= 1.1.1", "bottle", "netaddr", - "requests", "scp", "tosca-parser"] + "requests >= 2.19", "scp", "tosca-parser"] ) From ed57914f737cb9742752de89c8f8c9d2fb18591d Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Jun 2018 12:34:00 +0200 Subject: [PATCH 12/15] Remove unnecessary line --- docker/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 80b96d30b..f35176e6d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,9 +24,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y git gcc libmysq # Install pip optional libraries RUN pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage cheroot xmltodict -# Force requests to be version 2.11.1 to avoid SSL ca errors with proxy files -RUN pip install requests==2.11.1 - # Copy a ansible.cfg with correct minimum values COPY ansible.cfg /etc/ansible/ansible.cfg From 6d19dedba938c9f4ae5d05f2903160c6932c3b26 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Jun 2018 12:40:58 +0200 Subject: [PATCH 13/15] Improve #646 --- IM/InfrastructureManager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index f07409789..9d371baab 100644 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -342,7 +342,7 @@ def _compute_score(system_score, requested_radl): return concrete_system, score @staticmethod - def systems_with_vmrc(radl, auth): + def systems_with_vmrc(sel_inf, radl, auth): """ Concrete systems using VMRC NOTE: consider not-fake deploys (vm_number > 0) @@ -382,6 +382,7 @@ def systems_with_vmrc(radl, auth): vmrc_res = [s0 for vmrc in vmrc_list for s0 in vmrc.search_vm(s)] # Check that now the image URL is in the RADL if not s.getValue("disk.0.image.url") and not vmrc_res: + sel_inf.add_cont_msg("No VMI obtained from VMRC to system: " + system_id) raise Exception("No VMI obtained from VMRC to system: " + system_id) n = [s_without_apps.clone().applyFeatures(s0, conflict="other", missing="other") @@ -435,6 +436,8 @@ def sort_by_score(sel_inf, concrete_systems, cloud_list, deploy_groups, auth): if sorted_scored_clouds and sorted_scored_clouds[0]: deploys_group_cloud[id(deploy_group)] = sorted_scored_clouds[0][0] else: + sel_inf.configured = False + sel_inf.add_cont_msg("No cloud provider available") raise Exception("No cloud provider available") return deploys_group_cloud @@ -502,7 +505,7 @@ def AddResource(inf_id, radl_data, auth, context=True): break # Concrete systems using VMRC - systems_with_vmrc = InfrastructureManager.systems_with_vmrc(radl, auth) + systems_with_vmrc = InfrastructureManager.systems_with_vmrc(sel_inf, radl, auth) # Concrete systems with cloud providers and select systems with the greatest score # in every cloud From 54acb053f08803d0d1bd544a38096b0e09d7bf52 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Jun 2018 12:51:57 +0200 Subject: [PATCH 14/15] Set version 1.7.4 --- IM/__init__.py | 2 +- changelog | 7 +++++++ docker-devel/Dockerfile | 2 +- docker-py3/Dockerfile | 12 +++++------- docker/Dockerfile | 4 ++-- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/IM/__init__.py b/IM/__init__.py index 1d81b157d..80a2e9c58 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -19,5 +19,5 @@ 'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry', 'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPAdapter', 'uriparse', 'VirtualMachine', 'VMRC', 'xmlobject'] -__version__ = '1.7.3' +__version__ = '1.7.4' __author__ = 'Miguel Caballer' diff --git a/changelog b/changelog index ecc738689..05eeaabbc 100644 --- a/changelog +++ b/changelog @@ -422,3 +422,10 @@ IM 1.7.3: * Improve OpenStack network management. * Enable to set poolname in the TOSCA docs. * Error when trying to attach a storage in OCCI (OpenStack sites). + +IM 1.7.4: + * Fix error OCCI connector does not correctly AlterVM adding/removing ips. + * Add VERIFI_SSL var. + * Improve error messages in Async call. + + \ No newline at end of file diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 8f28d5c88..c6dd34e37 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -2,7 +2,7 @@ FROM grycap/jenkins:ubuntu16.04-im ARG BRANCH=devel MAINTAINER Miguel Caballer -LABEL version="1.7.3" +LABEL version="1.7.4" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index dd2847f23..47f4f5930 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM ubuntu:16.04 LABEL maintainer="Miguel Caballer " -LABEL version="1.7.3" +LABEL version="1.7.4" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 @@ -15,15 +15,13 @@ RUN pip3 install pyOpenSSL --upgrade -I RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage cheroot xmltodict # Install IM -RUN apt-get update && apt-get install --no-install-recommends -y gcc libssl-dev libffi-dev libsqlite3-dev && \ - pip3 install IM==1.7.3 && \ - apt-get remove -y gcc libssl-dev libffi-dev libsqlite3-dev python-dev python-pip && \ +RUN apt-get update && apt-get install --no-install-recommends -y gcc git libssl-dev libffi-dev libsqlite3-dev && \ + cd /tmp && git clone https://github.com/micafer/libcloud && cd libcloud && pip3 install /tmp/libcloud && \ + pip3 install IM==1.7.4 && \ + apt-get remove -y gcc git libssl-dev libffi-dev libsqlite3-dev python-dev python-pip && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* -# Force requests to be version 2.11.1 to avoid SSL ca errors with proxy files -RUN pip3 install requests==2.11.1 - # Copy a ansible.cfg with correct minimum values COPY ansible.cfg /etc/ansible/ansible.cfg diff --git a/docker/Dockerfile b/docker/Dockerfile index f35176e6d..5e52f175b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM ubuntu:18.04 LABEL maintainer="Miguel Caballer " -LABEL version="1.7.3" +LABEL version="1.7.4" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y git gcc libmysq pip install MySQL-python && \ # Install my version until PR #1215 is accepted cd /tmp && git clone https://github.com/micafer/libcloud && cd libcloud && pip install /tmp/libcloud && \ - pip install IM==1.7.3 && \ + pip install IM==1.7.4 && \ apt-get purge -y git gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev python-dev python-pip && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ From b7bb1d890b671617286ab1bd4391771aeaaaeee9 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Jun 2018 13:14:43 +0200 Subject: [PATCH 15/15] Do not return error in ctxt in case of off or failed vms --- contextualization/ctxt_agent.py | 2 +- contextualization/ctxt_agent_dist.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contextualization/ctxt_agent.py b/contextualization/ctxt_agent.py index a8877d372..0326f6302 100644 --- a/contextualization/ctxt_agent.py +++ b/contextualization/ctxt_agent.py @@ -477,7 +477,7 @@ def contextualize_vm(general_conf_data, vm_conf_data): if not ctxt_vm: CtxtAgent.logger.error("No VM to Contextualize!") - res_data['OK'] = False + res_data['OK'] = True return res_data for task in vm_conf_data['tasks']: diff --git a/contextualization/ctxt_agent_dist.py b/contextualization/ctxt_agent_dist.py index 4ff3fe390..c0edb0027 100755 --- a/contextualization/ctxt_agent_dist.py +++ b/contextualization/ctxt_agent_dist.py @@ -692,7 +692,7 @@ def contextualize_vm(general_conf_data, vm_conf_data, ctxt_vm, local): if not ctxt_vm: CtxtAgent.logger.error("No VM to Contextualize!") - res_data['OK'] = False + res_data['OK'] = True return res_data for task in vm_conf_data['tasks']: