Merge pull request #198 from indigo-dc/devel

Devel

IM/SSHRetry.py

@@ -16,67 +16,82 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from IM.retry import retry
-from IM.SSH import SSH
+from IM.SSH import SSH, AuthenticationException
+import paramiko
 
 
 class SSHRetry(SSH):
     """ SSH class decorated to perform a number of retries """
-    TRIES = 3
+    TRIES = 5
     DELAY = 3
     BACKOFF = 2
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def execute(self, command, timeout=None):
         return SSH.execute(self, command, timeout)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_get(self, src, dest):
         return SSH.sftp_get(self, src, dest)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_get_files(self, src, dest):
         return SSH.sftp_get_files(self, src, dest)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_put_files(self, files):
         return SSH.sftp_put_files(self, files)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_put(self, src, dest):
         return SSH.sftp_put(self, src, dest)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_put_dir(self, src, dest):
         return SSH.sftp_put_dir(self, src, dest)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_get_dir(self, src, dest):
         return SSH.sftp_get_dir(self, src, dest)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_put_content(self, content, dest):
         return SSH.sftp_put_content(self, content, dest)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_mkdir(self, directory):
         return SSH.sftp_mkdir(self, directory)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_list(self, directory):
         return SSH.sftp_list(self, directory)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_list_attr(self, directory):
         return SSH.sftp_list_attr(self, directory)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def getcwd(self):
         return SSH.getcwd(self)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_remove(self, path):
         return SSH.sftp_remove(self, path)
 
-    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    @retry(Exception, (AuthenticationException, paramiko.AuthenticationException),
+           tries=TRIES, delay=DELAY, backoff=BACKOFF)
     def sftp_chmod(self, path, mode):
         return SSH.sftp_chmod(self, path, mode)

IM/retry.py

@@ -2,7 +2,7 @@
 from functools import wraps
 
 
-def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None, quiet=True):
+def retry(ExceptionToCheck, ExceptionToAvoid, tries=4, delay=3, backoff=2, logger=None, quiet=True):
     """Retry calling the decorated function using an exponential backoff.
 
     http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
@@ -11,6 +11,9 @@ def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None, quiet=True
     :param ExceptionToCheck: the exception to check. may be a tuple of
         exceptions to check
     :type ExceptionToCheck: Exception or tuple
+    :param ExceptionToAvoid: the exception to avoid to check. may be a tuple of
+        exceptions to check
+    :type ExceptionToAvoid: Exception or tuple
     :param tries: number of times to try (not retry) before giving up
     :type tries: int
     :param delay: initial delay between retries in seconds
@@ -31,6 +34,8 @@ def f_retry(*args, **kwargs):
             while mtries > 1:
                 try:
                     return f(*args, **kwargs)
+                except ExceptionToAvoid as a:
+                    raise(a)
                 except ExceptionToCheck as e:
                     if not quiet:
                         msg = "%s, Retrying in %d seconds..." % (

IM/tosca/Tosca.py

@@ -245,7 +245,7 @@ def _add_node_nets(self, node, radl, system, nodetemplates):
                         elif network_name.endswith(".PRIVATE"):
                             parts = network_name.split(".")
                             net_provider_id = ".".join(parts[:-1])
-                        else:
+                        elif network_name != "PRIVATE":
                             # assume that is a private one
                             net_provider_id = network_name
                     if cap_props and "dns_name" in cap_props:

contextualization/basic.yml

@@ -38,3 +38,6 @@
     - name: Add the authorized_key to the nodes again
       lineinfile: dest="/home/{{ansible_user}}/.ssh/authorized_keys" line="{{ lookup('file', pk_file) }}2"
       ignore_errors: yes
+
+    - name: Gather Facts
+      setup: 

contextualization/ctxt_agent_dist.py

@@ -554,7 +554,7 @@ def replace_vm_ip(vm_data):
             if vm['id'] == vm_data['id']:
                 vm['ctxt_ip'] = vm_data['ctxt_ip']
 
-        with open(CtxtAgent.CONF_DATA_FILENAME, 'w+') as f:
+        with open(CtxtAgent.CONF_DATA_FILENAME + ".rep", 'w+') as f:
             json.dump(general_conf_data, f, indent=2)
 
         # Now in the ansible inventory
@@ -708,6 +708,10 @@ def contextualize_vm(general_conf_data, vm_conf_data, ctxt_vm, local):
                             ssh_client = CtxtAgent.get_ssh(ctxt_vm, True, CtxtAgent.PK_FILE)
                             ssh_client.sftp_mkdir(cache_dir)
                             ssh_client.sftp_put_dir(cache_dir, cache_dir)
+
+                            CtxtAgent.logger.info("Copy ansible roles to: %s" % ctxt_vm['ip'])
+                            ssh_client.sftp_mkdir(general_conf_data['conf_dir'] + "/roles")
+                            ssh_client.sftp_put_dir("/etc/ansible/roles", general_conf_data['conf_dir'] + "/roles")
                         except:
                             CtxtAgent.logger.exception("Error copying cache to VM: " + ctxt_vm['ip'])
                     else:
@@ -817,8 +821,9 @@ def contextualize_vm(general_conf_data, vm_conf_data, ctxt_vm, local):
                         if local:
                             # this step is not needed in windows systems
                             CtxtAgent.set_ansible_connection_local(general_conf_data, ctxt_vm)
-                            # Install ansible modules
-                            playbook = CtxtAgent.install_ansible_modules(general_conf_data, playbook)
+                            if ctxt_vm['master']:
+                                # Install ansible modules
+                                playbook = CtxtAgent.install_ansible_modules(general_conf_data, playbook)
                             ansible_thread = CtxtAgent.LaunchAnsiblePlaybook(CtxtAgent.logger,
                                                                              vm_conf_data['remote_dir'],
                                                                              playbook, ctxt_vm, 2,
@@ -871,8 +876,18 @@ def run(general_conf_file, vm_conf_file, local):
         CtxtAgent.CONF_DATA_FILENAME = os.path.abspath(general_conf_file)
         CtxtAgent.VM_CONF_DATA_FILENAME = os.path.abspath(vm_conf_file)
 
-        with open(CtxtAgent.CONF_DATA_FILENAME) as f:
-            general_conf_data = json.load(f)
+        # if we have the .rep file, read it instead
+        if os.path.isfile(CtxtAgent.CONF_DATA_FILENAME + ".rep"):
+            try:
+                with open(CtxtAgent.CONF_DATA_FILENAME + ".rep") as f:
+                    general_conf_data = json.load(f)
+            except:
+                print("Error loading .rep file, using original one.")
+                with open(CtxtAgent.CONF_DATA_FILENAME) as f:
+                    general_conf_data = json.load(f)
+        else:
+            with open(CtxtAgent.CONF_DATA_FILENAME) as f:
+                general_conf_data = json.load(f)
         with open(vm_conf_file) as f:
             vm_conf_data = json.load(f)
 

test/unit/test_ctxt_agent_dist.py

@@ -268,7 +268,7 @@ def test_90_replace_vm_ip(self):
         vm_data['ctxt_ip'] = "10.0.0.2"
         CtxtAgent.replace_vm_ip(vm_data)
 
-        with open("/tmp/gen_data.json", "r") as f:
+        with open("/tmp/gen_data.json.rep", "r") as f:
             general_conf_data = json.load(f)
         for vm in general_conf_data['vms']:
             if vm['id'] == vm_data['id']: