From f0e042c423fec70643187e8568ea39ba3b433c93 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 20 Oct 2018 21:18:42 -0400 Subject: [PATCH 01/12] Support SSH configuration file --- nix/eval-machine-info.nix | 2 +- nix/options.nix | 6 ++++++ nix/ssh-tunnel.nix | 6 ++++++ nixops/backends/__init__.py | 18 +++++++++++++++--- nixops/deployment.py | 1 + 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/nix/eval-machine-info.nix b/nix/eval-machine-info.nix index ceeab5f8d..7c93f69a9 100644 --- a/nix/eval-machine-info.nix +++ b/nix/eval-machine-info.nix @@ -303,7 +303,7 @@ rec { machines = flip mapAttrs nodes (n: v': let v = scrubOptionValue v'; in - { inherit (v.config.deployment) targetEnv targetPort targetHost encryptedLinksTo storeKeysOnMachine alwaysActivate owners keys hasFastConnection; + { inherit (v.config.deployment) targetEnv targetPort targetHost sshConfigOptionsFile encryptedLinksTo storeKeysOnMachine alwaysActivate owners keys hasFastConnection; nixosRelease = v.config.system.nixos.release or v.config.system.nixosRelease or (removeSuffix v.config.system.nixosVersionSuffix v.config.system.nixosVersion); azure = optionalAttrs (v.config.deployment.targetEnv == "azure") v.config.deployment.azure; ec2 = optionalAttrs (v.config.deployment.targetEnv == "ec2") v.config.deployment.ec2; diff --git a/nix/options.nix b/nix/options.nix index 0866c3ab8..d8fb2083f 100644 --- a/nix/options.nix +++ b/nix/options.nix @@ -65,6 +65,12 @@ in ''; }; + deployment.sshConfigOptionsFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Arbitrary SSH configuration options file."; + }; + deployment.alwaysActivate = mkOption { type = types.bool; default = true; diff --git a/nix/ssh-tunnel.nix b/nix/ssh-tunnel.nix index c44c80569..aa2b690fa 100644 --- a/nix/ssh-tunnel.nix +++ b/nix/ssh-tunnel.nix @@ -30,6 +30,11 @@ with lib; type = types.int; description = "Port number that SSH listens to on the remote machine."; }; + sshConfigOptionsFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Arbitrary SSH configuration option file."; + }; privateKey = mkOption { type = types.path; description = "Path to the private key file used to connect to the remote machine."; @@ -91,6 +96,7 @@ with lib; + " -o PermitLocalCommand=yes" + " -o ServerAliveInterval=20" + " -o LocalCommand='${localCommand}'" + + (if v.sshConfigOptionsFile == null then "" else " -F ${v.sshConfigOptionsFile}") + " -w ${toString v.localTunnel}:${toString v.remoteTunnel}" + " ${v.target} -p ${toString v.targetPort}" + " '${remoteCommand}'"; diff --git a/nixops/backends/__init__.py b/nixops/backends/__init__.py index cbfd4734d..0d73d7ef0 100644 --- a/nixops/backends/__init__.py +++ b/nixops/backends/__init__.py @@ -16,6 +16,11 @@ def __init__(self, xml, config={}): self.encrypted_links_to = set([e.get("value") for e in xml.findall("attrs/attr[@name='encryptedLinksTo']/list/string")]) self.store_keys_on_machine = xml.find("attrs/attr[@name='storeKeysOnMachine']/bool").get("value") == "true" self.ssh_port = int(xml.find("attrs/attr[@name='targetPort']/int").get("value")) + ssh_options_file_elem = xml.find("attrs/attr[@name='sshConfigOptionsFile']/string") + if ssh_options_file_elem: + self.ssh_config_options_file = ssh_options_file_elem.get('value') + else: + self.ssh_config_options_file = None self.always_activate = xml.find("attrs/attr[@name='alwaysActivate']/bool").get("value") == "true" self.owners = [e.get("value") for e in xml.findall("attrs/attr[@name='owners']/list/string")] self.has_fast_connection = xml.find("attrs/attr[@name='hasFastConnection']/bool").get("value") == "true" @@ -44,6 +49,7 @@ class MachineState(nixops.resources.ResourceState): has_fast_connection = nixops.util.attr_property("hasFastConnection", False, bool) ssh_pinged = nixops.util.attr_property("sshPinged", False, bool) ssh_port = nixops.util.attr_property("targetPort", 22, int) + ssh_config_options_file = nixops.util.attr_property("sshConfigOptionsFile", "", str) public_vpn_key = nixops.util.attr_property("publicVpnKey", None) store_keys_on_machine = nixops.util.attr_property("storeKeysOnMachine", False, bool) keys = nixops.util.attr_property("keys", {}, 'json') @@ -275,11 +281,17 @@ def get_ssh_name(self): assert False def get_ssh_flags(self, scp=False): - if scp: - return ["-P", str(self.ssh_port)] + if self.ssh_config_options_file == "": + options = [] else: - return ["-p", str(self.ssh_port)] + options = [ + "-F", self.ssh_config_options_file + ] + if scp: + return options + ["-P", str(self.ssh_port)] + else: + return options + ["-p", str(self.ssh_port)] def get_ssh_password(self): return None diff --git a/nixops/deployment.py b/nixops/deployment.py index a767806dc..1a01a5603 100644 --- a/nixops/deployment.py +++ b/nixops/deployment.py @@ -468,6 +468,7 @@ def do_machine(m): ('networking', 'p2pTunnels', 'ssh', m2.name): { 'target': '{0}-unencrypted'.format(m2.name), 'targetPort': m2.ssh_port, + 'sshConfigOptionsFile': m2.ssh_config_options_file, 'localTunnel': local_tunnel, 'remoteTunnel': remote_tunnel, 'localIPv4': local_ipv4, From 37eb32faf767c7a36aadefcec54bb16306acdd37 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 20 Oct 2018 22:07:43 -0400 Subject: [PATCH 02/12] fixups --- nixops/backends/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nixops/backends/__init__.py b/nixops/backends/__init__.py index 0d73d7ef0..0e576fd63 100644 --- a/nixops/backends/__init__.py +++ b/nixops/backends/__init__.py @@ -16,8 +16,9 @@ def __init__(self, xml, config={}): self.encrypted_links_to = set([e.get("value") for e in xml.findall("attrs/attr[@name='encryptedLinksTo']/list/string")]) self.store_keys_on_machine = xml.find("attrs/attr[@name='storeKeysOnMachine']/bool").get("value") == "true" self.ssh_port = int(xml.find("attrs/attr[@name='targetPort']/int").get("value")) - ssh_options_file_elem = xml.find("attrs/attr[@name='sshConfigOptionsFile']/string") - if ssh_options_file_elem: + ssh_options_file_elem = xml.find("attrs/attr[@name='sshConfigOptionsFile']/path") + + if ssh_options_file_elem is not None: self.ssh_config_options_file = ssh_options_file_elem.get('value') else: self.ssh_config_options_file = None @@ -49,7 +50,7 @@ class MachineState(nixops.resources.ResourceState): has_fast_connection = nixops.util.attr_property("hasFastConnection", False, bool) ssh_pinged = nixops.util.attr_property("sshPinged", False, bool) ssh_port = nixops.util.attr_property("targetPort", 22, int) - ssh_config_options_file = nixops.util.attr_property("sshConfigOptionsFile", "", str) + ssh_config_options_file = nixops.util.attr_property("sshConfigOptionsFile", None, str) public_vpn_key = nixops.util.attr_property("publicVpnKey", None) store_keys_on_machine = nixops.util.attr_property("storeKeysOnMachine", False, bool) keys = nixops.util.attr_property("keys", {}, 'json') @@ -92,6 +93,7 @@ def set_common_state(self, defn): self.store_keys_on_machine = defn.store_keys_on_machine self.keys = defn.keys self.ssh_port = defn.ssh_port + self.ssh_config_options_file = defn.ssh_config_options_file self.has_fast_connection = defn.has_fast_connection if not self.has_fast_connection: self.ssh.enable_compression() @@ -281,12 +283,12 @@ def get_ssh_name(self): assert False def get_ssh_flags(self, scp=False): - if self.ssh_config_options_file == "": - options = [] - else: + if self.ssh_config_options_file: options = [ "-F", self.ssh_config_options_file ] + else: + options = [] if scp: return options + ["-P", str(self.ssh_port)] From 90617cd4d7ffdb04127883225c30416cb6ccf6c6 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sun, 21 Oct 2018 07:26:17 -0400 Subject: [PATCH 03/12] Wait for SSH instead of a TCP port --- nixops/backends/__init__.py | 2 +- nixops/ssh_util.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/nixops/backends/__init__.py b/nixops/backends/__init__.py index 0e576fd63..1b4e7f009 100644 --- a/nixops/backends/__init__.py +++ b/nixops/backends/__init__.py @@ -317,7 +317,7 @@ def wait_for_ssh(self, check=False): """Wait until the SSH port is open on this machine.""" if self.ssh_pinged and (not check or self._ssh_pinged_this_time): return self.log_start("waiting for SSH...") - nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, callback=lambda: self.log_continue(".")) + self.ssh.wait_for_ssh(callback=lambda: self.log_continue(".")) self.log_end("") if self.state != self.RESCUE: self.state = self.UP diff --git a/nixops/ssh_util.py b/nixops/ssh_util.py index 9813cab02..054923475 100644 --- a/nixops/ssh_util.py +++ b/nixops/ssh_util.py @@ -299,5 +299,20 @@ def run_command(self, command, flags=[], timeout=None, logged=True, else: return res + def wait_for_ssh(self, user=None, timeout=-1, callback=None): + """Wait until the remote's SSH is up.""" + n = 0 + while True: + try: + self.run_command('true', timeout=5, user=user) + return True + except nixops.ssh_util.SSHConnectionFailed: + n = n + 1 + if timeout != -1 and n >= timeout: break + if callback: callback() + raise Exception("timed out waiting for SSH on ‘{1}’".format( + self._get_target(user))) + + def enable_compression(self): self._compress = True From e65f7c8c46b79155cc2cc403a6fc5c110e621def Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sun, 21 Oct 2018 08:37:31 -0400 Subject: [PATCH 04/12] Use wait_for_ssh for verifying ssh is back up, but not on down for forced reboots --- nixops/backends/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/backends/__init__.py b/nixops/backends/__init__.py index 1b4e7f009..bda78705e 100644 --- a/nixops/backends/__init__.py +++ b/nixops/backends/__init__.py @@ -198,10 +198,10 @@ def reboot_sync(self, hard=False): """Reboot this machine and wait until it's up again.""" self.reboot(hard=hard) self.log_start("waiting for the machine to finish rebooting...") + # !!! TODO nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, open=False, callback=lambda: self.log_continue(".")) self.log_continue("[down]") - nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, callback=lambda: self.log_continue(".")) - self.log_end("[up]") + self.wait_for_ssh() self.state = self.UP self.ssh_pinged = True self._ssh_pinged_this_time = True From 0192a61182c96ae3e3e38c66aa5b93aa9c7727e2 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 11 Dec 2018 21:48:53 -0500 Subject: [PATCH 05/12] Finish up patching away TCP checks, use SSH directly --- nixops/backends/__init__.py | 7 +++++-- nixops/backends/azure_vm.py | 1 - nixops/backends/hetzner.py | 15 +++++++++------ nixops/backends/none.py | 2 +- nixops/ssh_util.py | 12 ++++++++++-- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/nixops/backends/__init__.py b/nixops/backends/__init__.py index bda78705e..c1a86e7b0 100644 --- a/nixops/backends/__init__.py +++ b/nixops/backends/__init__.py @@ -198,8 +198,8 @@ def reboot_sync(self, hard=False): """Reboot this machine and wait until it's up again.""" self.reboot(hard=hard) self.log_start("waiting for the machine to finish rebooting...") - # !!! TODO - nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, open=False, callback=lambda: self.log_continue(".")) + while self.try_ssh(): + self.log_continue(".") self.log_continue("[down]") self.wait_for_ssh() self.state = self.UP @@ -313,6 +313,9 @@ def address_to(self, r): """Return the IP address to be used to access resource "r" from this machine.""" return r.public_ipv4 + def try_ssh(self): + return self.ssh.try_ssh() + def wait_for_ssh(self, check=False): """Wait until the SSH port is open on this machine.""" if self.ssh_pinged and (not check or self._ssh_pinged_this_time): return diff --git a/nixops/backends/azure_vm.py b/nixops/backends/azure_vm.py index 403a25ffc..03c68e781 100644 --- a/nixops/backends/azure_vm.py +++ b/nixops/backends/azure_vm.py @@ -15,7 +15,6 @@ import nixops from nixops import known_hosts -from nixops.util import wait_for_tcp_port, ping_tcp_port from nixops.util import attr_property, create_key_pair, generate_random_string, check_wait from nixops.nix_expr import Call, RawValue diff --git a/nixops/backends/hetzner.py b/nixops/backends/hetzner.py index 6e7f9a0ac..704cc9ed6 100644 --- a/nixops/backends/hetzner.py +++ b/nixops/backends/hetzner.py @@ -9,7 +9,6 @@ from hetzner.robot import Robot from nixops import known_hosts -from nixops.util import wait_for_tcp_port, ping_tcp_port from nixops.util import attr_property, create_key_pair, xml_expr_to_python from nixops.ssh_util import SSHCommandFailed from nixops.backends import MachineDefinition, MachineState @@ -187,10 +186,12 @@ def _wait_for_rescue(self, ip): # so only wait for the reboot to finish when deploying real # systems. self.log_start("waiting for rescue system...") - dotlog = lambda: self.log_continue(".") # NOQA - wait_for_tcp_port(ip, 22, open=False, callback=dotlog) + while self.try_ssh(): + self.log_continue(".") self.log_continue("[down]") - wait_for_tcp_port(ip, 22, callback=dotlog) + + while not self.try_ssh(): + self.log_continue(".") self.log_end("[up]") self.state = self.RESCUE @@ -650,7 +651,9 @@ def _wait_stop(self): """ self.log_start("waiting for system to shutdown... ") dotlog = lambda: self.log_continue(".") # NOQA - wait_for_tcp_port(self.main_ipv4, 22, open=False, callback=dotlog) + while self.try_ssh(): + self.log_continue(".") + self.log_continue("[down]") self.state = self.STOPPED @@ -684,7 +687,7 @@ def _check(self, res): return if self.state in (self.STOPPED, self.STOPPING): - res.is_up = ping_tcp_port(self.main_ipv4, 22) + res.is_up = self.try_ssh() if not res.is_up: self.state = self.STOPPED res.is_reachable = False diff --git a/nixops/backends/none.py b/nixops/backends/none.py index 7222eb777..7ec3a17ae 100644 --- a/nixops/backends/none.py +++ b/nixops/backends/none.py @@ -88,7 +88,7 @@ def _check(self, res): res.exists = False return res.exists = True - res.is_up = nixops.util.ping_tcp_port(self.target_host, self.ssh_port) + res.is_up = self.try_ssh() if res.is_up: MachineState._check(self, res) diff --git a/nixops/ssh_util.py b/nixops/ssh_util.py index 054923475..674845061 100644 --- a/nixops/ssh_util.py +++ b/nixops/ssh_util.py @@ -299,12 +299,20 @@ def run_command(self, command, flags=[], timeout=None, logged=True, else: return res - def wait_for_ssh(self, user=None, timeout=-1, callback=None): + def try_ssh(self, user=None, timeout=-1): + try: + self.run_command('true', timeout=1, user=user, flags=['-q']) + return True + except nixops.ssh_util.SSHConnectionFailed: + return False + + + def wait_for_ssh(self, user=None, attempts=-1, timeout=5, callback=None): """Wait until the remote's SSH is up.""" n = 0 while True: try: - self.run_command('true', timeout=5, user=user) + self.run_command('true', timeout=5, user=user, flags=['-q']) return True except nixops.ssh_util.SSHConnectionFailed: n = n + 1 From d9c55c684187112a5819d6db6a5c129a25bd6665 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 8 Jun 2019 14:10:57 -0400 Subject: [PATCH 06/12] try_ssh: drop timeout option --- nixops/ssh_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixops/ssh_util.py b/nixops/ssh_util.py index 674845061..f66f01625 100644 --- a/nixops/ssh_util.py +++ b/nixops/ssh_util.py @@ -299,7 +299,7 @@ def run_command(self, command, flags=[], timeout=None, logged=True, else: return res - def try_ssh(self, user=None, timeout=-1): + def try_ssh(self, user=None): try: self.run_command('true', timeout=1, user=user, flags=['-q']) return True From bb09606db99fc96bac522fb4bb4a4ab892e5dcce Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 8 Jun 2019 14:14:22 -0400 Subject: [PATCH 07/12] wait_for_ssh: clean up attempts, timeout --- nixops/ssh_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nixops/ssh_util.py b/nixops/ssh_util.py index f66f01625..b718a80c8 100644 --- a/nixops/ssh_util.py +++ b/nixops/ssh_util.py @@ -309,14 +309,14 @@ def try_ssh(self, user=None): def wait_for_ssh(self, user=None, attempts=-1, timeout=5, callback=None): """Wait until the remote's SSH is up.""" - n = 0 + attempt = 0 while True: try: - self.run_command('true', timeout=5, user=user, flags=['-q']) + self.run_command('true', timeout=timeout, user=user, flags=['-q']) return True except nixops.ssh_util.SSHConnectionFailed: - n = n + 1 - if timeout != -1 and n >= timeout: break + attempt += 1 + if attempts != -1 and attempt >= attempts: break if callback: callback() raise Exception("timed out waiting for SSH on ‘{1}’".format( self._get_target(user))) From cdefcaa41e84a036ed7a563ef51069161ce78950 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 8 Jun 2019 14:15:39 -0400 Subject: [PATCH 08/12] Try_ssh: teach about timeouts --- nixops/ssh_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/ssh_util.py b/nixops/ssh_util.py index b718a80c8..9a2aac6b4 100644 --- a/nixops/ssh_util.py +++ b/nixops/ssh_util.py @@ -299,9 +299,9 @@ def run_command(self, command, flags=[], timeout=None, logged=True, else: return res - def try_ssh(self, user=None): + def try_ssh(self, user=None, timeout=1): try: - self.run_command('true', timeout=1, user=user, flags=['-q']) + self.run_command('true', timeout=timeout, user=user, flags=['-q']) return True except nixops.ssh_util.SSHConnectionFailed: return False From bf1f0bfab98769683ff63ad75fa147121538b5ec Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 8 Jun 2019 14:15:49 -0400 Subject: [PATCH 09/12] wait_for_ssh: define in terms of try_ssh --- nixops/ssh_util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nixops/ssh_util.py b/nixops/ssh_util.py index 9a2aac6b4..18e80abcd 100644 --- a/nixops/ssh_util.py +++ b/nixops/ssh_util.py @@ -311,10 +311,9 @@ def wait_for_ssh(self, user=None, attempts=-1, timeout=5, callback=None): """Wait until the remote's SSH is up.""" attempt = 0 while True: - try: - self.run_command('true', timeout=timeout, user=user, flags=['-q']) + if self.try_ssh(user=user, timeout=timeout) return True - except nixops.ssh_util.SSHConnectionFailed: + else attempt += 1 if attempts != -1 and attempt >= attempts: break if callback: callback() From f2418991f3e09e368f6bb81aad2b9cc3c484fb87 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 8 Jun 2019 14:21:59 -0400 Subject: [PATCH 10/12] =?UTF-8?q?Wait=5Ffor=5Fssh:=20make=20=C2=ABup=C2=BB?= =?UTF-8?q?=20configurable=20so=20it=20can=20wait=20for=20a=20down=20SSH?= =?UTF-8?q?=20server=20too?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nixops/ssh_util.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nixops/ssh_util.py b/nixops/ssh_util.py index 18e80abcd..e144e1167 100644 --- a/nixops/ssh_util.py +++ b/nixops/ssh_util.py @@ -306,12 +306,11 @@ def try_ssh(self, user=None, timeout=1): except nixops.ssh_util.SSHConnectionFailed: return False - - def wait_for_ssh(self, user=None, attempts=-1, timeout=5, callback=None): - """Wait until the remote's SSH is up.""" + def wait_for_ssh(self, user=None, attempts=-1, timeout=5, callback=None, up=True): + """Wait until the remote's SSH is up or down based on the «up» parameter.""" attempt = 0 while True: - if self.try_ssh(user=user, timeout=timeout) + if self.try_ssh(user=user, timeout=timeout) == up return True else attempt += 1 @@ -320,6 +319,5 @@ def wait_for_ssh(self, user=None, attempts=-1, timeout=5, callback=None): raise Exception("timed out waiting for SSH on ‘{1}’".format( self._get_target(user))) - def enable_compression(self): self._compress = True From d5bb60c45372faba348444006c8deb57e367de58 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 8 Jun 2019 14:23:31 -0400 Subject: [PATCH 11/12] hetzner: use wait_for_ssh instead of try_ssh in a loop --- nixops/backends/hetzner.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nixops/backends/hetzner.py b/nixops/backends/hetzner.py index 704cc9ed6..1e534c306 100644 --- a/nixops/backends/hetzner.py +++ b/nixops/backends/hetzner.py @@ -186,12 +186,11 @@ def _wait_for_rescue(self, ip): # so only wait for the reboot to finish when deploying real # systems. self.log_start("waiting for rescue system...") - while self.try_ssh(): - self.log_continue(".") + dotlog = lambda: self.log_continue(".") # NOQA + self.wait_for_ssh(callback=dotlog, up=False) self.log_continue("[down]") - while not self.try_ssh(): - self.log_continue(".") + self.wait_for_ssh(callback=dotlog, up=True) self.log_end("[up]") self.state = self.RESCUE From 975f87f4384d0c0d4175bceba57b7eebd351481a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 8 Jun 2019 14:31:46 -0400 Subject: [PATCH 12/12] ssh_util: Fixup python syntax --- nixops/ssh_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixops/ssh_util.py b/nixops/ssh_util.py index e144e1167..4e779b4c6 100644 --- a/nixops/ssh_util.py +++ b/nixops/ssh_util.py @@ -310,9 +310,9 @@ def wait_for_ssh(self, user=None, attempts=-1, timeout=5, callback=None, up=True """Wait until the remote's SSH is up or down based on the «up» parameter.""" attempt = 0 while True: - if self.try_ssh(user=user, timeout=timeout) == up + if self.try_ssh(user=user, timeout=timeout) == up: return True - else + else: attempt += 1 if attempts != -1 and attempt >= attempts: break if callback: callback()