From d945e564052ffc06ede59c78f99344f9fd3e571c Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 12 May 2014 16:06:40 -0500 Subject: [PATCH] Add retries to orchestra.connection.connect() This is an attempt to fix: http://tracker.ceph.com/issues/8314 Signed-off-by: Zack Cerza --- teuthology/orchestra/connection.py | 15 ++++++++++++++- teuthology/orchestra/test/test_connection.py | 8 +++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/teuthology/orchestra/connection.py b/teuthology/orchestra/connection.py index 9317d695bd2..c1f51a7e0ee 100644 --- a/teuthology/orchestra/connection.py +++ b/teuthology/orchestra/connection.py @@ -4,7 +4,12 @@ Connection utilities import base64 import paramiko import os +import logging + from ..config import config +from ..contextutil import safe_while + +log = logging.getLogger(__name__) def split_user(user_at_host): @@ -85,7 +90,15 @@ def connect(user_at_host, host_key=None, keep_alive=False, if opt_name in opts: connect_args[arg_name] = opts[opt_name] + log.info(connect_args) + # just let the exceptions bubble up to caller - ssh.connect(**connect_args) + with safe_while(sleep=1, action='connect to ' + host) as proceed: + while proceed(): + try: + ssh.connect(**connect_args) + break + except paramiko.AuthenticationException: + log.exception("Error connecting to {host}".format(host=host)) ssh.get_transport().set_keepalive(keep_alive) return ssh diff --git a/teuthology/orchestra/test/test_connection.py b/teuthology/orchestra/test/test_connection.py index 951a6742340..5dd3861860f 100644 --- a/teuthology/orchestra/test/test_connection.py +++ b/teuthology/orchestra/test/test_connection.py @@ -1,13 +1,15 @@ -from teuthology import config - import fudge +from teuthology import config from .util import assert_raises - from .. import connection class TestConnection(object): + def setup(self): + import time + time.sleep = lambda s: True + def clear_config(self): config.config.teuthology_yaml = '' config.config.load_files()