mirror of
https://github.com/ceph/ceph
synced 2024-12-28 22:43:29 +00:00
138 lines
4.3 KiB
Python
138 lines
4.3 KiB
Python
import contextlib
|
|
import sys
|
|
import logging
|
|
import time
|
|
import itertools
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
@contextlib.contextmanager
|
|
def nested(*managers):
|
|
"""
|
|
Like contextlib.nested but takes callables returning context
|
|
managers, to avoid the major reason why contextlib.nested was
|
|
deprecated.
|
|
|
|
This version also logs any exceptions early, much like run_tasks,
|
|
to ease debugging. TODO combine nested and run_tasks.
|
|
"""
|
|
exits = []
|
|
vars = []
|
|
exc = (None, None, None)
|
|
try:
|
|
for mgr_fn in managers:
|
|
mgr = mgr_fn()
|
|
exit = mgr.__exit__
|
|
enter = mgr.__enter__
|
|
vars.append(enter())
|
|
exits.append(exit)
|
|
yield vars
|
|
except Exception:
|
|
log.exception('Saw exception from nested tasks')
|
|
exc = sys.exc_info()
|
|
finally:
|
|
while exits:
|
|
exit = exits.pop()
|
|
try:
|
|
if exit(*exc):
|
|
exc = (None, None, None)
|
|
except Exception:
|
|
exc = sys.exc_info()
|
|
if exc != (None, None, None):
|
|
# Don't rely on sys.exc_info() still containing
|
|
# the right information. Another exception may
|
|
# have been raised and caught by an exit method
|
|
raise exc[0], exc[1], exc[2]
|
|
|
|
|
|
class MaxWhileTries(Exception):
|
|
pass
|
|
|
|
|
|
class safe_while(object):
|
|
"""
|
|
A context manager to remove boiler plate code that deals with `while` loops
|
|
that need a given number of tries and some seconds to sleep between each
|
|
one of those tries.
|
|
|
|
The most simple example possible will try 10 times sleeping for 6 seconds:
|
|
|
|
>>> from teuthology.contexutil import safe_while
|
|
>>> with safe_while() as proceed:
|
|
... while proceed():
|
|
... # repetitive code here
|
|
... print "hello world"
|
|
...
|
|
Traceback (most recent call last):
|
|
...
|
|
MaxWhileTries: reached maximum tries (5) after waiting for 75 seconds
|
|
|
|
Yes, this adds yet another level of indentation but it allows you to
|
|
implement while loops exactly the same as before with just 1 more
|
|
indentation level and one extra call. Everything else stays the same,
|
|
code-wise. So adding this helper to existing code is simpler.
|
|
|
|
:param sleep: The amount of time to sleep between tries. Default 6
|
|
:param increment: The amount to add to the sleep value on each try.
|
|
Default 0.
|
|
:param tries: The amount of tries before giving up. Default 10.
|
|
:param action: The name of the action being attempted. Default none.
|
|
:param _raise: Whether to raise an exception (or log a warning).
|
|
Default True.
|
|
:param _sleeper: The function to use to sleep. Only used for testing.
|
|
Default time.sleep
|
|
"""
|
|
|
|
def __init__(self, sleep=6, increment=0, tries=10, action=None,
|
|
_raise=True, _sleeper=None):
|
|
self.sleep = sleep
|
|
self.increment = increment
|
|
self.tries = tries
|
|
self.counter = 0
|
|
self.sleep_current = sleep
|
|
self.action = action
|
|
self._raise = _raise
|
|
self.sleeper = _sleeper or time.sleep
|
|
|
|
def _make_error_msg(self):
|
|
"""
|
|
Sum the total number of seconds we waited while providing the number
|
|
of tries we attempted
|
|
"""
|
|
total_seconds_waiting = sum(
|
|
itertools.islice(
|
|
itertools.count(self.sleep, self.increment),
|
|
self.tries
|
|
)
|
|
)
|
|
msg = 'reached maximum tries ({tries})' + \
|
|
' after waiting for {total} seconds'
|
|
if self.action:
|
|
msg = "'{action}'" + msg
|
|
|
|
msg = msg.format(
|
|
action=self.action,
|
|
tries=self.tries,
|
|
total=total_seconds_waiting,
|
|
)
|
|
return msg
|
|
|
|
def __call__(self):
|
|
self.counter += 1
|
|
if self.counter > self.tries:
|
|
error_msg = self._make_error_msg()
|
|
if self._raise:
|
|
raise MaxWhileTries(error_msg)
|
|
else:
|
|
log.warning(error_msg)
|
|
return False
|
|
self.sleeper(self.sleep_current)
|
|
self.sleep_current += self.increment
|
|
return True
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
return False
|