mgr/dashboard: Set timeout in RestClient calls

Set a default timeout of 45 seconds to all REST client calls. This can be customized via 'ceph dashboard set-rest-requests-timeout <seconds>'. Currently the REST client is only used by the RGW controller.

Signed-off-by: Volker Theile <vtheile@suse.com>

doc/mgr/dashboard.rst

@@ -190,7 +190,7 @@ commands::
 
 
 Enabling the Object Gateway management frontend
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To use the Object Gateway management functionality of the dashboard, you will
 need to provide the login credentials of a user with the ``system`` flag
@@ -232,6 +232,13 @@ exist and you may find yourself in the situation that you have to use them::
   $ ceph dashboard set-rgw-api-admin-resource <admin_resource>
   $ ceph dashboard set-rgw-api-user-id <user_id>
 
+If the Object Gateway takes too long to process requests and the dashboard runs
+into timeouts, then you can set the timeout value to your needs::
+
+  $ ceph dashboard set-rest-requests-timeout <seconds>
+
+The default value is 45 seconds.
+
 Enabling the Embedding of Grafana Dashboards
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

src/pybind/mgr/dashboard/rest_client.py

@@ -13,11 +13,12 @@
 """
 from __future__ import absolute_import
 
+from .settings import Settings
 from .tools import build_url
 import inspect
 import re
 import requests
-from requests.exceptions import ConnectionError, InvalidURL
+from requests.exceptions import ConnectionError, InvalidURL, Timeout
 from . import logger
 
 try:
@@ -26,6 +27,18 @@ except ImportError:
     from urllib3.exceptions import SSLError
 
 
+class TimeoutRequestsSession(requests.Session):
+    """
+    Set timeout argument for all requests if this is not already done.
+    """
+    def request(self, *args, **kwargs):
+        if ((args[8] if len(args) > 8 else None) is None) \
+                and kwargs.get('timeout') is None:
+            if Settings.REST_REQUESTS_TIMEOUT > 0:
+                kwargs['timeout'] = Settings.REST_REQUESTS_TIMEOUT
+        return super(TimeoutRequestsSession, self).request(*args, **kwargs)
+
+
 class RequestException(Exception):
     def __init__(self,
                  message,
@@ -315,7 +328,7 @@ class RestClient(object):
         logger.debug("REST service base URL: %s", self.base_url)
         self.headers = {'Accept': 'application/json'}
         self.auth = auth
-        self.session = requests.Session()
+        self.session = TimeoutRequestsSession()
 
     def _login(self, request=None):
         pass
@@ -465,6 +478,12 @@ class RestClient(object):
             logger.exception("%s REST API failed %s: %s", self.client_name,
                              method.upper(), str(ex))
             raise RequestException(str(ex))
+        except Timeout as ex:
+            msg = "{} REST API {} timed out after {} seconds (url={}).".format(
+                self.client_name, ex.request.method, Settings.REST_REQUESTS_TIMEOUT,
+                ex.request.url)
+            logger.exception(msg)
+            raise RequestException(msg)
 
     @staticmethod
     def api(path, **api_kwargs):

src/pybind/mgr/dashboard/settings.py

@@ -19,6 +19,7 @@ class Options(object):
         GRAFANA_API_PORT = (3000, int)
     """
     ENABLE_BROWSABLE_API = (True, bool)
+    REST_REQUESTS_TIMEOUT = (45, int)
 
     # RGW settings
     RGW_API_HOST = ('', str)

src/pybind/mgr/dashboard/tests/test_rest_client.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+import unittest
+
+from mock import patch
+from .. import mgr
+from ..rest_client import RestClient
+
+
+class RestClientTest(unittest.TestCase):
+    def setUp(self):
+        settings = {'REST_REQUESTS_TIMEOUT': 45}
+        mgr.get_config.side_effect = settings.get
+
+    def test_timeout_auto_set(self):
+        with patch('requests.Session.request') as mock_request:
+            rest_client = RestClient('localhost', 8000)
+            rest_client.session.request('GET', '/test')
+            mock_request.assert_called_with('GET', '/test', timeout=45)
+
+    def test_timeout_auto_set_arg(self):
+        with patch('requests.Session.request') as mock_request:
+            rest_client = RestClient('localhost', 8000)
+            rest_client.session.request(
+                'GET', '/test', None, None, None, None,
+                None, None, None)
+            mock_request.assert_called_with(
+                'GET', '/test', None, None, None, None,
+                None, None, None, timeout=45)
+
+    def test_timeout_no_auto_set_kwarg(self):
+        with patch('requests.Session.request') as mock_request:
+            rest_client = RestClient('localhost', 8000)
+            rest_client.session.request('GET', '/test', timeout=20)
+            mock_request.assert_called_with('GET', '/test', timeout=20)
+
+    def test_timeout_no_auto_set_arg(self):
+        with patch('requests.Session.request') as mock_request:
+            rest_client = RestClient('localhost', 8000)
+            rest_client.session.request(
+                'GET', '/test', None, None, None, None,
+                None, None, 40)
+            mock_request.assert_called_with(
+                'GET', '/test', None, None, None, None,
+                None, None, 40)