github: give an explicit error message when rate limited

also update tests.

nvchecker/core.py

@@ -176,6 +176,8 @@ class Source:
         self.on_exception(name, result)
       elif result is not None:
         self.print_version_update(name, result)
+      else:
+        self.on_no_result(name)
 
     await filler_fu
 
@@ -194,6 +196,9 @@ class Source:
   def on_update(self, name, version, oldver):
     pass
 
+  def on_no_result(self, name, oldver):
+    pass
+
   def on_exception(self, name, exc):
     pass
 

nvchecker/source/aiohttp_httpclient.py

@@ -8,9 +8,10 @@ connector = aiohttp.TCPConnector(limit=20)
 __all__ = ['session', 'HTTPError']
 
 class HTTPError(Exception):
-    def __init__(self, code, message):
+    def __init__(self, code, message, response):
         self.code = code
         self.message = message
+        self.response = response
 
 class BetterClientSession(aiohttp.ClientSession):
     async def _request(self, *args, **kwargs):
@@ -20,7 +21,7 @@ class BetterClientSession(aiohttp.ClientSession):
         res = await super(BetterClientSession, self)._request(
             *args, **kwargs)
         if res.status >= 400:
-            raise HTTPError(res.status, res.reason)
+            raise HTTPError(res.status, res.reason, res)
         return res
 
 session = BetterClientSession(connector=connector, read_timeout=10, conn_timeout=5)

nvchecker/source/github.py

@@ -3,11 +3,12 @@
 
 import os
 import re
+import time
 from functools import partial
 
 import structlog
 
-from . import session
+from . import session, HTTPError
 from ..sortversion import sort_version_keys
 
 logger = structlog.get_logger(logger_name=__name__)
@@ -17,6 +18,12 @@ GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest'
 GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/tags'
 
 async def get_version(name, conf, **kwargs):
+  try:
+    return await get_version_real(name, conf, **kwargs)
+  except HTTPError as e:
+    check_ratelimit(e, name)
+
+async def get_version_real(name, conf, **kwargs):
   repo = conf.get('github')
   br = conf.get('branch')
   use_latest_release = conf.getboolean('use_latest_release', False)
@@ -81,9 +88,9 @@ async def max_tag(
 
   while True:
     async with getter(url) as res:
-      links = res.headers.get('Link')
       logger.debug('X-RateLimit-Remaining',
-                   n=res.headers.get('X-RateLimit-Remaining'))
+                    n=res.headers.get('X-RateLimit-Remaining'))
+      links = res.headers.get('Link')
       data = await res.json()
 
     data = [tag["name"] for tag in data if tag["name"] not in ignored_tags]
@@ -112,3 +119,16 @@ def get_next_page_url(links):
     return
 
   return next_link[0].split('>', 1)[0][1:]
+
+def check_ratelimit(exc, name):
+  res = exc.response
+  n = int(res.headers.get('X-RateLimit-Remaining'))
+  if n == 0:
+    reset = int(res.headers.get('X-RateLimit-Reset'))
+    logger.error('rate limited, resetting at %s. '
+                 'Or get an API token to increase the allowance if not yet'
+                 % time.ctime(reset),
+                 name = name,
+                 reset = reset)
+  else:
+    raise

tests/conftest.py

@@ -14,6 +14,9 @@ class TestSource(Source):
     def on_update(self, name, version, oldver):
         self._future.set_result(version)
 
+    def on_no_result(self, name):
+        self._future.set_result(None)
+
     def on_exception(self, name, exc):
         self._future.set_exception(exc)
 

tests/test_keyfile.py

@@ -1,23 +1,37 @@
 # MIT licensed
 # Copyright (c) 2018 lilydjwg <lilydjwg@gmail.com>, et al.
 
+import os
 import tempfile
+import contextlib
 
 from nvchecker.source import HTTPError
 
 import pytest
 pytestmark = [pytest.mark.asyncio]
 
+@contextlib.contextmanager
+def unset_github_token_env():
+  token = os.environ.get('NVCHECKER_GITHUB_TOKEN')
+  try:
+    if token:
+      del os.environ['NVCHECKER_GITHUB_TOKEN']
+    yield token
+  finally:
+    if token:
+      os.environ['NVCHECKER_GITHUB_TOKEN'] = token
+
 async def test_keyfile_missing(run_source):
   test_conf = '''\
 [example]
 github = harry-sanabria/ReleaseTestRepo
 '''
 
-  assert await run_source(test_conf) == '20140122.012101'
+  assert await run_source(test_conf) in ['20140122.012101', None]
 
 async def test_keyfile_invalid(run_source):
-  with tempfile.NamedTemporaryFile(mode='w+') as f:
+  with tempfile.NamedTemporaryFile(mode='w') as f, \
+    unset_github_token_env():
     f.write('''\
 [keys]
 github = xxx
@@ -32,6 +46,32 @@ keyfile = {f.name}
       '''
 
     try:
-      await run_source(test_conf)
+      version = await run_source(test_conf)
+      assert version is None # out of allowance
+      return
     except HTTPError as e:
       assert e.code == 401
+      return
+
+    raise Exception('expected 401 response')
+
+@pytest.mark.skipif('NVCHECKER_GITHUB_TOKEN' not in os.environ,
+                    reason='no key given')
+async def test_keyfile_valid(run_source):
+  with tempfile.NamedTemporaryFile(mode='w') as f, \
+    unset_github_token_env() as token:
+    f.write(f'''\
+[keys]
+github = {token}
+            ''')
+    f.flush()
+
+    test_conf = f'''\
+[example]
+github = harry-sanabria/ReleaseTestRepo
+
+[__config__]
+keyfile = {f.name}
+      '''
+
+    assert await run_source(test_conf) == '20140122.012101'