From 3da3e356fabd21082b3a33ffbe97c60f7ac42f59 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Tue, 8 May 2018 19:15:36 +0800 Subject: [PATCH] github: give an explicit error message when rate limited also update tests. --- nvchecker/core.py | 5 +++ nvchecker/source/aiohttp_httpclient.py | 5 +-- nvchecker/source/github.py | 26 +++++++++++++-- tests/conftest.py | 3 ++ tests/test_keyfile.py | 46 ++++++++++++++++++++++++-- 5 files changed, 77 insertions(+), 8 deletions(-) diff --git a/nvchecker/core.py b/nvchecker/core.py index 7f578eb..a63ef47 100644 --- a/nvchecker/core.py +++ b/nvchecker/core.py @@ -176,6 +176,8 @@ class Source: self.on_exception(name, result) elif result is not None: self.print_version_update(name, result) + else: + self.on_no_result(name) await filler_fu @@ -194,6 +196,9 @@ class Source: def on_update(self, name, version, oldver): pass + def on_no_result(self, name, oldver): + pass + def on_exception(self, name, exc): pass diff --git a/nvchecker/source/aiohttp_httpclient.py b/nvchecker/source/aiohttp_httpclient.py index 85cff0a..a950e17 100644 --- a/nvchecker/source/aiohttp_httpclient.py +++ b/nvchecker/source/aiohttp_httpclient.py @@ -8,9 +8,10 @@ connector = aiohttp.TCPConnector(limit=20) __all__ = ['session', 'HTTPError'] class HTTPError(Exception): - def __init__(self, code, message): + def __init__(self, code, message, response): self.code = code self.message = message + self.response = response class BetterClientSession(aiohttp.ClientSession): async def _request(self, *args, **kwargs): @@ -20,7 +21,7 @@ class BetterClientSession(aiohttp.ClientSession): res = await super(BetterClientSession, self)._request( *args, **kwargs) if res.status >= 400: - raise HTTPError(res.status, res.reason) + raise HTTPError(res.status, res.reason, res) return res session = BetterClientSession(connector=connector, read_timeout=10, conn_timeout=5) diff --git a/nvchecker/source/github.py b/nvchecker/source/github.py index b13409a..c1c755c 100644 --- a/nvchecker/source/github.py +++ b/nvchecker/source/github.py @@ -3,11 +3,12 @@ import os import re +import time from functools import partial import structlog -from . import session +from . import session, HTTPError from ..sortversion import sort_version_keys logger = structlog.get_logger(logger_name=__name__) @@ -17,6 +18,12 @@ GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest' GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/tags' async def get_version(name, conf, **kwargs): + try: + return await get_version_real(name, conf, **kwargs) + except HTTPError as e: + check_ratelimit(e, name) + +async def get_version_real(name, conf, **kwargs): repo = conf.get('github') br = conf.get('branch') use_latest_release = conf.getboolean('use_latest_release', False) @@ -81,9 +88,9 @@ async def max_tag( while True: async with getter(url) as res: - links = res.headers.get('Link') logger.debug('X-RateLimit-Remaining', - n=res.headers.get('X-RateLimit-Remaining')) + n=res.headers.get('X-RateLimit-Remaining')) + links = res.headers.get('Link') data = await res.json() data = [tag["name"] for tag in data if tag["name"] not in ignored_tags] @@ -112,3 +119,16 @@ def get_next_page_url(links): return return next_link[0].split('>', 1)[0][1:] + +def check_ratelimit(exc, name): + res = exc.response + n = int(res.headers.get('X-RateLimit-Remaining')) + if n == 0: + reset = int(res.headers.get('X-RateLimit-Reset')) + logger.error('rate limited, resetting at %s. ' + 'Or get an API token to increase the allowance if not yet' + % time.ctime(reset), + name = name, + reset = reset) + else: + raise diff --git a/tests/conftest.py b/tests/conftest.py index d5c9a75..6af862f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,9 @@ class TestSource(Source): def on_update(self, name, version, oldver): self._future.set_result(version) + def on_no_result(self, name): + self._future.set_result(None) + def on_exception(self, name, exc): self._future.set_exception(exc) diff --git a/tests/test_keyfile.py b/tests/test_keyfile.py index c237b57..0dcef9d 100644 --- a/tests/test_keyfile.py +++ b/tests/test_keyfile.py @@ -1,23 +1,37 @@ # MIT licensed # Copyright (c) 2018 lilydjwg , et al. +import os import tempfile +import contextlib from nvchecker.source import HTTPError import pytest pytestmark = [pytest.mark.asyncio] +@contextlib.contextmanager +def unset_github_token_env(): + token = os.environ.get('NVCHECKER_GITHUB_TOKEN') + try: + if token: + del os.environ['NVCHECKER_GITHUB_TOKEN'] + yield token + finally: + if token: + os.environ['NVCHECKER_GITHUB_TOKEN'] = token + async def test_keyfile_missing(run_source): test_conf = '''\ [example] github = harry-sanabria/ReleaseTestRepo ''' - assert await run_source(test_conf) == '20140122.012101' + assert await run_source(test_conf) in ['20140122.012101', None] async def test_keyfile_invalid(run_source): - with tempfile.NamedTemporaryFile(mode='w+') as f: + with tempfile.NamedTemporaryFile(mode='w') as f, \ + unset_github_token_env(): f.write('''\ [keys] github = xxx @@ -32,6 +46,32 @@ keyfile = {f.name} ''' try: - await run_source(test_conf) + version = await run_source(test_conf) + assert version is None # out of allowance + return except HTTPError as e: assert e.code == 401 + return + + raise Exception('expected 401 response') + +@pytest.mark.skipif('NVCHECKER_GITHUB_TOKEN' not in os.environ, + reason='no key given') +async def test_keyfile_valid(run_source): + with tempfile.NamedTemporaryFile(mode='w') as f, \ + unset_github_token_env() as token: + f.write(f'''\ +[keys] +github = {token} + ''') + f.flush() + + test_conf = f'''\ +[example] +github = harry-sanabria/ReleaseTestRepo + +[__config__] +keyfile = {f.name} + ''' + + assert await run_source(test_conf) == '20140122.012101'