From 51e2348d5599f8584ce7c9876239290941535ba9 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Thu, 4 Jul 2019 22:34:07 +0800 Subject: [PATCH 1/4] Install libgnutls-dev Well, installing packages for some builds only seems complicated. https://github.com/travis-ci/travis-ci/issues/3505 --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index b009407..a5c1f13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,3 +28,8 @@ matrix: # doesn't work well, see https://travis-ci.org/lilydjwg/nvchecker/jobs/376326582 - python: pypy3.5 env: DEPS=aiohttp + +addons: + apt: + packages: + - libgnutls-dev From 8720d59f79a9fb79e3231466c7ddb8b4ec16d618 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Thu, 4 Jul 2019 23:04:58 +0800 Subject: [PATCH 2/4] Use the latest pypy3 Ref: https://github.com/pyenv/pyenv/tree/master/plugins/python-build/share/python-build --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5c1f13..4d90b3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: - "3.6" - "3.7" - "nightly" - - "pypy3.5" + - "pypy3.6-7.1.1" install: pip install -U $DEPS pytest pytest-asyncio flaky structlog script: pytest env: @@ -26,7 +26,7 @@ matrix: fast_finish: true allow_failures: # doesn't work well, see https://travis-ci.org/lilydjwg/nvchecker/jobs/376326582 - - python: pypy3.5 + - python: pypy3.6-7.1.1 env: DEPS=aiohttp addons: From 629e82ac8a2ce7776c2e80354868d28e7ac8a664 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Fri, 5 Jul 2019 09:20:36 +0800 Subject: [PATCH 3/4] Add default User-Agent crates.io requires it since https://github.com/rust-lang/crates.io/pull/1696 --- nvchecker/source/aiohttp_httpclient.py | 4 ++++ nvchecker/source/github.py | 1 - nvchecker/source/httpclient.py | 1 + nvchecker/source/tornado_httpclient.py | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 nvchecker/source/httpclient.py diff --git a/nvchecker/source/aiohttp_httpclient.py b/nvchecker/source/aiohttp_httpclient.py index 4fd8b4f..bc73107 100644 --- a/nvchecker/source/aiohttp_httpclient.py +++ b/nvchecker/source/aiohttp_httpclient.py @@ -4,6 +4,8 @@ import atexit import asyncio import aiohttp +from .httpclient import DEFAULT_USER_AGENT + connector = aiohttp.TCPConnector(limit=20) __all__ = ['session', 'HTTPError', 'NetworkErrors'] @@ -19,6 +21,8 @@ class BetterClientSession(aiohttp.ClientSession): if hasattr(self, "nv_config") and self.nv_config.get("proxy"): kwargs.setdefault("proxy", self.nv_config.get("proxy")) + kwargs.setdefault("headers", {}).setdefault('User-Agent', DEFAULT_USER_AGENT) + res = await super(BetterClientSession, self)._request( *args, **kwargs) if res.status >= 400: diff --git a/nvchecker/source/github.py b/nvchecker/source/github.py index 6ecd499..e19b804 100644 --- a/nvchecker/source/github.py +++ b/nvchecker/source/github.py @@ -46,7 +46,6 @@ async def get_version_real(name, conf, **kwargs): url += '?' + urlencode(parameters) headers = { 'Accept': 'application/vnd.github.quicksilver-preview+json', - 'User-Agent': 'lilydjwg/nvchecker', } if 'NVCHECKER_GITHUB_TOKEN' in os.environ: headers['Authorization'] = 'token %s' % os.environ['NVCHECKER_GITHUB_TOKEN'] diff --git a/nvchecker/source/httpclient.py b/nvchecker/source/httpclient.py new file mode 100644 index 0000000..a1666e1 --- /dev/null +++ b/nvchecker/source/httpclient.py @@ -0,0 +1 @@ +DEFAULT_USER_AGENT = 'lilydjwg/nvchecker' diff --git a/nvchecker/source/tornado_httpclient.py b/nvchecker/source/tornado_httpclient.py index dcf4ae8..74ab246 100644 --- a/nvchecker/source/tornado_httpclient.py +++ b/nvchecker/source/tornado_httpclient.py @@ -15,6 +15,8 @@ try: except ImportError: pycurl = None +from .httpclient import DEFAULT_USER_AGENT + __all__ = ['session', 'HTTPError', 'NetworkErrors'] client = AsyncHTTPClient() @@ -51,6 +53,7 @@ class Session: q = urlencode(params) url += '?' + q + kwargs.setdefault("headers", {}).setdefault('User-Agent', DEFAULT_USER_AGENT) r = HTTPRequest(url, **kwargs) return ResponseManager(r) From 851e141f3d5bcea2a8447790c6624a5c0c05cd98 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Sat, 6 Jul 2019 13:29:01 +0800 Subject: [PATCH 4/4] Use pytest-httpbin and add a test for default User-Agent --- .travis.yml | 2 +- setup.py | 1 + tests/test_regex.py | 14 +++++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d90b3f..8e25359 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ python: - "3.7" - "nightly" - "pypy3.6-7.1.1" -install: pip install -U $DEPS pytest pytest-asyncio flaky structlog +install: pip install -U $DEPS pytest pytest-asyncio pytest-httpbin flaky structlog script: pytest env: global: diff --git a/setup.py b/setup.py index 8853f61..8446e8f 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ setup( tests_require = [ 'pytest', 'pytest-asyncio', + 'pytest-httpbin', 'flaky', ], entry_points = { diff --git a/tests/test_regex.py b/tests/test_regex.py index 0c1ac72..ee57c8a 100644 --- a/tests/test_regex.py +++ b/tests/test_regex.py @@ -4,12 +4,16 @@ import pytest pytestmark = pytest.mark.asyncio -@pytest.mark.skipif(True, - reason='httpbin is overloaded?') -async def test_regex_httpbin(get_version): +async def test_regex_httpbin_default_user_agent(get_version, httpbin): assert await get_version("example", { - "url": "https://httpbin.org/get", - "regex": r'"User-Agent": "(\w+)"', + "url": httpbin.url + "/get", + "regex": r'"User-Agent":\s*"([^"]+)"', + }) == "lilydjwg/nvchecker" + +async def test_regex_httpbin(get_version, httpbin): + assert await get_version("example", { + "url": httpbin.url + "/get", + "regex": r'"User-Agent":\s*"([^"]+)"', "user_agent": "Meow", }) == "Meow"