From 95150fa8e934f54dd5131a55bc729abfad507f6c Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 20 Aug 2020 14:56:40 +0800 Subject: [PATCH] port github source and tests (the last one!) `include_tags_pattern` and `ignored_tags` removed, see #99 --- .gitignore | 1 + README.rst | 7 -- nvchecker-old/source/github.py | 213 --------------------------------- nvchecker/util.py | 6 - nvchecker_source/github.py | 149 +++++++++++++++++++++++ nvchecker_source/gitlab.py | 3 +- tests-old/test_github.py | 54 --------- tests-old/test_keyfile.py | 77 ------------ tests/conftest.py | 10 +- tests/test_github.py | 78 ++++++++++++ 10 files changed, 236 insertions(+), 362 deletions(-) delete mode 100644 nvchecker-old/source/github.py create mode 100644 nvchecker_source/github.py delete mode 100644 tests-old/test_github.py delete mode 100644 tests-old/test_keyfile.py create mode 100644 tests/test_github.py diff --git a/.gitignore b/.gitignore index ae1f056..31d9843 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ __pycache__/ *.pyo .travis.pub .pytest_cache/ +keyfile.toml diff --git a/README.rst b/README.rst index a1b3ae3..9982092 100644 --- a/README.rst +++ b/README.rst @@ -314,13 +314,6 @@ use_max_tag lightweight ones, and return the largest one sorted by the ``sort_version_key`` option. Will return the tag name instead of date. -max_page - How many pages do we search for the max tag? Default is 1. This works when - ``use_max_tag`` is set. - - However, with current API in use, GitHub seems to always return all data in - one page, making this option obsolete. - proxy The HTTP proxy to use. The format is ``host:port``, e.g. ``localhost:8087``. diff --git a/nvchecker-old/source/github.py b/nvchecker-old/source/github.py deleted file mode 100644 index 828283b..0000000 --- a/nvchecker-old/source/github.py +++ /dev/null @@ -1,213 +0,0 @@ -# MIT licensed -# Copyright (c) 2013-2018 lilydjwg , et al. - -import os -import re -import time -from urllib.parse import urlencode -from functools import partial - -import structlog - -from . import session, HTTPError - -logger = structlog.get_logger(logger_name=__name__) - -GITHUB_URL = 'https://api.github.com/repos/%s/commits' -GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest' -# https://developer.github.com/v3/git/refs/#get-all-references -GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/git/refs/tags' -GITHUB_GRAPHQL_URL = 'https://api.github.com/graphql' - -async def get_version(name, conf, **kwargs): - try: - return await get_version_real(name, conf, **kwargs) - except HTTPError as e: - check_ratelimit(e, name) - -QUERY_LATEST_TAG = ''' -{{ - repository(name: "{name}", owner: "{owner}") {{ - refs(refPrefix: "refs/tags/", first: 1, - query: "{query}", - orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{ - edges {{ - node {{ - name - }} - }} - }} - }} -}} -''' - -async def get_latest_tag(name, conf, token): - repo = conf.get('github') - query = conf.get('query', '') - owner, reponame = repo.split('/') - headers = { - 'Authorization': 'bearer %s' % token, - 'Content-Type': 'application/json', - } - q = QUERY_LATEST_TAG.format( - owner = owner, - name = reponame, - query = query, - ) - async with session.post( - GITHUB_GRAPHQL_URL, - headers = headers, - json = {'query': q}, - ) as res: - j = await res.json() - - refs = j['data']['repository']['refs']['edges'] - if not refs: - logger.error('no tag found', name=name) - return - - return refs[0]['node']['name'] - -def get_token(kwargs): - token = os.environ.get('NVCHECKER_GITHUB_TOKEN') - if token: - return token - - if 'keyman' not in kwargs: - return None - - token = kwargs['keyman'].get_key('github') - return token - -async def get_version_real(name, conf, **kwargs): - token = get_token(kwargs) - - use_latest_tag = conf.getboolean('use_latest_tag', False) - if use_latest_tag: - if not token: - logger.error('token not given but it is required', - name = name) - return - return await get_latest_tag(name, conf, token) - - repo = conf.get('github') - br = conf.get('branch') - path = conf.get('path') - use_latest_release = conf.getboolean('use_latest_release', False) - use_max_tag = conf.getboolean('use_max_tag', False) - include_tags_pattern = conf.get("include_tags_pattern", "") - ignored_tags = conf.get("ignored_tags", "").split() - if use_latest_release: - url = GITHUB_LATEST_RELEASE % repo - elif use_max_tag: - url = GITHUB_MAX_TAG % repo - else: - url = GITHUB_URL % repo - parameters = {} - if br: - parameters['sha'] = br - if path: - parameters['path'] = path - url += '?' + urlencode(parameters) - headers = { - 'Accept': 'application/vnd.github.quicksilver-preview+json', - } - if token: - headers['Authorization'] = 'token %s' % token - - kwargs = {} - if conf.get('proxy'): - kwargs["proxy"] = conf.get("proxy") - - if use_max_tag: - return await max_tag(partial( - session.get, headers=headers, **kwargs), - url, name, ignored_tags, include_tags_pattern, - max_page = conf.getint("max_page", 1), - ) - - async with session.get(url, headers=headers, **kwargs) as res: - logger.debug('X-RateLimit-Remaining', - n=res.headers.get('X-RateLimit-Remaining')) - data = await res.json() - - if use_latest_release: - if 'tag_name' not in data: - logger.error('No tag found in upstream repository.', - name=name) - return - version = data['tag_name'] - - else: - # YYYYMMDD.HHMMSS - version = data[0]['commit']['committer']['date'] \ - .rstrip('Z').replace('-', '').replace(':', '').replace('T', '.') - - return version - -async def max_tag( - getter, url, name, ignored_tags, include_tags_pattern, max_page, -): - # paging is needed - tags = [] - - for _ in range(max_page): - async with getter(url) as res: - logger.debug('X-RateLimit-Remaining', - n=res.headers.get('X-RateLimit-Remaining')) - links = res.headers.get('Link') - j = await res.json() - - data = [] - - for ref in j: - tag = ref['ref'].split('/', 2)[-1] - if tag in ignored_tags: - continue - data.append(tag) - - if include_tags_pattern: - data = [x for x in data - if re.search(include_tags_pattern, x)] - if data: - tags += data - - next_page_url = get_next_page_url(links) - if not next_page_url: - break - else: - url = next_page_url - - if not tags: - logger.error('No tag found in upstream repository.', - name=name, - include_tags_pattern=include_tags_pattern) - return tags - -def get_next_page_url(links): - if not links: - return - - links = links.split(', ') - next_link = [x for x in links if x.endswith('rel="next"')] - if not next_link: - return - - return next_link[0].split('>', 1)[0][1:] - -def check_ratelimit(exc, name): - res = exc.response - if not res: - raise - - # default -1 is used to re-raise the exception - n = int(res.headers.get('X-RateLimit-Remaining', -1)) - if n == 0: - reset = int(res.headers.get('X-RateLimit-Reset')) - logger.error('rate limited, resetting at %s. ' - 'Or get an API token to increase the allowance if not yet' - % time.ctime(reset), - name = name, - reset = reset) - else: - raise diff --git a/nvchecker/util.py b/nvchecker/util.py index 738ad5d..066b631 100644 --- a/nvchecker/util.py +++ b/nvchecker/util.py @@ -40,12 +40,6 @@ class KeyManager: keys = {} self.keys = keys - @classmethod - def from_str(cls, toml_str: str) -> KeyManager: - self = cls(None) - self.keys = toml.loads(toml_str)['keys'] - return self - def get_key(self, name: str) -> Optional[str]: return self.keys.get(name) diff --git a/nvchecker_source/github.py b/nvchecker_source/github.py new file mode 100644 index 0000000..36414b7 --- /dev/null +++ b/nvchecker_source/github.py @@ -0,0 +1,149 @@ +# MIT licensed +# Copyright (c) 2013-2020 lilydjwg , et al. + +import time +from urllib.parse import urlencode +from typing import Tuple + +import structlog + +from nvchecker.api import ( + VersionResult, Entry, AsyncCache, KeyManager, + TemporaryError, session, GetVersionError, +) + +logger = structlog.get_logger(logger_name=__name__) + +GITHUB_URL = 'https://api.github.com/repos/%s/commits' +GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest' +# https://developer.github.com/v3/git/refs/#get-all-references +GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/git/refs/tags' +GITHUB_GRAPHQL_URL = 'https://api.github.com/graphql' + +async def get_version(name, conf, **kwargs): + try: + return await get_version_real(name, conf, **kwargs) + except TemporaryError as e: + check_ratelimit(e, name) + +QUERY_LATEST_TAG = ''' +{{ + repository(name: "{name}", owner: "{owner}") {{ + refs(refPrefix: "refs/tags/", first: 1, + query: "{query}", + orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{ + edges {{ + node {{ + name + }} + }} + }} + }} +}} +''' + +async def get_latest_tag(key: Tuple[str, str, str]) -> str: + repo, query, token = key + owner, reponame = repo.split('/') + headers = { + 'Authorization': 'bearer %s' % token, + 'Content-Type': 'application/json', + } + q = QUERY_LATEST_TAG.format( + owner = owner, + name = reponame, + query = query, + ) + + res = await session.post( + GITHUB_GRAPHQL_URL, + headers = headers, + json = {'query': q}, + ) + j = res.json() + + refs = j['data']['repository']['refs']['edges'] + if not refs: + raise GetVersionError('no tag found') + + return refs[0]['node']['name'] + +async def get_version_real( + name: str, conf: Entry, *, + cache: AsyncCache, keymanager: KeyManager, + **kwargs, +) -> VersionResult: + repo = conf['github'] + + # Load token from config + token = conf.get('token') + # Load token from keyman + if token is None: + token = keymanager.get_key('github') + + use_latest_tag = conf.get('use_latest_tag', False) + if use_latest_tag: + if not token: + raise GetVersionError('token not given but it is required') + + query = conf.get('query', '') + return await cache.get((repo, query, token), get_latest_tag) # type: ignore + + br = conf.get('branch') + path = conf.get('path') + use_latest_release = conf.get('use_latest_release', False) + use_max_tag = conf.get('use_max_tag', False) + if use_latest_release: + url = GITHUB_LATEST_RELEASE % repo + elif use_max_tag: + url = GITHUB_MAX_TAG % repo + else: + url = GITHUB_URL % repo + parameters = {} + if br: + parameters['sha'] = br + if path: + parameters['path'] = path + url += '?' + urlencode(parameters) + headers = { + 'Accept': 'application/vnd.github.quicksilver-preview+json', + } + if token: + headers['Authorization'] = 'token %s' % token + + data = await cache.get_json(url, headers = headers) + + if use_max_tag: + tags = [ref['ref'].split('/', 2)[-1] for ref in data] + if not tags: + raise GetVersionError('No tag found in upstream repository.') + return tags + + if use_latest_release: + if 'tag_name' not in data: + raise GetVersionError('No release found in upstream repository.') + version = data['tag_name'] + + else: + # YYYYMMDD.HHMMSS + version = data[0]['commit']['committer']['date'] \ + .rstrip('Z').replace('-', '').replace(':', '').replace('T', '.') + + return version + +def check_ratelimit(exc, name): + res = exc.response + if not res: + raise + + # default -1 is used to re-raise the exception + n = int(res.headers.get('X-RateLimit-Remaining', -1)) + if n == 0: + reset = int(res.headers.get('X-RateLimit-Reset')) + logger.error('rate limited, resetting at %s. ' + 'Or get an API token to increase the allowance if not yet' + % time.ctime(reset), + name = name, + reset = reset) + else: + raise diff --git a/nvchecker_source/gitlab.py b/nvchecker_source/gitlab.py index bcc9151..0a22ece 100644 --- a/nvchecker_source/gitlab.py +++ b/nvchecker_source/gitlab.py @@ -24,6 +24,7 @@ async def get_version(name, conf, **kwargs): async def get_version_real( name: str, conf: Entry, *, cache: AsyncCache, keymanager: KeyManager, + **kwargs, ) -> VersionResult: repo = urllib.parse.quote_plus(conf['gitlab']) br = conf.get('branch', 'master') @@ -39,7 +40,7 @@ async def get_version_real( token = conf.get('token') # Load token from keyman if token is None: - key_name = 'gitlab_' + host.lower().replace('.', '_').replace("/", "_") + key_name = 'gitlab_' + host.lower() token = keymanager.get_key(key_name) # Set private token if token exists. diff --git a/tests-old/test_github.py b/tests-old/test_github.py deleted file mode 100644 index 3a59a4a..0000000 --- a/tests-old/test_github.py +++ /dev/null @@ -1,54 +0,0 @@ -# MIT licensed -# Copyright (c) 2013-2020 lilydjwg , et al. - -import os -import re -import pytest -pytestmark = [pytest.mark.asyncio, - pytest.mark.needs_net, - pytest.mark.skipif("NVCHECKER_GITHUB_TOKEN" not in os.environ, - reason="requires NVCHECKER_GITHUB_TOKEN, or it fails too much")] - -async def test_github(get_version): - assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo"}) == "20140122.012101" - -async def test_github_default_not_master(get_version): - assert await get_version("example", {"github": "MariaDB/server"}) is not None - -async def test_github_latest_release(get_version): - assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_latest_release": 1}) == "release3" - -async def test_github_max_tag(get_version): - assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_max_tag": 1}) == "second_release" - -async def test_github_max_tag_with_ignored_tags(get_version): - assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_max_tag": 1, "ignored_tags": "second_release release3"}) == "first_release" - -async def test_github_max_tag_with_ignored(get_version): - assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_max_tag": 1, "ignored": "second_release release3"}) == "first_release" - -async def test_github_with_path(get_version): - assert await get_version("example", {"github": "petronny/ReleaseTestRepo", "path": "test_directory"}) == "20140122.012101" - -async def test_github_with_path_and_branch(get_version): - assert await get_version("example", {"github": "petronny/ReleaseTestRepo", "branch": "test", "path": "test_directory/test_directory"}) == "20190128.113201" - -async def test_github_max_tag_with_include_old(get_version): - version = await get_version("example", { - "github": "EFForg/https-everywhere", - "use_max_tag": 1, - "include_tags_pattern": r"^chrome-\d", - }) - assert re.match(r'chrome-[\d.]+', version) - -async def test_github_max_tag_with_include(get_version): - version = await get_version("example", { - "github": "EFForg/https-everywhere", - "use_max_tag": 1, - "include_regex": r"chrome-\d.*", - }) - assert re.match(r'chrome-[\d.]+', version) - -async def test_github_latest_tag(get_version): - assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_latest_tag": 1}) == "release3" - diff --git a/tests-old/test_keyfile.py b/tests-old/test_keyfile.py deleted file mode 100644 index 8b61b85..0000000 --- a/tests-old/test_keyfile.py +++ /dev/null @@ -1,77 +0,0 @@ -# MIT licensed -# Copyright (c) 2018 lilydjwg , et al. - -import os -import tempfile -import contextlib - -from nvchecker.source import HTTPError - -import pytest -pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net] - -@contextlib.contextmanager -def unset_github_token_env(): - token = os.environ.get('NVCHECKER_GITHUB_TOKEN') - try: - if token: - del os.environ['NVCHECKER_GITHUB_TOKEN'] - yield token - finally: - if token: - os.environ['NVCHECKER_GITHUB_TOKEN'] = token - -async def test_keyfile_missing(run_str): - test_conf = '''\ -[example] -github = harry-sanabria/ReleaseTestRepo -''' - - assert await run_str(test_conf) in ['20140122.012101', None] - -async def test_keyfile_invalid(run_str): - with tempfile.NamedTemporaryFile(mode='w') as f, \ - unset_github_token_env(): - f.write('''\ -[keys] -github = xxx - ''') - f.flush() - test_conf = '''\ -[example] -github = harry-sanabria/ReleaseTestRepo - -[__config__] -keyfile = {name} -'''.format(name=f.name) - - try: - version = await run_str(test_conf, clear_cache=True) - assert version is None # out of allowance - return - except HTTPError as e: - assert e.code == 401 - return - - raise Exception('expected 401 response') - -@pytest.mark.skipif('NVCHECKER_GITHUB_TOKEN' not in os.environ, - reason='no key given') -async def test_keyfile_valid(run_str): - with tempfile.NamedTemporaryFile(mode='w') as f, \ - unset_github_token_env() as token: - f.write('''\ -[keys] -github = {token} - '''.format(token=token)) - f.flush() - - test_conf = '''\ -[example] -github = harry-sanabria/ReleaseTestRepo - -[__config__] -keyfile = {name} - '''.format(name=f.name) - - assert await run_str(test_conf) == '20140122.012101' diff --git a/tests/conftest.py b/tests/conftest.py index 5b7f07c..964b8e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,8 @@ import asyncio import structlog -from typing import Optional +import os +from pathlib import Path import toml import pytest @@ -14,12 +15,13 @@ from nvchecker.util import Entries, VersData, RawResult async def run( entries: Entries, max_concurrency: int = 20, - keys_toml: Optional[str] = None, ) -> VersData: token_q = core.token_queue(max_concurrency) result_q: asyncio.Queue[RawResult] = asyncio.Queue() - if keys_toml: - keymanager = core.KeyManager.from_str(keys_toml) + keyfile = os.environ.get('KEYFILE') + if keyfile: + filepath = Path(keyfile) + keymanager = core.KeyManager(filepath) else: keymanager = core.KeyManager(None) diff --git a/tests/test_github.py b/tests/test_github.py new file mode 100644 index 0000000..a874bda --- /dev/null +++ b/tests/test_github.py @@ -0,0 +1,78 @@ +# MIT licensed +# Copyright (c) 2013-2020 lilydjwg , et al. + +import os +import re + +import pytest + +pytestmark = [pytest.mark.asyncio, + pytest.mark.needs_net, + pytest.mark.skipif("KEYFILE" not in os.environ, + reason="requires KEYFILE, or it fails too much")] + +async def test_github(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + }) == "20140122.012101" + +async def test_github_default_not_master(get_version): + assert await get_version("example", { + "source": "github", + "github": "MariaDB/server", + }) is not None + +async def test_github_latest_release(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + "use_latest_release": True, + }) == "release3" + +async def test_github_max_tag(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + "use_max_tag": True, + }) == "second_release" + +async def test_github_max_tag_with_ignored(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + "use_max_tag": True, + "ignored": "second_release release3", + }) == "first_release" + +async def test_github_with_path(get_version): + assert await get_version("example", { + "source": "github", + "github": "petronny/ReleaseTestRepo", + "path": "test_directory", + }) == "20140122.012101" + +async def test_github_with_path_and_branch(get_version): + assert await get_version("example", { + "source": "github", + "github": "petronny/ReleaseTestRepo", + "branch": "test", + "path": "test_directory/test_directory", + }) == "20190128.113201" + +async def test_github_max_tag_with_include(get_version): + version = await get_version("example", { + "source": "github", + "github": "EFForg/https-everywhere", + "use_max_tag": True, + "include_regex": r"chrome-\d.*", + }) + assert re.match(r'chrome-[\d.]+', version) + +async def test_github_latest_tag(get_version): + assert await get_version("example", { + "source": "github", + "github": "harry-sanabria/ReleaseTestRepo", + "use_latest_tag": True, + }) == "release3" +