diff --git a/nvchecker/api.py b/nvchecker/api.py index c705766..bcdc10a 100644 --- a/nvchecker/api.py +++ b/nvchecker/api.py @@ -1,7 +1,7 @@ # MIT licensed # Copyright (c) 2020 lilydjwg , et al. -from .httpclient import session # type: ignore +from .httpclient import session, TemporaryError # type: ignore from .core import GetVersionError from .util import ( Entry, BaseWorker, RawResult, VersionResult, diff --git a/nvchecker/httpclient/__init__.py b/nvchecker/httpclient/__init__.py index dd1f6dd..78ae39c 100644 --- a/nvchecker/httpclient/__init__.py +++ b/nvchecker/httpclient/__init__.py @@ -23,3 +23,5 @@ m = __import__('%s_httpclient' % which, globals(), locals(), level=1) __all__ = m.__all__ for x in __all__: globals()[x] = getattr(m, x) + +from .base import TemporaryError diff --git a/nvchecker_source/gitea.py b/nvchecker_source/gitea.py index 5ccc5aa..4280744 100644 --- a/nvchecker_source/gitea.py +++ b/nvchecker_source/gitea.py @@ -1,6 +1,8 @@ # MIT licensed # Copyright (c) 2013-2020 lilydjwg , et al. +from __future__ import annotations + import urllib.parse GITEA_URL = 'https://%s/api/v1/repos/%s/commits?sha=%s' diff --git a/nvchecker-old/source/gitlab.py b/nvchecker_source/gitlab.py similarity index 58% rename from nvchecker-old/source/gitlab.py rename to nvchecker_source/gitlab.py index a0bbc9e..bcc9151 100644 --- a/nvchecker-old/source/gitlab.py +++ b/nvchecker_source/gitlab.py @@ -1,30 +1,34 @@ # MIT licensed -# Copyright (c) 2013-2018 lilydjwg , et al. +# Copyright (c) 2013-2020 lilydjwg , et al. -import os import urllib.parse import structlog -from . import session, HTTPError - -logger = structlog.get_logger(logger_name=__name__) +from nvchecker.api import ( + VersionResult, Entry, AsyncCache, KeyManager, + TemporaryError, +) GITLAB_URL = 'https://%s/api/v4/projects/%s/repository/commits?ref_name=%s' GITLAB_MAX_TAG = 'https://%s/api/v4/projects/%s/repository/tags' +logger = structlog.get_logger(logger_name=__name__) + async def get_version(name, conf, **kwargs): try: return await get_version_real(name, conf, **kwargs) - except HTTPError as e: + except TemporaryError as e: check_ratelimit(e, name) -async def get_version_real(name, conf, **kwargs): - repo = urllib.parse.quote_plus(conf.get('gitlab')) +async def get_version_real( + name: str, conf: Entry, *, + cache: AsyncCache, keymanager: KeyManager, +) -> VersionResult: + repo = urllib.parse.quote_plus(conf['gitlab']) br = conf.get('branch', 'master') host = conf.get('host', "gitlab.com") - use_max_tag = conf.getboolean('use_max_tag', False) - ignored_tags = conf.get("ignored_tags", "").split() + use_max_tag = conf.get('use_max_tag', False) if use_max_tag: url = GITLAB_MAX_TAG % (host, repo) @@ -33,24 +37,19 @@ async def get_version_real(name, conf, **kwargs): # Load token from config token = conf.get('token') - # Load token from environ - if token is None: - env_name = "NVCHECKER_GITLAB_TOKEN_" + host.upper().replace(".", "_").replace("/", "_") - token = os.environ.get(env_name) # Load token from keyman - if token is None and 'keyman' in kwargs: + if token is None: key_name = 'gitlab_' + host.lower().replace('.', '_').replace("/", "_") - token = kwargs['keyman'].get_key(key_name) + token = keymanager.get_key(key_name) # Set private token if token exists. headers = {} if token: headers["PRIVATE-TOKEN"] = token - async with session.get(url, headers=headers) as res: - data = await res.json() + data = await cache.get_json(url, headers = headers) if use_max_tag: - version = [tag["name"] for tag in data if tag["name"] not in ignored_tags] + version = [tag["name"] for tag in data] else: version = data[0]['created_at'].split('T', 1)[0].replace('-', '') return version @@ -63,8 +62,8 @@ def check_ratelimit(exc, name): # default -1 is used to re-raise the exception n = int(res.headers.get('RateLimit-Remaining', -1)) if n == 0: - logger.error('rate limited, resetting at (unknown). ' - 'Or get an API token to increase the allowance if not yet', + logger.error('gitlab rate limited. Wait some time ' + 'or get an API token to increase the allowance if not yet', name = name) else: raise diff --git a/tests-old/test_gitlab.py b/tests-old/test_gitlab.py deleted file mode 100644 index aa8cc3b..0000000 --- a/tests-old/test_gitlab.py +++ /dev/null @@ -1,34 +0,0 @@ -# MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. - -import os -import pytest -pytestmark = [pytest.mark.asyncio, - pytest.mark.needs_net, - pytest.mark.skipif("NVCHECKER_GITLAB_TOKEN_GITLAB_COM" not in os.environ, - reason="requires NVCHECKER_GITLAB_TOKEN_GITLAB_COM")] - -async def test_gitlab(get_version): - ver = await get_version("example", - {"gitlab": "gitlab-org/gitlab-test"}) - assert len(ver) == 8 - assert ver.isdigit() - -async def test_gitlab_max_tag(get_version): - assert await get_version("example", {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1}) == "v1.1.1" - -async def test_gitlab_max_tag_with_ignored_tags(get_version): - assert await get_version("example", {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, "ignored_tags": "v1.1.0 v1.1.1"}) == "v1.0.0" - -async def test_gitlab_max_tag_with_include(get_version): - assert await get_version("example", { - "gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, - "include_regex": r'v1\.0.*', - }) == "v1.0.0" - -async def test_gitlab_max_tag_with_ignored(get_version): - assert await get_version("example", { - "gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, - "ignored": "v1.1.0 v1.1.1", - }) == "v1.0.0" - diff --git a/tests-old/test_gitlab_local.py b/tests-old/test_gitlab_local.py deleted file mode 100644 index 53f4e96..0000000 --- a/tests-old/test_gitlab_local.py +++ /dev/null @@ -1,39 +0,0 @@ -# MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. - -import os -import pytest -import contextlib -pytestmark = [pytest.mark.asyncio, - pytest.mark.needs_net, - pytest.mark.skipif(os.environ.get('TRAVIS') == 'true', - reason="rate-limited per IP")] - -@contextlib.contextmanager -def unset_gitlab_token_env(): - token = os.environ.get('NVCHECKER_GITLAB_TOKEN_GITLAB_COM') - try: - if token: - del os.environ['NVCHECKER_GITLAB_TOKEN_GITLAB_COM'] - yield token - finally: - if token: - os.environ['NVCHECKER_GITLAB_TOKEN_GITLAB_COM'] = token - -async def test_gitlab(get_version): - with unset_gitlab_token_env(): - ver = await get_version("example", - {"gitlab": "gitlab-org/gitlab-test"}) - assert len(ver) == 8 - assert ver.isdigit() - -async def test_gitlab_max_tag(get_version): - with unset_gitlab_token_env(): - assert await get_version("example", {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1}) == "v1.1.1" - -async def test_gitlab_max_tag_with_ignored_tags(get_version): - with unset_gitlab_token_env(): - ver = await get_version("example", - {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, "ignored_tags": "v1.1.0 v1.1.1"}) - assert ver == "v1.0.0" - diff --git a/tests/test_gitlab.py b/tests/test_gitlab.py new file mode 100644 index 0000000..85d269b --- /dev/null +++ b/tests/test_gitlab.py @@ -0,0 +1,37 @@ +# MIT licensed +# Copyright (c) 2013-2020 lilydjwg , et al. + +import pytest +pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net] + +async def test_gitlab(get_version): + ver = await get_version("example", { + "source": "gitlab", + "gitlab": "gitlab-org/gitlab-test", + }) + assert len(ver) == 8 + assert ver.isdigit() + +async def test_gitlab_max_tag(get_version): + assert await get_version("example", { + "source": "gitlab", + "gitlab": "gitlab-org/gitlab-test", + "use_max_tag": True, + }) == "v1.1.1" + +async def test_gitlab_max_tag_with_include(get_version): + assert await get_version("example", { + "source": "gitlab", + "gitlab": "gitlab-org/gitlab-test", + "use_max_tag": True, + "include_regex": r'v1\.0.*', + }) == "v1.0.0" + +async def test_gitlab_max_tag_with_ignored(get_version): + assert await get_version("example", { + "source": "gitlab", + "gitlab": "gitlab-org/gitlab-test", + "use_max_tag": True, + "ignored": "v1.1.0 v1.1.1", + }) == "v1.0.0" +