port gitlab source

also remove `ignored_tags` as per #99

nvchecker/api.py

@@ -1,7 +1,7 @@
 # MIT licensed
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-from .httpclient import session # type: ignore
+from .httpclient import session, TemporaryError # type: ignore
 from .core import GetVersionError
 from .util import (
   Entry, BaseWorker, RawResult, VersionResult,

nvchecker/httpclient/__init__.py

@@ -23,3 +23,5 @@ m = __import__('%s_httpclient' % which, globals(), locals(), level=1)
 __all__ = m.__all__
 for x in __all__:
   globals()[x] = getattr(m, x)
+
+from .base import TemporaryError

nvchecker_source/gitea.py

@@ -1,6 +1,8 @@
 # MIT licensed
 # Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
+from __future__ import annotations
+
 import urllib.parse
 
 GITEA_URL = 'https://%s/api/v1/repos/%s/commits?sha=%s'

nvchecker_source/gitlab.py

@@ -1,30 +1,34 @@
 # MIT licensed
-# Copyright (c) 2013-2018 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-import os
 import urllib.parse
 
 import structlog
 
-from . import session, HTTPError
-
-logger = structlog.get_logger(logger_name=__name__)
+from nvchecker.api import (
+  VersionResult, Entry, AsyncCache, KeyManager,
+  TemporaryError,
+)
 
 GITLAB_URL = 'https://%s/api/v4/projects/%s/repository/commits?ref_name=%s'
 GITLAB_MAX_TAG = 'https://%s/api/v4/projects/%s/repository/tags'
 
+logger = structlog.get_logger(logger_name=__name__)
+
 async def get_version(name, conf, **kwargs):
   try:
     return await get_version_real(name, conf, **kwargs)
-  except HTTPError as e:
+  except TemporaryError as e:
     check_ratelimit(e, name)
 
-async def get_version_real(name, conf, **kwargs):
-  repo = urllib.parse.quote_plus(conf.get('gitlab'))
+async def get_version_real(
+  name: str, conf: Entry, *,
+  cache: AsyncCache, keymanager: KeyManager,
+) -> VersionResult:
+  repo = urllib.parse.quote_plus(conf['gitlab'])
   br = conf.get('branch', 'master')
   host = conf.get('host', "gitlab.com")
-  use_max_tag = conf.getboolean('use_max_tag', False)
-  ignored_tags = conf.get("ignored_tags", "").split()
+  use_max_tag = conf.get('use_max_tag', False)
 
   if use_max_tag:
     url = GITLAB_MAX_TAG % (host, repo)
@@ -33,24 +37,19 @@ async def get_version_real(name, conf, **kwargs):
 
   # Load token from config
   token = conf.get('token')
-  # Load token from environ
-  if token is None:
-    env_name = "NVCHECKER_GITLAB_TOKEN_" + host.upper().replace(".", "_").replace("/", "_")
-    token = os.environ.get(env_name)
   # Load token from keyman
-  if token is None and 'keyman' in kwargs:
+  if token is None:
     key_name = 'gitlab_' + host.lower().replace('.', '_').replace("/", "_")
-    token = kwargs['keyman'].get_key(key_name)
+    token = keymanager.get_key(key_name)
 
   # Set private token if token exists.
   headers = {}
   if token:
     headers["PRIVATE-TOKEN"] = token
 
-  async with session.get(url, headers=headers) as res:
-    data = await res.json()
+  data = await cache.get_json(url, headers = headers)
   if use_max_tag:
-    version = [tag["name"] for tag in data if tag["name"] not in ignored_tags]
+    version = [tag["name"] for tag in data]
   else:
     version = data[0]['created_at'].split('T', 1)[0].replace('-', '')
   return version
@@ -63,8 +62,8 @@ def check_ratelimit(exc, name):
   # default -1 is used to re-raise the exception
   n = int(res.headers.get('RateLimit-Remaining', -1))
   if n == 0:
-    logger.error('rate limited, resetting at (unknown). '
-                 'Or get an API token to increase the allowance if not yet',
+    logger.error('gitlab rate limited. Wait some time '
+                 'or get an API token to increase the allowance if not yet',
                  name = name)
   else:
     raise

tests-old/test_gitlab.py

@@ -1,34 +0,0 @@
-# MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
-
-import os
-import pytest
-pytestmark = [pytest.mark.asyncio,
-              pytest.mark.needs_net,
-              pytest.mark.skipif("NVCHECKER_GITLAB_TOKEN_GITLAB_COM" not in os.environ,
-                                 reason="requires NVCHECKER_GITLAB_TOKEN_GITLAB_COM")]
-
-async def test_gitlab(get_version):
-    ver = await get_version("example",
-                            {"gitlab": "gitlab-org/gitlab-test"})
-    assert len(ver) == 8
-    assert ver.isdigit()
-
-async def test_gitlab_max_tag(get_version):
-    assert await get_version("example", {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1}) == "v1.1.1"
-
-async def test_gitlab_max_tag_with_ignored_tags(get_version):
-    assert await get_version("example", {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, "ignored_tags": "v1.1.0 v1.1.1"}) == "v1.0.0"
-
-async def test_gitlab_max_tag_with_include(get_version):
-    assert await get_version("example", {
-        "gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1,
-        "include_regex": r'v1\.0.*',
-    }) == "v1.0.0"
-
-async def test_gitlab_max_tag_with_ignored(get_version):
-    assert await get_version("example", {
-        "gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1,
-        "ignored": "v1.1.0 v1.1.1",
-    }) == "v1.0.0"
-

tests-old/test_gitlab_local.py

@@ -1,39 +0,0 @@
-# MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
-
-import os
-import pytest
-import contextlib
-pytestmark = [pytest.mark.asyncio,
-              pytest.mark.needs_net,
-              pytest.mark.skipif(os.environ.get('TRAVIS') == 'true',
-                                 reason="rate-limited per IP")]
-
-@contextlib.contextmanager
-def unset_gitlab_token_env():
-  token = os.environ.get('NVCHECKER_GITLAB_TOKEN_GITLAB_COM')
-  try:
-    if token:
-      del os.environ['NVCHECKER_GITLAB_TOKEN_GITLAB_COM']
-    yield token
-  finally:
-    if token:
-      os.environ['NVCHECKER_GITLAB_TOKEN_GITLAB_COM'] = token
-
-async def test_gitlab(get_version):
-    with unset_gitlab_token_env():
-        ver = await get_version("example",
-                                {"gitlab": "gitlab-org/gitlab-test"})
-        assert len(ver) == 8
-        assert ver.isdigit()
-
-async def test_gitlab_max_tag(get_version):
-    with unset_gitlab_token_env():
-        assert await get_version("example", {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1}) == "v1.1.1"
-
-async def test_gitlab_max_tag_with_ignored_tags(get_version):
-    with unset_gitlab_token_env():
-        ver = await get_version("example",
-                                {"gitlab": "gitlab-org/gitlab-test", "use_max_tag": 1, "ignored_tags": "v1.1.0 v1.1.1"})
-        assert ver == "v1.0.0"
-

tests/test_gitlab.py

@@ -0,0 +1,37 @@
+# MIT licensed
+# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
+
+import pytest
+pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
+
+async def test_gitlab(get_version):
+    ver = await get_version("example", {
+        "source": "gitlab",
+        "gitlab": "gitlab-org/gitlab-test",
+    })
+    assert len(ver) == 8
+    assert ver.isdigit()
+
+async def test_gitlab_max_tag(get_version):
+    assert await get_version("example", {
+        "source": "gitlab",
+        "gitlab": "gitlab-org/gitlab-test",
+        "use_max_tag": True,
+    }) == "v1.1.1"
+
+async def test_gitlab_max_tag_with_include(get_version):
+    assert await get_version("example", {
+        "source": "gitlab",
+        "gitlab": "gitlab-org/gitlab-test",
+        "use_max_tag": True,
+        "include_regex": r'v1\.0.*',
+    }) == "v1.0.0"
+
+async def test_gitlab_max_tag_with_ignored(get_version):
+    assert await get_version("example", {
+        "source": "gitlab",
+        "gitlab": "gitlab-org/gitlab-test",
+        "use_max_tag": True,
+        "ignored": "v1.1.0 v1.1.1",
+    }) == "v1.0.0"
+