From f957acc756563a90e849dd290cdc843d454753f4 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 6 Oct 2020 01:40:04 +0800 Subject: [PATCH 1/3] Use my formal English name for previous contributions --- nvchecker_source/android_sdk.py | 2 +- tests/test_android_sdk.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nvchecker_source/android_sdk.py b/nvchecker_source/android_sdk.py index 1ff6e5d..b101caa 100644 --- a/nvchecker_source/android_sdk.py +++ b/nvchecker_source/android_sdk.py @@ -1,6 +1,6 @@ # MIT licensed # Copyright (c) 2020 lilydjwg , et al. -# Copyright (c) 2017 Yen Chi Hsuan +# Copyright (c) 2017 Chih-Hsuan Yen import os import re diff --git a/tests/test_android_sdk.py b/tests/test_android_sdk.py index de8b95d..ffcb953 100644 --- a/tests/test_android_sdk.py +++ b/tests/test_android_sdk.py @@ -1,6 +1,6 @@ # MIT licensed # Copyright (c) 2020 lilydjwg , et al. -# Copyright (c) 2017 Yen Chi Hsuan +# Copyright (c) 2017 Chih-Hsuan Yen import pytest pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net] From 8d83d7ac6683577769fe8d1a3bb615bf1be21ced Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 6 Oct 2020 21:35:15 +0800 Subject: [PATCH 2/3] Add exception HTTPError for HTTP 4xx errors --- nvchecker/api.py | 2 +- nvchecker/httpclient/__init__.py | 2 +- nvchecker/httpclient/aiohttp_httpclient.py | 11 +++++++---- nvchecker/httpclient/base.py | 8 ++++++-- nvchecker/httpclient/httpx_httpclient.py | 11 +++++++---- nvchecker/httpclient/tornado_httpclient.py | 11 +++++++---- 6 files changed, 29 insertions(+), 16 deletions(-) diff --git a/nvchecker/api.py b/nvchecker/api.py index a05019c..dc31324 100644 --- a/nvchecker/api.py +++ b/nvchecker/api.py @@ -1,7 +1,7 @@ # MIT licensed # Copyright (c) 2020 lilydjwg , et al. -from .httpclient import session, TemporaryError +from .httpclient import session, TemporaryError, HTTPError from .util import ( Entry, BaseWorker, RawResult, VersionResult, AsyncCache, KeyManager, GetVersionError, diff --git a/nvchecker/httpclient/__init__.py b/nvchecker/httpclient/__init__.py index 6c5ae14..3f8cf1e 100644 --- a/nvchecker/httpclient/__init__.py +++ b/nvchecker/httpclient/__init__.py @@ -3,7 +3,7 @@ from typing import Optional -from .base import TemporaryError +from .base import TemporaryError, HTTPError class Proxy: _obj = None diff --git a/nvchecker/httpclient/aiohttp_httpclient.py b/nvchecker/httpclient/aiohttp_httpclient.py index fe538ad..d32dbfd 100644 --- a/nvchecker/httpclient/aiohttp_httpclient.py +++ b/nvchecker/httpclient/aiohttp_httpclient.py @@ -8,7 +8,7 @@ from typing import Optional, Dict import structlog import aiohttp -from .base import BaseSession, TemporaryError, Response +from .base import BaseSession, TemporaryError, Response, HTTPError __all__ = ['session'] @@ -54,10 +54,13 @@ class AiohttpSession(BaseSession): ) as e: raise TemporaryError(599, repr(e), e) + err_cls: Optional[type] = None if res.status >= 500: - raise TemporaryError(res.status, res.reason, res) - else: - res.raise_for_status() + err_cls = TemporaryError + elif res.status >= 400: + err_cls = HTTPError + if err_cls is not None: + raise err_cls(res.status, res.reason, res) body = await res.content.read() return Response(body) diff --git a/nvchecker/httpclient/base.py b/nvchecker/httpclient/base.py index 8115fd4..cd038c8 100644 --- a/nvchecker/httpclient/base.py +++ b/nvchecker/httpclient/base.py @@ -86,10 +86,14 @@ class BaseSession: ''':meta private:''' raise NotImplementedError -class TemporaryError(Exception): - '''A temporary error (e.g. network error) happens.''' +class BaseHTTPError(Exception): def __init__(self, code, message, response): self.code = code self.message = message self.response = response +class TemporaryError(BaseHTTPError): + '''A temporary error (e.g. network error) happens.''' + +class HTTPError(BaseHTTPError): + ''' An HTTP 4xx error happens ''' diff --git a/nvchecker/httpclient/httpx_httpclient.py b/nvchecker/httpclient/httpx_httpclient.py index acfc5e5..4071fa9 100644 --- a/nvchecker/httpclient/httpx_httpclient.py +++ b/nvchecker/httpclient/httpx_httpclient.py @@ -6,7 +6,7 @@ from typing import Dict, Optional import httpx -from .base import BaseSession, TemporaryError, Response +from .base import BaseSession, TemporaryError, Response, HTTPError __all__ = ['session'] @@ -42,14 +42,17 @@ class HttpxSession(BaseSession): headers = headers, params = params, ) + err_cls: Optional[type] = None if r.status_code >= 500: - raise TemporaryError( + err_cls = TemporaryError + elif r.status_code >= 400: + err_cls = HTTPError + if err_cls is not None: + raise err_cls( r.status_code, r.reason_phrase, r, ) - else: - r.raise_for_status() except httpx.TransportError as e: raise TemporaryError(599, repr(e), e) diff --git a/nvchecker/httpclient/tornado_httpclient.py b/nvchecker/httpclient/tornado_httpclient.py index fbd77b3..aeba048 100644 --- a/nvchecker/httpclient/tornado_httpclient.py +++ b/nvchecker/httpclient/tornado_httpclient.py @@ -12,7 +12,7 @@ try: except ImportError: pycurl = None # type: ignore -from .base import BaseSession, TemporaryError, Response +from .base import BaseSession, TemporaryError, Response, HTTPError __all__ = ['session'] @@ -74,12 +74,15 @@ class TornadoSession(BaseSession): r = HTTPRequest(url, **kwargs) res = await AsyncHTTPClient().fetch( r, raise_error=False) + err_cls: Optional[type] = None if res.code >= 500: - raise TemporaryError( + err_cls = TemporaryError + elif res.code >= 400: + err_cls = HTTPError + if err_cls is not None: + raise err_cls( res.code, res.reason, res ) - else: - res.rethrow() return Response(res.body) From 75e72c11b3ccaa8d375f6c7c7b94dfa35a0de5c1 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 6 Oct 2020 01:38:14 +0800 Subject: [PATCH 3/3] Add container source plugin Closes https://github.com/lilydjwg/nvchecker/issues/59 --- docs/usage.rst | 16 +++++++ nvchecker_source/container.py | 83 +++++++++++++++++++++++++++++++++++ tests/test_container.py | 12 +++++ 3 files changed, 111 insertions(+) create mode 100644 nvchecker_source/container.py create mode 100644 tests/test_container.py diff --git a/docs/usage.rst b/docs/usage.rst index 51a2940..4cb94ee 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -662,6 +662,22 @@ git This source returns tags and supports :ref:`list options`. +Check container registry +~~~~~~~~~~~~~~~~~~~~~~~~ +:: + + source = "container" + +This enables you to check tags of images on a container registry like Docker. + +container + The path for the container image. For official Docker images, use namespace ``library/`` (e.g. ``library/python``). + +registry + The container registry host. Default: ``docker.io`` + +This source returns tags and supports :ref:`list options`. + Manually updating ~~~~~~~~~~~~~~~~~ :: diff --git a/nvchecker_source/container.py b/nvchecker_source/container.py new file mode 100644 index 0000000..e8bb0f3 --- /dev/null +++ b/nvchecker_source/container.py @@ -0,0 +1,83 @@ +# MIT licensed +# Copyright (c) 2020 Chih-Hsuan Yen + +from typing import Dict, List, NamedTuple, Optional, Tuple +from urllib.request import parse_http_list + +from nvchecker.api import session, HTTPError + +class AuthInfo(NamedTuple): + service: Optional[str] + realm: str + +def parse_www_authenticate_header(header: str) -> Tuple[str, Dict[str, str]]: + ''' + Parse WWW-Authenticate header used in OAuth2 authentication for container + registries. This is NOT RFC-compliant! + + Simplified from http.parse_www_authenticate_header in Werkzeug (BSD license) + ''' + auth_type, auth_info = header.split(None, 1) + result = {} + for item in parse_http_list(auth_info): + name, value = item.split("=", 1) + if value[:1] == value[-1:] == '"': + value = value[1:-1] + result[name] = value + return auth_type, result + +# Inspired by https://stackoverflow.com/a/51921869 +# Reference: https://github.com/containers/image/blob/v5.6.0/docker/docker_client.go + +class UnsupportedAuthenticationError(NotImplementedError): + def __init__(self): + super().__init__('Only Bearer authentication supported for now') + +async def get_registry_auth_info(registry_host: str) -> AuthInfo: + auth_service = auth_realm = None + + try: + await session.get(f'https://{registry_host}/v2/') + raise UnsupportedAuthenticationError # No authentication needed + except HTTPError as e: + if e.code != 401: + raise + + auth_type, auth_info = parse_www_authenticate_header(e.response.headers['WWW-Authenticate']) + if auth_type.lower() != 'bearer': + raise UnsupportedAuthenticationError + + # Although 'service' is needed as per https://docs.docker.com/registry/spec/auth/token/, + # ghcr.io (GitHub container registry) does not provide it + auth_service = auth_info.get('service') + auth_realm = auth_info['realm'] + + return AuthInfo(auth_service, auth_realm) + +async def get_container_tags(info: Tuple[str, str, AuthInfo]) -> List[str]: + image_path, registry_host, auth_info = info + + auth_params = { + 'scope': f'repository:{image_path}:pull', + } + if auth_info.service: + auth_params['service'] = auth_info.service + res = await session.get(auth_info.realm, params=auth_params) + token = res.json()['token'] + + res = await session.get(f'https://{registry_host}/v2/{image_path}/tags/list', headers={ + 'Authorization': f'Bearer {token}', + 'Accept': 'application/json', + }) + return res.json()['tags'] + +async def get_version(name, conf, *, cache, **kwargs): + image_path = conf.get('container', name) + registry_host = conf.get('registry', 'docker.io') + if registry_host == 'docker.io': + registry_host = 'registry-1.docker.io' + + auth_info = await cache.get(registry_host, get_registry_auth_info) + + key = image_path, registry_host, auth_info + return await cache.get(key, get_container_tags) diff --git a/tests/test_container.py b/tests/test_container.py new file mode 100644 index 0000000..fb7aedf --- /dev/null +++ b/tests/test_container.py @@ -0,0 +1,12 @@ +# MIT licensed +# Copyright (c) 2020 Chih-Hsuan Yen + +import pytest +pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net] + +async def test_container(get_version): + assert await get_version("hello-world", { + "source": "container", + "container": "library/hello-world", + "include_regex": "linux", + }) == "linux"