From 903b414183792dceb6338e249a58a03e0d0dfe05 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 10 Oct 2018 17:03:20 +0800 Subject: [PATCH] cache result during a run so that different entries can refer to a same software without much overhead. This is needed because entries may be generated, and only nvchecker can tell if two entries are expecting the same result because the name may or may not be relevant. Closes #81. --- nvchecker/get_version.py | 17 ++++++++++++++++- nvchecker/source/__init__.py | 7 +++++++ nvchecker/source/archpkg.py | 4 +++- nvchecker/source/aur.py | 4 +++- nvchecker/source/cpan.py | 2 +- nvchecker/source/cratesio.py | 7 ++++--- nvchecker/source/debianpkg.py | 4 +++- nvchecker/source/gems.py | 2 +- nvchecker/source/hackage.py | 2 +- nvchecker/source/npm.py | 2 +- nvchecker/source/packagist.py | 2 +- nvchecker/source/pacman.py | 4 +++- nvchecker/source/pypi.py | 2 +- nvchecker/source/simple_json.py | 6 ++++-- nvchecker/source/ubuntupkg.py | 4 +++- tests/test_anitya.py | 2 +- 16 files changed, 53 insertions(+), 18 deletions(-) diff --git a/nvchecker/get_version.py b/nvchecker/get_version.py index b8a75b5..cd79670 100644 --- a/nvchecker/get_version.py +++ b/nvchecker/get_version.py @@ -38,15 +38,27 @@ def substitute_version(version, name, conf): # No substitution rules found. Just return the original version string. return version +_cache = {} + async def get_version(name, conf, **kwargs): for key in handler_precedence: if key in conf: - func = import_module('.source.' + key, __package__).get_version + mod = import_module('.source.' + key, __package__) + func = mod.get_version + get_cacheable_conf = getattr(mod, 'get_cacheable_conf', lambda name, conf: conf) break else: logger.error('no idea to get version info.', name=name) return + cacheable_conf = get_cacheable_conf(name, conf) + cache_key = tuple(sorted(cacheable_conf.items())) + if cache_key in _cache: + version = _cache[cache_key] + logger.debug('cache hit', name=name, + cache_key=cache_key, cached=version) + return version + version = await func(name, conf, **kwargs) if version: version = version.replace('\n', ' ') @@ -54,4 +66,7 @@ async def get_version(name, conf, **kwargs): version = substitute_version(version, name, conf) except (ValueError, re.error): logger.exception('error occurred in version substitutions', name=name) + + if version is not None: + _cache[cache_key] = version return version diff --git a/nvchecker/source/__init__.py b/nvchecker/source/__init__.py index 024d9b0..ae79e12 100644 --- a/nvchecker/source/__init__.py +++ b/nvchecker/source/__init__.py @@ -19,3 +19,10 @@ m = __import__('%s_httpclient' % which, globals(), locals(), level=1) __all__ = m.__all__ for x in __all__: globals()[x] = getattr(m, x) + +def conf_cacheable_with_name(key): + def get_cacheable_conf(name, conf): + conf = dict(conf) + conf[key] = conf.get(key) or name + return conf + return get_cacheable_conf diff --git a/nvchecker/source/archpkg.py b/nvchecker/source/archpkg.py index b57bf16..cb6a192 100644 --- a/nvchecker/source/archpkg.py +++ b/nvchecker/source/archpkg.py @@ -3,12 +3,14 @@ import structlog -from . import session +from . import session, conf_cacheable_with_name logger = structlog.get_logger(logger_name=__name__) URL = 'https://www.archlinux.org/packages/search/json/' +get_cacheable_conf = conf_cacheable_with_name('archpkg') + async def get_version(name, conf, **kwargs): pkg = conf.get('archpkg') or name strip_release = conf.getboolean('strip-release', False) diff --git a/nvchecker/source/aur.py b/nvchecker/source/aur.py index fc4b0eb..1ec378e 100644 --- a/nvchecker/source/aur.py +++ b/nvchecker/source/aur.py @@ -4,12 +4,14 @@ import structlog from datetime import datetime -from . import session +from . import session, conf_cacheable_with_name logger = structlog.get_logger(logger_name=__name__) AUR_URL = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg[]=' +get_cacheable_conf = conf_cacheable_with_name('aur') + async def get_version(name, conf, **kwargs): aurname = conf.get('aur') or name use_last_modified = conf.getboolean('use_last_modified', False) diff --git a/nvchecker/source/cpan.py b/nvchecker/source/cpan.py index 22c3888..9fbfbe7 100644 --- a/nvchecker/source/cpan.py +++ b/nvchecker/source/cpan.py @@ -9,7 +9,7 @@ CPAN_URL = 'https://fastapi.metacpan.org/release/%s' def _version_from_json(data): return str(data['version']) -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( CPAN_URL, 'cpan', _version_from_json, diff --git a/nvchecker/source/cratesio.py b/nvchecker/source/cratesio.py index 5449665..9aea99c 100644 --- a/nvchecker/source/cratesio.py +++ b/nvchecker/source/cratesio.py @@ -1,11 +1,12 @@ # MIT licensed -# Copyright (c) 2013-2017 lilydjwg , et al. +# Copyright (c) 2013-2018 lilydjwg , et al. -import os -from . import session +from . import session, conf_cacheable_with_name API_URL = 'https://crates.io/api/v1/crates/%s' +get_cacheable_conf = conf_cacheable_with_name('cratesio') + async def get_version(name, conf, **kwargs): name = conf.get('cratesio') or name async with session.get(API_URL % name) as res: diff --git a/nvchecker/source/debianpkg.py b/nvchecker/source/debianpkg.py index d5ff0da..336315a 100644 --- a/nvchecker/source/debianpkg.py +++ b/nvchecker/source/debianpkg.py @@ -3,12 +3,14 @@ import structlog -from . import session +from . import session, conf_cacheable_with_name logger = structlog.get_logger(logger_name=__name__) URL = 'https://sources.debian.org/api/src/%(pkgname)s/?suite=%(suite)s' +get_cacheable_conf = conf_cacheable_with_name('debianpkg') + async def get_version(name, conf, **kwargs): pkg = conf.get('debianpkg') or name strip_release = conf.getboolean('strip-release', False) diff --git a/nvchecker/source/gems.py b/nvchecker/source/gems.py index 2d79bce..308e56d 100644 --- a/nvchecker/source/gems.py +++ b/nvchecker/source/gems.py @@ -8,7 +8,7 @@ GEMS_URL = 'https://rubygems.org/api/v1/versions/%s.json' def _version_from_json(data): return data[0]['number'] -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( GEMS_URL, 'gems', _version_from_json, diff --git a/nvchecker/source/hackage.py b/nvchecker/source/hackage.py index 178c94d..79ced66 100644 --- a/nvchecker/source/hackage.py +++ b/nvchecker/source/hackage.py @@ -8,7 +8,7 @@ HACKAGE_URL = 'https://hackage.haskell.org/package/%s/preferred.json' def _version_from_json(data): return data['normal-version'][0] -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( HACKAGE_URL, 'hackage', _version_from_json, diff --git a/nvchecker/source/npm.py b/nvchecker/source/npm.py index e046197..256c4b4 100644 --- a/nvchecker/source/npm.py +++ b/nvchecker/source/npm.py @@ -8,7 +8,7 @@ NPM_URL = 'https://registry.npmjs.org/%s' def _version_from_json(data): return data['dist-tags']['latest'] -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( NPM_URL, 'npm', _version_from_json, diff --git a/nvchecker/source/packagist.py b/nvchecker/source/packagist.py index 7e2397a..dbb4d6d 100644 --- a/nvchecker/source/packagist.py +++ b/nvchecker/source/packagist.py @@ -11,7 +11,7 @@ def _version_from_json(data): if len(data): return max(data, key=lambda version: data[version]["time"]) -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( PACKAGIST_URL, 'packagist', _version_from_json, diff --git a/nvchecker/source/pacman.py b/nvchecker/source/pacman.py index aa487a4..65de504 100644 --- a/nvchecker/source/pacman.py +++ b/nvchecker/source/pacman.py @@ -1,7 +1,9 @@ # MIT licensed # Copyright (c) 2013-2017 lilydjwg , et al. -from . import cmd +from . import cmd, conf_cacheable_with_name + +get_cacheable_conf = conf_cacheable_with_name('debianpkg') async def get_version(name, conf, **kwargs): referree = conf.get('pacman') or name diff --git a/nvchecker/source/pypi.py b/nvchecker/source/pypi.py index 6915e15..4b2ff69 100644 --- a/nvchecker/source/pypi.py +++ b/nvchecker/source/pypi.py @@ -8,7 +8,7 @@ PYPI_URL = 'https://pypi.python.org/pypi/%s/json' def _version_from_json(data): return data['info']['version'] -get_version = simple_json( +get_version, get_cacheable_conf = simple_json( PYPI_URL, 'pypi', _version_from_json, diff --git a/nvchecker/source/simple_json.py b/nvchecker/source/simple_json.py index 5561fa5..0f7f879 100644 --- a/nvchecker/source/simple_json.py +++ b/nvchecker/source/simple_json.py @@ -1,7 +1,7 @@ # MIT licensed # Copyright (c) 2013-2017 lilydjwg , et al. -from . import session +from . import session, conf_cacheable_with_name def simple_json(urlpat, confkey, version_from_json): @@ -17,4 +17,6 @@ def simple_json(urlpat, confkey, version_from_json): version = version_from_json(data) return version - return get_version + get_cacheable_conf = conf_cacheable_with_name(confkey) + + return get_version, get_cacheable_conf diff --git a/nvchecker/source/ubuntupkg.py b/nvchecker/source/ubuntupkg.py index 98c7083..a72be41 100644 --- a/nvchecker/source/ubuntupkg.py +++ b/nvchecker/source/ubuntupkg.py @@ -3,12 +3,14 @@ import structlog -from . import session +from . import session, conf_cacheable_with_name logger = structlog.get_logger(logger_name=__name__) URL = 'https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name=%s&exact_match=true' +get_cacheable_conf = conf_cacheable_with_name('ubuntupkg') + async def get_version(name, conf, **kwargs): pkg = conf.get('ubuntupkg') or name strip_release = conf.getboolean('strip-release', False) diff --git a/tests/test_anitya.py b/tests/test_anitya.py index 8ec7943..cf0a818 100644 --- a/tests/test_anitya.py +++ b/tests/test_anitya.py @@ -5,4 +5,4 @@ import pytest pytestmark = pytest.mark.asyncio async def test_anitya(get_version): - assert await get_version("shutter", {"anitya": "fedora/shutter"}) == "0.94" + assert await get_version("shutter", {"anitya": "fedora/shutter"}) == "0.94.2"