start porting tests to new version

.travis.yml

@@ -3,13 +3,11 @@ dist: bionic
 language: python
 cache: pip
 python:
-  - "3.5"
-  - "3.6"
   - "3.7"
   - "3.8"
   - "nightly"
   - "pypy3.6-7.3.1"
-install: pip install -U $DEPS pytest pytest-asyncio pytest-httpbin flaky structlog
+install: pip install -U $DEPS pytest pytest-asyncio pytest-httpbin flaky structlog toml
 script: pytest
 env:
   global:
@@ -21,6 +19,7 @@ env:
     - DEPS=aiohttp
     - DEPS="tornado pycurl"
     - DEPS=tornado
+    - DEPS="httpx>=0.14.0"
 jobs:
   fast_finish: true
   allow_failures:

mypy.ini

@@ -11,3 +11,6 @@ ignore_missing_imports = True
 
 [mypy-pyalpm]
 ignore_missing_imports = True
+
+[mypy-flaky]
+ignore_missing_imports = True

nvchecker/util.py

@@ -40,6 +40,12 @@ class KeyManager:
       keys = {}
     self.keys = keys
 
+  @classmethod
+  def from_str(cls, toml_str: str) -> KeyManager:
+    self = cls(None)
+    self.keys = toml.loads(toml_str)['keys']
+    return self
+
   def get_key(self, name: str) -> Optional[str]:
     return self.keys.get(name)
 

nvchecker_source/none.py

@@ -12,5 +12,5 @@ class Worker(BaseWorker):
     exc = GetVersionError('no source specified')
     async with self.acquire_token():
       for name, conf in self.tasks:
-        self.result_q.put(
+        await self.result_q.put(
           RawResult(name, exc, conf))

tests-old/test_keyfile.py

@@ -21,15 +21,15 @@ def unset_github_token_env():
     if token:
       os.environ['NVCHECKER_GITHUB_TOKEN'] = token
 
-async def test_keyfile_missing(run_source):
+async def test_keyfile_missing(run_str):
   test_conf = '''\
 [example]
 github = harry-sanabria/ReleaseTestRepo
 '''
 
-  assert await run_source(test_conf) in ['20140122.012101', None]
+  assert await run_str(test_conf) in ['20140122.012101', None]
 
-async def test_keyfile_invalid(run_source):
+async def test_keyfile_invalid(run_str):
   with tempfile.NamedTemporaryFile(mode='w') as f, \
     unset_github_token_env():
     f.write('''\
@@ -46,7 +46,7 @@ keyfile = {name}
 '''.format(name=f.name)
 
     try:
-      version = await run_source(test_conf, clear_cache=True)
+      version = await run_str(test_conf, clear_cache=True)
       assert version is None # out of allowance
       return
     except HTTPError as e:
@@ -57,7 +57,7 @@ keyfile = {name}
 
 @pytest.mark.skipif('NVCHECKER_GITHUB_TOKEN' not in os.environ,
                     reason='no key given')
-async def test_keyfile_valid(run_source):
+async def test_keyfile_valid(run_str):
   with tempfile.NamedTemporaryFile(mode='w') as f, \
     unset_github_token_env() as token:
     f.write('''\
@@ -74,4 +74,4 @@ github = harry-sanabria/ReleaseTestRepo
 keyfile = {name}
       '''.format(name=f.name)
 
-    assert await run_source(test_conf) == '20140122.012101'
+    assert await run_str(test_conf) == '20140122.012101'

tests/conftest.py

@@ -1,59 +1,63 @@
 # MIT licensed
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-import configparser
-import pytest
 import asyncio
-import io
 import structlog
+from typing import Optional
 
-from nvchecker.get_version import get_version as _get_version
-from nvchecker.get_version import _cache
-from nvchecker.core import Source
+import toml
+import pytest
 
-class TestSource(Source):
-  def __init__(self, future, *args, **kwargs):
-    super().__init__(*args, **kwargs)
-    self._future = future
+from nvchecker import core
+from nvchecker import __main__ as main
+from nvchecker.util import Entries, VersData, RawResult
 
-  def on_update(self, name, version, oldver):
-    self._future.set_result(version)
+async def run(
+  entries: Entries, max_concurrency: int = 20,
+  keys_toml: Optional[str] = None,
+) -> VersData:
+  token_q = core.token_queue(max_concurrency)
+  result_q: asyncio.Queue[RawResult] = asyncio.Queue()
+  if keys_toml:
+    keymanager = core.KeyManager.from_str(keys_toml)
+  else:
+    keymanager = core.KeyManager(None)
 
-  def on_no_result(self, name):
-    self._future.set_result(None)
+  futures = core.dispatch(
+    entries, token_q, result_q,
+    keymanager, 1,
+  )
 
-  def on_exception(self, name, exc):
-    self._future.set_exception(exc)
+  oldvers: VersData = {}
+  result_coro = core.process_result(oldvers, result_q)
+  runner_coro = core.run_tasks(futures)
 
-@pytest.fixture(scope="module")
-async def run_source():
-  async def __call__(conf, *, clear_cache=False):
-    if clear_cache:
-      _cache.clear()
-
-    future = asyncio.Future()
-    file = io.StringIO(conf)
-    file.name = '<StringIO>'
-
-    s = TestSource(future, file)
-    await s.check()
-    return await future
-
-  return __call__
+  return await main.run(result_coro, runner_coro)
 
 @pytest.fixture(scope="module")
 async def get_version():
   async def __call__(name, config):
+    entries = {name: config}
+    newvers = await run(entries)
+    return newvers[name]
 
-    if isinstance(config, dict):
-      _config = configparser.ConfigParser(
-        dict_type=dict, allow_no_value=True,
-        interpolation=None,
-      )
-      _config.read_dict({name: config})
-      config = _config[name]
+  return __call__
 
-    return await _get_version(name, config)
+@pytest.fixture(scope="module")
+async def run_str():
+  async def __call__(str):
+    entries = toml.loads(str)
+    newvers = await run(entries)
+    return newvers.popitem()[1]
+
+  return __call__
+
+@pytest.fixture(scope="module")
+async def run_str_multi():
+  async def __call__(str):
+    entries = toml.loads(str)
+    newvers = await run(entries)
+    return newvers
 
   return __call__
 

tests/test_aur.py

@@ -1,5 +1,5 @@
 # MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import os
 
@@ -12,12 +12,20 @@ pytestmark = [pytest.mark.asyncio,
 
 @flaky(max_runs=10)
 async def test_aur(get_version):
-    assert await get_version("ssed", {"aur": None}) == "3.62-2"
+    assert await get_version("ssed", {
+        "source": "aur",
+    }) == "3.62-2"
 
 @flaky(max_runs=10)
 async def test_aur_strip_release(get_version):
-    assert await get_version("ssed", {"aur": None, "strip-release": 1}) == "3.62"
+    assert await get_version("ssed", {
+        "source": "aur",
+        "strip_release": 1,
+    }) == "3.62"
 
 @flaky(max_runs=10)
 async def test_aur_use_last_modified(get_version):
-    assert await get_version("ssed", {"aur": None, 'use_last_modified': True}) == "3.62-2-20150725052412"
+    assert await get_version("ssed", {
+        "source": "aur",
+        'use_last_modified': True,
+    }) == "3.62-2-20150725052412"

tests/test_cache.py

@@ -1,12 +1,18 @@
 # MIT licensed
-# Copyright (c) 2018 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import pytest
 pytestmark = pytest.mark.asyncio
 
-async def test_cache(get_version):
-  a = await get_version("a", {"cmd": "date +%%Y-%%m-%%d"})
-  b = await get_version("b", {"cmd": "date +%%Y-%%m-%%d"})
-  c = await get_version("c", {"cmd": "date"})
-  assert a == b
-  assert a != c
+async def test_cache(run_str_multi):
+  conf = r'''
+[cache-1]
+source = "cmd"
+cmd = "echo $RANDOM"
+
+[cache-2]
+source = "cmd"
+cmd = "echo $RANDOM"'''
+
+  r = await run_str_multi(conf)
+  assert r['cache-1'] == r['cache-2']

tests/test_cmd.py

@@ -1,21 +1,28 @@
 # MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import time
 import pytest
 pytestmark = pytest.mark.asyncio
 
 async def test_cmd(get_version):
-    assert await get_version("example", {"cmd": "echo Meow"}) == "Meow"
+    assert await get_version("example", {
+        "source": "cmd",
+        "cmd": "echo Meow",
+    }) == "Meow"
 
 async def test_cmd_complex(get_version):
-    assert await get_version("example", {"cmd": "echo Meow | sed 's/meow/woof/i'"}) == "woof"
+    assert await get_version("example", {
+        "source": "cmd",
+        "cmd": "echo Meow | sed 's/meow/woof/i'",
+    }) == "woof"
 
-async def test_cmd_with_percent(run_source):
+async def test_cmd_with_percent(run_str):
     test_conf = '''\
 [example]
-cmd = date +%Y-%m-%d'''
-    date = await run_source(test_conf)
+source = "cmd"
+cmd = "date +%Y-%m-%d"'''
+    date = await run_str(test_conf)
     expected = time.strftime('%Y-%m-%d')
     assert date == expected