From 10f6b1561eac6ac105720b9e515f642010a11f10 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Mon, 17 Aug 2020 15:11:14 +0800
Subject: [PATCH] start porting tests to new version

---
 .travis.yml                               |  5 +-
 mypy.ini                                  |  3 +
 nvchecker/util.py                         |  6 ++
 nvchecker_source/none.py                  |  2 +-
 {tests => tests-old}/test_android.py      |  0
 {tests => tests-old}/test_anitya.py       |  0
 {tests => tests-old}/test_archpkg.py      |  0
 {tests => tests-old}/test_bitbucket.py    |  0
 {tests => tests-old}/test_cpan.py         |  0
 {tests => tests-old}/test_cratesio.py     |  0
 {tests => tests-old}/test_debianpkg.py    |  0
 {tests => tests-old}/test_gems.py         |  0
 {tests => tests-old}/test_gitea.py        |  0
 {tests => tests-old}/test_github.py       |  0
 {tests => tests-old}/test_gitlab.py       |  0
 {tests => tests-old}/test_gitlab_local.py |  0
 {tests => tests-old}/test_hackage.py      |  0
 {tests => tests-old}/test_keyfile.py      | 12 ++--
 {tests => tests-old}/test_manual.py       |  0
 {tests => tests-old}/test_npm.py          |  0
 {tests => tests-old}/test_packagist.py    |  0
 {tests => tests-old}/test_pacman.py       |  0
 {tests => tests-old}/test_proxy.py        |  0
 {tests => tests-old}/test_pypi.py         |  0
 {tests => tests-old}/test_regex.py        |  0
 {tests => tests-old}/test_repology.py     |  0
 {tests => tests-old}/test_sparkle.py      |  0
 {tests => tests-old}/test_substitute.py   |  0
 {tests => tests-old}/test_ubuntupkg.py    |  0
 {tests => tests-old}/test_vcs.py          |  0
 tests/conftest.py                         | 82 ++++++++++++-----------
 tests/test_aur.py                         | 16 +++--
 tests/test_cache.py                       | 20 ++++--
 tests/test_cmd.py                         | 19 ++++--
 34 files changed, 99 insertions(+), 66 deletions(-)
 rename {tests => tests-old}/test_android.py (100%)
 rename {tests => tests-old}/test_anitya.py (100%)
 rename {tests => tests-old}/test_archpkg.py (100%)
 rename {tests => tests-old}/test_bitbucket.py (100%)
 rename {tests => tests-old}/test_cpan.py (100%)
 rename {tests => tests-old}/test_cratesio.py (100%)
 rename {tests => tests-old}/test_debianpkg.py (100%)
 rename {tests => tests-old}/test_gems.py (100%)
 rename {tests => tests-old}/test_gitea.py (100%)
 rename {tests => tests-old}/test_github.py (100%)
 rename {tests => tests-old}/test_gitlab.py (100%)
 rename {tests => tests-old}/test_gitlab_local.py (100%)
 rename {tests => tests-old}/test_hackage.py (100%)
 rename {tests => tests-old}/test_keyfile.py (82%)
 rename {tests => tests-old}/test_manual.py (100%)
 rename {tests => tests-old}/test_npm.py (100%)
 rename {tests => tests-old}/test_packagist.py (100%)
 rename {tests => tests-old}/test_pacman.py (100%)
 rename {tests => tests-old}/test_proxy.py (100%)
 rename {tests => tests-old}/test_pypi.py (100%)
 rename {tests => tests-old}/test_regex.py (100%)
 rename {tests => tests-old}/test_repology.py (100%)
 rename {tests => tests-old}/test_sparkle.py (100%)
 rename {tests => tests-old}/test_substitute.py (100%)
 rename {tests => tests-old}/test_ubuntupkg.py (100%)
 rename {tests => tests-old}/test_vcs.py (100%)

diff --git a/.travis.yml b/.travis.yml
index 843a433..4d41ab2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,13 +3,11 @@ dist: bionic
 language: python
 cache: pip
 python:
-  - "3.5"
-  - "3.6"
   - "3.7"
   - "3.8"
   - "nightly"
   - "pypy3.6-7.3.1"
-install: pip install -U $DEPS pytest pytest-asyncio pytest-httpbin flaky structlog
+install: pip install -U $DEPS pytest pytest-asyncio pytest-httpbin flaky structlog toml
 script: pytest
 env:
   global:
@@ -21,6 +19,7 @@ env:
     - DEPS=aiohttp
     - DEPS="tornado pycurl"
     - DEPS=tornado
+    - DEPS="httpx>=0.14.0"
 jobs:
   fast_finish: true
   allow_failures:
diff --git a/mypy.ini b/mypy.ini
index f3f5e64..7b96531 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -11,3 +11,6 @@ ignore_missing_imports = True
 
 [mypy-pyalpm]
 ignore_missing_imports = True
+
+[mypy-flaky]
+ignore_missing_imports = True
diff --git a/nvchecker/util.py b/nvchecker/util.py
index 377acd6..010710a 100644
--- a/nvchecker/util.py
+++ b/nvchecker/util.py
@@ -40,6 +40,12 @@ class KeyManager:
       keys = {}
     self.keys = keys
 
+  @classmethod
+  def from_str(cls, toml_str: str) -> KeyManager:
+    self = cls(None)
+    self.keys = toml.loads(toml_str)['keys']
+    return self
+
   def get_key(self, name: str) -> Optional[str]:
     return self.keys.get(name)
 
diff --git a/nvchecker_source/none.py b/nvchecker_source/none.py
index 6bcda99..5427acd 100644
--- a/nvchecker_source/none.py
+++ b/nvchecker_source/none.py
@@ -12,5 +12,5 @@ class Worker(BaseWorker):
     exc = GetVersionError('no source specified')
     async with self.acquire_token():
       for name, conf in self.tasks:
-        self.result_q.put(
+        await self.result_q.put(
           RawResult(name, exc, conf))
diff --git a/tests/test_android.py b/tests-old/test_android.py
similarity index 100%
rename from tests/test_android.py
rename to tests-old/test_android.py
diff --git a/tests/test_anitya.py b/tests-old/test_anitya.py
similarity index 100%
rename from tests/test_anitya.py
rename to tests-old/test_anitya.py
diff --git a/tests/test_archpkg.py b/tests-old/test_archpkg.py
similarity index 100%
rename from tests/test_archpkg.py
rename to tests-old/test_archpkg.py
diff --git a/tests/test_bitbucket.py b/tests-old/test_bitbucket.py
similarity index 100%
rename from tests/test_bitbucket.py
rename to tests-old/test_bitbucket.py
diff --git a/tests/test_cpan.py b/tests-old/test_cpan.py
similarity index 100%
rename from tests/test_cpan.py
rename to tests-old/test_cpan.py
diff --git a/tests/test_cratesio.py b/tests-old/test_cratesio.py
similarity index 100%
rename from tests/test_cratesio.py
rename to tests-old/test_cratesio.py
diff --git a/tests/test_debianpkg.py b/tests-old/test_debianpkg.py
similarity index 100%
rename from tests/test_debianpkg.py
rename to tests-old/test_debianpkg.py
diff --git a/tests/test_gems.py b/tests-old/test_gems.py
similarity index 100%
rename from tests/test_gems.py
rename to tests-old/test_gems.py
diff --git a/tests/test_gitea.py b/tests-old/test_gitea.py
similarity index 100%
rename from tests/test_gitea.py
rename to tests-old/test_gitea.py
diff --git a/tests/test_github.py b/tests-old/test_github.py
similarity index 100%
rename from tests/test_github.py
rename to tests-old/test_github.py
diff --git a/tests/test_gitlab.py b/tests-old/test_gitlab.py
similarity index 100%
rename from tests/test_gitlab.py
rename to tests-old/test_gitlab.py
diff --git a/tests/test_gitlab_local.py b/tests-old/test_gitlab_local.py
similarity index 100%
rename from tests/test_gitlab_local.py
rename to tests-old/test_gitlab_local.py
diff --git a/tests/test_hackage.py b/tests-old/test_hackage.py
similarity index 100%
rename from tests/test_hackage.py
rename to tests-old/test_hackage.py
diff --git a/tests/test_keyfile.py b/tests-old/test_keyfile.py
similarity index 82%
rename from tests/test_keyfile.py
rename to tests-old/test_keyfile.py
index b512f96..8b61b85 100644
--- a/tests/test_keyfile.py
+++ b/tests-old/test_keyfile.py
@@ -21,15 +21,15 @@ def unset_github_token_env():
     if token:
       os.environ['NVCHECKER_GITHUB_TOKEN'] = token
 
-async def test_keyfile_missing(run_source):
+async def test_keyfile_missing(run_str):
   test_conf = '''\
 [example]
 github = harry-sanabria/ReleaseTestRepo
 '''
 
-  assert await run_source(test_conf) in ['20140122.012101', None]
+  assert await run_str(test_conf) in ['20140122.012101', None]
 
-async def test_keyfile_invalid(run_source):
+async def test_keyfile_invalid(run_str):
   with tempfile.NamedTemporaryFile(mode='w') as f, \
     unset_github_token_env():
     f.write('''\
@@ -46,7 +46,7 @@ keyfile = {name}
 '''.format(name=f.name)
 
     try:
-      version = await run_source(test_conf, clear_cache=True)
+      version = await run_str(test_conf, clear_cache=True)
       assert version is None # out of allowance
       return
     except HTTPError as e:
@@ -57,7 +57,7 @@ keyfile = {name}
 
 @pytest.mark.skipif('NVCHECKER_GITHUB_TOKEN' not in os.environ,
                     reason='no key given')
-async def test_keyfile_valid(run_source):
+async def test_keyfile_valid(run_str):
   with tempfile.NamedTemporaryFile(mode='w') as f, \
     unset_github_token_env() as token:
     f.write('''\
@@ -74,4 +74,4 @@ github = harry-sanabria/ReleaseTestRepo
 keyfile = {name}
       '''.format(name=f.name)
 
-    assert await run_source(test_conf) == '20140122.012101'
+    assert await run_str(test_conf) == '20140122.012101'
diff --git a/tests/test_manual.py b/tests-old/test_manual.py
similarity index 100%
rename from tests/test_manual.py
rename to tests-old/test_manual.py
diff --git a/tests/test_npm.py b/tests-old/test_npm.py
similarity index 100%
rename from tests/test_npm.py
rename to tests-old/test_npm.py
diff --git a/tests/test_packagist.py b/tests-old/test_packagist.py
similarity index 100%
rename from tests/test_packagist.py
rename to tests-old/test_packagist.py
diff --git a/tests/test_pacman.py b/tests-old/test_pacman.py
similarity index 100%
rename from tests/test_pacman.py
rename to tests-old/test_pacman.py
diff --git a/tests/test_proxy.py b/tests-old/test_proxy.py
similarity index 100%
rename from tests/test_proxy.py
rename to tests-old/test_proxy.py
diff --git a/tests/test_pypi.py b/tests-old/test_pypi.py
similarity index 100%
rename from tests/test_pypi.py
rename to tests-old/test_pypi.py
diff --git a/tests/test_regex.py b/tests-old/test_regex.py
similarity index 100%
rename from tests/test_regex.py
rename to tests-old/test_regex.py
diff --git a/tests/test_repology.py b/tests-old/test_repology.py
similarity index 100%
rename from tests/test_repology.py
rename to tests-old/test_repology.py
diff --git a/tests/test_sparkle.py b/tests-old/test_sparkle.py
similarity index 100%
rename from tests/test_sparkle.py
rename to tests-old/test_sparkle.py
diff --git a/tests/test_substitute.py b/tests-old/test_substitute.py
similarity index 100%
rename from tests/test_substitute.py
rename to tests-old/test_substitute.py
diff --git a/tests/test_ubuntupkg.py b/tests-old/test_ubuntupkg.py
similarity index 100%
rename from tests/test_ubuntupkg.py
rename to tests-old/test_ubuntupkg.py
diff --git a/tests/test_vcs.py b/tests-old/test_vcs.py
similarity index 100%
rename from tests/test_vcs.py
rename to tests-old/test_vcs.py
diff --git a/tests/conftest.py b/tests/conftest.py
index 89f5381..42848a8 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,59 +1,63 @@
 # MIT licensed
 # Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
-import configparser
-import pytest
 import asyncio
-import io
 import structlog
+from typing import Optional
 
-from nvchecker.get_version import get_version as _get_version
-from nvchecker.get_version import _cache
-from nvchecker.core import Source
+import toml
+import pytest
 
-class TestSource(Source):
-  def __init__(self, future, *args, **kwargs):
-    super().__init__(*args, **kwargs)
-    self._future = future
+from nvchecker import core
+from nvchecker import __main__ as main
+from nvchecker.util import Entries, VersData, RawResult
 
-  def on_update(self, name, version, oldver):
-    self._future.set_result(version)
+async def run(
+  entries: Entries, max_concurrency: int = 20,
+  keys_toml: Optional[str] = None,
+) -> VersData:
+  token_q = core.token_queue(max_concurrency)
+  result_q: asyncio.Queue[RawResult] = asyncio.Queue()
+  if keys_toml:
+    keymanager = core.KeyManager.from_str(keys_toml)
+  else:
+    keymanager = core.KeyManager(None)
 
-  def on_no_result(self, name):
-    self._future.set_result(None)
+  futures = core.dispatch(
+    entries, token_q, result_q,
+    keymanager, 1,
+  )
 
-  def on_exception(self, name, exc):
-    self._future.set_exception(exc)
+  oldvers: VersData = {}
+  result_coro = core.process_result(oldvers, result_q)
+  runner_coro = core.run_tasks(futures)
 
-@pytest.fixture(scope="module")
-async def run_source():
-  async def __call__(conf, *, clear_cache=False):
-    if clear_cache:
-      _cache.clear()
-
-    future = asyncio.Future()
-    file = io.StringIO(conf)
-    file.name = '<StringIO>'
-
-    s = TestSource(future, file)
-    await s.check()
-    return await future
-
-  return __call__
+  return await main.run(result_coro, runner_coro)
 
 @pytest.fixture(scope="module")
 async def get_version():
   async def __call__(name, config):
+    entries = {name: config}
+    newvers = await run(entries)
+    return newvers[name]
 
-    if isinstance(config, dict):
-      _config = configparser.ConfigParser(
-        dict_type=dict, allow_no_value=True,
-        interpolation=None,
-      )
-      _config.read_dict({name: config})
-      config = _config[name]
+  return __call__
 
-    return await _get_version(name, config)
+@pytest.fixture(scope="module")
+async def run_str():
+  async def __call__(str):
+    entries = toml.loads(str)
+    newvers = await run(entries)
+    return newvers.popitem()[1]
+
+  return __call__
+
+@pytest.fixture(scope="module")
+async def run_str_multi():
+  async def __call__(str):
+    entries = toml.loads(str)
+    newvers = await run(entries)
+    return newvers
 
   return __call__
 
diff --git a/tests/test_aur.py b/tests/test_aur.py
index 784304c..5a171da 100644
--- a/tests/test_aur.py
+++ b/tests/test_aur.py
@@ -1,5 +1,5 @@
 # MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import os
 
@@ -12,12 +12,20 @@ pytestmark = [pytest.mark.asyncio,
 
 @flaky(max_runs=10)
 async def test_aur(get_version):
-    assert await get_version("ssed", {"aur": None}) == "3.62-2"
+    assert await get_version("ssed", {
+        "source": "aur",
+    }) == "3.62-2"
 
 @flaky(max_runs=10)
 async def test_aur_strip_release(get_version):
-    assert await get_version("ssed", {"aur": None, "strip-release": 1}) == "3.62"
+    assert await get_version("ssed", {
+        "source": "aur",
+        "strip_release": 1,
+    }) == "3.62"
 
 @flaky(max_runs=10)
 async def test_aur_use_last_modified(get_version):
-    assert await get_version("ssed", {"aur": None, 'use_last_modified': True}) == "3.62-2-20150725052412"
+    assert await get_version("ssed", {
+        "source": "aur",
+        'use_last_modified': True,
+    }) == "3.62-2-20150725052412"
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 17ecf81..d20bd1d 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -1,12 +1,18 @@
 # MIT licensed
-# Copyright (c) 2018 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import pytest
 pytestmark = pytest.mark.asyncio
 
-async def test_cache(get_version):
-  a = await get_version("a", {"cmd": "date +%%Y-%%m-%%d"})
-  b = await get_version("b", {"cmd": "date +%%Y-%%m-%%d"})
-  c = await get_version("c", {"cmd": "date"})
-  assert a == b
-  assert a != c
+async def test_cache(run_str_multi):
+  conf = r'''
+[cache-1]
+source = "cmd"
+cmd = "echo $RANDOM"
+
+[cache-2]
+source = "cmd"
+cmd = "echo $RANDOM"'''
+
+  r = await run_str_multi(conf)
+  assert r['cache-1'] == r['cache-2']
diff --git a/tests/test_cmd.py b/tests/test_cmd.py
index 326f33d..5f8a464 100644
--- a/tests/test_cmd.py
+++ b/tests/test_cmd.py
@@ -1,21 +1,28 @@
 # MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import time
 import pytest
 pytestmark = pytest.mark.asyncio
 
 async def test_cmd(get_version):
-    assert await get_version("example", {"cmd": "echo Meow"}) == "Meow"
+    assert await get_version("example", {
+        "source": "cmd",
+        "cmd": "echo Meow",
+    }) == "Meow"
 
 async def test_cmd_complex(get_version):
-    assert await get_version("example", {"cmd": "echo Meow | sed 's/meow/woof/i'"}) == "woof"
+    assert await get_version("example", {
+        "source": "cmd",
+        "cmd": "echo Meow | sed 's/meow/woof/i'",
+    }) == "woof"
 
-async def test_cmd_with_percent(run_source):
+async def test_cmd_with_percent(run_str):
     test_conf = '''\
 [example]
-cmd = date +%Y-%m-%d'''
-    date = await run_source(test_conf)
+source = "cmd"
+cmd = "date +%Y-%m-%d"'''
+    date = await run_str(test_conf)
     expected = time.strftime('%Y-%m-%d')
     assert date == expected