port tools.py and change record files to use json format

because special characters like spaces broke the old format.

.gitignore

@@ -1,4 +1,3 @@
-records/
 *.egg-info/
 __pycache__/
 /build/

README.rst

@@ -126,30 +126,11 @@ There are several backward-incompatible changes from the previous 1.x version.
 2. The configuration file format has been changed from ini to `toml`_. You can use the ``scripts/ini2toml`` script in this repo to convert your old configuration files. However, comments and formatting will be lost.
 3. Several options have been renamed. ``max_concurrent`` to ``max_concurrency``, and all option names have their ``-`` be replaced with ``_``.
 4. All software configuration tables need a ``source`` option to specify which source is to be used rather than being figured out from option names in use. This enables additional source plugins to be discovered.
+5. The version record files have been changed to use JSON format (the old format will be converted on writing).
 
 Version Record Files
 ====================
-Version record files record which version of the software you know or is available. They are simple key-value pairs of ``(name, version)`` separated by a space::
-
-  fcitx 4.2.7
-  google-chrome 27.0.1453.93-200836
-  vim 7.3.1024
-
-Say you've got a version record file called ``old_ver.txt`` which records all your watched software and their versions, as well as some configuration entries. To update it using ``nvchecker``::
-
-  nvchecker -c source.toml
-
-See what are updated with ``nvcmp``::
-
-  nvcmp -c source.toml
-
-Manually compare the two files for updates (assuming they are sorted alphabetically; files generated by ``nvchecker`` are already sorted)::
-
-  comm -13 old_ver.txt new_ver.txt
-  # or say that in English:
-  comm -13 old_ver.txt new_ver.txt | awk '{print $1 " has updated to version " $2 "."}'
-  # show both old and new versions
-  join old_ver.txt new_ver.txt | awk '$2 != $3'
+Version record files record which version of the software you know or is available. They are a simple JSON object mapping software names to known versions.
 
 The ``nvtake`` Command
 ----------------------
@@ -159,6 +140,14 @@ This helps when you have known (and processed) some of the updated software, but
 
 This command will help most if you specify where you version record files are in your config file. See below for how to use a config file.
 
+The ``nvcmp`` Command
+----------------------
+This command compares the ``newver`` file with the ``oldver`` one and prints out any differences as updates, e.g.::
+
+    $ nvcmp -c sample_source.toml
+    Sparkle Test App None -> 2.0
+    test 0.0 -> 0.1
+
 Configuration Files
 ===================
 The software version source files are in `toml`_ format. The *key name* is the name of the software. Following fields are used to tell nvchecker how to determine the current version of that software.

nvchecker/__main__.py

@@ -30,16 +30,11 @@ def main() -> None:
   if core.process_common_arguments(args):
     return
 
-  if not args.file:
-    try:
-      file = open(core.get_default_config())
-    except FileNotFoundError:
-      sys.exit('version configuration file not given and default does not exist')
-  else:
-    file = args.file
-
-  entries, options = core.load_file(
-    file, use_keymanager=bool(args.keyfile))
+  try:
+    entries, options = core.load_file(
+      args.file, use_keymanager=bool(args.keyfile))
+  except FileNotFoundError:
+    sys.exit('version configuration file not given and default does not exist')
 
   if args.keyfile:
     keymanager = KeyManager(Path(args.keyfile))

nvchecker/core.py

@@ -10,7 +10,7 @@ from asyncio import Queue
 import logging
 import argparse
 from typing import (
-  TextIO, Tuple, NamedTuple, Optional, List, Union,
+  Tuple, NamedTuple, Optional, List, Union,
   cast, Dict, Awaitable, Sequence,
 )
 import types
@@ -18,6 +18,7 @@ from pathlib import Path
 from importlib import import_module
 import re
 import contextvars
+import json
 
 import structlog
 import toml
@@ -51,9 +52,11 @@ def add_common_arguments(parser: argparse.ArgumentParser) -> None:
                       help='specify fd to send json logs to. stdout by default')
   parser.add_argument('-V', '--version', action='store_true',
                       help='show version and exit')
+  default_config = get_default_config()
   parser.add_argument('-c', '--file',
-                      metavar='FILE', type=open,
-                      help='software version configuration file [default: %s]' % get_default_config())
+                      metavar='FILE', type=str,
+                      default=default_config,
+                      help='software version configuration file [default: %s]' % default_config)
 
 def process_common_arguments(args: argparse.Namespace) -> bool:
   '''return True if should stop'''
@@ -109,23 +112,26 @@ def safe_overwrite(fname: str, data: Union[bytes, str], *,
   os.rename(tmpname, fname)
 
 def read_verfile(file: Path) -> VersData:
-  v = {}
   try:
     with open(file) as f:
-      for l in f:
-        name, ver = l.rstrip().split(None, 1)
-        v[name] = ver
+      data = f.read()
   except FileNotFoundError:
-    pass
+    return {}
+
+  try:
+    v = json.loads(data)
+  except json.decoder.JSONDecodeError:
+    # old format
+    v = {}
+    for l in data.splitlines():
+      name, ver = l.rstrip().split(None, 1)
+      v[name] = ver
+
   return v
 
 def write_verfile(file: Path, versions: VersData) -> None:
-  # sort using only alphanums, as done by the sort command,
-  # and needed by comm command
-  data = ['%s %s\n' % item
-          for item in sorted(versions.items(), key=lambda i: (''.join(filter(str.isalnum, i[0])), i[1]))]
-  safe_overwrite(
-    str(file), ''.join(data), method='writelines')
+  data = json.dumps(versions, ensure_ascii=False) + '\n'
+  safe_overwrite(str(file), data)
 
 class Options(NamedTuple):
   ver_files: Optional[Tuple[Path, Path]]
@@ -134,7 +140,7 @@ class Options(NamedTuple):
   keymanager: KeyManager
 
 def load_file(
-  file: TextIO, *,
+  file: str, *,
   use_keymanager: bool,
 ) -> Tuple[Entries, Options]:
   config = toml.load(file)
@@ -143,7 +149,7 @@ def load_file(
 
   if '__config__' in config:
     c = config.pop('__config__')
-    d = Path(file.name).parent
+    d = Path(file).parent
 
     if 'oldver' in c and 'newver' in c:
       oldver_s = os.path.expandvars(

nvchecker/tools.py

@@ -1,9 +1,8 @@
 # vim: se sw=2:
 # MIT licensed
-# Copyright (c) 2013-2017 lilydjwg <lilydjwg@gmail.com>, et al.
+# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
 
 import sys
-import os
 import argparse
 import structlog
 
@@ -11,7 +10,7 @@ from . import core
 
 logger = structlog.get_logger(logger_name=__name__)
 
-def take():
+def take() -> None:
   parser = argparse.ArgumentParser(description='update version records of nvchecker')
   core.add_common_arguments(parser)
   parser.add_argument('--all', action='store_true',
@@ -24,15 +23,19 @@ def take():
   if core.process_common_arguments(args):
     return
 
-  s = core.Source(args.file)
-  if not s.oldver or not s.newver:
+  opt = core.load_file(args.file, use_keymanager=False)[1]
+  if opt.ver_files is None:
     logger.critical(
-      "doesn't have both 'oldver' and 'newver' set.", source=s,
+      "doesn't have 'oldver' and 'newver' set.",
+      source=args.file,
     )
     sys.exit(2)
+  else:
+    oldverf = opt.ver_files[0]
+    newverf = opt.ver_files[1]
 
-  oldvers = core.read_verfile(s.oldver)
-  newvers = core.read_verfile(s.newver)
+  oldvers = core.read_verfile(oldverf)
+  newvers = core.read_verfile(newverf)
 
   if args.all:
     oldvers.update(newvers)
@@ -51,21 +54,33 @@ def take():
         sys.exit(2)
 
   try:
-      os.rename(s.oldver, s.oldver + '~')
+    oldverf.rename(
+      oldverf.with_name(oldverf.name + '~'),
+    )
   except FileNotFoundError:
-      pass
-  core.write_verfile(s.oldver, oldvers)
+    pass
+  core.write_verfile(oldverf, oldvers)
 
-def cmp():
+def cmp() -> None:
   parser = argparse.ArgumentParser(description='compare version records of nvchecker')
   core.add_common_arguments(parser)
   args = parser.parse_args()
   if core.process_common_arguments(args):
     return
 
-  s = core.Source(args.file)
-  oldvers = core.read_verfile(s.oldver) if s.oldver else {}
-  newvers = core.read_verfile(s.newver)
+  opt = core.load_file(args.file, use_keymanager=False)[1]
+  if opt.ver_files is None:
+    logger.critical(
+      "doesn't have 'oldver' and 'newver' set.",
+      source=args.file,
+    )
+    sys.exit(2)
+  else:
+    oldverf = opt.ver_files[0]
+    newverf = opt.ver_files[1]
+
+  oldvers = core.read_verfile(oldverf)
+  newvers = core.read_verfile(newverf)
   for name, newver in sorted(newvers.items()):
     oldver = oldvers.get(name, None)
     if oldver != newver:

sample_source.toml

@@ -1,6 +1,6 @@
 [__config__]
-oldver = "old_ver.txt"
-newver = "new_ver.txt"
+oldver = "old_ver.json"
+newver = "new_ver.json"
 
 [vim]
 source = "regex"