feat: add jq source (#261)

feat: add jq source to parse json

.github/workflows/tests.yaml

@@ -47,7 +47,7 @@ jobs:
         sudo ln -s /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt
     # werkzeug is pinned for httpbin compatibility https://github.com/postmanlabs/httpbin/issues/673
     - name: Install Python deps
-      run: pip install -U ${{ matrix.deps }} pytest 'pytest-asyncio>=0.23' pytest-httpbin pytest-rerunfailures structlog tomli platformdirs lxml 'werkzeug<2.1' awesomeversion
+      run: pip install -U ${{ matrix.deps }} pytest 'pytest-asyncio>=0.23' pytest-httpbin pytest-rerunfailures structlog tomli platformdirs lxml jq 'werkzeug<2.1' awesomeversion
     - name: Decrypt keys
       env:
         KEY: ${{ secrets.KEY }}

docs/usage.rst

@@ -326,6 +326,31 @@ post_data_type
    An additional dependency "lxml" is required.
    You can use ``pip install 'nvchecker[htmlparser]'``.
 
+Search with an JSON Parser (jq)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+  source = "jq"
+
+Send an HTTP request and search through the body with a specific ``jq`` filter.
+
+url
+  The URL of the HTTP request.
+
+filter
+  An ``jq`` filter used to find the version string.
+
+post_data
+  (*Optional*) When present, a ``POST`` request (instead of a ``GET``) will be used. The value should be a string containing the full body of the request. The encoding of the string can be specified using the ``post_data_type`` option.
+
+post_data_type
+  (*Optional*) Specifies the ``Content-Type`` of the request body (``post_data``). By default, this is ``application/json``.
+
+This source supports :ref:`list options`.
+
+.. note::
+   An additional dependency "jq" is required.
+
 Find with a Command
 ~~~~~~~~~~~~~~~~~~~
 ::

mypy.ini

@@ -23,3 +23,6 @@ ignore_missing_imports = True
 
 [mypy-tomllib]
 ignore_missing_imports = True
+
+[mypy-jq]
+ignore_missing_imports = True

nvchecker_source/jq.py

@@ -0,0 +1,42 @@
+# MIT licensed
+# Copyright (c) 2024 Rocket Aaron <i@rocka.me>, et al.
+
+import json
+import jq
+
+from nvchecker.api import session, GetVersionError
+
+async def get_version(name, conf, *, cache, **kwargs):
+  key = tuple(sorted(conf.items()))
+  return await cache.get(key, get_version_impl)
+
+async def get_version_impl(info):
+  conf = dict(info)
+
+  try:
+    program = jq.compile(conf.get('filter', '.'))
+  except ValueError as e:
+    raise GetVersionError('bad jq filter', exc_info=e)
+
+  data = conf.get('post_data')
+  if data is None:
+    res = await session.get(conf['url'])
+  else:
+    res = await session.post(conf['url'], body = data, headers = {
+        'Content-Type': conf.get('post_data_type', 'application/json')
+      })
+
+  try:
+    obj = json.loads(res.body)
+  except json.decoder.JSONDecodeError as e:
+    raise GetVersionError('bad json string', exc_info=e)
+
+  try:
+    version = program.input(obj).all()
+    if version == [None] and not conf.get('missing_ok', False):
+      raise GetVersionError('version string not found.')
+    version = [str(v) for v in version]
+  except ValueError as e:
+    raise GetVersionError('failed to filter json', exc_info=e)
+
+  return version

setup.cfg

@@ -63,6 +63,8 @@ pypi =
   packaging
 htmlparser =
   lxml
+jq =
+  jq
 
 [options.entry_points]
 console_scripts =

tests/test_jq.py

@@ -0,0 +1,33 @@
+# MIT licensed
+# Copyright (c) 2024 Rocket Aaron <i@rocka.me>, et al.
+
+import pytest
+
+jq_available = True
+try:
+  import jq
+except jq:
+  jq_available = False
+
+pytestmark = [
+  pytest.mark.asyncio(scope="session"),
+  pytest.mark.needs_net,
+  pytest.mark.skipif(not jq_available, reason="needs jq"),
+]
+
+async def test_jq(get_version):
+    ver = await get_version("aur", {
+        "source": "jq",
+        "url": "https://aur.archlinux.org/rpc/v5/info?arg[]=nvchecker-git"
+    })
+    ver = ver.strip()
+    assert ver.startswith("{")
+    assert ver.endswith("}")
+
+async def test_jq_filter(get_version):
+    ver = await get_version("aur", {
+        "source": "jq",
+        "url": "https://aur.archlinux.org/rpc/v5/info?arg[]=nvchecker-git",
+        "filter": '.results[0].PackageBase',
+    })
+    assert ver == "nvchecker-git"