port github source and tests (the last one!)

`include_tags_pattern` and `ignored_tags` removed, see #99
This commit is contained in:
lilydjwg 2020-08-20 14:56:40 +08:00
parent 61a67a4a5b
commit 95150fa8e9
10 changed files with 236 additions and 362 deletions

1
.gitignore vendored
View File

@ -8,3 +8,4 @@ __pycache__/
*.pyo
.travis.pub
.pytest_cache/
keyfile.toml

View File

@ -314,13 +314,6 @@ use_max_tag
lightweight ones, and return the largest one sorted by the
``sort_version_key`` option. Will return the tag name instead of date.
max_page
How many pages do we search for the max tag? Default is 1. This works when
``use_max_tag`` is set.
However, with current API in use, GitHub seems to always return all data in
one page, making this option obsolete.
proxy
The HTTP proxy to use. The format is ``host:port``, e.g. ``localhost:8087``.

View File

@ -1,213 +0,0 @@
# MIT licensed
# Copyright (c) 2013-2018 lilydjwg <lilydjwg@gmail.com>, et al.
import os
import re
import time
from urllib.parse import urlencode
from functools import partial
import structlog
from . import session, HTTPError
logger = structlog.get_logger(logger_name=__name__)
GITHUB_URL = 'https://api.github.com/repos/%s/commits'
GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest'
# https://developer.github.com/v3/git/refs/#get-all-references
GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/git/refs/tags'
GITHUB_GRAPHQL_URL = 'https://api.github.com/graphql'
async def get_version(name, conf, **kwargs):
try:
return await get_version_real(name, conf, **kwargs)
except HTTPError as e:
check_ratelimit(e, name)
QUERY_LATEST_TAG = '''
{{
repository(name: "{name}", owner: "{owner}") {{
refs(refPrefix: "refs/tags/", first: 1,
query: "{query}",
orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{
edges {{
node {{
name
}}
}}
}}
}}
}}
'''
async def get_latest_tag(name, conf, token):
repo = conf.get('github')
query = conf.get('query', '')
owner, reponame = repo.split('/')
headers = {
'Authorization': 'bearer %s' % token,
'Content-Type': 'application/json',
}
q = QUERY_LATEST_TAG.format(
owner = owner,
name = reponame,
query = query,
)
async with session.post(
GITHUB_GRAPHQL_URL,
headers = headers,
json = {'query': q},
) as res:
j = await res.json()
refs = j['data']['repository']['refs']['edges']
if not refs:
logger.error('no tag found', name=name)
return
return refs[0]['node']['name']
def get_token(kwargs):
token = os.environ.get('NVCHECKER_GITHUB_TOKEN')
if token:
return token
if 'keyman' not in kwargs:
return None
token = kwargs['keyman'].get_key('github')
return token
async def get_version_real(name, conf, **kwargs):
token = get_token(kwargs)
use_latest_tag = conf.getboolean('use_latest_tag', False)
if use_latest_tag:
if not token:
logger.error('token not given but it is required',
name = name)
return
return await get_latest_tag(name, conf, token)
repo = conf.get('github')
br = conf.get('branch')
path = conf.get('path')
use_latest_release = conf.getboolean('use_latest_release', False)
use_max_tag = conf.getboolean('use_max_tag', False)
include_tags_pattern = conf.get("include_tags_pattern", "")
ignored_tags = conf.get("ignored_tags", "").split()
if use_latest_release:
url = GITHUB_LATEST_RELEASE % repo
elif use_max_tag:
url = GITHUB_MAX_TAG % repo
else:
url = GITHUB_URL % repo
parameters = {}
if br:
parameters['sha'] = br
if path:
parameters['path'] = path
url += '?' + urlencode(parameters)
headers = {
'Accept': 'application/vnd.github.quicksilver-preview+json',
}
if token:
headers['Authorization'] = 'token %s' % token
kwargs = {}
if conf.get('proxy'):
kwargs["proxy"] = conf.get("proxy")
if use_max_tag:
return await max_tag(partial(
session.get, headers=headers, **kwargs),
url, name, ignored_tags, include_tags_pattern,
max_page = conf.getint("max_page", 1),
)
async with session.get(url, headers=headers, **kwargs) as res:
logger.debug('X-RateLimit-Remaining',
n=res.headers.get('X-RateLimit-Remaining'))
data = await res.json()
if use_latest_release:
if 'tag_name' not in data:
logger.error('No tag found in upstream repository.',
name=name)
return
version = data['tag_name']
else:
# YYYYMMDD.HHMMSS
version = data[0]['commit']['committer']['date'] \
.rstrip('Z').replace('-', '').replace(':', '').replace('T', '.')
return version
async def max_tag(
getter, url, name, ignored_tags, include_tags_pattern, max_page,
):
# paging is needed
tags = []
for _ in range(max_page):
async with getter(url) as res:
logger.debug('X-RateLimit-Remaining',
n=res.headers.get('X-RateLimit-Remaining'))
links = res.headers.get('Link')
j = await res.json()
data = []
for ref in j:
tag = ref['ref'].split('/', 2)[-1]
if tag in ignored_tags:
continue
data.append(tag)
if include_tags_pattern:
data = [x for x in data
if re.search(include_tags_pattern, x)]
if data:
tags += data
next_page_url = get_next_page_url(links)
if not next_page_url:
break
else:
url = next_page_url
if not tags:
logger.error('No tag found in upstream repository.',
name=name,
include_tags_pattern=include_tags_pattern)
return tags
def get_next_page_url(links):
if not links:
return
links = links.split(', ')
next_link = [x for x in links if x.endswith('rel="next"')]
if not next_link:
return
return next_link[0].split('>', 1)[0][1:]
def check_ratelimit(exc, name):
res = exc.response
if not res:
raise
# default -1 is used to re-raise the exception
n = int(res.headers.get('X-RateLimit-Remaining', -1))
if n == 0:
reset = int(res.headers.get('X-RateLimit-Reset'))
logger.error('rate limited, resetting at %s. '
'Or get an API token to increase the allowance if not yet'
% time.ctime(reset),
name = name,
reset = reset)
else:
raise

View File

@ -40,12 +40,6 @@ class KeyManager:
keys = {}
self.keys = keys
@classmethod
def from_str(cls, toml_str: str) -> KeyManager:
self = cls(None)
self.keys = toml.loads(toml_str)['keys']
return self
def get_key(self, name: str) -> Optional[str]:
return self.keys.get(name)

149
nvchecker_source/github.py Normal file
View File

@ -0,0 +1,149 @@
# MIT licensed
# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
import time
from urllib.parse import urlencode
from typing import Tuple
import structlog
from nvchecker.api import (
VersionResult, Entry, AsyncCache, KeyManager,
TemporaryError, session, GetVersionError,
)
logger = structlog.get_logger(logger_name=__name__)
GITHUB_URL = 'https://api.github.com/repos/%s/commits'
GITHUB_LATEST_RELEASE = 'https://api.github.com/repos/%s/releases/latest'
# https://developer.github.com/v3/git/refs/#get-all-references
GITHUB_MAX_TAG = 'https://api.github.com/repos/%s/git/refs/tags'
GITHUB_GRAPHQL_URL = 'https://api.github.com/graphql'
async def get_version(name, conf, **kwargs):
try:
return await get_version_real(name, conf, **kwargs)
except TemporaryError as e:
check_ratelimit(e, name)
QUERY_LATEST_TAG = '''
{{
repository(name: "{name}", owner: "{owner}") {{
refs(refPrefix: "refs/tags/", first: 1,
query: "{query}",
orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{
edges {{
node {{
name
}}
}}
}}
}}
}}
'''
async def get_latest_tag(key: Tuple[str, str, str]) -> str:
repo, query, token = key
owner, reponame = repo.split('/')
headers = {
'Authorization': 'bearer %s' % token,
'Content-Type': 'application/json',
}
q = QUERY_LATEST_TAG.format(
owner = owner,
name = reponame,
query = query,
)
res = await session.post(
GITHUB_GRAPHQL_URL,
headers = headers,
json = {'query': q},
)
j = res.json()
refs = j['data']['repository']['refs']['edges']
if not refs:
raise GetVersionError('no tag found')
return refs[0]['node']['name']
async def get_version_real(
name: str, conf: Entry, *,
cache: AsyncCache, keymanager: KeyManager,
**kwargs,
) -> VersionResult:
repo = conf['github']
# Load token from config
token = conf.get('token')
# Load token from keyman
if token is None:
token = keymanager.get_key('github')
use_latest_tag = conf.get('use_latest_tag', False)
if use_latest_tag:
if not token:
raise GetVersionError('token not given but it is required')
query = conf.get('query', '')
return await cache.get((repo, query, token), get_latest_tag) # type: ignore
br = conf.get('branch')
path = conf.get('path')
use_latest_release = conf.get('use_latest_release', False)
use_max_tag = conf.get('use_max_tag', False)
if use_latest_release:
url = GITHUB_LATEST_RELEASE % repo
elif use_max_tag:
url = GITHUB_MAX_TAG % repo
else:
url = GITHUB_URL % repo
parameters = {}
if br:
parameters['sha'] = br
if path:
parameters['path'] = path
url += '?' + urlencode(parameters)
headers = {
'Accept': 'application/vnd.github.quicksilver-preview+json',
}
if token:
headers['Authorization'] = 'token %s' % token
data = await cache.get_json(url, headers = headers)
if use_max_tag:
tags = [ref['ref'].split('/', 2)[-1] for ref in data]
if not tags:
raise GetVersionError('No tag found in upstream repository.')
return tags
if use_latest_release:
if 'tag_name' not in data:
raise GetVersionError('No release found in upstream repository.')
version = data['tag_name']
else:
# YYYYMMDD.HHMMSS
version = data[0]['commit']['committer']['date'] \
.rstrip('Z').replace('-', '').replace(':', '').replace('T', '.')
return version
def check_ratelimit(exc, name):
res = exc.response
if not res:
raise
# default -1 is used to re-raise the exception
n = int(res.headers.get('X-RateLimit-Remaining', -1))
if n == 0:
reset = int(res.headers.get('X-RateLimit-Reset'))
logger.error('rate limited, resetting at %s. '
'Or get an API token to increase the allowance if not yet'
% time.ctime(reset),
name = name,
reset = reset)
else:
raise

View File

@ -24,6 +24,7 @@ async def get_version(name, conf, **kwargs):
async def get_version_real(
name: str, conf: Entry, *,
cache: AsyncCache, keymanager: KeyManager,
**kwargs,
) -> VersionResult:
repo = urllib.parse.quote_plus(conf['gitlab'])
br = conf.get('branch', 'master')
@ -39,7 +40,7 @@ async def get_version_real(
token = conf.get('token')
# Load token from keyman
if token is None:
key_name = 'gitlab_' + host.lower().replace('.', '_').replace("/", "_")
key_name = 'gitlab_' + host.lower()
token = keymanager.get_key(key_name)
# Set private token if token exists.

View File

@ -1,54 +0,0 @@
# MIT licensed
# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
import os
import re
import pytest
pytestmark = [pytest.mark.asyncio,
pytest.mark.needs_net,
pytest.mark.skipif("NVCHECKER_GITHUB_TOKEN" not in os.environ,
reason="requires NVCHECKER_GITHUB_TOKEN, or it fails too much")]
async def test_github(get_version):
assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo"}) == "20140122.012101"
async def test_github_default_not_master(get_version):
assert await get_version("example", {"github": "MariaDB/server"}) is not None
async def test_github_latest_release(get_version):
assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_latest_release": 1}) == "release3"
async def test_github_max_tag(get_version):
assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_max_tag": 1}) == "second_release"
async def test_github_max_tag_with_ignored_tags(get_version):
assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_max_tag": 1, "ignored_tags": "second_release release3"}) == "first_release"
async def test_github_max_tag_with_ignored(get_version):
assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_max_tag": 1, "ignored": "second_release release3"}) == "first_release"
async def test_github_with_path(get_version):
assert await get_version("example", {"github": "petronny/ReleaseTestRepo", "path": "test_directory"}) == "20140122.012101"
async def test_github_with_path_and_branch(get_version):
assert await get_version("example", {"github": "petronny/ReleaseTestRepo", "branch": "test", "path": "test_directory/test_directory"}) == "20190128.113201"
async def test_github_max_tag_with_include_old(get_version):
version = await get_version("example", {
"github": "EFForg/https-everywhere",
"use_max_tag": 1,
"include_tags_pattern": r"^chrome-\d",
})
assert re.match(r'chrome-[\d.]+', version)
async def test_github_max_tag_with_include(get_version):
version = await get_version("example", {
"github": "EFForg/https-everywhere",
"use_max_tag": 1,
"include_regex": r"chrome-\d.*",
})
assert re.match(r'chrome-[\d.]+', version)
async def test_github_latest_tag(get_version):
assert await get_version("example", {"github": "harry-sanabria/ReleaseTestRepo", "use_latest_tag": 1}) == "release3"

View File

@ -1,77 +0,0 @@
# MIT licensed
# Copyright (c) 2018 lilydjwg <lilydjwg@gmail.com>, et al.
import os
import tempfile
import contextlib
from nvchecker.source import HTTPError
import pytest
pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net]
@contextlib.contextmanager
def unset_github_token_env():
token = os.environ.get('NVCHECKER_GITHUB_TOKEN')
try:
if token:
del os.environ['NVCHECKER_GITHUB_TOKEN']
yield token
finally:
if token:
os.environ['NVCHECKER_GITHUB_TOKEN'] = token
async def test_keyfile_missing(run_str):
test_conf = '''\
[example]
github = harry-sanabria/ReleaseTestRepo
'''
assert await run_str(test_conf) in ['20140122.012101', None]
async def test_keyfile_invalid(run_str):
with tempfile.NamedTemporaryFile(mode='w') as f, \
unset_github_token_env():
f.write('''\
[keys]
github = xxx
''')
f.flush()
test_conf = '''\
[example]
github = harry-sanabria/ReleaseTestRepo
[__config__]
keyfile = {name}
'''.format(name=f.name)
try:
version = await run_str(test_conf, clear_cache=True)
assert version is None # out of allowance
return
except HTTPError as e:
assert e.code == 401
return
raise Exception('expected 401 response')
@pytest.mark.skipif('NVCHECKER_GITHUB_TOKEN' not in os.environ,
reason='no key given')
async def test_keyfile_valid(run_str):
with tempfile.NamedTemporaryFile(mode='w') as f, \
unset_github_token_env() as token:
f.write('''\
[keys]
github = {token}
'''.format(token=token))
f.flush()
test_conf = '''\
[example]
github = harry-sanabria/ReleaseTestRepo
[__config__]
keyfile = {name}
'''.format(name=f.name)
assert await run_str(test_conf) == '20140122.012101'

View File

@ -3,7 +3,8 @@
import asyncio
import structlog
from typing import Optional
import os
from pathlib import Path
import toml
import pytest
@ -14,12 +15,13 @@ from nvchecker.util import Entries, VersData, RawResult
async def run(
entries: Entries, max_concurrency: int = 20,
keys_toml: Optional[str] = None,
) -> VersData:
token_q = core.token_queue(max_concurrency)
result_q: asyncio.Queue[RawResult] = asyncio.Queue()
if keys_toml:
keymanager = core.KeyManager.from_str(keys_toml)
keyfile = os.environ.get('KEYFILE')
if keyfile:
filepath = Path(keyfile)
keymanager = core.KeyManager(filepath)
else:
keymanager = core.KeyManager(None)

78
tests/test_github.py Normal file
View File

@ -0,0 +1,78 @@
# MIT licensed
# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.
import os
import re
import pytest
pytestmark = [pytest.mark.asyncio,
pytest.mark.needs_net,
pytest.mark.skipif("KEYFILE" not in os.environ,
reason="requires KEYFILE, or it fails too much")]
async def test_github(get_version):
assert await get_version("example", {
"source": "github",
"github": "harry-sanabria/ReleaseTestRepo",
}) == "20140122.012101"
async def test_github_default_not_master(get_version):
assert await get_version("example", {
"source": "github",
"github": "MariaDB/server",
}) is not None
async def test_github_latest_release(get_version):
assert await get_version("example", {
"source": "github",
"github": "harry-sanabria/ReleaseTestRepo",
"use_latest_release": True,
}) == "release3"
async def test_github_max_tag(get_version):
assert await get_version("example", {
"source": "github",
"github": "harry-sanabria/ReleaseTestRepo",
"use_max_tag": True,
}) == "second_release"
async def test_github_max_tag_with_ignored(get_version):
assert await get_version("example", {
"source": "github",
"github": "harry-sanabria/ReleaseTestRepo",
"use_max_tag": True,
"ignored": "second_release release3",
}) == "first_release"
async def test_github_with_path(get_version):
assert await get_version("example", {
"source": "github",
"github": "petronny/ReleaseTestRepo",
"path": "test_directory",
}) == "20140122.012101"
async def test_github_with_path_and_branch(get_version):
assert await get_version("example", {
"source": "github",
"github": "petronny/ReleaseTestRepo",
"branch": "test",
"path": "test_directory/test_directory",
}) == "20190128.113201"
async def test_github_max_tag_with_include(get_version):
version = await get_version("example", {
"source": "github",
"github": "EFForg/https-everywhere",
"use_max_tag": True,
"include_regex": r"chrome-\d.*",
})
assert re.match(r'chrome-[\d.]+', version)
async def test_github_latest_tag(get_version):
assert await get_version("example", {
"source": "github",
"github": "harry-sanabria/ReleaseTestRepo",
"use_latest_tag": True,
}) == "release3"