JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

discourse: update.py: Improve version handling, use pinned plugins

Browse Source 
Add a DiscourseVersion class which handles Discourse's version
numbering properly when sorting - beta versions are sorted lower than
their respective release versions. It can also return both its version
number and equivalent git tag, removing the need for `rev2version` and
manually adding `v` to the front.

Using DiscourseVersion instead of LooseVersion, we can list all
current version number tags from the `discourse` repo and sort them
correctly, giving us the latest one, regardless of type; i.e. we don't
have to filter for only release versions or beta versions anymore.

This also implements the plugin pinning algorithm laid out here:
https://meta.discourse.org/t/pinning-plugin-and-theme-versions-for-older-discourse-installs/156971
to make sure we don't upgrade plugins further than what's compatible
with our currently packaged Discourse version. While it likely won't
matter much most of the time if we continue packaging the beta
versions, it could be helpful if we decide to go back to packaging
release versions or if we run into issues with future upgrades. In
that case, the plugins could still be updated safely even though we're
not on the latest version of Discourse.
This commit is contained in:
talyz 2021-12-02 11:55:07 +01:00
parent ab042d6452
commit 4fb343c87e
No known key found for this signature in database
GPG Key ID: 2DED2151F4671A2B
1 changed files with 109 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
pkgs/servers/web-apps/discourse/update.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env nix-shell
#! nix-shell -i python3 -p bundix bundler nix-update nix-universal-prefetch python3 python3Packages.requests python3Packages.click python3Packages.click-log
from __future__ import annotations
import click
import click_log
@ -13,16 +14,70 @@ import stat
import json
import requests
import textwrap
from functools import total_ordering
from distutils.version import LooseVersion
from itertools import zip_longest
from pathlib import Path
from typing import Iterable
from typing import Union, Iterable
logger = logging.getLogger(__name__)
@total_ordering
class DiscourseVersion:
"""Represents a Discourse style version number and git tag.
This takes either a tag or version string as input and
extrapolates the other. Sorting is implemented to work as expected
in regard to A.B.C.betaD version numbers - 2.0.0.beta1 is
considered lower than 2.0.0.
"""
tag: str = ""
version: str = ""
split_version: Iterable[Union[None, int, str]] = []
def __init__(self, version: str):
"""Take either a tag or version number, calculate the other."""
if version.startswith('v'):
self.tag = version
self.version = version.lstrip('v')
else:
self.tag = 'v' + version
self.version = version
self.split_version = LooseVersion(self.version).version
def __eq__(self, other: DiscourseVersion):
"""Versions are equal when their individual parts are."""
return self.split_version == other.split_version
def __gt__(self, other: DiscourseVersion):
"""Check if this version is greater than the other.
Goes through the parts of the version numbers from most to
least significant, only continuing on to the next if the
numbers are equal and no decision can be made. If one version
ends in 'betaX' and the other doesn't, all else being equal,
the one without 'betaX' is considered greater, since it's the
release version.
"""
for (this_ver, other_ver) in zip_longest(self.split_version, other.split_version):
if this_ver == other_ver:
continue
if type(this_ver) is int and type(other_ver) is int:
return this_ver > other_ver
elif 'beta' in [this_ver, other_ver]:
# release version (None) is greater than beta
return this_ver is None
else:
return False
class DiscourseRepo:
version_regex = re.compile(r'^v\d+\.\d+\.\d+.beta\d+$')
version_regex = re.compile(r'^v\d+\.\d+\.\d+(\.beta\d+)?$')
_latest_commit_sha = None
def __init__(self, owner: str = 'discourse', repo: str = 'discourse'):
@ -30,15 +85,14 @@ class DiscourseRepo:
self.repo = repo
@property
def tags(self) -> Iterable[str]:
def versions(self) -> Iterable[str]:
r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/git/refs/tags').json()
tags = [x['ref'].replace('refs/tags/', '') for x in r]
# filter out versions not matching version_regex
versions = list(filter(self.version_regex.match, tags))
# sort, but ignore v for sorting comparisons
versions.sort(key=lambda x: LooseVersion(x.replace('v', '')), reverse=True)
versions = filter(self.version_regex.match, tags)
versions = [DiscourseVersion(x) for x in versions]
versions.sort(reverse=True)
return versions
@property
@ -50,24 +104,15 @@ class DiscourseRepo:
return self._latest_commit_sha
@staticmethod
def rev2version(tag: str) -> str:
"""
normalize a tag to a version number.
This obviously isn't very smart if we don't pass something that looks like a tag
:param tag: the tag to normalize
:return: a normalized version number
"""
# strip v prefix
return re.sub(r'^v', '', tag)
def get_file(self, filepath, rev):
"""returns file contents at a given rev :param filepath: the path to
the file, relative to the repo root :param rev: the rev to
fetch at :return:
"""
return requests.get(f'https://raw.githubusercontent.com/{self.owner}/{self.repo}/{rev}/{filepath}').text
r = requests.get(f'https://raw.githubusercontent.com/{self.owner}/{self.repo}/{rev}/{filepath}')
r.raise_for_status()
return r.text
def _call_nix_update(pkg, version):
@ -89,13 +134,13 @@ def _get_current_package_version(pkg: str):
return _nix_eval(f'{pkg}.version')
def _diff_file(filepath: str, old_version: str, new_version: str):
def _diff_file(filepath: str, old_version: DiscourseVersion, new_version: DiscourseVersion):
repo = DiscourseRepo()
current_dir = Path(__file__).parent
old = repo.get_file(filepath, 'v' + old_version)
new = repo.get_file(filepath, 'v' + new_version)
old = repo.get_file(filepath, old_version.tag)
new = repo.get_file(filepath, new_version.tag)
if old == new:
click.secho(f'{filepath} is unchanged', fg='green')
@ -111,7 +156,7 @@ def _diff_file(filepath: str, old_version: str, new_version: str):
text=True
)
click.secho(f'Diff for {filepath} ({old_version} -> {new_version}):', fg='bright_blue', bold=True)
click.secho(f'Diff for {filepath} ({old_version.version} -> {new_version.version}):', fg='bright_blue', bold=True)
click.echo(diff_proc.stdout + '\n')
return
@ -153,10 +198,10 @@ def print_diffs(rev, reverse):
"""
if rev == 'latest':
repo = DiscourseRepo()
rev = repo.tags[0]
rev = repo.versions[0].tag
old_version = _get_current_package_version('discourse')
new_version = DiscourseRepo.rev2version(rev)
old_version = DiscourseVersion(_get_current_package_version('discourse'))
new_version = DiscourseVersion(rev)
if reverse:
old_version, new_version = new_version, old_version
@ -170,30 +215,32 @@ def print_diffs(rev, reverse):
def update(rev):
"""Update gem files and version.
REV should be the git rev to update to ('vX.Y.Z') or 'latest';
defaults to 'latest'.
REV should be the git rev to update to ('vX.Y.Z[.betaA]') or
'latest'; defaults to 'latest'.
"""
repo = DiscourseRepo()
if rev == 'latest':
rev = repo.tags[0]
logger.debug(f"Using rev {rev}")
version = repo.versions[0]
else:
version = DiscourseVersion(rev)
version = repo.rev2version(rev)
logger.debug(f"Using version {version}")
logger.debug(f"Using rev {version.tag}")
logger.debug(f"Using version {version.version}")
rubyenv_dir = Path(__file__).parent / "rubyEnv"
for fn in ['Gemfile.lock', 'Gemfile']:
with open(rubyenv_dir / fn, 'w') as f:
f.write(repo.get_file(fn, rev))
f.write(repo.get_file(fn, version.tag))
subprocess.check_output(['bundle', 'lock'], cwd=rubyenv_dir)
_remove_platforms(rubyenv_dir)
subprocess.check_output(['bundix'], cwd=rubyenv_dir)
_call_nix_update('discourse', repo.rev2version(rev))
_call_nix_update('discourse', version.version)
@cli.command()
def update_plugins():
@ -229,13 +276,34 @@ def update_plugins():
repo = DiscourseRepo(owner=owner, repo=repo_name)
# implement the plugin pinning algorithm laid out here:
# https://meta.discourse.org/t/pinning-plugin-and-theme-versions-for-older-discourse-installs/156971
# this makes sure we don't upgrade plugins to revisions that
# are incompatible with the packaged Discourse version
try:
compatibility_spec = repo.get_file('.discourse-compatibility', repo.latest_commit_sha)
versions = [(DiscourseVersion(discourse_version), plugin_rev.strip(' '))
for [discourse_version, plugin_rev]
in [line.split(':')
for line
in compatibility_spec.splitlines()]]
discourse_version = DiscourseVersion(_get_current_package_version('discourse'))
versions = list(filter(lambda ver: ver[0] >= discourse_version, versions))
if versions == []:
rev = repo.latest_commit_sha
else:
rev = versions[0][1]
print(rev)
except requests.exceptions.HTTPError:
rev = repo.latest_commit_sha
filename = _nix_eval(f'builtins.unsafeGetAttrPos "src" discourse.plugins.{name}')
if filename is None:
filename = Path(__file__).parent / 'plugins' / name / 'default.nix'
filename.parent.mkdir()
has_ruby_deps = False
for line in repo.get_file('plugin.rb', repo.latest_commit_sha).splitlines():
for line in repo.get_file('plugin.rb', rev).splitlines():
if 'gem ' in line:
has_ruby_deps = True
break
@ -277,7 +345,7 @@ def update_plugins():
prev_commit_sha = _nix_eval(f'discourse.plugins.{name}.src.rev')
if prev_commit_sha == repo.latest_commit_sha:
if prev_commit_sha == rev:
click.echo(f'Plugin {name} is already at the latest revision')
continue
@ -286,14 +354,14 @@ def update_plugins():
'nix-universal-prefetch', fetcher,
'--owner', owner,
'--repo', repo_name,
'--rev', repo.latest_commit_sha,
'--rev', rev,
], text=True).strip("\n")
click.echo(f"Update {name}, {prev_commit_sha} -> {repo.latest_commit_sha} in {filename}")
click.echo(f"Update {name}, {prev_commit_sha} -> {rev} in {filename}")
with open(filename, 'r+') as f:
content = f.read()
content = content.replace(prev_commit_sha, repo.latest_commit_sha)
content = content.replace(prev_commit_sha, rev)
content = content.replace(prev_hash, new_hash)
f.seek(0)
f.write(content)
@ -303,14 +371,14 @@ def update_plugins():
gemfile = rubyenv_dir / "Gemfile"
version_file_regex = re.compile(r'.*File\.expand_path\("\.\./(.*)", __FILE__\)')
gemfile_text = ''
for line in repo.get_file('plugin.rb', repo.latest_commit_sha).splitlines():
for line in repo.get_file('plugin.rb', rev).splitlines():
if 'gem ' in line:
gemfile_text = gemfile_text + line + os.linesep
version_file_match = version_file_regex.match(line)
if version_file_match is not None:
filename = version_file_match.groups()[0]
content = repo.get_file(filename, repo.latest_commit_sha)
content = repo.get_file(filename, rev)
with open(rubyenv_dir / filename, 'w') as f:
f.write(content)