78f336b21d
when updating data.json, truncate file after reading and before writing again, or we otherwise might end up with broken json
236 lines
8.0 KiB
Python
Executable File
236 lines
8.0 KiB
Python
Executable File
#!/usr/bin/env nix-shell
|
|
#! nix-shell -i python3 -p bundix common-updater-scripts nix nix-prefetch-git python3 python3Packages.requests python3Packages.lxml python3Packages.click python3Packages.click-log
|
|
|
|
import click
|
|
import click_log
|
|
import os
|
|
import re
|
|
import logging
|
|
import subprocess
|
|
import json
|
|
import pathlib
|
|
from typing import Iterable
|
|
|
|
import requests
|
|
from xml.etree import ElementTree
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class GitLabRepo:
|
|
def __init__(self, owner: str, repo: str):
|
|
self.owner = owner
|
|
self.repo = repo
|
|
|
|
@property
|
|
def url(self):
|
|
return f"https://gitlab.com/{self.owner}/{self.repo}"
|
|
|
|
@property
|
|
def tags(self) -> Iterable[str]:
|
|
r = requests.get(self.url + "/tags?format=atom", stream=True)
|
|
|
|
tree = ElementTree.fromstring(r.content)
|
|
return sorted((e.text for e in tree.findall(
|
|
'{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}title')), reverse=True)
|
|
|
|
def get_git_hash(self, rev: str):
|
|
out = subprocess.check_output(['nix-prefetch-git', self.url, rev])
|
|
j = json.loads(out)
|
|
return j['sha256']
|
|
|
|
def get_deb_url(self, flavour: str, version: str, arch: str = 'amd64') -> str:
|
|
"""
|
|
gitlab builds debian packages, which we currently need as we don't build the frontend on our own
|
|
this returns the url of a given flavour, version and arch
|
|
:param flavour: 'ce' or 'ee'
|
|
:param version: a version, without 'v' prefix and '-ee' suffix
|
|
:param arch: amd64
|
|
:return: url of the debian package
|
|
"""
|
|
if self.owner != "gitlab-org" or self.repo not in ['gitlab-ce', 'gitlab-ee']:
|
|
raise Exception(f"don't know how to get deb_url for {self.url}")
|
|
return f"https://packages.gitlab.com/gitlab/gitlab-{flavour}/packages" + \
|
|
f"/debian/stretch/gitlab-{flavour}_{version}-{flavour}.0_{arch}.deb/download.deb"
|
|
|
|
def get_deb_hash(self, flavour: str, version: str) -> str:
|
|
out = subprocess.check_output(['nix-prefetch-url', self.get_deb_url(flavour, version)])
|
|
return out.decode('utf-8').strip()
|
|
|
|
@staticmethod
|
|
def rev2version(tag: str) -> str:
|
|
"""
|
|
normalize a tag to a version number.
|
|
This obviously isn't very smart if we don't pass something that looks like a tag
|
|
:param tag: the tag to normalize
|
|
:return: a normalized version number
|
|
"""
|
|
# strip v prefix
|
|
version = re.sub(r"^v", '', tag)
|
|
# strip -ee suffix
|
|
return re.sub(r"-ee$", '', version)
|
|
|
|
def get_file(self, filepath, rev):
|
|
"""
|
|
returns file contents at a given rev
|
|
:param filepath: the path to the file, relative to the repo root
|
|
:param rev: the rev to fetch at
|
|
:return:
|
|
"""
|
|
return requests.get(self.url + f"/raw/{rev}/{filepath}").text
|
|
|
|
def get_data(self, rev, flavour):
|
|
version = self.rev2version(rev)
|
|
|
|
passthru = {v: self.get_file(v, rev).strip() for v in ['GITALY_SERVER_VERSION', 'GITLAB_PAGES_VERSION',
|
|
'GITLAB_SHELL_VERSION', 'GITLAB_WORKHORSE_VERSION']}
|
|
return dict(version=self.rev2version(rev),
|
|
repo_hash=self.get_git_hash(rev),
|
|
deb_hash=self.get_deb_hash(flavour, version),
|
|
deb_url=self.get_deb_url(flavour, version),
|
|
owner=self.owner,
|
|
repo=self.repo,
|
|
rev=rev,
|
|
passthru=passthru)
|
|
|
|
|
|
def _flavour2gitlabrepo(flavour: str):
|
|
if flavour not in ['ce', 'ee']:
|
|
raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")
|
|
|
|
owner = 'gitlab-org'
|
|
repo = 'gitlab-' + flavour
|
|
|
|
return GitLabRepo(owner, repo)
|
|
|
|
|
|
def _update_data_json(filename: str, repo: GitLabRepo, rev: str, flavour: str):
|
|
flavour_data = repo.get_data(rev, flavour)
|
|
|
|
if not os.path.exists(filename):
|
|
with open(filename, 'w') as f:
|
|
json.dump({flavour: flavour_data}, f, indent=2)
|
|
else:
|
|
with open(filename, 'r+') as f:
|
|
data = json.load(f)
|
|
data[flavour] = flavour_data
|
|
f.seek(0)
|
|
f.truncate()
|
|
json.dump(data, f, indent=2)
|
|
|
|
|
|
def _get_data_json():
|
|
data_file_path = pathlib.Path(__file__).parent / 'data.json'
|
|
with open(data_file_path, 'r') as f:
|
|
return json.load(f)
|
|
|
|
|
|
def _call_update_source_version(pkg, version):
|
|
"""calls update-source-version from nixpkgs root dir"""
|
|
nixpkgs_path = pathlib.Path(__file__).parent / '../../../../'
|
|
return subprocess.check_output(['update-source-version', pkg, version], cwd=nixpkgs_path)
|
|
|
|
|
|
@click_log.simple_verbosity_option(logger)
|
|
@click.group()
|
|
def cli():
|
|
pass
|
|
|
|
|
|
@cli.command('update-data')
|
|
@click.option('--rev', default='latest', help='The rev to use, \'latest\' points to the latest (stable) tag')
|
|
@click.argument('flavour')
|
|
def update_data(rev: str, flavour: str):
|
|
"""Update data.nix for a selected flavour"""
|
|
r = _flavour2gitlabrepo(flavour)
|
|
|
|
if rev == 'latest':
|
|
# filter out pre and re releases
|
|
rev = next(filter(lambda x: not ('rc' in x or x.endswith('pre')), r.tags))
|
|
logger.debug(f"Using rev {rev}")
|
|
|
|
version = r.rev2version(rev)
|
|
logger.debug(f"Using version {version}")
|
|
|
|
data_file_path = pathlib.Path(__file__).parent / 'data.json'
|
|
|
|
_update_data_json(filename=data_file_path.as_posix(),
|
|
repo=r,
|
|
rev=rev,
|
|
flavour=flavour)
|
|
|
|
|
|
@cli.command('update-rubyenv')
|
|
@click.argument('flavour')
|
|
def update_rubyenv(flavour):
|
|
"""Update rubyEnv-${flavour}"""
|
|
if flavour not in ['ce', 'ee']:
|
|
raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")
|
|
|
|
r = _flavour2gitlabrepo(flavour)
|
|
rubyenv_dir = pathlib.Path(__file__).parent / f"rubyEnv-{flavour}"
|
|
|
|
# load rev from data.json
|
|
data = _get_data_json()
|
|
rev = data[flavour]['rev']
|
|
|
|
for fn in ['Gemfile.lock', 'Gemfile']:
|
|
with open(rubyenv_dir / fn, 'w') as f:
|
|
f.write(r.get_file(fn, rev))
|
|
|
|
subprocess.check_output(['bundix'], cwd=rubyenv_dir)
|
|
|
|
|
|
@cli.command('update-gitaly')
|
|
def update_gitaly():
|
|
"""Update gitaly"""
|
|
data = _get_data_json()
|
|
gitaly_server_version = data['ce']['passthru']['GITALY_SERVER_VERSION']
|
|
r = GitLabRepo('gitlab-org', 'gitaly')
|
|
rubyenv_dir = pathlib.Path(__file__).parent / 'gitaly'
|
|
|
|
for fn in ['Gemfile.lock', 'Gemfile']:
|
|
with open(rubyenv_dir / fn, 'w') as f:
|
|
f.write(r.get_file(f"ruby/{fn}", f"v{gitaly_server_version}"))
|
|
|
|
subprocess.check_output(['bundix'], cwd=rubyenv_dir)
|
|
# currently broken, as `gitaly.meta.position` returns
|
|
# pkgs/development/go-modules/generic/default.nix
|
|
# so update-source-version doesn't know where to update hashes
|
|
# _call_update_source_version('gitaly', gitaly_server_version)
|
|
gitaly_hash = r.get_git_hash(f"v{gitaly_server_version}")
|
|
click.echo(f"Please update gitaly/default.nix to version {gitaly_server_version} and hash {gitaly_hash}")
|
|
|
|
|
|
|
|
@cli.command('update-gitlab-shell')
|
|
def update_gitlab_shell():
|
|
"""Update gitlab-shell"""
|
|
data = _get_data_json()
|
|
gitlab_shell_version = data['ce']['passthru']['GITLAB_SHELL_VERSION']
|
|
_call_update_source_version('gitlab-shell', gitlab_shell_version)
|
|
|
|
|
|
@cli.command('update-gitlab-workhorse')
|
|
def update_gitlab_workhorse():
|
|
"""Update gitlab-shell"""
|
|
data = _get_data_json()
|
|
gitlab_workhorse_version = data['ce']['passthru']['GITLAB_WORKHORSE_VERSION']
|
|
_call_update_source_version('gitlab-workhorse', gitlab_workhorse_version)
|
|
|
|
|
|
@cli.command('update-all')
|
|
@click.pass_context
|
|
def update_all(ctx):
|
|
"""Update gitlab ce and ee data.nix and rubyenvs to the latest stable release"""
|
|
for flavour in ['ce', 'ee']:
|
|
ctx.invoke(update_data, rev='latest', flavour=flavour)
|
|
ctx.invoke(update_rubyenv, flavour=flavour)
|
|
ctx.invoke(update_gitaly)
|
|
ctx.invoke(update_gitlab_shell)
|
|
ctx.invoke(update_gitlab_workhorse)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
cli()
|