afa3abf632
GitLab recently restructured their repos; whereas previously they had one gitlab-ce and one gitlab-ee repo, they're now one and the same. All proprietary components are put into the ee subdirectory - removing it gives us the foss / community version of GitLab. For more info, see https://about.gitlab.com/2019/02/21/merging-ce-and-ee-codebases/ This gives us the opportunity to simplify things quite a bit, since we don't have to keep track of two separate versions of either the base data or rubyEnv.
244 lines
7.7 KiB
Python
Executable File
244 lines
7.7 KiB
Python
Executable File
#!/usr/bin/env nix-shell
|
|
#! nix-shell -i python3 -p bundix common-updater-scripts nix nix-prefetch-git python3 python3Packages.requests python3Packages.lxml python3Packages.click python3Packages.click-log vgo2nix yarn2nix-moretea.yarn2nix
|
|
|
|
import click
|
|
import click_log
|
|
import os
|
|
import re
|
|
import logging
|
|
import subprocess
|
|
import json
|
|
import pathlib
|
|
from distutils.version import LooseVersion
|
|
from typing import Iterable
|
|
|
|
import requests
|
|
from xml.etree import ElementTree
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class GitLabRepo:
|
|
version_regex = re.compile(r"^v\d+\.\d+\.\d+(\-rc\d+)?(\-ee)?")
|
|
def __init__(self, owner: str = 'gitlab-org', repo: str = 'gitlab'):
|
|
self.owner = owner
|
|
self.repo = repo
|
|
|
|
@property
|
|
def url(self):
|
|
return f"https://gitlab.com/{self.owner}/{self.repo}"
|
|
|
|
@property
|
|
def tags(self) -> Iterable[str]:
|
|
r = requests.get(self.url + "/tags?format=atom", stream=True)
|
|
|
|
tree = ElementTree.fromstring(r.content)
|
|
versions = [e.text for e in tree.findall('{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}title')]
|
|
# filter out versions not matching version_regex
|
|
versions = list(filter(self.version_regex.match, versions))
|
|
|
|
# sort, but ignore v and -ee for sorting comparisons
|
|
versions.sort(key=lambda x: LooseVersion(x.replace("v", "").replace("-ee", "")), reverse=True)
|
|
return versions
|
|
|
|
def get_git_hash(self, rev: str):
|
|
out = subprocess.check_output(['nix-prefetch-git', self.url, rev])
|
|
j = json.loads(out)
|
|
return j['sha256']
|
|
|
|
@staticmethod
|
|
def rev2version(tag: str) -> str:
|
|
"""
|
|
normalize a tag to a version number.
|
|
This obviously isn't very smart if we don't pass something that looks like a tag
|
|
:param tag: the tag to normalize
|
|
:return: a normalized version number
|
|
"""
|
|
# strip v prefix
|
|
version = re.sub(r"^v", '', tag)
|
|
# strip -ee suffix
|
|
return re.sub(r"-ee$", '', version)
|
|
|
|
def get_file(self, filepath, rev):
|
|
"""
|
|
returns file contents at a given rev
|
|
:param filepath: the path to the file, relative to the repo root
|
|
:param rev: the rev to fetch at
|
|
:return:
|
|
"""
|
|
return requests.get(self.url + f"/raw/{rev}/{filepath}").text
|
|
|
|
def get_data(self, rev):
|
|
version = self.rev2version(rev)
|
|
|
|
passthru = {v: self.get_file(v, rev).strip() for v in ['GITALY_SERVER_VERSION', 'GITLAB_PAGES_VERSION',
|
|
'GITLAB_SHELL_VERSION', 'GITLAB_WORKHORSE_VERSION']}
|
|
return dict(version=self.rev2version(rev),
|
|
repo_hash=self.get_git_hash(rev),
|
|
owner=self.owner,
|
|
repo=self.repo,
|
|
rev=rev,
|
|
passthru=passthru)
|
|
|
|
|
|
def _get_data_json():
|
|
data_file_path = pathlib.Path(__file__).parent / 'data.json'
|
|
with open(data_file_path, 'r') as f:
|
|
return json.load(f)
|
|
|
|
|
|
def _call_update_source_version(pkg, version):
|
|
"""calls update-source-version from nixpkgs root dir"""
|
|
nixpkgs_path = pathlib.Path(__file__).parent / '../../../../'
|
|
return subprocess.check_output(['update-source-version', pkg, version], cwd=nixpkgs_path)
|
|
|
|
|
|
@click_log.simple_verbosity_option(logger)
|
|
@click.group()
|
|
def cli():
|
|
pass
|
|
|
|
|
|
@cli.command('update-data')
|
|
@click.option('--rev', default='latest', help='The rev to use, \'latest\' points to the latest (stable) tag')
|
|
def update_data(rev: str):
|
|
"""Update data.nix"""
|
|
repo = GitLabRepo()
|
|
|
|
if rev == 'latest':
|
|
# filter out pre and re releases
|
|
rev = next(filter(lambda x: not ('rc' in x or x.endswith('pre')), repo.tags))
|
|
logger.debug(f"Using rev {rev}")
|
|
|
|
version = repo.rev2version(rev)
|
|
logger.debug(f"Using version {version}")
|
|
|
|
data_file_path = pathlib.Path(__file__).parent / 'data.json'
|
|
|
|
data = repo.get_data(rev)
|
|
|
|
with open(data_file_path.as_posix(), 'w') as f:
|
|
json.dump(data, f, indent=2)
|
|
|
|
|
|
@cli.command('update-rubyenv')
|
|
def update_rubyenv():
|
|
"""Update rubyEnv"""
|
|
repo = GitLabRepo()
|
|
rubyenv_dir = pathlib.Path(__file__).parent / f"rubyEnv"
|
|
|
|
# load rev from data.json
|
|
data = _get_data_json()
|
|
rev = data['rev']
|
|
|
|
for fn in ['Gemfile.lock', 'Gemfile']:
|
|
with open(rubyenv_dir / fn, 'w') as f:
|
|
f.write(repo.get_file(fn, rev))
|
|
|
|
subprocess.check_output(['bundix'], cwd=rubyenv_dir)
|
|
|
|
|
|
@cli.command('update-yarnpkgs')
|
|
def update_yarnpkgs():
|
|
"""Update yarnPkgs"""
|
|
|
|
repo = GitLabRepo()
|
|
yarnpkgs_dir = pathlib.Path(__file__).parent
|
|
|
|
# load rev from data.json
|
|
data = _get_data_json()
|
|
rev = data['rev']
|
|
|
|
with open(yarnpkgs_dir / 'yarn.lock', 'w') as f:
|
|
f.write(repo.get_file('yarn.lock', rev))
|
|
|
|
with open(yarnpkgs_dir / 'yarnPkgs.nix', 'w') as f:
|
|
subprocess.run(['yarn2nix'], cwd=yarnpkgs_dir, check=True, stdout=f)
|
|
|
|
os.unlink(yarnpkgs_dir / 'yarn.lock')
|
|
|
|
|
|
@cli.command('update-gitaly')
|
|
def update_gitaly():
|
|
"""Update gitaly"""
|
|
data = _get_data_json()
|
|
gitaly_server_version = data['passthru']['GITALY_SERVER_VERSION']
|
|
repo = GitLabRepo(repo='gitaly')
|
|
gitaly_dir = pathlib.Path(__file__).parent / 'gitaly'
|
|
|
|
for fn in ['Gemfile.lock', 'Gemfile']:
|
|
with open(gitaly_dir / fn, 'w') as f:
|
|
f.write(repo.get_file(f"ruby/{fn}", f"v{gitaly_server_version}"))
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
with open(gitaly_dir / fn, 'w') as f:
|
|
f.write(repo.get_file(fn, f"v{gitaly_server_version}"))
|
|
|
|
subprocess.check_output(['bundix'], cwd=gitaly_dir)
|
|
subprocess.check_output(['vgo2nix'], cwd=gitaly_dir)
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
os.unlink(gitaly_dir / fn)
|
|
|
|
_call_update_source_version('gitaly', gitaly_server_version)
|
|
|
|
|
|
@cli.command('update-gitlab-shell')
|
|
def update_gitlab_shell():
|
|
"""Update gitlab-shell"""
|
|
data = _get_data_json()
|
|
gitlab_shell_version = data['passthru']['GITLAB_SHELL_VERSION']
|
|
_call_update_source_version('gitlab-shell', gitlab_shell_version)
|
|
|
|
repo = GitLabRepo(repo='gitlab-shell')
|
|
gitlab_shell_dir = pathlib.Path(__file__).parent / 'gitlab-shell'
|
|
|
|
for fn in ['Gemfile.lock', 'Gemfile']:
|
|
with open(gitlab_shell_dir / fn, 'w') as f:
|
|
f.write(repo.get_file(fn, f"v{gitlab_shell_version}"))
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
with open(gitlab_shell_dir / fn, 'w') as f:
|
|
f.write(repo.get_file(f"go/{fn}", f"v{gitlab_shell_version}"))
|
|
|
|
subprocess.check_output(['bundix'], cwd=gitlab_shell_dir)
|
|
subprocess.check_output(['vgo2nix'], cwd=gitlab_shell_dir)
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
os.unlink(gitlab_shell_dir / fn)
|
|
|
|
|
|
@cli.command('update-gitlab-workhorse')
|
|
def update_gitlab_workhorse():
|
|
"""Update gitlab-workhorse"""
|
|
data = _get_data_json()
|
|
gitlab_workhorse_version = data['passthru']['GITLAB_WORKHORSE_VERSION']
|
|
_call_update_source_version('gitlab-workhorse', gitlab_workhorse_version)
|
|
|
|
repo = GitLabRepo('gitlab-org', 'gitlab-workhorse')
|
|
gitlab_workhorse_dir = pathlib.Path(__file__).parent / 'gitlab-workhorse'
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
with open(gitlab_workhorse_dir / fn, 'w') as f:
|
|
f.write(repo.get_file(fn, f"v{gitlab_workhorse_version}"))
|
|
|
|
subprocess.check_output(['vgo2nix'], cwd=gitlab_workhorse_dir)
|
|
|
|
for fn in ['go.mod', 'go.sum']:
|
|
os.unlink(gitlab_workhorse_dir / fn)
|
|
|
|
@cli.command('update-all')
|
|
@click.pass_context
|
|
def update_all(ctx):
|
|
"""Update all gitlab components to the latest stable release"""
|
|
ctx.invoke(update_data, rev='latest')
|
|
ctx.invoke(update_rubyenv)
|
|
ctx.invoke(update_yarnpkgs)
|
|
ctx.invoke(update_gitaly)
|
|
ctx.invoke(update_gitlab_shell)
|
|
ctx.invoke(update_gitlab_workhorse)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
cli()
|