JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vimPlugins: Automate redirect updates in update.py

Browse Source 
Many of the plugins in vim-plugin-names are out of date and redirect to
their new github repos. This commit adds a flag to automatically update
these and defines a process for committing these updates into nixpkgs.
This commit is contained in:
ryneeverett 2020-03-21 04:27:09 +00:00
parent 11315a3e27
commit d4874a7ee6
2 changed files with 86 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/languages-frameworks/vim.section.md
View File

@ -263,7 +263,7 @@ Sometimes plugins require an override that must be changed when the plugin is up
To add a new plugin:
1. run `./update.py` and create a commit named "vimPlugins: Update",
1. run `./update.py --update-redirects` and create a commit named "vimPlugins: Update",
2. add the new plugin to [vim-plugin-names](/pkgs/misc/vim-plugins/vim-plugin-names) and add overrides if required to [overrides.nix](/pkgs/misc/vim-plugins/overrides.nix),
3. run `./update.py` again and create a commit named "vimPlugins.[name]: init at [version]" (where `name` and `version` can be found in [generated.nix](/pkgs/misc/vim-plugins/generated.nix)), and
4. create a pull request.

100
pkgs/misc/vim-plugins/update.py
View File

@ -9,13 +9,16 @@
# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
import argparse
import fileinput
import functools
import http
import json
import os
import subprocess
import sys
import traceback
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime
@ -71,9 +74,11 @@ def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: floa
class Repo:
def __init__(self, owner: str, name: str) -> None:
def __init__(self, owner: str, name: str, alias: str) -> None:
self.owner = owner
self.name = name
self.alias = alias
self.redirect: Dict[str, str] = {}
def url(self, path: str) -> str:
return urljoin(f"https://github.com/{self.owner}/{self.name}/", path)
@ -96,7 +101,9 @@ class Repo:
@retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
def latest_commit(self) -> Tuple[str, datetime]:
with urllib.request.urlopen(self.url("commits/master.atom"), timeout=10) as req:
commit_url = self.url("commits/master.atom")
with urllib.request.urlopen(commit_url, timeout=10) as req:
self.check_for_redirect(commit_url, req)
xml = req.read()
root = ET.fromstring(xml)
latest_entry = root.find(ATOM_ENTRY)
@ -111,6 +118,22 @@ class Repo:
updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ")
return Path(str(url.path)).name, updated
def check_for_redirect(self, url: str, req: http.client.HTTPResponse):
response_url = req.geturl()
if url != response_url:
new_owner, new_name = (
urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
)
end_line = f" as {self.alias}\n" if self.alias else "\n"
plugin_line = "{owner}/{name}" + end_line
old_plugin = plugin_line.format(owner=self.owner, name=self.name)
new_plugin = plugin_line.format(owner=new_owner, name=new_name)
self.redirect[old_plugin] = new_plugin
if new_name != self.name:
print(f"Plugin Name Changed: {self.name} -> {new_name}")
def prefetch_git(self, ref: str) -> str:
data = subprocess.check_output(
["nix-prefetch-git", "--fetch-submodules", self.url(""), ref]
@ -197,15 +220,17 @@ def get_current_plugins() -> List[Plugin]:
return plugins
def prefetch_plugin(user: str, repo_name: str, alias: str, cache: "Cache") -> Plugin:
repo = Repo(user, repo_name)
def prefetch_plugin(
user: str, repo_name: str, alias: str, cache: "Cache"
) -> Tuple[Plugin, Dict[str, str]]:
repo = Repo(user, repo_name, alias)
commit, date = repo.latest_commit()
has_submodules = repo.has_submodules()
cached_plugin = cache[commit]
if cached_plugin is not None:
cached_plugin.name = alias or repo_name
cached_plugin.date = date
return cached_plugin
return cached_plugin, repo.redirect
print(f"prefetch {user}/{repo_name}")
if has_submodules:
@ -213,7 +238,10 @@ def prefetch_plugin(user: str, repo_name: str, alias: str, cache: "Cache") -> Pl
else:
sha256 = repo.prefetch_github(commit)
return Plugin(alias or repo_name, commit, has_submodules, sha256, date=date)
return (
Plugin(alias or repo_name, commit, has_submodules, sha256, date=date),
repo.redirect,
)
def print_download_error(plugin: str, ex: Exception):
@ -227,20 +255,22 @@ def print_download_error(plugin: str, ex: Exception):
def check_results(
results: List[Tuple[str, str, Union[Exception, Plugin]]]
) -> List[Tuple[str, str, Plugin]]:
results: List[Tuple[str, str, Union[Exception, Plugin], Dict[str, str]]]
) -> Tuple[List[Tuple[str, str, Plugin]], Dict[str, str]]:
failures: List[Tuple[str, Exception]] = []
plugins = []
for (owner, name, result) in results:
redirects: Dict[str, str] = {}
for (owner, name, result, redirect) in results:
if isinstance(result, Exception):
failures.append((name, result))
else:
plugins.append((owner, name, result))
redirects.update(redirect)
print(f"{len(results) - len(failures)} plugins were checked", end="")
if len(failures) == 0:
print()
return plugins
return plugins, redirects
else:
print(f", {len(failures)} plugin(s) could not be downloaded:\n")
@ -328,15 +358,15 @@ class Cache:
def prefetch(
args: Tuple[str, str, str], cache: Cache
) -> Tuple[str, str, Union[Exception, Plugin]]:
) -> Tuple[str, str, Union[Exception, Plugin], dict]:
assert len(args) == 3
owner, repo, alias = args
try:
plugin = prefetch_plugin(owner, repo, alias, cache)
plugin, redirect = prefetch_plugin(owner, repo, alias, cache)
cache[plugin.commit] = plugin
return (owner, repo, plugin)
return (owner, repo, plugin, redirect)
except Exception as e:
return (owner, repo, e)
return (owner, repo, e, {})
header = (
@ -386,6 +416,31 @@ in lib.fix' (lib.extends overrides packages)
print(f"updated {outfile}")
def update_redirects(input_file: Path, output_file: Path, redirects: dict):
with fileinput.input(input_file, inplace=True) as f:
for line in f:
print(redirects.get(line, line), end="")
print(
f"""\
Redirects have been detected and {input_file} has been updated. Please take the
following steps:
1. Go ahead and commit just the updated expressions as you intended to do:
git add {output_file}
git commit -m "vimPlugins: Update"
2. If any of the plugin names were changed, add the old names as aliases in
aliases.nix
3. Make sure the updated {input_file} is still correctly sorted:
sort -udf ./vim-plugin-names > sorted && mv sorted vim-plugin-names
4. Run this script again so these changes will be reflected in the
generated expressions (no need to use the --update-redirects flag again):
./update.py
5. Commit {input_file} along with aliases and generated expressions:
git add {output_file} {input_file} aliases.nix
git commit -m "vimPlugins: Update redirects"
"""
)
def parse_args():
parser = argparse.ArgumentParser(
description=(
@ -407,6 +462,12 @@ def parse_args():
default=DEFAULT_OUT,
help="Filename to save generated nix code",
)
parser.add_argument(
"--update-redirects",
dest="update_redirects",
action="store_true",
help="Update input file if repos have been redirected.",
)
return parser.parse_args()
@ -428,10 +489,19 @@ def main() -> None:
finally:
cache.store()
plugins = check_results(results)
plugins, redirects = check_results(results)
generate_nix(plugins, args.outfile)
if redirects:
if args.update_redirects:
update_redirects(args.input_file, args.outfile, redirects)
else:
print(
"Outdated vim-plugin-names found. Please run with "
"--update-redirects flag."
)
if __name__ == "__main__":
main()