maintainers/scripts: add kde2nix tooling
This will be used to generate data for KDE 6 packages.
This commit is contained in:
parent
2248bdfbbc
commit
3a52750ac7
31
maintainers/scripts/kde/collect-licenses.sh
Executable file
31
maintainers/scripts/kde/collect-licenses.sh
Executable file
@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p gnutar jq reuse
|
||||
set -eu
|
||||
cd "$(dirname "$(readlink -f "$0")")"/../../..
|
||||
|
||||
TMPDIR=$(mktemp -d)
|
||||
trap 'rm -rf $TMPDIR' EXIT
|
||||
|
||||
echo "# Prebuilding sources..."
|
||||
nix-build -A kdePackages.sources --no-link || true
|
||||
|
||||
echo "# Evaluating sources..."
|
||||
declare -A sources
|
||||
eval "$(nix-instantiate --eval -A kdePackages.sources --json --strict | jq 'to_entries[] | "sources[" + .key + "]=" + .value' -r)"
|
||||
|
||||
echo "# Collecting licenses..."
|
||||
for k in "${!sources[@]}"; do
|
||||
echo "- Processing $k..."
|
||||
|
||||
if [ ! -f "${sources[$k]}" ]; then
|
||||
echo "Not found!"
|
||||
continue
|
||||
fi
|
||||
|
||||
mkdir "$TMPDIR/$k"
|
||||
tar -C "$TMPDIR/$k" -xf "${sources[$k]}"
|
||||
|
||||
(cd "$TMPDIR/$k"; reuse lint --json) | jq --arg name "$k" '{$name: .summary.used_licenses | sort}' -c > "$TMPDIR/$k.json"
|
||||
done
|
||||
|
||||
jq -s 'add' -S "$TMPDIR"/*.json > pkgs/kde/generated/licenses.json
|
13
maintainers/scripts/kde/collect-logs.sh
Executable file
13
maintainers/scripts/kde/collect-logs.sh
Executable file
@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p gnused jq
|
||||
set -eu
|
||||
cd "$(dirname "$(readlink -f "$0")")"/../../..
|
||||
|
||||
mkdir -p logs
|
||||
for name in $(nix-env -qaP -f . -A kdePackages --json | jq -r 'to_entries[] | .key' | sed s/kdePackages.//); do
|
||||
echo "Processing ${name}..."
|
||||
path=$(nix eval ".#kdePackages.${name}.outPath" --json --option warn-dirty false | jq -r)
|
||||
if [ -n "${path}" ]; then
|
||||
nix-store --read-log "${path}" > "logs/${name}.log" || true
|
||||
fi
|
||||
done
|
36
maintainers/scripts/kde/collect-metadata.py
Executable file
36
maintainers/scripts/kde/collect-metadata.py
Executable file
@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.click ps.pyyaml ])"
|
||||
import pathlib
|
||||
|
||||
import click
|
||||
|
||||
import utils
|
||||
|
||||
@click.command
|
||||
@click.argument(
|
||||
"repo-metadata",
|
||||
type=click.Path(
|
||||
exists=True,
|
||||
file_okay=False,
|
||||
resolve_path=True,
|
||||
path_type=pathlib.Path,
|
||||
),
|
||||
)
|
||||
@click.option(
|
||||
"--nixpkgs",
|
||||
type=click.Path(
|
||||
exists=True,
|
||||
file_okay=False,
|
||||
resolve_path=True,
|
||||
writable=True,
|
||||
path_type=pathlib.Path,
|
||||
),
|
||||
default=pathlib.Path(__file__).parent.parent.parent.parent
|
||||
)
|
||||
def main(repo_metadata: pathlib.Path, nixpkgs: pathlib.Path):
|
||||
metadata = utils.KDERepoMetadata.from_repo_metadata_checkout(repo_metadata)
|
||||
out_dir = nixpkgs / "pkgs/kde/generated"
|
||||
metadata.write_json(out_dir)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main() # type: ignore
|
127
maintainers/scripts/kde/collect-missing-deps.py
Executable file
127
maintainers/scripts/kde/collect-missing-deps.py
Executable file
@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i python3 -p python3
|
||||
import pathlib
|
||||
|
||||
OK_MISSING = {
|
||||
# we don't use precompiled QML
|
||||
'Qt6QuickCompiler',
|
||||
'Qt6QmlCompilerPlusPrivate',
|
||||
# usually used for version numbers
|
||||
'Git',
|
||||
# useless by itself, will warn if something else is not found
|
||||
'PkgConfig',
|
||||
# license verification
|
||||
'ReuseTool',
|
||||
# dev only
|
||||
'ClangFormat',
|
||||
# doesn't exist
|
||||
'Qt6X11Extras',
|
||||
}
|
||||
|
||||
OK_MISSING_BY_PACKAGE = {
|
||||
"angelfish": {
|
||||
"Qt6Feedback", # we don't have it
|
||||
},
|
||||
"attica": {
|
||||
"Python3", # only used for license checks
|
||||
},
|
||||
"discover": {
|
||||
"rpm-ostree-1", # we don't have rpm-ostree (duh)
|
||||
"Snapd", # we don't have snaps and probably never will
|
||||
},
|
||||
"elisa": {
|
||||
"UPNPQT", # upstream says it's broken
|
||||
},
|
||||
"extra-cmake-modules": {
|
||||
"Sphinx", # only used for docs, bloats closure size
|
||||
"QCollectionGenerator"
|
||||
},
|
||||
"kio-extras-kf5": {
|
||||
"KDSoapWSDiscoveryClient", # actually vendored on KF5 version
|
||||
},
|
||||
"kitinerary": {
|
||||
"OsmTools", # used for map data updates, we use prebuilt
|
||||
},
|
||||
"kosmindoormap": {
|
||||
"OsmTools", # same
|
||||
"Protobuf",
|
||||
},
|
||||
"kpty": {
|
||||
"UTEMPTER", # we don't have it and it probably wouldn't work anyway
|
||||
},
|
||||
"kpublictransport": {
|
||||
"OsmTools", # same
|
||||
"PolyClipping",
|
||||
"Protobuf",
|
||||
},
|
||||
"krfb": {
|
||||
"Qt6XkbCommonSupport", # not real
|
||||
},
|
||||
"kuserfeedback": {
|
||||
"Qt6Svg", # all used for backend console stuff we don't ship
|
||||
"QmlLint",
|
||||
"Qt6Charts",
|
||||
"FLEX",
|
||||
"BISON",
|
||||
"Php",
|
||||
"PhpUnit",
|
||||
},
|
||||
"kwin": {
|
||||
"display-info", # newer versions identify as libdisplay-info
|
||||
},
|
||||
"mlt": {
|
||||
"Qt5", # intentionally disabled
|
||||
"SWIG",
|
||||
},
|
||||
"plasma-desktop": {
|
||||
"scim", # upstream is dead, not packaged in Nixpkgs
|
||||
},
|
||||
"powerdevil": {
|
||||
"DDCUtil", # cursed, intentionally disabled
|
||||
},
|
||||
"pulseaudio-qt": {
|
||||
"Qt6Qml", # tests only
|
||||
"Qt6Quick",
|
||||
},
|
||||
"syntax-highlighting": {
|
||||
"XercesC", # only used for extra validation at build time
|
||||
}
|
||||
}
|
||||
|
||||
def main():
|
||||
here = pathlib.Path(__file__).parent.parent.parent.parent
|
||||
logs = (here / "logs").glob("*.log")
|
||||
|
||||
for log in sorted(logs):
|
||||
pname = log.stem
|
||||
|
||||
missing = []
|
||||
is_in_block = False
|
||||
with log.open(errors="replace") as fd:
|
||||
for line in fd:
|
||||
line = line.strip()
|
||||
if line.startswith("-- No package '"):
|
||||
package = line.removeprefix("-- No package '").removesuffix("' found")
|
||||
missing.append(package)
|
||||
if line == "-- The following OPTIONAL packages have not been found:" or line == "-- The following RECOMMENDED packages have not been found:":
|
||||
is_in_block = True
|
||||
elif line.startswith("--") and is_in_block:
|
||||
is_in_block = False
|
||||
elif line.startswith("*") and is_in_block:
|
||||
package = line.removeprefix("* ")
|
||||
missing.append(package)
|
||||
|
||||
missing = {
|
||||
package
|
||||
for package in missing
|
||||
if not any(package.startswith(i) for i in OK_MISSING | OK_MISSING_BY_PACKAGE.get(pname, set()))
|
||||
}
|
||||
|
||||
if missing:
|
||||
print(pname + ":")
|
||||
for line in missing:
|
||||
print(" -", line)
|
||||
print()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
113
maintainers/scripts/kde/generate-sources.py
Executable file
113
maintainers/scripts/kde/generate-sources.py
Executable file
@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.beautifulsoup4 ps.click ps.httpx ps.jinja2 ps.pyyaml ])
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import pathlib
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import bs4
|
||||
import click
|
||||
import httpx
|
||||
import jinja2
|
||||
|
||||
import utils
|
||||
|
||||
|
||||
LEAF_TEMPLATE = jinja2.Template('''
|
||||
{mkKdeDerivation}:
|
||||
mkKdeDerivation {
|
||||
pname = "{{ pname }}";
|
||||
}
|
||||
'''.strip())
|
||||
|
||||
ROOT_TEMPLATE = jinja2.Template('''
|
||||
{callPackage}: {
|
||||
{%- for p in packages %}
|
||||
{{ p }} = callPackage ./{{ p }} {};
|
||||
{%- endfor %}
|
||||
}
|
||||
'''.strip());
|
||||
|
||||
def to_sri(hash):
|
||||
raw = binascii.unhexlify(hash)
|
||||
b64 = base64.b64encode(raw).decode()
|
||||
return f"sha256-{b64}"
|
||||
|
||||
|
||||
@click.command
|
||||
@click.argument(
|
||||
"set",
|
||||
type=click.Choice(["frameworks", "gear", "plasma"]),
|
||||
required=True
|
||||
)
|
||||
@click.argument(
|
||||
"version",
|
||||
type=str,
|
||||
required=True
|
||||
)
|
||||
@click.option(
|
||||
"--nixpkgs",
|
||||
type=click.Path(
|
||||
exists=True,
|
||||
file_okay=False,
|
||||
resolve_path=True,
|
||||
writable=True,
|
||||
path_type=pathlib.Path,
|
||||
),
|
||||
default=pathlib.Path(__file__).parent.parent.parent.parent
|
||||
)
|
||||
def main(set: str, version: str, nixpkgs: pathlib.Path):
|
||||
root_dir = nixpkgs / "pkgs/kde"
|
||||
set_dir = root_dir / set
|
||||
generated_dir = root_dir / "generated"
|
||||
metadata = utils.KDERepoMetadata.from_json(generated_dir)
|
||||
|
||||
set_url = {
|
||||
"frameworks": "kf",
|
||||
"gear": "releases",
|
||||
"plasma": "plasma",
|
||||
}[set]
|
||||
|
||||
sources = httpx.get(f"https://kde.org/info/sources/source-{set_url}-{version}.html")
|
||||
sources.raise_for_status()
|
||||
bs = bs4.BeautifulSoup(sources.text, features="html.parser")
|
||||
|
||||
results = {}
|
||||
for item in bs.select("tr")[1:]:
|
||||
link = item.select_one("td:nth-child(1) a")
|
||||
assert link
|
||||
|
||||
hash = item.select_one("td:nth-child(3) tt")
|
||||
assert hash
|
||||
|
||||
project_name, version = link.text.rsplit("-", maxsplit=1)
|
||||
if project_name not in metadata.projects_by_name:
|
||||
print(f"Warning: unknown tarball: {project_name}")
|
||||
|
||||
results[project_name] = {
|
||||
"version": version,
|
||||
"url": "mirror://kde" + urlparse(link.attrs["href"]).path,
|
||||
"hash": to_sri(hash.text)
|
||||
}
|
||||
|
||||
pkg_dir = set_dir / project_name
|
||||
pkg_file = pkg_dir / "default.nix"
|
||||
if not pkg_file.exists():
|
||||
print(f"Generated new package: {set}/{project_name}")
|
||||
pkg_dir.mkdir(parents=True, exist_ok=True)
|
||||
with pkg_file.open("w") as fd:
|
||||
fd.write(LEAF_TEMPLATE.render(pname=project_name) + "\n")
|
||||
|
||||
set_dir.mkdir(parents=True, exist_ok=True)
|
||||
with (set_dir / "default.nix").open("w") as fd:
|
||||
fd.write(ROOT_TEMPLATE.render(packages=results.keys()) + "\n")
|
||||
|
||||
sources_dir = generated_dir / "sources"
|
||||
sources_dir.mkdir(parents=True, exist_ok=True)
|
||||
with (sources_dir / f"{set}.json").open("w") as fd:
|
||||
json.dump(results, fd, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main() # type: ignore
|
185
maintainers/scripts/kde/utils.py
Normal file
185
maintainers/scripts/kde/utils.py
Normal file
@ -0,0 +1,185 @@
|
||||
import collections
|
||||
import dataclasses
|
||||
import functools
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
|
||||
class DataclassEncoder(json.JSONEncoder):
|
||||
def default(self, it):
|
||||
if dataclasses.is_dataclass(it):
|
||||
return dataclasses.asdict(it)
|
||||
return super().default(it)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Project:
|
||||
name: str
|
||||
description: str | None
|
||||
project_path: str
|
||||
repo_path: str | None
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.name)
|
||||
|
||||
@classmethod
|
||||
def from_yaml(cls, path: pathlib.Path):
|
||||
data = yaml.safe_load(path.open())
|
||||
return cls(
|
||||
name=data["identifier"],
|
||||
description=data["description"],
|
||||
project_path=data["projectpath"],
|
||||
repo_path=data["repopath"]
|
||||
)
|
||||
|
||||
|
||||
def get_git_commit(path: pathlib.Path):
|
||||
return subprocess.check_output(["git", "-C", path, "rev-parse", "--short", "HEAD"]).decode().strip()
|
||||
|
||||
|
||||
def validate_unique(projects: list[Project], attr: str):
|
||||
seen = set()
|
||||
for item in projects:
|
||||
attr_value = getattr(item, attr)
|
||||
if attr_value in seen:
|
||||
raise Exception(f"Duplicate {attr}: {attr_value}")
|
||||
seen.add(attr_value)
|
||||
|
||||
|
||||
THIRD_PARTY = {
|
||||
"third-party/appstream": "appstream-qt",
|
||||
"third-party/cmark": "cmark",
|
||||
"third-party/gpgme": "gpgme",
|
||||
"third-party/kdsoap": "kdsoap",
|
||||
"third-party/libaccounts-qt": "accounts-qt",
|
||||
"third-party/libgpg-error": "libgpg-error",
|
||||
"third-party/libquotient": "libquotient",
|
||||
"third-party/packagekit-qt": "packagekit-qt",
|
||||
"third-party/poppler": "poppler",
|
||||
"third-party/qcoro": "qcoro",
|
||||
"third-party/qmltermwidget": "qmltermwidget",
|
||||
"third-party/qtkeychain": "qtkeychain",
|
||||
"third-party/signond": "signond",
|
||||
"third-party/taglib": "taglib",
|
||||
"third-party/wayland-protocols": "wayland-protocols",
|
||||
"third-party/wayland": "wayland",
|
||||
"third-party/zxing-cpp": "zxing-cpp",
|
||||
}
|
||||
|
||||
IGNORE = {
|
||||
"kdesupport/phonon-directshow",
|
||||
"kdesupport/phonon-mmf",
|
||||
"kdesupport/phonon-mplayer",
|
||||
"kdesupport/phonon-quicktime",
|
||||
"kdesupport/phonon-waveout",
|
||||
"kdesupport/phonon-xine"
|
||||
}
|
||||
|
||||
WARNED = set()
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class KDERepoMetadata:
|
||||
version: str
|
||||
projects: list[Project]
|
||||
dep_graph: dict[Project, set[Project]]
|
||||
|
||||
@functools.cached_property
|
||||
def projects_by_name(self):
|
||||
return {p.name: p for p in self.projects}
|
||||
|
||||
@functools.cached_property
|
||||
def projects_by_path(self):
|
||||
return {p.project_path: p for p in self.projects}
|
||||
|
||||
def try_lookup_package(self, path):
|
||||
if path in IGNORE:
|
||||
return None
|
||||
project = self.projects_by_path.get(path)
|
||||
if project is None and path not in WARNED:
|
||||
WARNED.add(path)
|
||||
print(f"Warning: unknown project {path}")
|
||||
return project
|
||||
|
||||
@classmethod
|
||||
def from_repo_metadata_checkout(cls, repo_metadata: pathlib.Path):
|
||||
projects = [
|
||||
Project.from_yaml(metadata_file)
|
||||
for metadata_file in repo_metadata.glob("projects-invent/**/metadata.yaml")
|
||||
] + [
|
||||
Project(id, None, project_path, None)
|
||||
for project_path, id in THIRD_PARTY.items()
|
||||
]
|
||||
|
||||
validate_unique(projects, "name")
|
||||
validate_unique(projects, "project_path")
|
||||
|
||||
self = cls(
|
||||
version=get_git_commit(repo_metadata),
|
||||
projects=projects,
|
||||
dep_graph={},
|
||||
)
|
||||
|
||||
dep_specs = [
|
||||
"dependency-data-common",
|
||||
"dependency-data-kf6-qt6"
|
||||
]
|
||||
dep_graph = collections.defaultdict(set)
|
||||
|
||||
for spec in dep_specs:
|
||||
spec_path = repo_metadata / "dependencies" / spec
|
||||
for line in spec_path.open():
|
||||
line = line.strip()
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
if not line:
|
||||
continue
|
||||
|
||||
dependent, dependency = line.split(": ")
|
||||
|
||||
dependent = self.try_lookup_package(dependent)
|
||||
if dependent is None:
|
||||
continue
|
||||
|
||||
dependency = self.try_lookup_package(dependency)
|
||||
if dependency is None:
|
||||
continue
|
||||
|
||||
dep_graph[dependent].add(dependency)
|
||||
|
||||
self.dep_graph = dep_graph
|
||||
|
||||
return self
|
||||
|
||||
def write_json(self, root: pathlib.Path):
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with (root / "projects.json").open("w") as fd:
|
||||
json.dump(self.projects_by_name, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
|
||||
|
||||
with (root / "dependencies.json").open("w") as fd:
|
||||
deps = {k.name: sorted(dep.name for dep in v) for k, v in self.dep_graph.items()}
|
||||
json.dump({"version": self.version, "dependencies": deps}, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, root: pathlib.Path):
|
||||
projects = [
|
||||
Project(**v) for v in json.load((root / "projects.json").open()).values()
|
||||
]
|
||||
|
||||
deps = json.load((root / "dependencies.json").open())
|
||||
self = cls(
|
||||
version=deps["version"],
|
||||
projects=projects,
|
||||
dep_graph={},
|
||||
)
|
||||
|
||||
dep_graph = collections.defaultdict(set)
|
||||
for dependent, dependencies in deps["dependencies"].items():
|
||||
for dependency in dependencies:
|
||||
dep_graph[self.projects_by_name[dependent]].add(self.projects_by_name[dependency])
|
||||
|
||||
self.dep_graph = dep_graph
|
||||
return self
|
Loading…
Reference in New Issue
Block a user