maintainers/scripts: add kde2nix tooling

This will be used to generate data for KDE 6 packages.

maintainers/scripts/kde/collect-licenses.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p gnutar jq reuse
+set -eu
+cd "$(dirname "$(readlink -f "$0")")"/../../..
+
+TMPDIR=$(mktemp -d)
+trap 'rm -rf $TMPDIR' EXIT
+
+echo "# Prebuilding sources..."
+nix-build -A kdePackages.sources --no-link || true
+
+echo "# Evaluating sources..."
+declare -A sources
+eval "$(nix-instantiate --eval -A kdePackages.sources --json --strict | jq 'to_entries[] | "sources[" + .key + "]=" + .value' -r)"
+
+echo "# Collecting licenses..."
+for k in "${!sources[@]}"; do
+    echo "- Processing $k..."
+
+    if [ ! -f "${sources[$k]}" ]; then
+        echo "Not found!"
+        continue
+    fi
+
+    mkdir "$TMPDIR/$k"
+    tar -C "$TMPDIR/$k" -xf "${sources[$k]}"
+
+    (cd "$TMPDIR/$k"; reuse lint --json) | jq --arg name "$k" '{$name: .summary.used_licenses | sort}' -c > "$TMPDIR/$k.json"
+done
+
+jq -s 'add' -S "$TMPDIR"/*.json > pkgs/kde/generated/licenses.json

maintainers/scripts/kde/collect-logs.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p gnused jq
+set -eu
+cd "$(dirname "$(readlink -f "$0")")"/../../..
+
+mkdir -p logs
+for name in $(nix-env -qaP -f . -A kdePackages --json | jq -r 'to_entries[] | .key' | sed s/kdePackages.//); do
+    echo "Processing ${name}..."
+    path=$(nix eval ".#kdePackages.${name}.outPath" --json --option warn-dirty false | jq -r)
+    if [ -n "${path}" ]; then
+        nix-store --read-log "${path}" > "logs/${name}.log" || true
+    fi
+done

maintainers/scripts/kde/collect-metadata.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.click ps.pyyaml ])"
+import pathlib
+
+import click
+
+import utils
+
+@click.command
+@click.argument(
+    "repo-metadata",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        path_type=pathlib.Path,
+    ),
+)
+@click.option(
+    "--nixpkgs",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        writable=True,
+        path_type=pathlib.Path,
+    ),
+    default=pathlib.Path(__file__).parent.parent.parent.parent
+)
+def main(repo_metadata: pathlib.Path, nixpkgs: pathlib.Path):
+    metadata = utils.KDERepoMetadata.from_repo_metadata_checkout(repo_metadata)
+    out_dir = nixpkgs / "pkgs/kde/generated"
+    metadata.write_json(out_dir)
+
+if __name__ == "__main__":
+    main()  # type: ignore

maintainers/scripts/kde/collect-missing-deps.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3
+import pathlib
+
+OK_MISSING = {
+    # we don't use precompiled QML
+    'Qt6QuickCompiler',
+    'Qt6QmlCompilerPlusPrivate',
+    # usually used for version numbers
+    'Git',
+    # useless by itself, will warn if something else is not found
+    'PkgConfig',
+    # license verification
+    'ReuseTool',
+    # dev only
+    'ClangFormat',
+    # doesn't exist
+    'Qt6X11Extras',
+}
+
+OK_MISSING_BY_PACKAGE = {
+    "angelfish": {
+        "Qt6Feedback",  # we don't have it
+    },
+    "attica": {
+        "Python3",  # only used for license checks
+    },
+    "discover": {
+        "rpm-ostree-1",  # we don't have rpm-ostree (duh)
+        "Snapd",  # we don't have snaps and probably never will
+    },
+    "elisa": {
+        "UPNPQT",  # upstream says it's broken
+    },
+    "extra-cmake-modules": {
+        "Sphinx",  # only used for docs, bloats closure size
+        "QCollectionGenerator"
+    },
+    "kio-extras-kf5": {
+        "KDSoapWSDiscoveryClient",  # actually vendored on KF5 version
+    },
+    "kitinerary": {
+        "OsmTools",  # used for map data updates, we use prebuilt
+    },
+    "kosmindoormap": {
+        "OsmTools",  # same
+        "Protobuf",
+    },
+    "kpty": {
+        "UTEMPTER",  # we don't have it and it probably wouldn't work anyway
+    },
+    "kpublictransport": {
+        "OsmTools",  # same
+        "PolyClipping",
+        "Protobuf",
+    },
+    "krfb": {
+        "Qt6XkbCommonSupport",  # not real
+    },
+    "kuserfeedback": {
+        "Qt6Svg",  # all used for backend console stuff we don't ship
+        "QmlLint",
+        "Qt6Charts",
+        "FLEX",
+        "BISON",
+        "Php",
+        "PhpUnit",
+    },
+    "kwin": {
+        "display-info",  # newer versions identify as libdisplay-info
+    },
+    "mlt": {
+        "Qt5",  # intentionally disabled
+        "SWIG",
+    },
+    "plasma-desktop": {
+        "scim",  # upstream is dead, not packaged in Nixpkgs
+    },
+    "powerdevil": {
+        "DDCUtil",  # cursed, intentionally disabled
+    },
+    "pulseaudio-qt": {
+        "Qt6Qml",  # tests only
+        "Qt6Quick",
+    },
+    "syntax-highlighting": {
+        "XercesC",  # only used for extra validation at build time
+    }
+}
+
+def main():
+    here = pathlib.Path(__file__).parent.parent.parent.parent
+    logs = (here / "logs").glob("*.log")
+
+    for log in sorted(logs):
+        pname = log.stem
+
+        missing = []
+        is_in_block = False
+        with log.open(errors="replace") as fd:
+            for line in fd:
+                line = line.strip()
+                if line.startswith("--   No package '"):
+                    package = line.removeprefix("--   No package '").removesuffix("' found")
+                    missing.append(package)
+                if line == "-- The following OPTIONAL packages have not been found:" or line == "-- The following RECOMMENDED packages have not been found:":
+                    is_in_block = True
+                elif line.startswith("--") and is_in_block:
+                    is_in_block = False
+                elif line.startswith("*") and is_in_block:
+                    package = line.removeprefix("* ")
+                    missing.append(package)
+
+        missing = {
+            package
+            for package in missing
+            if not any(package.startswith(i) for i in OK_MISSING | OK_MISSING_BY_PACKAGE.get(pname, set()))
+        }
+
+        if missing:
+            print(pname + ":")
+            for line in missing:
+                print("  -", line)
+            print()
+
+if __name__ == '__main__':
+    main()

maintainers/scripts/kde/generate-sources.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.beautifulsoup4 ps.click ps.httpx ps.jinja2 ps.pyyaml ])
+import base64
+import binascii
+import json
+import pathlib
+from urllib.parse import urlparse
+
+import bs4
+import click
+import httpx
+import jinja2
+
+import utils
+
+
+LEAF_TEMPLATE = jinja2.Template('''
+{mkKdeDerivation}:
+mkKdeDerivation {
+  pname = "{{ pname }}";
+}
+'''.strip())
+
+ROOT_TEMPLATE = jinja2.Template('''
+{callPackage}: {
+  {%- for p in packages %}
+  {{ p }} = callPackage ./{{ p }} {};
+  {%- endfor %}
+}
+'''.strip());
+
+def to_sri(hash):
+    raw = binascii.unhexlify(hash)
+    b64 = base64.b64encode(raw).decode()
+    return f"sha256-{b64}"
+
+
+@click.command
+@click.argument(
+    "set",
+    type=click.Choice(["frameworks", "gear", "plasma"]),
+    required=True
+)
+@click.argument(
+    "version",
+    type=str,
+    required=True
+)
+@click.option(
+    "--nixpkgs",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        writable=True,
+        path_type=pathlib.Path,
+    ),
+    default=pathlib.Path(__file__).parent.parent.parent.parent
+)
+def main(set: str, version: str, nixpkgs: pathlib.Path):
+    root_dir = nixpkgs / "pkgs/kde"
+    set_dir = root_dir / set
+    generated_dir = root_dir / "generated"
+    metadata = utils.KDERepoMetadata.from_json(generated_dir)
+
+    set_url = {
+        "frameworks": "kf",
+        "gear": "releases",
+        "plasma": "plasma",
+    }[set]
+
+    sources = httpx.get(f"https://kde.org/info/sources/source-{set_url}-{version}.html")
+    sources.raise_for_status()
+    bs = bs4.BeautifulSoup(sources.text, features="html.parser")
+
+    results = {}
+    for item in bs.select("tr")[1:]:
+        link = item.select_one("td:nth-child(1) a")
+        assert link
+
+        hash = item.select_one("td:nth-child(3) tt")
+        assert hash
+
+        project_name, version = link.text.rsplit("-", maxsplit=1)
+        if project_name not in metadata.projects_by_name:
+            print(f"Warning: unknown tarball: {project_name}")
+
+        results[project_name] = {
+            "version": version,
+            "url": "mirror://kde" + urlparse(link.attrs["href"]).path,
+            "hash": to_sri(hash.text)
+        }
+
+        pkg_dir = set_dir / project_name
+        pkg_file = pkg_dir / "default.nix"
+        if not pkg_file.exists():
+            print(f"Generated new package: {set}/{project_name}")
+            pkg_dir.mkdir(parents=True, exist_ok=True)
+            with pkg_file.open("w") as fd:
+                fd.write(LEAF_TEMPLATE.render(pname=project_name) + "\n")
+
+    set_dir.mkdir(parents=True, exist_ok=True)
+    with (set_dir / "default.nix").open("w") as fd:
+        fd.write(ROOT_TEMPLATE.render(packages=results.keys()) + "\n")
+
+    sources_dir = generated_dir / "sources"
+    sources_dir.mkdir(parents=True, exist_ok=True)
+    with (sources_dir / f"{set}.json").open("w") as fd:
+        json.dump(results, fd, indent=2)
+
+
+if __name__ == "__main__":
+    main()  # type: ignore

maintainers/scripts/kde/utils.py

@@ -0,0 +1,185 @@
+import collections
+import dataclasses
+import functools
+import json
+import pathlib
+import subprocess
+
+import yaml
+
+class DataclassEncoder(json.JSONEncoder):
+    def default(self, it):
+        if dataclasses.is_dataclass(it):
+            return dataclasses.asdict(it)
+        return super().default(it)
+
+
+@dataclasses.dataclass
+class Project:
+    name: str
+    description: str | None
+    project_path: str
+    repo_path: str | None
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    @classmethod
+    def from_yaml(cls, path: pathlib.Path):
+        data = yaml.safe_load(path.open())
+        return cls(
+            name=data["identifier"],
+            description=data["description"],
+            project_path=data["projectpath"],
+            repo_path=data["repopath"]
+        )
+
+
+def get_git_commit(path: pathlib.Path):
+    return subprocess.check_output(["git", "-C", path, "rev-parse", "--short", "HEAD"]).decode().strip()
+
+
+def validate_unique(projects: list[Project], attr: str):
+    seen = set()
+    for item in projects:
+        attr_value = getattr(item, attr)
+        if attr_value in seen:
+            raise Exception(f"Duplicate {attr}: {attr_value}")
+        seen.add(attr_value)
+
+
+THIRD_PARTY = {
+    "third-party/appstream": "appstream-qt",
+    "third-party/cmark": "cmark",
+    "third-party/gpgme": "gpgme",
+    "third-party/kdsoap": "kdsoap",
+    "third-party/libaccounts-qt": "accounts-qt",
+    "third-party/libgpg-error": "libgpg-error",
+    "third-party/libquotient": "libquotient",
+    "third-party/packagekit-qt": "packagekit-qt",
+    "third-party/poppler": "poppler",
+    "third-party/qcoro": "qcoro",
+    "third-party/qmltermwidget": "qmltermwidget",
+    "third-party/qtkeychain": "qtkeychain",
+    "third-party/signond": "signond",
+    "third-party/taglib": "taglib",
+    "third-party/wayland-protocols": "wayland-protocols",
+    "third-party/wayland": "wayland",
+    "third-party/zxing-cpp": "zxing-cpp",
+}
+
+IGNORE = {
+    "kdesupport/phonon-directshow",
+    "kdesupport/phonon-mmf",
+    "kdesupport/phonon-mplayer",
+    "kdesupport/phonon-quicktime",
+    "kdesupport/phonon-waveout",
+    "kdesupport/phonon-xine"
+}
+
+WARNED = set()
+
+
+@dataclasses.dataclass
+class KDERepoMetadata:
+    version: str
+    projects: list[Project]
+    dep_graph: dict[Project, set[Project]]
+
+    @functools.cached_property
+    def projects_by_name(self):
+        return {p.name: p for p in self.projects}
+
+    @functools.cached_property
+    def projects_by_path(self):
+        return {p.project_path: p for p in self.projects}
+
+    def try_lookup_package(self, path):
+        if path in IGNORE:
+            return None
+        project = self.projects_by_path.get(path)
+        if project is None and path not in WARNED:
+            WARNED.add(path)
+            print(f"Warning: unknown project {path}")
+        return project
+
+    @classmethod
+    def from_repo_metadata_checkout(cls, repo_metadata: pathlib.Path):
+        projects = [
+            Project.from_yaml(metadata_file)
+            for metadata_file in repo_metadata.glob("projects-invent/**/metadata.yaml")
+        ] + [
+            Project(id, None, project_path, None)
+            for project_path, id in THIRD_PARTY.items()
+        ]
+
+        validate_unique(projects, "name")
+        validate_unique(projects, "project_path")
+
+        self = cls(
+            version=get_git_commit(repo_metadata),
+            projects=projects,
+            dep_graph={},
+        )
+
+        dep_specs = [
+            "dependency-data-common",
+            "dependency-data-kf6-qt6"
+        ]
+        dep_graph = collections.defaultdict(set)
+
+        for spec in dep_specs:
+            spec_path = repo_metadata / "dependencies" / spec
+            for line in spec_path.open():
+                line = line.strip()
+                if line.startswith("#"):
+                    continue
+                if not line:
+                    continue
+
+                dependent, dependency = line.split(": ")
+
+                dependent = self.try_lookup_package(dependent)
+                if dependent is None:
+                    continue
+
+                dependency = self.try_lookup_package(dependency)
+                if dependency is None:
+                    continue
+
+                dep_graph[dependent].add(dependency)
+
+        self.dep_graph = dep_graph
+
+        return self
+
+    def write_json(self, root: pathlib.Path):
+        root.mkdir(parents=True, exist_ok=True)
+
+        with (root / "projects.json").open("w") as fd:
+            json.dump(self.projects_by_name, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
+
+        with (root / "dependencies.json").open("w") as fd:
+            deps = {k.name: sorted(dep.name for dep in v) for k, v in self.dep_graph.items()}
+            json.dump({"version": self.version, "dependencies": deps}, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
+
+    @classmethod
+    def from_json(cls, root: pathlib.Path):
+        projects = [
+            Project(**v) for v in json.load((root / "projects.json").open()).values()
+        ]
+
+        deps = json.load((root / "dependencies.json").open())
+        self = cls(
+            version=deps["version"],
+            projects=projects,
+            dep_graph={},
+        )
+
+        dep_graph = collections.defaultdict(set)
+        for dependent, dependencies in deps["dependencies"].items():
+            for dependency in dependencies:
+                dep_graph[self.projects_by_name[dependent]].add(self.projects_by_name[dependency])
+
+        self.dep_graph = dep_graph
+        return self