From c4b05347015db3eb8ad2f6f67e5b6f5308c3c9f0 Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Sat, 22 Apr 2023 20:22:39 +0100 Subject: [PATCH 1/2] texlive: use version info from tlpdb instead of hardcoding --- pkgs/test/texlive/default.nix | 4 +- .../typesetting/tex/texlive/UPGRADING.md | 58 ++++++--------- pkgs/tools/typesetting/tex/texlive/bin.nix | 2 +- .../tools/typesetting/tex/texlive/combine.nix | 4 +- .../tools/typesetting/tex/texlive/default.nix | 70 ++++++++++++------- pkgs/tools/typesetting/tex/texlive/tl2nix.sed | 11 +++ pkgs/tools/typesetting/tex/texlive/tlpdb.nix | 5 ++ 7 files changed, 88 insertions(+), 66 deletions(-) diff --git a/pkgs/test/texlive/default.nix b/pkgs/test/texlive/default.nix index cbfa0c45e8fc..6aac195d512b 100644 --- a/pkgs/test/texlive/default.nix +++ b/pkgs/test/texlive/default.nix @@ -2,9 +2,9 @@ { - tlpdb-nix = runCommand "texlive-test-tlpdb-nix" { + tlpdbNix = runCommand "texlive-test-tlpdb-nix" { nixpkgsTlpdbNix = ../../tools/typesetting/tex/texlive/tlpdb.nix; - tlpdbNix = texlive.tlpdb-nix; + tlpdbNix = texlive.tlpdb.nix; } '' mkdir -p "$out" diff --git a/pkgs/tools/typesetting/tex/texlive/UPGRADING.md b/pkgs/tools/typesetting/tex/texlive/UPGRADING.md index acad0646d2b9..a5b7000e52fb 100644 --- a/pkgs/tools/typesetting/tex/texlive/UPGRADING.md +++ b/pkgs/tools/typesetting/tex/texlive/UPGRADING.md @@ -1,62 +1,46 @@ # Notes on maintaining/upgrading -## Upgrading texlive.bin +## Upgrading `texlive.bin` -texlive contains a few binaries, defined in bin.nix and released once a year. +`texlive` contains a few binaries, defined in `bin.nix` and released once a year. In order to reduce closure size for users who just need a few of them, we split it into -packages such as core, core-big, xvdi, etc. This requires making assumptions +packages such as `core`, `core-big`, `xdvi`, etc. This requires making assumptions about dependencies between the projects that may change between releases; if you upgrade you may have to do some work here. - ## Updating the package set -texlive contains several thousand packages from CTAN, defined in pkgs.nix. +`texlive` contains several thousand packages from CTAN, defined in `tlpdb.nix`. The CTAN mirrors are not version-controlled and continuously moving, with more than 100 updates per month. -To create a consistent and reproducible package set in nixpkgs, we snapshot CTAN -and generate nix expressions for all packages in texlive at that point. - -We mirror CTAN sources of this snapshot on community-operated servers and on IPFS. - -To upgrade the package snapshot, follow this process: - - -### Snapshot sources and texlive package database - -Mirror the current CTAN archive to our mirror(s) and IPFS (URLs in `default.nix`). -See https://tug.org/texlive/acquire-mirror.html for instructions. +To create a consistent and reproducible package set in nixpkgs, we generate nix +expressions for all packages in TeX Live at a certain day. +To upgrade the package snapshot, follow this process. ### Upgrade package information from texlive package database -First, edit `default.nix` as follows. - -If upgrading to a daily snapshot: -- change `snapshot.year`, `snapshot.month`, `snapshot.day`; -- ensure `urlPrefixes` uses the https://texlive.info/tlnet-archive mirror; -- ensure `texlive.extraVersion` uses the `snapshot` info. - -If upgrading to a final release: -- upgrade `texlive.bin` first; -- ensure `urlPrefixes` uses the historic mirrors; -- ensure `texlive.extraVersion` is `"-final"`. - -Then upgrade `tlpdb.hash` to match the new hash of `texlive.tlpdb.xz` and run +Update `version` in `default.nix` with the day of the new snapshot, the new TeX +Live year, and the final status of the snapshot. Then update +`texlive.tlpdbxz.hash` to match the new hash of `texlive.tlpdb.xz` and run ```bash -nix-build ../../../../.. -A texlive.tlpdb-nix --no-out-link +nix-build ../../../../.. -A texlive.tlpdb.nix --no-out-link ``` -This will download the daily snapshot of the CTAN package database -`texlive.tlpdb.xz` and extract the relevant package info (including version -numbers and sha512 hashes) for the selected upstream distribution. Then replace -`tlpdb.nix` with the generated file. +This will download either the daily or the final snapshot of the TeX Live +package database `texlive.tlpdb.xz` and extract the relevant package info +(including version numbers and sha512 hashes) for the selected upstream +distribution. -The test `pkgs.tests.texlive.tlpdb-nix` verifies that the file `tlpdb.nix` +Finally, replace `tlpdb.nix` with the generated file. Note that if the +`version` info does not match the metadata of `tlpdb.nix` (as found in the +`00texlive.config` package), TeX Live packages will not evaluate. + +The test `pkgs.tests.texlive.tlpdbNix` verifies that the file `tlpdb.nix` in Nixpkgs matches the one that generated from `texlive.tlpdb.xz`. ### Build packages locally and generate fix hashes @@ -81,4 +65,4 @@ mv fixedHashes-new.nix fixedHashes.nix ### Commit changes -Commit the updated `pkgs.nix` and `fixedHashes.nix` to the repository. +Commit the updated `tlpdb.nix` and `fixedHashes.nix` to the repository. diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix index 77c195962b3a..9ea739100e01 100644 --- a/pkgs/tools/typesetting/tex/texlive/bin.nix +++ b/pkgs/tools/typesetting/tex/texlive/bin.nix @@ -14,7 +14,7 @@ let withSystemLibs = map (libname: "--with-system-${libname}"); - year = "2022"; + year = toString ((import ./tlpdb.nix)."00texlive.config").year; version = year; # keep names simple for now common = { diff --git a/pkgs/tools/typesetting/tex/texlive/combine.nix b/pkgs/tools/typesetting/tex/texlive/combine.nix index 1b67c62a7eeb..7da3811e2488 100644 --- a/pkgs/tools/typesetting/tex/texlive/combine.nix +++ b/pkgs/tools/typesetting/tex/texlive/combine.nix @@ -10,8 +10,8 @@ let pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ] // { # include a fake "core" package core.pkgs = [ - (bin.core.out // { pname = "core"; version = "0"; tlType = "bin"; }) - (bin.core.doc // { pname = "core"; version = "0"; tlType = "doc"; }) + (bin.core.out // { pname = "core"; tlType = "bin"; }) + (bin.core.doc // { pname = "core"; tlType = "doc"; }) ]; }; pkgList = rec { diff --git a/pkgs/tools/typesetting/tex/texlive/default.nix b/pkgs/tools/typesetting/tex/texlive/default.nix index cbdedc9310c3..12e2e25e8806 100644 --- a/pkgs/tools/typesetting/tex/texlive/default.nix +++ b/pkgs/tools/typesetting/tex/texlive/default.nix @@ -28,9 +28,13 @@ let ghostscript = ghostscript_headless; }; + tlpdb = import ./tlpdb.nix; + + tlpdbVersion = tlpdb."00texlive.config"; + # the set of TeX Live packages, collections, and schemes; using upstream naming tl = let - orig = import ./tlpdb.nix; + orig = removeAttrs tlpdb [ "00texlive.config" ]; overridden = orig // { # overrides of texlive.tlpdb @@ -59,7 +63,7 @@ let # build Data.tlpdb.lua (part of the 'tlType == "run"' package) postUnpack = '' if [[ -f "$out"/scripts/texdoc/texdoc.tlu ]]; then - unxz --stdout "${tlpdb}" > texlive.tlpdb + unxz --stdout "${tlpdbxz}" > texlive.tlpdb # create dummy doc file to ensure that texdoc does not return an error mkdir -p support/texdoc @@ -109,11 +113,15 @@ let ( bin.${pname} // { tlType = "bin"; } ); }; - # for daily snapshots - snapshot = { - year = "2022"; - month = "12"; - day = "27"; + version = { + # day of the snapshot being taken + year = "2023"; + month = "03"; + day = "19"; + # TeX Live version + texliveYear = 2022; + # final (historic) release or snapshot + final = true; }; # The tarballs on CTAN mirrors for the current release are constantly @@ -121,27 +129,30 @@ let # need to be used instead. Ideally, for the release branches of NixOS we # should be switching to the tlnet-final versions # (https://tug.org/historic/). - urlPrefixes = [ - # tlnet-final snapshot - "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/archive" - "ftp://tug.org/texlive/historic/${bin.texliveYear}/tlnet-final/archive" - - # Daily snapshots hosted by one of the texlive release managers - #"https://texlive.info/tlnet-archive/${snapshot.year}/${snapshot.month}/${snapshot.day}/tlnet/archive" + urlPrefixes = with version; lib.optionals final [ + # tlnet-final snapshot; used when texlive.tlpdb is frozen + # the TeX Live yearly freeze typically happens in mid-March + "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${toString texliveYear}/tlnet-final" + "ftp://tug.org/texlive/historic/${toString texliveYear}/tlnet-final" + ] ++ [ + # daily snapshots hosted by one of the texlive release managers; + # used for non-final snapshots and as fallback for final snapshots that have not reached yet the historic mirrors + # please note that this server is not meant for large scale deployment and should be avoided on release branches + # https://tug.org/pipermail/tex-live/2019-November/044456.html + "https://texlive.info/tlnet-archive/${year}/${month}/${day}/tlnet" ]; - tlpdb = fetchurl { - # use the same mirror(s) as urlPrefixes above - urls = map (up: "${up}/../tlpkg/texlive.tlpdb.xz") urlPrefixes; + tlpdbxz = fetchurl { + urls = map (up: "${up}/tlpkg/texlive.tlpdb.xz") urlPrefixes; hash = "sha256-vm7DmkH/h183pN+qt1p1wZ6peT2TcMk/ae0nCXsCoMw="; }; - tlpdb-nix = runCommand "tlpdb.nix" { - inherit tlpdb; + tlpdbNix = runCommand "tlpdb.nix" { + inherit tlpdbxz; tl2nix = ./tl2nix.sed; } '' - xzcat "$tlpdb" | sed -rn -f "$tl2nix" | uniq > "$out" + xzcat "$tlpdbxz" | sed -rn -f "$tl2nix" | uniq > "$out" ''; # create a derivation that contains an unpacked upstream TL package @@ -153,7 +164,7 @@ let fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes urls = args.urls or (if args ? url then [ args.url ] else - map (up: "${up}/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes)); + map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes)); in runCommand "texlive-${tlName}" ( { @@ -198,9 +209,21 @@ let operator = { pkg, ... }: pkgListToSets (pkg.tlDeps or []); }); + assertions = + lib.assertMsg (tlpdbVersion.year == version.texliveYear) "TeX Live year in texlive does not match tlpdb.nix, refusing to evaluate" && + lib.assertMsg (tlpdbVersion.frozen == version.final) "TeX Live final status in texlive does not match tlpdb.nix, refusing to evaluate"; + in tl // { - inherit bin combine tlpdb-nix; + + tlpdb = { + # nested in an attribute set to prevent them from appearing in search + nix = tlpdbNix; + xz = tlpdbxz; + }; + + bin = assert assertions; bin; + combine = assert assertions; combine; # Pre-defined combined packages for TeX Live schemes, # to make nix-env usage more comfortable and build selected on Hydra. @@ -215,8 +238,7 @@ in (combine { ${pname} = attrs; extraName = "combined" + lib.removePrefix "scheme" pname; - #extraVersion = "-final"; - extraVersion = ".${snapshot.year}${snapshot.month}${snapshot.day}"; + extraVersion = with version; if final then "-final" else ".${year}${month}${day}"; }) ) { inherit (tl) diff --git a/pkgs/tools/typesetting/tex/texlive/tl2nix.sed b/pkgs/tools/typesetting/tex/texlive/tl2nix.sed index 506843c0952a..31194565cfef 100644 --- a/pkgs/tools/typesetting/tex/texlive/tl2nix.sed +++ b/pkgs/tools/typesetting/tex/texlive/tl2nix.sed @@ -2,6 +2,17 @@ 1i{ # no indentation $a} +# extract repository metadata +/^name 00texlive\.config$/,/^$/{ + s/^name (.*)$/"\1" = {/p + /^$/,1i}; + + s!^depend frozen/0$! frozen = false;!p + s!^depend frozen/1$! frozen = true;!p + s!^depend release/(.*)$! year = \1;!p + s!^depend revision/(.*)$! revision = \1;!p +} + # form an attrmap per package # ignore packages whose name contains "." (such as binaries) /^name ([^.]+)$/,/^$/{ diff --git a/pkgs/tools/typesetting/tex/texlive/tlpdb.nix b/pkgs/tools/typesetting/tex/texlive/tlpdb.nix index a3d0f48c9224..f5e78a3704a6 100644 --- a/pkgs/tools/typesetting/tex/texlive/tlpdb.nix +++ b/pkgs/tools/typesetting/tex/texlive/tlpdb.nix @@ -1,4 +1,9 @@ { # no indentation +"00texlive.config" = { + frozen = true; + year = 2022; + revision = 66590; +}; "12many" = { revision = 15878; stripPrefix = 0; From 671f7556b2363b18c4bea0777a6a344e1eb4ce3a Mon Sep 17 00:00:00 2001 From: Vincenzo Mantova <1962985+xworld21@users.noreply.github.com> Date: Sat, 22 Apr 2023 20:22:39 +0100 Subject: [PATCH 2/2] texlive.texdoc: add tlpdb revision to version --- pkgs/tools/typesetting/tex/texlive/default.nix | 2 ++ pkgs/tools/typesetting/tex/texlive/fixedHashes.nix | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/tools/typesetting/tex/texlive/default.nix b/pkgs/tools/typesetting/tex/texlive/default.nix index 12e2e25e8806..9e436bc99101 100644 --- a/pkgs/tools/typesetting/tex/texlive/default.nix +++ b/pkgs/tools/typesetting/tex/texlive/default.nix @@ -60,6 +60,8 @@ let }; texdoc = orig.texdoc // { + version = orig.texdoc.version + "-tlpdb-" + (toString tlpdbVersion.revision); + # build Data.tlpdb.lua (part of the 'tlType == "run"' package) postUnpack = '' if [[ -f "$out"/scripts/texdoc/texdoc.tlu ]]; then diff --git a/pkgs/tools/typesetting/tex/texlive/fixedHashes.nix b/pkgs/tools/typesetting/tex/texlive/fixedHashes.nix index d83949b77cda..266800a03bb6 100644 --- a/pkgs/tools/typesetting/tex/texlive/fixedHashes.nix +++ b/pkgs/tools/typesetting/tex/texlive/fixedHashes.nix @@ -797,8 +797,8 @@ "texdiff.doc-0.4"="0f7q7cszy6ca6fmwzi7fn6p72szgzyj9fldvzz1rk3ywkhsnpakr"; "texdirflatten-1.3"="0q2m138gpk9cvfp19jvsw746mkxa613c2204qfxw1bxw8b454d7m"; "texdirflatten.doc-1.3"="19iiznql82a34d2yhzqcp339zszbhfz9fh0h4mb11696mj8rx3v7"; -"texdoc-4.0.1"="1ph4mzrlri8daxg7j4n15s3b9dj58ni4fdqhpvqxwy6736fiw130"; -"texdoc.doc-4.0.1"="1dpz5ni9xdbwfy4m12j3v407ywbddmy3j2dhzwrdsyj7wq3impmg"; +"texdoc-4.0.1-tlpdb-66590"="1ph4mzrlri8daxg7j4n15s3b9dj58ni4fdqhpvqxwy6736fiw130"; +"texdoc.doc-4.0.1-tlpdb-66590"="1dpz5ni9xdbwfy4m12j3v407ywbddmy3j2dhzwrdsyj7wq3impmg"; "texdoctk-0.6.0"="13j0c5m12if8jx8sp7hlirz33y6l2jk8lvnms7dalz52mxz8kq4h"; "texdoctk.doc-0.6.0"="1k41z2lbfjsar5rl47xnk1adbcnnb4w97bzbgf2rnjhimd1gn9hq"; "texfot-1.48"="19aswiyr722n9misvg9ajdsgpn1ghmic2xfmr4w0w00sai5sh5rx";