Merge pull request #206763 from xworld21/texlive-centralize-version
texlive: use version info from tlpdb instead of hardcoding
This commit is contained in:
Dmitry Kalinkin
GitHub
commit
5c27dc43d2
@ -2,9 +2,9 @@
|
||||
|
||||
{
|
||||
|
||||
tlpdb-nix = runCommand "texlive-test-tlpdb-nix" {
|
||||
tlpdbNix = runCommand "texlive-test-tlpdb-nix" {
|
||||
nixpkgsTlpdbNix = ../../tools/typesetting/tex/texlive/tlpdb.nix;
|
||||
tlpdbNix = texlive.tlpdb-nix;
|
||||
tlpdbNix = texlive.tlpdb.nix;
|
||||
}
|
||||
''
|
||||
mkdir -p "$out"
|
||||
|
||||
@ -1,62 +1,46 @@
|
||||
# Notes on maintaining/upgrading
|
||||
|
||||
## Upgrading texlive.bin
|
||||
## Upgrading `texlive.bin`
|
||||
|
||||
texlive contains a few binaries, defined in bin.nix and released once a year.
|
||||
`texlive` contains a few binaries, defined in `bin.nix` and released once a year.
|
||||
|
||||
In order to reduce closure size for users who just need a few of them, we split it into
|
||||
packages such as core, core-big, xvdi, etc. This requires making assumptions
|
||||
packages such as `core`, `core-big`, `xdvi`, etc. This requires making assumptions
|
||||
about dependencies between the projects that may change between releases; if
|
||||
you upgrade you may have to do some work here.
|
||||
|
||||
|
||||
## Updating the package set
|
||||
|
||||
texlive contains several thousand packages from CTAN, defined in pkgs.nix.
|
||||
`texlive` contains several thousand packages from CTAN, defined in `tlpdb.nix`.
|
||||
|
||||
The CTAN mirrors are not version-controlled and continuously moving,
|
||||
with more than 100 updates per month.
|
||||
|
||||
To create a consistent and reproducible package set in nixpkgs, we snapshot CTAN
|
||||
and generate nix expressions for all packages in texlive at that point.
|
||||
|
||||
We mirror CTAN sources of this snapshot on community-operated servers and on IPFS.
|
||||
|
||||
To upgrade the package snapshot, follow this process:
|
||||
|
||||
|
||||
### Snapshot sources and texlive package database
|
||||
|
||||
Mirror the current CTAN archive to our mirror(s) and IPFS (URLs in `default.nix`).
|
||||
See https://tug.org/texlive/acquire-mirror.html for instructions.
|
||||
To create a consistent and reproducible package set in nixpkgs, we generate nix
|
||||
expressions for all packages in TeX Live at a certain day.
|
||||
|
||||
To upgrade the package snapshot, follow this process.
|
||||
|
||||
### Upgrade package information from texlive package database
|
||||
|
||||
First, edit `default.nix` as follows.
|
||||
|
||||
If upgrading to a daily snapshot:
|
||||
- change `snapshot.year`, `snapshot.month`, `snapshot.day`;
|
||||
- ensure `urlPrefixes` uses the https://texlive.info/tlnet-archive mirror;
|
||||
- ensure `texlive.extraVersion` uses the `snapshot` info.
|
||||
|
||||
If upgrading to a final release:
|
||||
- upgrade `texlive.bin` first;
|
||||
- ensure `urlPrefixes` uses the historic mirrors;
|
||||
- ensure `texlive.extraVersion` is `"-final"`.
|
||||
|
||||
Then upgrade `tlpdb.hash` to match the new hash of `texlive.tlpdb.xz` and run
|
||||
Update `version` in `default.nix` with the day of the new snapshot, the new TeX
|
||||
Live year, and the final status of the snapshot. Then update
|
||||
`texlive.tlpdbxz.hash` to match the new hash of `texlive.tlpdb.xz` and run
|
||||
|
||||
```bash
|
||||
nix-build ../../../../.. -A texlive.tlpdb-nix --no-out-link
|
||||
nix-build ../../../../.. -A texlive.tlpdb.nix --no-out-link
|
||||
```
|
||||
|
||||
This will download the daily snapshot of the CTAN package database
|
||||
`texlive.tlpdb.xz` and extract the relevant package info (including version
|
||||
numbers and sha512 hashes) for the selected upstream distribution. Then replace
|
||||
`tlpdb.nix` with the generated file.
|
||||
This will download either the daily or the final snapshot of the TeX Live
|
||||
package database `texlive.tlpdb.xz` and extract the relevant package info
|
||||
(including version numbers and sha512 hashes) for the selected upstream
|
||||
distribution.
|
||||
|
||||
The test `pkgs.tests.texlive.tlpdb-nix` verifies that the file `tlpdb.nix`
|
||||
Finally, replace `tlpdb.nix` with the generated file. Note that if the
|
||||
`version` info does not match the metadata of `tlpdb.nix` (as found in the
|
||||
`00texlive.config` package), TeX Live packages will not evaluate.
|
||||
|
||||
The test `pkgs.tests.texlive.tlpdbNix` verifies that the file `tlpdb.nix`
|
||||
in Nixpkgs matches the one that generated from `texlive.tlpdb.xz`.
|
||||
|
||||
### Build packages locally and generate fix hashes
|
||||
@ -81,4 +65,4 @@ mv fixedHashes-new.nix fixedHashes.nix
|
||||
|
||||
### Commit changes
|
||||
|
||||
Commit the updated `pkgs.nix` and `fixedHashes.nix` to the repository.
|
||||
Commit the updated `tlpdb.nix` and `fixedHashes.nix` to the repository.
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
let
|
||||
withSystemLibs = map (libname: "--with-system-${libname}");
|
||||
|
||||
year = "2022";
|
||||
year = toString ((import ./tlpdb.nix)."00texlive.config").year;
|
||||
version = year; # keep names simple for now
|
||||
|
||||
common = {
|
||||
|
||||
@ -10,8 +10,8 @@ let
|
||||
pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ] // {
|
||||
# include a fake "core" package
|
||||
core.pkgs = [
|
||||
(bin.core.out // { pname = "core"; version = "0"; tlType = "bin"; })
|
||||
(bin.core.doc // { pname = "core"; version = "0"; tlType = "doc"; })
|
||||
(bin.core.out // { pname = "core"; tlType = "bin"; })
|
||||
(bin.core.doc // { pname = "core"; tlType = "doc"; })
|
||||
];
|
||||
};
|
||||
pkgList = rec {
|
||||
|
||||
@ -28,9 +28,13 @@ let
|
||||
ghostscript = ghostscript_headless;
|
||||
};
|
||||
|
||||
tlpdb = import ./tlpdb.nix;
|
||||
|
||||
tlpdbVersion = tlpdb."00texlive.config";
|
||||
|
||||
# the set of TeX Live packages, collections, and schemes; using upstream naming
|
||||
tl = let
|
||||
orig = import ./tlpdb.nix;
|
||||
orig = removeAttrs tlpdb [ "00texlive.config" ];
|
||||
|
||||
overridden = orig // {
|
||||
# overrides of texlive.tlpdb
|
||||
@ -56,10 +60,12 @@ let
|
||||
};
|
||||
|
||||
texdoc = orig.texdoc // {
|
||||
version = orig.texdoc.version + "-tlpdb-" + (toString tlpdbVersion.revision);
|
||||
|
||||
# build Data.tlpdb.lua (part of the 'tlType == "run"' package)
|
||||
postUnpack = ''
|
||||
if [[ -f "$out"/scripts/texdoc/texdoc.tlu ]]; then
|
||||
unxz --stdout "${tlpdb}" > texlive.tlpdb
|
||||
unxz --stdout "${tlpdbxz}" > texlive.tlpdb
|
||||
|
||||
# create dummy doc file to ensure that texdoc does not return an error
|
||||
mkdir -p support/texdoc
|
||||
@ -109,11 +115,15 @@ let
|
||||
( bin.${pname} // { tlType = "bin"; } );
|
||||
};
|
||||
|
||||
# for daily snapshots
|
||||
snapshot = {
|
||||
year = "2022";
|
||||
month = "12";
|
||||
day = "27";
|
||||
version = {
|
||||
# day of the snapshot being taken
|
||||
year = "2023";
|
||||
month = "03";
|
||||
day = "19";
|
||||
# TeX Live version
|
||||
texliveYear = 2022;
|
||||
# final (historic) release or snapshot
|
||||
final = true;
|
||||
};
|
||||
|
||||
# The tarballs on CTAN mirrors for the current release are constantly
|
||||
@ -121,27 +131,30 @@ let
|
||||
# need to be used instead. Ideally, for the release branches of NixOS we
|
||||
# should be switching to the tlnet-final versions
|
||||
# (https://tug.org/historic/).
|
||||
urlPrefixes = [
|
||||
# tlnet-final snapshot
|
||||
"http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/archive"
|
||||
"ftp://tug.org/texlive/historic/${bin.texliveYear}/tlnet-final/archive"
|
||||
|
||||
# Daily snapshots hosted by one of the texlive release managers
|
||||
#"https://texlive.info/tlnet-archive/${snapshot.year}/${snapshot.month}/${snapshot.day}/tlnet/archive"
|
||||
urlPrefixes = with version; lib.optionals final [
|
||||
# tlnet-final snapshot; used when texlive.tlpdb is frozen
|
||||
# the TeX Live yearly freeze typically happens in mid-March
|
||||
"http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${toString texliveYear}/tlnet-final"
|
||||
"ftp://tug.org/texlive/historic/${toString texliveYear}/tlnet-final"
|
||||
] ++ [
|
||||
# daily snapshots hosted by one of the texlive release managers;
|
||||
# used for non-final snapshots and as fallback for final snapshots that have not reached yet the historic mirrors
|
||||
# please note that this server is not meant for large scale deployment and should be avoided on release branches
|
||||
# https://tug.org/pipermail/tex-live/2019-November/044456.html
|
||||
"https://texlive.info/tlnet-archive/${year}/${month}/${day}/tlnet"
|
||||
];
|
||||
|
||||
tlpdb = fetchurl {
|
||||
# use the same mirror(s) as urlPrefixes above
|
||||
urls = map (up: "${up}/../tlpkg/texlive.tlpdb.xz") urlPrefixes;
|
||||
tlpdbxz = fetchurl {
|
||||
urls = map (up: "${up}/tlpkg/texlive.tlpdb.xz") urlPrefixes;
|
||||
hash = "sha256-vm7DmkH/h183pN+qt1p1wZ6peT2TcMk/ae0nCXsCoMw=";
|
||||
};
|
||||
|
||||
tlpdb-nix = runCommand "tlpdb.nix" {
|
||||
inherit tlpdb;
|
||||
tlpdbNix = runCommand "tlpdb.nix" {
|
||||
inherit tlpdbxz;
|
||||
tl2nix = ./tl2nix.sed;
|
||||
}
|
||||
''
|
||||
xzcat "$tlpdb" | sed -rn -f "$tl2nix" | uniq > "$out"
|
||||
xzcat "$tlpdbxz" | sed -rn -f "$tl2nix" | uniq > "$out"
|
||||
'';
|
||||
|
||||
# create a derivation that contains an unpacked upstream TL package
|
||||
@ -153,7 +166,7 @@ let
|
||||
fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
|
||||
|
||||
urls = args.urls or (if args ? url then [ args.url ] else
|
||||
map (up: "${up}/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));
|
||||
map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));
|
||||
|
||||
in runCommand "texlive-${tlName}"
|
||||
( {
|
||||
@ -198,9 +211,21 @@ let
|
||||
operator = { pkg, ... }: pkgListToSets (pkg.tlDeps or []);
|
||||
});
|
||||
|
||||
assertions =
|
||||
lib.assertMsg (tlpdbVersion.year == version.texliveYear) "TeX Live year in texlive does not match tlpdb.nix, refusing to evaluate" &&
|
||||
lib.assertMsg (tlpdbVersion.frozen == version.final) "TeX Live final status in texlive does not match tlpdb.nix, refusing to evaluate";
|
||||
|
||||
in
|
||||
tl // {
|
||||
inherit bin combine tlpdb-nix;
|
||||
|
||||
tlpdb = {
|
||||
# nested in an attribute set to prevent them from appearing in search
|
||||
nix = tlpdbNix;
|
||||
xz = tlpdbxz;
|
||||
};
|
||||
|
||||
bin = assert assertions; bin;
|
||||
combine = assert assertions; combine;
|
||||
|
||||
# Pre-defined combined packages for TeX Live schemes,
|
||||
# to make nix-env usage more comfortable and build selected on Hydra.
|
||||
@ -215,8 +240,7 @@ in
|
||||
(combine {
|
||||
${pname} = attrs;
|
||||
extraName = "combined" + lib.removePrefix "scheme" pname;
|
||||
#extraVersion = "-final";
|
||||
extraVersion = ".${snapshot.year}${snapshot.month}${snapshot.day}";
|
||||
extraVersion = with version; if final then "-final" else ".${year}${month}${day}";
|
||||
})
|
||||
)
|
||||
{ inherit (tl)
|
||||
|
||||
@ -797,8 +797,8 @@
|
||||
"texdiff.doc-0.4"="0f7q7cszy6ca6fmwzi7fn6p72szgzyj9fldvzz1rk3ywkhsnpakr";
|
||||
"texdirflatten-1.3"="0q2m138gpk9cvfp19jvsw746mkxa613c2204qfxw1bxw8b454d7m";
|
||||
"texdirflatten.doc-1.3"="19iiznql82a34d2yhzqcp339zszbhfz9fh0h4mb11696mj8rx3v7";
|
||||
"texdoc-4.0.1"="1ph4mzrlri8daxg7j4n15s3b9dj58ni4fdqhpvqxwy6736fiw130";
|
||||
"texdoc.doc-4.0.1"="1dpz5ni9xdbwfy4m12j3v407ywbddmy3j2dhzwrdsyj7wq3impmg";
|
||||
"texdoc-4.0.1-tlpdb-66590"="1ph4mzrlri8daxg7j4n15s3b9dj58ni4fdqhpvqxwy6736fiw130";
|
||||
"texdoc.doc-4.0.1-tlpdb-66590"="1dpz5ni9xdbwfy4m12j3v407ywbddmy3j2dhzwrdsyj7wq3impmg";
|
||||
"texdoctk-0.6.0"="13j0c5m12if8jx8sp7hlirz33y6l2jk8lvnms7dalz52mxz8kq4h";
|
||||
"texdoctk.doc-0.6.0"="1k41z2lbfjsar5rl47xnk1adbcnnb4w97bzbgf2rnjhimd1gn9hq";
|
||||
"texfot-1.48"="19aswiyr722n9misvg9ajdsgpn1ghmic2xfmr4w0w00sai5sh5rx";
|
||||
|
||||
@ -2,6 +2,17 @@
|
||||
1i{ # no indentation
|
||||
$a}
|
||||
|
||||
# extract repository metadata
|
||||
/^name 00texlive\.config$/,/^$/{
|
||||
s/^name (.*)$/"\1" = {/p
|
||||
/^$/,1i};
|
||||
|
||||
s!^depend frozen/0$! frozen = false;!p
|
||||
s!^depend frozen/1$! frozen = true;!p
|
||||
s!^depend release/(.*)$! year = \1;!p
|
||||
s!^depend revision/(.*)$! revision = \1;!p
|
||||
}
|
||||
|
||||
# form an attrmap per package
|
||||
# ignore packages whose name contains "." (such as binaries)
|
||||
/^name ([^.]+)$/,/^$/{
|
||||
|
||||
@ -1,4 +1,9 @@
|
||||
{ # no indentation
|
||||
"00texlive.config" = {
|
||||
frozen = true;
|
||||
year = 2022;
|
||||
revision = 66590;
|
||||
};
|
||||
"12many" = {
|
||||
revision = 15878;
|
||||
stripPrefix = 0;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user