JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dockerTools: Calculate tarsum's on the fly

Browse Source 
Calculating the tarsum after creating a layer is inefficient, since
we have to read the tarball we've just written from the disk.

This commit simultaneously calculates the tarsum while creating the
tarball.
This commit is contained in:
Utku Demir 2020-05-06 21:17:47 +12:00
parent 69f6294724
commit f12346d493
No known key found for this signature in database
GPG Key ID: F3F8629C3E0BF60B
2 changed files with 29 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
pkgs/build-support/docker/default.nix
View File

@ -392,14 +392,10 @@ rec {
(cd layer; eval "$extraCommands")
fi
# Tar up the layer and throw it into 'layer.tar'.
# Tar up the layer and throw it into 'layer.tar', while calculating its checksum.
echo "Packing layer..."
mkdir $out
tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar .
# Compute a checksum of the tarball.
echo "Computing layer checksum..."
tarhash=$(tarsum < $out/layer.tar)
tarhash=$(tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum)
# Add a 'checksum' field to the JSON, with the value set to the
# checksum of the tarball.
@ -449,11 +445,7 @@ rec {
# Tar up the layer and throw it into 'layer.tar'.
echo "Packing layer..."
mkdir $out
tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar .
# Compute a checksum of the tarball.
echo "Computing layer checksum..."
tarhash=$(tarsum < $out/layer.tar)
tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum)
# Add a 'checksum' field to the JSON, with the value set to the
# checksum of the tarball.
@ -537,11 +529,10 @@ rec {
echo "Packing layer..."
mkdir -p $out
tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar .
tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf - . |
tee $out/layer.tar |
${tarsum}/bin/tarsum)
# Compute the tar checksum and add it to the output json.
echo "Computing checksum..."
tarhash=$(${tarsum}/bin/tarsum < $out/layer.tar)
cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json
# Indicate to docker that we're using schema version 1.0.
echo -n "1.0" > $out/VERSION

48
pkgs/build-support/docker/store-path-to-layer.sh
View File

@ -11,37 +11,35 @@ echo "Creating layer #$layerNumber for $@"
mkdir -p "$layerPath"
# Make sure /nix and /nix/store appear first in the archive.
#
# We create the directories here and use them because
# when there are other things being added to the
# nix store, tar could fail, saying,
# "tar: /nix/store: file changed as we read it"
mkdir -p nix/store
tar -cf "$layerPath/layer.tar" \
--mtime="@$SOURCE_DATE_EPOCH" \
--owner=0 --group=0 \
--transform='s,nix,/nix,' \
nix
# We change into the /nix/store in order to avoid a similar
# "file changed as we read it" error as above. Namely,
# if we use the absolute path of /nix/store/123-pkg
# and something new is added to the nix store while tar
# is running, it will detect a change to /nix/store and
# fail. Instead, if we cd into the nix store and copy
# the relative nix store path, tar will ignore changes
# to /nix/store. In order to create the correct structure
# in the tar file, we transform the relative nix store
# path to the absolute store path.
basename -a "$@" |
tar -C /nix/store -rpf "$layerPath/layer.tar" \
--verbatim-files-from --files-from - \
--hard-dereference --sort=name \
--mtime="@$SOURCE_DATE_EPOCH" \
--owner=0 --group=0 \
--transform="flags=rS;s,^,/nix/store/,"
# Compute a checksum of the tarball.
tarhash=$(tarsum < $layerPath/layer.tar)
# Then we change into the /nix/store in order to
# avoid a similar "file changed as we read it" error
# as above. Namely, if we use the absolute path of
# /nix/store/123-pkg and something new is added to the nix
# store while tar is running, it will detect a change to
# /nix/store and fail. Instead, if we cd into the nix store
# and copy the relative nix store path, tar will ignore
# changes to /nix/store. In order to create the correct
# structure in the tar file, we transform the relative nix
# store path to the absolute store path.
tarhash=$(
basename -a "$@" |
tar -cp nix \
-C /nix/store --verbatim-files-from --files-from - \
--hard-dereference --sort=name \
--mtime="@$SOURCE_DATE_EPOCH" \
--owner=0 --group=0 \
--transform 's,^nix(/|$),/nix/,' \
--transform 's,^[^/],/nix/store/\0,rS' |
tee "$layerPath/layer.tar" |
tarsum
)
# Add a 'checksum' field to the JSON, with the value set to the
# checksum of the tarball.