JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dockerTools: Calculate tarsum's on the fly

Browse Source 
Calculating the tarsum after creating a layer is inefficient, since
we have to read the tarball we've just written from the disk.

This commit simultaneously calculates the tarsum while creating the
tarball.
This commit is contained in:
Utku Demir 2020-05-06 21:17:47 +12:00
parent 69f6294724
commit f12346d493
No known key found for this signature in database
GPG Key ID: F3F8629C3E0BF60B
2 changed files with 29 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
pkgs/build-support/docker/default.nix
View File

@ -392,14 +392,10 @@ rec {
(cd layer; eval "$extraCommands") (cd layer; eval "$extraCommands")
fi fi
# Tar up the layer and throw it into 'layer.tar'. # Tar up the layer and throw it into 'layer.tar', while calculating its checksum.
echo "Packing layer..." echo "Packing layer..."
mkdir $out mkdir $out
tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar . tarhash=$(tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum)
# Compute a checksum of the tarball.
echo "Computing layer checksum..."
tarhash=$(tarsum < $out/layer.tar)
# Add a 'checksum' field to the JSON, with the value set to the # Add a 'checksum' field to the JSON, with the value set to the
# checksum of the tarball. # checksum of the tarball.
@ -449,11 +445,7 @@ rec {
# Tar up the layer and throw it into 'layer.tar'. # Tar up the layer and throw it into 'layer.tar'.
echo "Packing layer..." echo "Packing layer..."
mkdir $out mkdir $out
tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar . tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum)
# Compute a checksum of the tarball.
echo "Computing layer checksum..."
tarhash=$(tarsum < $out/layer.tar)
# Add a 'checksum' field to the JSON, with the value set to the # Add a 'checksum' field to the JSON, with the value set to the
# checksum of the tarball. # checksum of the tarball.
@ -537,11 +529,10 @@ rec {
echo "Packing layer..." echo "Packing layer..."
mkdir -p $out mkdir -p $out
tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar . tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf - . |
tee $out/layer.tar |
${tarsum}/bin/tarsum)
# Compute the tar checksum and add it to the output json.
echo "Computing checksum..."
tarhash=$(${tarsum}/bin/tarsum < $out/layer.tar)
cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json
# Indicate to docker that we're using schema version 1.0. # Indicate to docker that we're using schema version 1.0.
echo -n "1.0" > $out/VERSION echo -n "1.0" > $out/VERSION

42
pkgs/build-support/docker/store-path-to-layer.sh
View File

@ -11,37 +11,35 @@ echo "Creating layer #$layerNumber for $@"
mkdir -p "$layerPath" mkdir -p "$layerPath"
# Make sure /nix and /nix/store appear first in the archive. # Make sure /nix and /nix/store appear first in the archive.
#
# We create the directories here and use them because # We create the directories here and use them because
# when there are other things being added to the # when there are other things being added to the
# nix store, tar could fail, saying, # nix store, tar could fail, saying,
# "tar: /nix/store: file changed as we read it" # "tar: /nix/store: file changed as we read it"
mkdir -p nix/store mkdir -p nix/store
tar -cf "$layerPath/layer.tar" \
--mtime="@$SOURCE_DATE_EPOCH" \
--owner=0 --group=0 \
--transform='s,nix,/nix,' \
nix
# We change into the /nix/store in order to avoid a similar # Then we change into the /nix/store in order to
# "file changed as we read it" error as above. Namely, # avoid a similar "file changed as we read it" error
# if we use the absolute path of /nix/store/123-pkg # as above. Namely, if we use the absolute path of
# and something new is added to the nix store while tar # /nix/store/123-pkg and something new is added to the nix
# is running, it will detect a change to /nix/store and # store while tar is running, it will detect a change to
# fail. Instead, if we cd into the nix store and copy # /nix/store and fail. Instead, if we cd into the nix store
# the relative nix store path, tar will ignore changes # and copy the relative nix store path, tar will ignore
# to /nix/store. In order to create the correct structure # changes to /nix/store. In order to create the correct
# in the tar file, we transform the relative nix store # structure in the tar file, we transform the relative nix
# path to the absolute store path. # store path to the absolute store path.
basename -a "$@" | tarhash=$(
tar -C /nix/store -rpf "$layerPath/layer.tar" \ basename -a "$@" |
--verbatim-files-from --files-from - \ tar -cp nix \
-C /nix/store --verbatim-files-from --files-from - \
--hard-dereference --sort=name \ --hard-dereference --sort=name \
--mtime="@$SOURCE_DATE_EPOCH" \ --mtime="@$SOURCE_DATE_EPOCH" \
--owner=0 --group=0 \ --owner=0 --group=0 \
--transform="flags=rS;s,^,/nix/store/," --transform 's,^nix(/|$),/nix/,' \
--transform 's,^[^/],/nix/store/\0,rS' |
# Compute a checksum of the tarball. tee "$layerPath/layer.tar" |
tarhash=$(tarsum < $layerPath/layer.tar) tarsum
)
# Add a 'checksum' field to the JSON, with the value set to the # Add a 'checksum' field to the JSON, with the value set to the
# checksum of the tarball. # checksum of the tarball.