dockerTools: nix functions for manipulating docker images
This commit is contained in:
Luca Bruno
Luca Bruno
parent
a7468dd951
commit
4a4561ce24
@ -291,4 +291,340 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
|
||||
</para>
|
||||
</section>
|
||||
|
||||
<section xml:id="sec-pkgs-dockerTools">
|
||||
<title>pkgs.dockerTools</title>
|
||||
|
||||
<para>
|
||||
<varname>pkgs.dockerTools</varname> is a set of functions for creating and
|
||||
manipulating Docker images according to the
|
||||
<link xlink:href="https://github.com/docker/docker/blob/master/image/spec/v1.md#docker-image-specification-v100">
|
||||
Docker Image Specification v1.0.0
|
||||
</link>. Docker itself is not used to perform any of the operations done by these
|
||||
functions.
|
||||
</para>
|
||||
|
||||
<warning>
|
||||
<para>
|
||||
The <varname>dockerTools</varname> API is unstable and may be subject to
|
||||
backwards-incompatible changes in the future.
|
||||
</para>
|
||||
</warning>
|
||||
|
||||
<section xml:id="ssec-pkgs-dockerTools-buildImage">
|
||||
<title>buildImage</title>
|
||||
|
||||
<para>
|
||||
This function is analogous to the <command>docker build</command> command,
|
||||
in that can used to build a Docker-compatible repository tarball containing
|
||||
a single image with one or multiple layers. As such, the result
|
||||
is suitable for being loaded in Docker with <command>docker load</command>.
|
||||
</para>
|
||||
|
||||
<para>
|
||||
The parameters of <varname>buildImage</varname> with relative example values are
|
||||
described below:
|
||||
</para>
|
||||
|
||||
<example xml:id='ex-dockerTools-buildImage'><title>Docker build</title>
|
||||
<programlisting>
|
||||
buildImage {
|
||||
name = "redis"; <co xml:id='ex-dockerTools-buildImage-1' />
|
||||
tag = "latest"; <co xml:id='ex-dockerTools-buildImage-2' />
|
||||
|
||||
fromImage = someBaseImage; <co xml:id='ex-dockerTools-buildImage-3' />
|
||||
fromImageName = null; <co xml:id='ex-dockerTools-buildImage-4' />
|
||||
fromImageTag = "latest"; <co xml:id='ex-dockerTools-buildImage-5' />
|
||||
|
||||
contents = pkgs.redis; <co xml:id='ex-dockerTools-buildImage-6' />
|
||||
runAsRoot = '' <co xml:id='ex-dockerTools-buildImage-runAsRoot' />
|
||||
#!${stdenv.shell}
|
||||
mkdir -p /data
|
||||
'';
|
||||
|
||||
config = { <co xml:id='ex-dockerTools-buildImage-8' />
|
||||
Cmd = [ "/bin/redis-server" ];
|
||||
WorkingDir = "/data";
|
||||
Volumes = {
|
||||
"/data" = {};
|
||||
};
|
||||
};
|
||||
}
|
||||
</programlisting>
|
||||
</example>
|
||||
|
||||
<para>The above example will build a Docker image <literal>redis/latest</literal>
|
||||
from the given base image. Loading and running this image in Docker results in
|
||||
<literal>redis-server</literal> being started automatically.
|
||||
</para>
|
||||
|
||||
<calloutlist>
|
||||
<callout arearefs='ex-dockerTools-buildImage-1'>
|
||||
<para>
|
||||
<varname>name</varname> specifies the name of the resulting image.
|
||||
This is the only required argument for <varname>buildImage</varname>.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-buildImage-2'>
|
||||
<para>
|
||||
<varname>tag</varname> specifies the tag of the resulting image.
|
||||
By default it's <literal>latest</literal>.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-buildImage-3'>
|
||||
<para>
|
||||
<varname>fromImage</varname> is the repository tarball containing the base image.
|
||||
It must be a valid Docker image, such as exported by <command>docker save</command>.
|
||||
By default it's <literal>null</literal>, which can be seen as equivalent
|
||||
to <literal>FROM scratch</literal> of a <filename>Dockerfile</filename>.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-buildImage-4'>
|
||||
<para>
|
||||
<varname>fromImageName</varname> can be used to further specify
|
||||
the base image within the repository, in case it contains multiple images.
|
||||
By default it's <literal>null</literal>, in which case
|
||||
<varname>buildImage</varname> will peek the first image available
|
||||
in the repository.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-buildImage-5'>
|
||||
<para>
|
||||
<varname>fromImageTag</varname> can be used to further specify the tag
|
||||
of the base image within the repository, in case an image contains multiple tags.
|
||||
By default it's <literal>null</literal>, in which case
|
||||
<varname>buildImage</varname> will peek the first tag available for the base image.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-buildImage-6'>
|
||||
<para>
|
||||
<varname>contents</varname> is a derivation that will be copied in the new
|
||||
layer of the resulting image. This can be similarly seen as
|
||||
<command>ADD contents/ /</command> in a <filename>Dockerfile</filename>.
|
||||
By default it's <literal>null</literal>.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-buildImage-runAsRoot'>
|
||||
<para>
|
||||
<varname>runAsRoot</varname> is a bash script that will run as root
|
||||
in an environment that overlays the existing layers of the base image with
|
||||
the new resulting layer, including the previously copied
|
||||
<varname>contents</varname> derivation.
|
||||
This can be similarly seen as
|
||||
<command>RUN ...</command> in a <filename>Dockerfile</filename>.
|
||||
|
||||
<note>
|
||||
<para>
|
||||
Using this parameter requires the <literal>kvm</literal>
|
||||
device to be available.
|
||||
</para>
|
||||
</note>
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-buildImage-8'>
|
||||
<para>
|
||||
<varname>config</varname> is used to specify the configuration of the
|
||||
containers that will be started off the built image in Docker.
|
||||
The available options are listed in the
|
||||
<link xlink:href="https://github.com/docker/docker/blob/master/image/spec/v1.md#container-runconfig-field-descriptions">
|
||||
Docker Image Specification v1.0.0
|
||||
</link>.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
</calloutlist>
|
||||
|
||||
<para>
|
||||
After the new layer has been created, its closure
|
||||
(to which <varname>contents</varname>, <varname>config</varname> and
|
||||
<varname>runAsRoot</varname> contribute) will be copied in the layer itself.
|
||||
Only new dependencies that are not already in the existing layers will be copied.
|
||||
</para>
|
||||
|
||||
<para>
|
||||
At the end of the process, only one new single layer will be produced and
|
||||
added to the resulting image.
|
||||
</para>
|
||||
|
||||
<para>
|
||||
The resulting repository will only list the single image
|
||||
<varname>image/tag</varname>. In the case of <xref linkend='ex-dockerTools-buildImage'/>
|
||||
it would be <varname>redis/latest</varname>.
|
||||
</para>
|
||||
|
||||
<para>
|
||||
It is possible to inspect the arguments with which an image was built
|
||||
using its <varname>buildArgs</varname> attribute.
|
||||
</para>
|
||||
|
||||
</section>
|
||||
|
||||
<section xml:id="ssec-pkgs-dockerTools-fetchFromRegistry">
|
||||
<title>pullImage</title>
|
||||
|
||||
<para>
|
||||
This function is analogous to the <command>docker pull</command> command,
|
||||
in that can be used to fetch a Docker image from a Docker registry.
|
||||
Currently only registry <literal>v1</literal> is supported.
|
||||
By default <link xlink:href="https://hub.docker.com/">Docker Hub</link>
|
||||
is used to pull images.
|
||||
</para>
|
||||
|
||||
<para>
|
||||
Its parameters are described in the example below:
|
||||
</para>
|
||||
|
||||
<example xml:id='ex-dockerTools-pullImage'><title>Docker pull</title>
|
||||
<programlisting>
|
||||
pullImage {
|
||||
imageName = "debian"; <co xml:id='ex-dockerTools-pullImage-1' />
|
||||
imageTag = "jessie"; <co xml:id='ex-dockerTools-pullImage-2' />
|
||||
imageId = null; <co xml:id='ex-dockerTools-pullImage-3' />
|
||||
sha256 = "1bhw5hkz6chrnrih0ymjbmn69hyfriza2lr550xyvpdrnbzr4gk2"; <co xml:id='ex-dockerTools-pullImage-4' />
|
||||
|
||||
indexUrl = "https://index.docker.io"; <co xml:id='ex-dockerTools-pullImage-5' />
|
||||
registryUrl = "https://registry-1.docker.io";
|
||||
registryVersion = "v1";
|
||||
}
|
||||
</programlisting>
|
||||
</example>
|
||||
|
||||
<calloutlist>
|
||||
<callout arearefs='ex-dockerTools-pullImage-1'>
|
||||
<para>
|
||||
<varname>imageName</varname> specifies the name of the image to be downloaded,
|
||||
which can also include the registry namespace (e.g. <literal>library/debian</literal>).
|
||||
This argument is required.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-pullImage-2'>
|
||||
<para>
|
||||
<varname>imageTag</varname> specifies the tag of the image to be downloaded.
|
||||
By default it's <literal>latest</literal>.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-pullImage-3'>
|
||||
<para>
|
||||
<varname>imageId</varname>, if specified this exact image will be fetched, instead
|
||||
of <varname>imageName/imageTag</varname>. However, the resulting repository
|
||||
will still be named <varname>imageName/imageTag</varname>.
|
||||
By default it's <literal>null</literal>.
|
||||
</para>
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-pullImage-4'>
|
||||
<para>
|
||||
<varname>sha256</varname> is the checksum of the whole fetched image.
|
||||
This argument is required.
|
||||
</para>
|
||||
|
||||
<note>
|
||||
<para>The checksum is computed on the unpacked directory, not on the final tarball.</para>
|
||||
</note>
|
||||
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-dockerTools-pullImage-5'>
|
||||
<para>
|
||||
In the above example the default values are shown for the variables <varname>indexUrl</varname>,
|
||||
<varname>registryUrl</varname> and <varname>registryVersion</varname>.
|
||||
Hence by default the Docker.io registry is used to pull the images.
|
||||
</para>
|
||||
</callout>
|
||||
</calloutlist>
|
||||
|
||||
</section>
|
||||
|
||||
<section xml:id="ssec-pkgs-dockerTools-exportImage">
|
||||
<title>exportImage</title>
|
||||
|
||||
<para>
|
||||
This function is analogous to the <command>docker export</command> command,
|
||||
in that can used to flatten a Docker image that contains multiple layers.
|
||||
It is in fact the result of the merge of all the layers of the image.
|
||||
As such, the result is suitable for being imported in Docker
|
||||
with <command>docker import</command>.
|
||||
</para>
|
||||
|
||||
<note>
|
||||
<para>
|
||||
Using this function requires the <literal>kvm</literal>
|
||||
device to be available.
|
||||
</para>
|
||||
</note>
|
||||
|
||||
<para>
|
||||
The parameters of <varname>exportImage</varname> are the following:
|
||||
</para>
|
||||
|
||||
<example xml:id='ex-dockerTools-exportImage'><title>Docker export</title>
|
||||
<programlisting>
|
||||
exportImage {
|
||||
fromImage = someLayeredImage;
|
||||
fromImageName = null;
|
||||
fromImageTag = null;
|
||||
|
||||
name = someLayeredImage.name;
|
||||
}
|
||||
</programlisting>
|
||||
</example>
|
||||
|
||||
<para>
|
||||
The parameters relative to the base image have the same synopsis as
|
||||
described in <xref linkend='ssec-pkgs-dockerTools-buildImage'/>, except that
|
||||
<varname>fromImage</varname> is the only required argument in this case.
|
||||
</para>
|
||||
|
||||
<para>
|
||||
The <varname>name</varname> argument is the name of the derivation output,
|
||||
which defaults to <varname>fromImage.name</varname>.
|
||||
</para>
|
||||
</section>
|
||||
|
||||
<section xml:id="ssec-pkgs-dockerTools-shadowSetup">
|
||||
<title>shadowSetup</title>
|
||||
|
||||
<para>
|
||||
This constant string is a helper for setting up the base files for managing
|
||||
users and groups, only if such files don't exist already.
|
||||
It is suitable for being used in a
|
||||
<varname>runAsRoot</varname> <xref linkend='ex-dockerTools-buildImage-runAsRoot'/> script for cases like
|
||||
in the example below:
|
||||
</para>
|
||||
|
||||
<example xml:id='ex-dockerTools-shadowSetup'><title>Shadow base files</title>
|
||||
<programlisting>
|
||||
buildImage {
|
||||
name = "shadow-basic";
|
||||
|
||||
runAsRoot = ''
|
||||
#!${stdenv.shell}
|
||||
${shadowSetup}
|
||||
groupadd -r redis
|
||||
useradd -r -g redis redis
|
||||
mkdir /data
|
||||
chown redis:redis /data
|
||||
'';
|
||||
}
|
||||
</programlisting>
|
||||
</example>
|
||||
|
||||
<para>
|
||||
Creating base files like <literal>/etc/passwd</literal> or
|
||||
<literal>/etc/login.defs</literal> are necessary for shadow-utils to
|
||||
manipulate users and groups.
|
||||
</para>
|
||||
|
||||
</section>
|
||||
|
||||
</section>
|
||||
|
||||
</chapter>
|
||||
|
||||
365
pkgs/build-support/docker/default.nix
Normal file
365
pkgs/build-support/docker/default.nix
Normal file
@ -0,0 +1,365 @@
|
||||
{ stdenv, lib, callPackage, runCommand, writeReferencesToFile, writeText, vmTools, writeScript
|
||||
, docker, shadow, utillinux, coreutils, jshon, e2fsprogs, goPackages }:
|
||||
|
||||
# WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future.
|
||||
|
||||
rec {
|
||||
|
||||
pullImage = callPackage ./pull.nix {};
|
||||
|
||||
# We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash.
|
||||
# And we cannot untar it, because then we cannot preserve permissions ecc.
|
||||
tarsum = runCommand "tarsum" {
|
||||
buildInputs = [ goPackages.go ];
|
||||
} ''
|
||||
mkdir tarsum
|
||||
cd tarsum
|
||||
|
||||
cp ${./tarsum.go} tarsum.go
|
||||
export GOPATH=$(pwd)
|
||||
mkdir src
|
||||
ln -sT ${docker.src}/pkg/tarsum src/tarsum
|
||||
go build
|
||||
|
||||
cp tarsum $out
|
||||
'';
|
||||
|
||||
# buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM
|
||||
mergeDrvs = { drvs, onlyDeps ? false }:
|
||||
runCommand "merge-drvs" {
|
||||
inherit drvs onlyDeps;
|
||||
} ''
|
||||
if [ -n "$onlyDeps" ]; then
|
||||
echo $drvs > $out
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir $out
|
||||
for drv in $drvs; do
|
||||
echo Merging $drv
|
||||
if [ -d "$drv" ]; then
|
||||
cp -drf --preserve=mode -f $drv/* $out/
|
||||
else
|
||||
tar -C $out -xpf $drv || true
|
||||
fi
|
||||
done
|
||||
'';
|
||||
|
||||
mkTarball = { name ? "docker-tar", drv, onlyDeps ? false }:
|
||||
runCommand "${name}.tar.gz" rec {
|
||||
inherit drv onlyDeps;
|
||||
|
||||
drvClosure = writeReferencesToFile drv;
|
||||
|
||||
} ''
|
||||
while read dep; do
|
||||
echo Copying $dep
|
||||
dir="$(dirname "$dep")"
|
||||
mkdir -p "rootfs/$dir"
|
||||
cp -drf --preserve=mode $dep "rootfs/$dir/"
|
||||
done < "$drvClosure"
|
||||
|
||||
if [ -z "$onlyDeps" ]; then
|
||||
cp -drf --preserve=mode $drv/* rootfs/
|
||||
fi
|
||||
|
||||
tar -C rootfs/ -cpzf $out .
|
||||
'';
|
||||
|
||||
shellScript = text:
|
||||
writeScript "script.sh" ''
|
||||
#!${stdenv.shell}
|
||||
set -e
|
||||
export PATH=${coreutils}/bin:/bin
|
||||
|
||||
${text}
|
||||
'';
|
||||
|
||||
shadowSetup = ''
|
||||
export PATH=${shadow}/bin:$PATH
|
||||
mkdir -p /etc/pam.d
|
||||
if [ ! -f /etc/passwd ]; then
|
||||
echo "root:x:0:0::/root:/bin/sh" > /etc/passwd
|
||||
echo "root:!x:::::::" > /etc/shadow
|
||||
fi
|
||||
if [ ! -f /etc/group ]; then
|
||||
echo "root:x:0:" > /etc/group
|
||||
echo "root:x::" > /etc/gshadow
|
||||
fi
|
||||
if [ ! -f /etc/pam.d/other ]; then
|
||||
cat > /etc/pam.d/other <<EOF
|
||||
account sufficient pam_unix.so
|
||||
auth sufficient pam_rootok.so
|
||||
password requisite pam_unix.so nullok sha512
|
||||
session required pam_unix.so
|
||||
EOF
|
||||
fi
|
||||
if [ ! -f /etc/login.defs ]; then
|
||||
touch /etc/login.defs
|
||||
fi
|
||||
'';
|
||||
|
||||
# Append to tar instead of unpacking
|
||||
mergeTarballs = tarballs:
|
||||
runCommand "merge-tars" { inherit tarballs; } ''
|
||||
mkdir tmp
|
||||
for tb in $tarballs; do
|
||||
tar -C tmp -xkpf $tb
|
||||
done
|
||||
tar -C tmp -cpzf $out .
|
||||
'';
|
||||
|
||||
runWithOverlay = { name , fromImage ? null, fromImageName ? null, fromImageTag ? null
|
||||
, diskSize ? 1024, preMount ? "", postMount ? "", postUmount ? "" }:
|
||||
vmTools.runInLinuxVM (
|
||||
runCommand name {
|
||||
preVM = vmTools.createEmptyImage { size = diskSize; fullName = "docker-run-disk"; };
|
||||
|
||||
inherit fromImage fromImageName fromImageTag;
|
||||
|
||||
buildInputs = [ utillinux e2fsprogs jshon ];
|
||||
} ''
|
||||
rm -rf $out
|
||||
|
||||
mkdir disk
|
||||
mkfs /dev/${vmTools.hd}
|
||||
mount /dev/${vmTools.hd} disk
|
||||
cd disk
|
||||
|
||||
if [ -n "$fromImage" ]; then
|
||||
echo Unpacking base image
|
||||
mkdir image
|
||||
tar -C image -xpf "$fromImage"
|
||||
|
||||
if [ -z "$fromImageName" ]; then
|
||||
fromImageName=$(jshon -k < image/repositories|head -n1)
|
||||
fi
|
||||
if [ -z "$fromImageTag" ]; then
|
||||
fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1)
|
||||
fi
|
||||
parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories)
|
||||
fi
|
||||
|
||||
lowerdir=""
|
||||
while [ -n "$parentID" ]; do
|
||||
echo Unpacking layer $parentID
|
||||
mkdir -p image/$parentID/layer
|
||||
tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
|
||||
rm image/$parentID/layer.tar
|
||||
|
||||
find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
|
||||
|
||||
lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
|
||||
parentID=$(cat image/$parentID/json|(jshon -e parent -u 2>/dev/null || true))
|
||||
done
|
||||
|
||||
mkdir work
|
||||
mkdir layer
|
||||
mkdir mnt
|
||||
|
||||
${preMount}
|
||||
|
||||
if [ -n "$lowerdir" ]; then
|
||||
mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt
|
||||
else
|
||||
mount --bind layer mnt
|
||||
fi
|
||||
|
||||
${postMount}
|
||||
|
||||
umount mnt
|
||||
|
||||
pushd layer
|
||||
find . -type c -exec bash -c 'name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"' \;
|
||||
popd
|
||||
|
||||
${postUmount}
|
||||
'');
|
||||
|
||||
exportImage = { name ? fromImage.name, fromImage, fromImageName ? null, fromImageTag ? null, diskSize ? 1024 }:
|
||||
runWithOverlay {
|
||||
inherit name fromImage fromImageName fromImageTag diskSize;
|
||||
|
||||
postMount = ''
|
||||
echo Packing raw image
|
||||
tar -C mnt -czf $out .
|
||||
'';
|
||||
};
|
||||
|
||||
mkPureLayer = { baseJson, contents ? null, extraCommands ? "" }:
|
||||
runCommand "docker-layer" {
|
||||
inherit baseJson contents extraCommands;
|
||||
|
||||
buildInputs = [ jshon ];
|
||||
} ''
|
||||
mkdir layer
|
||||
if [ -n "$contents" ]; then
|
||||
echo Adding contents
|
||||
for c in $contents; do
|
||||
cp -drf $c/* layer/
|
||||
chmod -R ug+w layer/
|
||||
done
|
||||
fi
|
||||
|
||||
pushd layer
|
||||
${extraCommands}
|
||||
popd
|
||||
|
||||
echo Packing layer
|
||||
mkdir $out
|
||||
tar -C layer -cf $out/layer.tar .
|
||||
ts=$(${tarsum} < $out/layer.tar)
|
||||
cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
|
||||
echo -n "1.0" > $out/VERSION
|
||||
'';
|
||||
|
||||
mkRootLayer = { runAsRoot, baseJson, fromImage ? null, fromImageName ? null, fromImageTag ? null
|
||||
, diskSize ? 1024, contents ? null, extraCommands ? "" }:
|
||||
let runAsRootScript = writeScript "run-as-root.sh" runAsRoot;
|
||||
in runWithOverlay {
|
||||
name = "docker-layer";
|
||||
|
||||
inherit fromImage fromImageName fromImageTag diskSize;
|
||||
|
||||
preMount = lib.optionalString (contents != null) ''
|
||||
echo Adding contents
|
||||
for c in ${builtins.toString contents}; do
|
||||
cp -drf $c/* layer/
|
||||
chmod -R ug+w layer/
|
||||
done
|
||||
'';
|
||||
|
||||
postMount = ''
|
||||
mkdir -p mnt/{dev,proc,sys,nix/store}
|
||||
mount --rbind /dev mnt/dev
|
||||
mount --rbind /sys mnt/sys
|
||||
mount --rbind /nix/store mnt/nix/store
|
||||
|
||||
unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript}
|
||||
umount -R mnt/dev mnt/sys mnt/nix/store
|
||||
rmdir --ignore-fail-on-non-empty mnt/dev mnt/proc mnt/sys mnt/nix/store mnt/nix
|
||||
'';
|
||||
|
||||
postUmount = ''
|
||||
pushd layer
|
||||
${extraCommands}
|
||||
popd
|
||||
|
||||
echo Packing layer
|
||||
mkdir $out
|
||||
tar -C layer -cf $out/layer.tar .
|
||||
ts=$(${tarsum} < $out/layer.tar)
|
||||
cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
|
||||
echo -n "1.0" > $out/VERSION
|
||||
'';
|
||||
};
|
||||
|
||||
# 1. extract the base image
|
||||
# 2. create the layer
|
||||
# 3. add layer deps to the layer itself, diffing with the base image
|
||||
# 4. compute the layer id
|
||||
# 5. put the layer in the image
|
||||
# 6. repack the image
|
||||
buildImage = args@{ name, tag ? "latest"
|
||||
, fromImage ? null, fromImageName ? null, fromImageTag ? null
|
||||
, contents ? null, tarballs ? [], config ? null
|
||||
, runAsRoot ? null, diskSize ? 1024, extraCommands ? "" }:
|
||||
|
||||
let
|
||||
|
||||
baseJson = writeText "${name}-config.json" (builtins.toJSON {
|
||||
created = "1970-01-01T00:00:01Z";
|
||||
architecture = "amd64";
|
||||
os = "linux";
|
||||
config = config;
|
||||
});
|
||||
|
||||
layer = (if runAsRoot == null
|
||||
then mkPureLayer { inherit baseJson contents extraCommands; }
|
||||
else mkRootLayer { inherit baseJson fromImage fromImageName fromImageTag contents runAsRoot diskSize extraCommands; });
|
||||
depsTarball = mkTarball { name = "${name}-deps";
|
||||
drv = layer;
|
||||
onlyDeps = true; };
|
||||
|
||||
result = runCommand "${name}.tar.gz" {
|
||||
buildInputs = [ jshon ];
|
||||
|
||||
imageName = name;
|
||||
imageTag = tag;
|
||||
inherit fromImage baseJson;
|
||||
|
||||
mergedTarball = if tarballs == [] then depsTarball else mergeTarballs ([ depsTarball ] ++ tarballs);
|
||||
|
||||
passthru = {
|
||||
buildArgs = args;
|
||||
};
|
||||
} ''
|
||||
mkdir image
|
||||
touch baseFiles
|
||||
if [ -n "$fromImage" ]; then
|
||||
echo Unpacking base image
|
||||
tar -C image -xpf "$fromImage"
|
||||
|
||||
if [ -z "$fromImageName" ]; then
|
||||
fromImageName=$(jshon -k < image/repositories|head -n1)
|
||||
fi
|
||||
if [ -z "$fromImageTag" ]; then
|
||||
fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1)
|
||||
fi
|
||||
parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories)
|
||||
|
||||
for l in image/*/layer.tar; do
|
||||
tar -tf $l >> baseFiles
|
||||
done
|
||||
fi
|
||||
|
||||
chmod -R ug+rw image
|
||||
|
||||
mkdir temp
|
||||
cp ${layer}/* temp/
|
||||
chmod ug+w temp/*
|
||||
|
||||
echo Adding dependencies
|
||||
tar -tf temp/layer.tar >> baseFiles
|
||||
tar -tf "$mergedTarball" | grep -v ${layer} > layerFiles
|
||||
if [ "$(wc -l layerFiles|cut -d ' ' -f 1)" -gt 3 ]; then
|
||||
sed -i -e 's|^[\./]\+||' baseFiles layerFiles
|
||||
comm <(sort -n baseFiles|uniq) <(sort -n layerFiles|uniq) -1 -3 > newFiles
|
||||
mkdir deps
|
||||
pushd deps
|
||||
tar -xpf "$mergedTarball" --no-recursion --files-from ../newFiles 2>/dev/null || true
|
||||
tar -rf ../temp/layer.tar --no-recursion --files-from ../newFiles 2>/dev/null || true
|
||||
popd
|
||||
else
|
||||
echo No new deps, no diffing needed
|
||||
fi
|
||||
|
||||
echo Adding meta
|
||||
|
||||
if [ -n "$parentID" ]; then
|
||||
cat temp/json | jshon -s "$parentID" -i parent > tmpjson
|
||||
mv tmpjson temp/json
|
||||
fi
|
||||
|
||||
layerID=$(sha256sum temp/json|cut -d ' ' -f 1)
|
||||
size=$(stat --printf="%s" temp/layer.tar)
|
||||
cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson
|
||||
mv tmpjson temp/json
|
||||
|
||||
mv temp image/$layerID
|
||||
|
||||
jshon -n object \
|
||||
-n object -s "$layerID" -i "$imageTag" \
|
||||
-i "$imageName" > image/repositories
|
||||
|
||||
chmod -R a-w image
|
||||
|
||||
echo Cooking the image
|
||||
tar -C image -czf $out .
|
||||
'';
|
||||
|
||||
in
|
||||
|
||||
result;
|
||||
|
||||
}
|
||||
38
pkgs/build-support/docker/detjson.py
Normal file
38
pkgs/build-support/docker/detjson.py
Normal file
@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Deterministic layer json: https://github.com/docker/hub-feedback/issues/488
|
||||
|
||||
import sys
|
||||
reload(sys)
|
||||
sys.setdefaultencoding('UTF8')
|
||||
import json
|
||||
|
||||
# If any of the keys below are equal to a certain value
|
||||
# then we can delete it because it's the default value
|
||||
SAFEDELS = {
|
||||
"Size": 0,
|
||||
"config": {
|
||||
"ExposedPorts": None,
|
||||
"MacAddress": "",
|
||||
"NetworkDisabled": False,
|
||||
"PortSpecs": None,
|
||||
"VolumeDriver": ""
|
||||
}
|
||||
}
|
||||
SAFEDELS["container_config"] = SAFEDELS["config"]
|
||||
|
||||
def makedet(j, safedels):
|
||||
for k,v in safedels.items():
|
||||
if type(v) == dict:
|
||||
makedet(j[k], v)
|
||||
elif k in j and j[k] == v:
|
||||
del j[k]
|
||||
|
||||
def main():
|
||||
j = json.load(sys.stdin)
|
||||
makedet(j, SAFEDELS)
|
||||
json.dump(j, sys.stdout, sort_keys=True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
50
pkgs/build-support/docker/pull.nix
Normal file
50
pkgs/build-support/docker/pull.nix
Normal file
@ -0,0 +1,50 @@
|
||||
{ stdenv, lib, curl, jshon, python, runCommand }:
|
||||
|
||||
# Inspired and simplified version of fetchurl.
|
||||
# For simplicity we only support sha256.
|
||||
|
||||
# Currently only registry v1 is supported, compatible with Docker Hub.
|
||||
|
||||
{ imageName, imageTag ? "latest", imageId ? null
|
||||
, sha256, name ? "${imageName}-${imageTag}"
|
||||
, indexUrl ? "https://index.docker.io"
|
||||
, registryUrl ? "https://registry-1.docker.io"
|
||||
, registryVersion ? "v1"
|
||||
, curlOpts ? "" }:
|
||||
|
||||
let layer = stdenv.mkDerivation {
|
||||
inherit name imageName imageTag imageId
|
||||
indexUrl registryUrl registryVersion curlOpts;
|
||||
|
||||
builder = ./pull.sh;
|
||||
detjson = ./detjson.py;
|
||||
|
||||
buildInputs = [ curl jshon python ];
|
||||
|
||||
outputHashAlgo = "sha256";
|
||||
outputHash = sha256;
|
||||
outputHashMode = "recursive";
|
||||
|
||||
impureEnvVars = [
|
||||
# We borrow these environment variables from the caller to allow
|
||||
# easy proxy configuration. This is impure, but a fixed-output
|
||||
# derivation like fetchurl is allowed to do so since its result is
|
||||
# by definition pure.
|
||||
"http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
|
||||
|
||||
# This variable allows the user to pass additional options to curl
|
||||
"NIX_CURL_FLAGS"
|
||||
|
||||
# This variable allows overriding the timeout for connecting to
|
||||
# the hashed mirrors.
|
||||
"NIX_CONNECT_TIMEOUT"
|
||||
];
|
||||
|
||||
# Doing the download on a remote machine just duplicates network
|
||||
# traffic, so don't do that.
|
||||
preferLocalBuild = true;
|
||||
};
|
||||
|
||||
in runCommand "${name}.tar.gz" {} ''
|
||||
tar -C ${layer} -czf $out .
|
||||
''
|
||||
75
pkgs/build-support/docker/pull.sh
Normal file
75
pkgs/build-support/docker/pull.sh
Normal file
@ -0,0 +1,75 @@
|
||||
# Reference: docker src contrib/download-frozen-image.sh
|
||||
|
||||
source $stdenv/setup
|
||||
|
||||
# Curl flags to handle redirects, not use EPSV, handle cookies for
|
||||
# servers to need them during redirects, and work on SSL without a
|
||||
# certificate (this isn't a security problem because we check the
|
||||
# cryptographic hash of the output anyway).
|
||||
curl="curl \
|
||||
--location --max-redirs 20 \
|
||||
--retry 3 \
|
||||
--fail \
|
||||
--disable-epsv \
|
||||
--cookie-jar cookies \
|
||||
--insecure \
|
||||
$curlOpts \
|
||||
$NIX_CURL_FLAGS"
|
||||
|
||||
baseUrl="$registryUrl/$registryVersion"
|
||||
|
||||
fetchLayer() {
|
||||
local url="$1"
|
||||
local dest="$2"
|
||||
local curlexit=18;
|
||||
|
||||
# if we get error code 18, resume partial download
|
||||
while [ $curlexit -eq 18 ]; do
|
||||
# keep this inside an if statement, since on failure it doesn't abort the script
|
||||
if $curl -H "Authorization: Token $token" "$url" --output "$dest"; then
|
||||
return 0
|
||||
else
|
||||
curlexit=$?;
|
||||
fi
|
||||
done
|
||||
|
||||
return $curlexit
|
||||
}
|
||||
|
||||
token="$($curl -o /dev/null -D- -H 'X-Docker-Token: true' "$indexUrl/$registryVersion/repositories/$imageName/images" | grep X-Docker-Token | tr -d '\r' | cut -d ' ' -f 2)"
|
||||
|
||||
if [ -z "$token" ]; then
|
||||
echo "error: registry returned no token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# token="${token//\"/\\\"}"
|
||||
|
||||
if [ -z "$imageId" ]; then
|
||||
imageId="$($curl -H "Authorization: Token $token" "$baseUrl/repositories/$imageName/tags/$imageTag")"
|
||||
imageId="${imageId//\"/}"
|
||||
if [ -z "$imageId" ]; then
|
||||
echo "error: no image ID found for ${imageName}:${imageTag}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "found image ${imageName}:${imageTag}@$imageId"
|
||||
fi
|
||||
|
||||
mkdir -p $out
|
||||
|
||||
jshon -n object \
|
||||
-n object -s "$imageId" -i "$imageTag" \
|
||||
-i "$imageName" > $out/repositories
|
||||
|
||||
$curl -H "Authorization: Token $token" "$baseUrl/images/$imageId/ancestry" -o ancestry.json
|
||||
|
||||
layerIds=$(jshon -a -u < ancestry.json)
|
||||
for layerId in $layerIds; do
|
||||
echo "fetching layer $layerId"
|
||||
|
||||
mkdir "$out/$layerId"
|
||||
echo '1.0' > "$out/$layerId/VERSION"
|
||||
$curl -H "Authorization: Token $token" "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json"
|
||||
fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar"
|
||||
done
|
||||
24
pkgs/build-support/docker/tarsum.go
Normal file
24
pkgs/build-support/docker/tarsum.go
Normal file
@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"tarsum"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ts, err := tarsum.NewTarSum(os.Stdin, false, tarsum.Version1)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if _, err = io.Copy(ioutil.Discard, ts); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Println(ts.Sum(nil))
|
||||
}
|
||||
@ -289,6 +289,8 @@ let
|
||||
|
||||
cmark = callPackage ../development/libraries/cmark { };
|
||||
|
||||
dockerTools = callPackage ../build-support/docker { };
|
||||
|
||||
dotnetenv = callPackage ../build-support/dotnetenv {
|
||||
dotnetfx = dotnetfx40;
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user