Merge pull request #118018 from considerate/master

dockerTools: Implement merging of image tarballs

nixos/tests/docker-tools.nix

@@ -321,5 +321,48 @@ import ./make-test-python.nix ({ pkgs, ... }: {
         docker.succeed(
             "docker run --rm ${examples.layeredImageWithFakeRootCommands.imageName} sh -c 'stat -c '%u' /home/jane | grep -E ^1000$'"
         )
+
+    with subtest("Ensure docker load on merged images loads all of the constituent images"):
+        docker.succeed(
+            "docker load --input='${examples.mergedBashAndRedis}'"
+        )
+        docker.succeed(
+            "docker images --format '{{.Repository}}-{{.Tag}}' | grep -F '${examples.bash.imageName}-${examples.bash.imageTag}'"
+        )
+        docker.succeed(
+            "docker images --format '{{.Repository}}-{{.Tag}}' | grep -F '${examples.redis.imageName}-${examples.redis.imageTag}'"
+        )
+        docker.succeed("docker run --rm ${examples.bash.imageName} bash --version")
+        docker.succeed("docker run --rm ${examples.redis.imageName} redis-cli --version")
+        docker.succeed("docker rmi ${examples.bash.imageName}")
+        docker.succeed("docker rmi ${examples.redis.imageName}")
+
+    with subtest(
+        "Ensure docker load on merged images loads all of the constituent images (missing tags)"
+    ):
+        docker.succeed(
+            "docker load --input='${examples.mergedBashNoTagAndRedis}'"
+        )
+        docker.succeed(
+            "docker images --format '{{.Repository}}-{{.Tag}}' | grep -F '${examples.bashNoTag.imageName}-${examples.bashNoTag.imageTag}'"
+        )
+        docker.succeed(
+            "docker images --format '{{.Repository}}-{{.Tag}}' | grep -F '${examples.redis.imageName}-${examples.redis.imageTag}'"
+        )
+        # we need to explicitly specify the generated tag here
+        docker.succeed(
+            "docker run --rm ${examples.bashNoTag.imageName}:${examples.bashNoTag.imageTag} bash --version"
+        )
+        docker.succeed("docker run --rm ${examples.redis.imageName} redis-cli --version")
+        docker.succeed("docker rmi ${examples.bashNoTag.imageName}:${examples.bashNoTag.imageTag}")
+        docker.succeed("docker rmi ${examples.redis.imageName}")
+
+    with subtest("mergeImages preserves owners of the original images"):
+        docker.succeed(
+            "docker load --input='${examples.mergedBashFakeRoot}'"
+        )
+        docker.succeed(
+            "docker run --rm ${examples.layeredImageWithFakeRootCommands.imageName} sh -c 'stat -c '%u' /home/jane | grep -E ^1000$'"
+        )
   '';
 })

pkgs/build-support/docker/default.nix

@@ -686,6 +686,42 @@ rec {
     in
     result;
 
+  # Merge the tarballs of images built with buildImage into a single
+  # tarball that contains all images. Running `docker load` on the resulting
+  # tarball will load the images into the docker daemon.
+  mergeImages = images: runCommand "merge-docker-images"
+    {
+      inherit images;
+      nativeBuildInputs = [ pigz jq ];
+    } ''
+    mkdir image inputs
+    # Extract images
+    repos=()
+    manifests=()
+    for item in $images; do
+      name=$(basename $item)
+      mkdir inputs/$name
+      tar -I pigz -xf $item -C inputs/$name
+      if [ -f inputs/$name/repositories ]; then
+        repos+=(inputs/$name/repositories)
+      fi
+      if [ -f inputs/$name/manifest.json ]; then
+        manifests+=(inputs/$name/manifest.json)
+      fi
+    done
+    # Copy all layers from input images to output image directory
+    cp -R --no-clobber inputs/*/* image/
+    # Merge repositories objects and manifests
+    jq -s add "''${repos[@]}" > repositories
+    jq -s add "''${manifests[@]}" > manifest.json
+    # Replace output image repositories and manifest with merged versions
+    mv repositories image/repositories
+    mv manifest.json image/manifest.json
+    # Create tarball and gzip
+    tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nT > $out
+  '';
+
+
   # Provide a /etc/passwd and /etc/group that contain root and nobody.
   # Useful when packaging binaries that insist on using nss to look up
   # username/groups (like nginx).

pkgs/build-support/docker/examples.nix

@@ -497,4 +497,23 @@ rec {
       chown 1000 ./home/jane
     '';
   };
+
+  # tarball consisting of both bash and redis images
+  mergedBashAndRedis = pkgs.dockerTools.mergeImages [
+    bash
+    redis
+  ];
+
+  # tarball consisting of bash (without tag) and redis images
+  mergedBashNoTagAndRedis = pkgs.dockerTools.mergeImages [
+    bashNoTag
+    redis
+  ];
+
+  # tarball consisting of bash and layered image with different owner of the
+  # /home/jane directory
+  mergedBashFakeRoot = pkgs.dockerTools.mergeImages [
+    bash
+    layeredImageWithFakeRootCommands
+  ];
 }