From cc46362929dbb5d13a8ff8c4669ac3a8d47db028 Mon Sep 17 00:00:00 2001 From: Utku Demir Date: Sat, 4 Jul 2020 22:00:57 +1200 Subject: [PATCH 1/3] dockerTools: Support files directly under /nix/store Also makes sure that the files inside a layer added in a sorted order to make the results more deterministic. --- nixos/tests/docker-tools.nix | 6 ++++++ pkgs/build-support/docker/examples.nix | 10 ++++++++++ pkgs/build-support/docker/stream_layered_image.py | 8 ++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix index c48e5b079762..01f7d17f58f0 100644 --- a/nixos/tests/docker-tools.nix +++ b/nixos/tests/docker-tools.nix @@ -178,5 +178,11 @@ import ./make-test-python.nix ({ pkgs, ... }: { # This check may be loosened to allow an *empty* store rather than *no* store. docker.succeed("docker run --rm no-store-paths ls /") docker.fail("docker run --rm no-store-paths ls /nix/store") + + with subtest("Ensure buildLayeredImage supports files directly under /nix/store"): + docker.succeed( + "docker load --input='${pkgs.dockerTools.examples.filesInStore}'", + "docker run file-in-store |& grep 'some data'", + ) ''; }) diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index b040d830b31c..8b77d6011d38 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -335,4 +335,14 @@ rec { }; }; + # 19. Support files in the store on buildLayeredImage + # See: https://github.com/NixOS/nixpkgs/pull/91084#issuecomment-653496223 + filesInStore = pkgs.dockerTools.buildLayeredImage { + name = "file-in-store"; + tag = "latest"; + config.Cmd = [ + "${pkgs.coreutils}/bin/cat" + (pkgs.writeText "somefile" "some data") + ]; + }; } diff --git a/pkgs/build-support/docker/stream_layered_image.py b/pkgs/build-support/docker/stream_layered_image.py index 4348513338d7..609af30c8bb3 100644 --- a/pkgs/build-support/docker/stream_layered_image.py +++ b/pkgs/build-support/docker/stream_layered_image.py @@ -39,6 +39,7 @@ import json import hashlib import pathlib import tarfile +import itertools import threading from datetime import datetime from collections import namedtuple @@ -87,10 +88,9 @@ def archive_paths_to(obj, paths, mtime, add_nix, filter=None): tar.addfile(apply_filters(dir("/nix/store"))) for path in paths: - ti = tar.gettarinfo(os.path.join("/", path)) - tar.addfile(apply_filters(append_root(ti))) - - for filename in pathlib.Path(path).rglob("*"): + path = pathlib.Path(path) + files = itertools.chain([path], path.rglob("*")) + for filename in sorted(files): ti = append_root(tar.gettarinfo(filename)) # copy hardlinks as regular files From 2ae2186dfda22ee49b8ab1027643bd11909246d5 Mon Sep 17 00:00:00 2001 From: Utku Demir Date: Mon, 6 Jul 2020 16:42:03 +1200 Subject: [PATCH 2/3] dockerTools.streamLayeredImage: Correctly copy hardlinks --- pkgs/build-support/docker/stream_layered_image.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/build-support/docker/stream_layered_image.py b/pkgs/build-support/docker/stream_layered_image.py index 609af30c8bb3..9a13878a783d 100644 --- a/pkgs/build-support/docker/stream_layered_image.py +++ b/pkgs/build-support/docker/stream_layered_image.py @@ -97,6 +97,7 @@ def archive_paths_to(obj, paths, mtime, add_nix, filter=None): if ti.islnk(): ti.type = tarfile.REGTYPE ti.linkname = "" + ti.size = filename.stat().st_size ti = apply_filters(ti) if ti.isfile(): From 06db3319226b061d385521cdab74cd676254e9e9 Mon Sep 17 00:00:00 2001 From: Utku Demir Date: Mon, 6 Jul 2020 16:59:58 +1200 Subject: [PATCH 3/3] dockerTools: Verify nix-store contents on buildLayeredImage test --- nixos/tests/docker-tools.nix | 5 +++-- pkgs/build-support/docker/examples.nix | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix index 01f7d17f58f0..ad734a71f01f 100644 --- a/nixos/tests/docker-tools.nix +++ b/nixos/tests/docker-tools.nix @@ -179,10 +179,11 @@ import ./make-test-python.nix ({ pkgs, ... }: { docker.succeed("docker run --rm no-store-paths ls /") docker.fail("docker run --rm no-store-paths ls /nix/store") - with subtest("Ensure buildLayeredImage supports files directly under /nix/store"): + with subtest("Ensure buildLayeredImage does not change store path contents."): docker.succeed( "docker load --input='${pkgs.dockerTools.examples.filesInStore}'", - "docker run file-in-store |& grep 'some data'", + "docker run --rm file-in-store nix-store --verify --check-contents", + "docker run --rm file-in-store |& grep 'some data'", ) ''; }) diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 8b77d6011d38..068daa8df722 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -337,12 +337,21 @@ rec { # 19. Support files in the store on buildLayeredImage # See: https://github.com/NixOS/nixpkgs/pull/91084#issuecomment-653496223 - filesInStore = pkgs.dockerTools.buildLayeredImage { + filesInStore = pkgs.dockerTools.buildLayeredImageWithNixDb { name = "file-in-store"; tag = "latest"; - config.Cmd = [ - "${pkgs.coreutils}/bin/cat" - (pkgs.writeText "somefile" "some data") + contents = [ + pkgs.coreutils + pkgs.nix + (pkgs.writeScriptBin "myscript" '' + #!${pkgs.runtimeShell} + cat ${pkgs.writeText "somefile" "some data"} + '') ]; + config = { + Cmd = [ "myscript" ]; + # For some reason 'nix-store --verify' requires this environment variable + Env = [ "USER=root" ]; + }; }; }