From a5183762c5288361f7fb6b1f15c474ffa1a090dd Mon Sep 17 00:00:00 2001 From: Uli Baum Date: Sat, 22 Sep 2018 01:32:06 +0200 Subject: [PATCH 1/3] nixos/tests/installer: prevent race between parted and udev by combining all parted commands into a single parted call. This eliminates one cause of non-deterministic failure. --- nixos/tests/installer.nix | 82 +++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/nixos/tests/installer.nix b/nixos/tests/installer.nix index 610444f90e47..96845b79ad97 100644 --- a/nixos/tests/installer.nix +++ b/nixos/tests/installer.nix @@ -282,9 +282,9 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel msdos", - "parted --script /dev/vda -- mkpart primary linux-swap 1M 1024M", - "parted --script /dev/vda -- mkpart primary ext2 1024M -1s", + "parted --script /dev/vda -- mklabel msdos" + . " mkpart primary linux-swap 1M 1024M" + . " mkpart primary ext2 1024M -1s", "udevadm settle", "mkswap /dev/vda1 -L swap", "swapon -L swap", @@ -299,11 +299,11 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel gpt", - "parted --script /dev/vda -- mkpart ESP fat32 1M 50MiB", # /boot - "parted --script /dev/vda -- set 1 boot on", - "parted --script /dev/vda -- mkpart primary linux-swap 50MiB 1024MiB", - "parted --script /dev/vda -- mkpart primary ext2 1024MiB -1MiB", # / + "parted --script /dev/vda -- mklabel gpt" + . " mkpart ESP fat32 1M 50MiB" # /boot + . " set 1 boot on" + . " mkpart primary linux-swap 50MiB 1024MiB" + . " mkpart primary ext2 1024MiB -1MiB", # / "udevadm settle", "mkswap /dev/vda2 -L swap", "swapon -L swap", @@ -321,11 +321,11 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel gpt", - "parted --script /dev/vda -- mkpart ESP fat32 1M 50MiB", # /boot - "parted --script /dev/vda -- set 1 boot on", - "parted --script /dev/vda -- mkpart primary linux-swap 50MiB 1024MiB", - "parted --script /dev/vda -- mkpart primary ext2 1024MiB -1MiB", # / + "parted --script /dev/vda -- mklabel gpt" + . " mkpart ESP fat32 1M 50MiB" # /boot + . " set 1 boot on" + . " mkpart primary linux-swap 50MiB 1024MiB" + . " mkpart primary ext2 1024MiB -1MiB", # / "udevadm settle", "mkswap /dev/vda2 -L swap", "swapon -L swap", @@ -345,10 +345,10 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel msdos", - "parted --script /dev/vda -- mkpart primary ext2 1M 50MB", # /boot - "parted --script /dev/vda -- mkpart primary linux-swap 50MB 1024M", - "parted --script /dev/vda -- mkpart primary ext2 1024M -1s", # / + "parted --script /dev/vda -- mklabel msdos" + . " mkpart primary ext2 1M 50MB" # /boot + . " mkpart primary linux-swap 50MB 1024M" + . " mkpart primary ext2 1024M -1s", # / "udevadm settle", "mkswap /dev/vda2 -L swap", "swapon -L swap", @@ -366,10 +366,10 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel msdos", - "parted --script /dev/vda -- mkpart primary ext2 1M 50MB", # /boot - "parted --script /dev/vda -- mkpart primary linux-swap 50MB 1024M", - "parted --script /dev/vda -- mkpart primary ext2 1024M -1s", # / + "parted --script /dev/vda -- mklabel msdos" + . " mkpart primary ext2 1M 50MB" # /boot + . " mkpart primary linux-swap 50MB 1024M" + . " mkpart primary ext2 1024M -1s", # / "udevadm settle", "mkswap /dev/vda2 -L swap", "swapon -L swap", @@ -402,9 +402,9 @@ in { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel msdos", - "parted --script /dev/vda -- mkpart primary linux-swap 1M 1024M", - "parted --script /dev/vda -- mkpart primary 1024M -1s", + "parted --script /dev/vda -- mklabel msdos" + . " mkpart primary linux-swap 1M 1024M" + . " mkpart primary 1024M -1s", "udevadm settle", "mkswap /dev/vda1 -L swap", @@ -425,11 +425,11 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel msdos", - "parted --script /dev/vda -- mkpart primary 1M 2048M", # PV1 - "parted --script /dev/vda -- set 1 lvm on", - "parted --script /dev/vda -- mkpart primary 2048M -1s", # PV2 - "parted --script /dev/vda -- set 2 lvm on", + "parted --script /dev/vda -- mklabel msdos" + . " mkpart primary 1M 2048M" # PV1 + . " set 1 lvm on" + . " mkpart primary 2048M -1s" # PV2 + . " set 2 lvm on", "udevadm settle", "pvcreate /dev/vda1 /dev/vda2", "vgcreate MyVolGroup /dev/vda1 /dev/vda2", @@ -447,10 +447,10 @@ in { luksroot = makeInstallerTest "luksroot" { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel msdos", - "parted --script /dev/vda -- mkpart primary ext2 1M 50MB", # /boot - "parted --script /dev/vda -- mkpart primary linux-swap 50M 1024M", - "parted --script /dev/vda -- mkpart primary 1024M -1s", # LUKS + "parted --script /dev/vda -- mklabel msdos" + . " mkpart primary ext2 1M 50MB" # /boot + . " mkpart primary linux-swap 50M 1024M" + . " mkpart primary 1024M -1s", # LUKS "udevadm settle", "mkswap /dev/vda2 -L swap", "swapon -L swap", @@ -481,11 +481,11 @@ in { filesystemEncryptedWithKeyfile = makeInstallerTest "filesystemEncryptedWithKeyfile" { createPartitions = '' $machine->succeed( - "parted --script /dev/vda mklabel msdos", - "parted --script /dev/vda -- mkpart primary ext2 1M 50MB", # /boot - "parted --script /dev/vda -- mkpart primary linux-swap 50M 1024M", - "parted --script /dev/vda -- mkpart primary 1024M 1280M", # LUKS with keyfile - "parted --script /dev/vda -- mkpart primary 1280M -1s", + "parted --script /dev/vda -- mklabel msdos" + . " mkpart primary ext2 1M 50MB" # /boot + . " mkpart primary linux-swap 50M 1024M" + . " mkpart primary 1024M 1280M" # LUKS with keyfile + . " mkpart primary 1280M -1s", "udevadm settle", "mkswap /dev/vda2 -L swap", "swapon -L swap", @@ -555,9 +555,9 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/sda mklabel msdos", - "parted --script /dev/sda -- mkpart primary linux-swap 1M 1024M", - "parted --script /dev/sda -- mkpart primary ext2 1024M -1s", + "parted --script /dev/sda -- mklabel msdos" + . " mkpart primary linux-swap 1M 1024M" + . " mkpart primary ext2 1024M -1s", "udevadm settle", "mkswap /dev/sda1 -L swap", "swapon -L swap", From c46677fec2d3264f6a8f8b1929288806ea4f91aa Mon Sep 17 00:00:00 2001 From: Uli Baum Date: Sat, 22 Sep 2018 12:22:17 +0200 Subject: [PATCH 2/3] nixos/tests/installer: use flock for all parted calls to further reduce risk of race with udev, like util-linux recommends for sfdisk: https://github.com/karelzak/util-linux/blob/v2.32/disk-utils/sfdisk.8#L71 --- nixos/tests/installer.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nixos/tests/installer.nix b/nixos/tests/installer.nix index 96845b79ad97..dc8edeab4e6c 100644 --- a/nixos/tests/installer.nix +++ b/nixos/tests/installer.nix @@ -282,7 +282,7 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel msdos" + "flock /dev/vda parted --script /dev/vda -- mklabel msdos" . " mkpart primary linux-swap 1M 1024M" . " mkpart primary ext2 1024M -1s", "udevadm settle", @@ -299,7 +299,7 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel gpt" + "flock /dev/vda parted --script /dev/vda -- mklabel gpt" . " mkpart ESP fat32 1M 50MiB" # /boot . " set 1 boot on" . " mkpart primary linux-swap 50MiB 1024MiB" @@ -321,7 +321,7 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel gpt" + "flock /dev/vda parted --script /dev/vda -- mklabel gpt" . " mkpart ESP fat32 1M 50MiB" # /boot . " set 1 boot on" . " mkpart primary linux-swap 50MiB 1024MiB" @@ -345,7 +345,7 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel msdos" + "flock /dev/vda parted --script /dev/vda -- mklabel msdos" . " mkpart primary ext2 1M 50MB" # /boot . " mkpart primary linux-swap 50MB 1024M" . " mkpart primary ext2 1024M -1s", # / @@ -366,7 +366,7 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel msdos" + "flock /dev/vda parted --script /dev/vda -- mklabel msdos" . " mkpart primary ext2 1M 50MB" # /boot . " mkpart primary linux-swap 50MB 1024M" . " mkpart primary ext2 1024M -1s", # / @@ -402,7 +402,7 @@ in { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel msdos" + "flock /dev/vda parted --script /dev/vda -- mklabel msdos" . " mkpart primary linux-swap 1M 1024M" . " mkpart primary 1024M -1s", "udevadm settle", @@ -425,7 +425,7 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel msdos" + "flock /dev/vda parted --script /dev/vda -- mklabel msdos" . " mkpart primary 1M 2048M" # PV1 . " set 1 lvm on" . " mkpart primary 2048M -1s" # PV2 @@ -447,7 +447,7 @@ in { luksroot = makeInstallerTest "luksroot" { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel msdos" + "flock /dev/vda parted --script /dev/vda -- mklabel msdos" . " mkpart primary ext2 1M 50MB" # /boot . " mkpart primary linux-swap 50M 1024M" . " mkpart primary 1024M -1s", # LUKS @@ -481,7 +481,7 @@ in { filesystemEncryptedWithKeyfile = makeInstallerTest "filesystemEncryptedWithKeyfile" { createPartitions = '' $machine->succeed( - "parted --script /dev/vda -- mklabel msdos" + "flock /dev/vda parted --script /dev/vda -- mklabel msdos" . " mkpart primary ext2 1M 50MB" # /boot . " mkpart primary linux-swap 50M 1024M" . " mkpart primary 1024M 1280M" # LUKS with keyfile @@ -520,7 +520,7 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/vda --" + "flock /dev/vda parted --script /dev/vda --" . " mklabel msdos" . " mkpart primary ext2 1M 100MB" # /boot . " mkpart extended 100M -1s" @@ -555,7 +555,7 @@ in { { createPartitions = '' $machine->succeed( - "parted --script /dev/sda -- mklabel msdos" + "flock /dev/sda parted --script /dev/sda -- mklabel msdos" . " mkpart primary linux-swap 1M 1024M" . " mkpart primary ext2 1024M -1s", "udevadm settle", From 7dd6a5192d5dec03acb2490188f021f46460ec2d Mon Sep 17 00:00:00 2001 From: Uli Baum Date: Sat, 22 Sep 2018 12:29:10 +0200 Subject: [PATCH 3/3] nixos/tests/installer: stop udev queue before calling mdadm In the swraid test, temporarily stop udev queue execution while creating mdraid devices to prevent a race with udev, see https://groups.google.com/forum/#!topic/scylladb-dev/u87yHgo3ylU --- nixos/tests/installer.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/tests/installer.nix b/nixos/tests/installer.nix index dc8edeab4e6c..3f9fa0e6016c 100644 --- a/nixos/tests/installer.nix +++ b/nixos/tests/installer.nix @@ -531,8 +531,10 @@ in { "udevadm settle", "ls -l /dev/vda* >&2", "cat /proc/partitions >&2", + "udevadm control --stop-exec-queue", "mdadm --create --force /dev/md0 --metadata 1.2 --level=raid1 --raid-devices=2 /dev/vda5 /dev/vda6", "mdadm --create --force /dev/md1 --metadata 1.2 --level=raid1 --raid-devices=2 /dev/vda7 /dev/vda8", + "udevadm control --start-exec-queue", "udevadm settle", "mkswap -f /dev/md1 -L swap", "swapon -L swap",