Merge pull request #144679 from hercules-ci/fix-issue-144613-nixosTest-wait-stdout

nixosTest: document wait for stdout behavior, fix #144613
This commit is contained in:
Bernardo Meurer 2021-11-04 22:57:06 -07:00 committed by GitHub
commit 86373221ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 83 additions and 45 deletions

View File

@ -159,6 +159,10 @@ The following methods are available on machine objects:
`execute`
: Execute a shell command, returning a list `(status, stdout)`.
If the command detaches, it must close stdout, as `execute` will wait
for this to consume all output reliably. This can be achieved by
redirecting stdout to stderr `>&2`, to `/dev/console`, `/dev/null` or
a file.
Takes an optional parameter `check_return` that defaults to `True`.
Setting this parameter to `False` will not check for the return code
and return -1 instead. This can be used for commands that shut down
@ -179,6 +183,8 @@ The following methods are available on machine objects:
- Dereferencing unset variables fail the command.
- It will wait for stdout to be closed. See `execute`.
`fail`
: Like `succeed`, but raising an exception if the command returns a zero

View File

@ -266,7 +266,12 @@ start_all()
<listitem>
<para>
Execute a shell command, returning a list
<literal>(status, stdout)</literal>. Takes an optional
<literal>(status, stdout)</literal>. If the command detaches,
it must close stdout, as <literal>execute</literal> will wait
for this to consume all output reliably. This can be achieved
by redirecting stdout to stderr <literal>&gt;&amp;2</literal>,
to <literal>/dev/console</literal>,
<literal>/dev/null</literal> or a file. Takes an optional
parameter <literal>check_return</literal> that defaults to
<literal>True</literal>. Setting this parameter to
<literal>False</literal> will not check for the return code
@ -306,6 +311,12 @@ start_all()
Dereferencing unset variables fail the command.
</para>
</listitem>
<listitem>
<para>
It will wait for stdout to be closed. See
<literal>execute</literal>.
</para>
</listitem>
</itemizedlist>
</listitem>
</varlistentry>

View File

@ -423,6 +423,23 @@
<section xml:id="sec-release-21.11-incompatibilities">
<title>Backward Incompatibilities</title>
<itemizedlist>
<listitem>
<para>
The NixOS VM test framework,
<literal>pkgs.nixosTest</literal>/<literal>make-test-python.nix</literal>,
now requires non-terminating commands such as
<literal>succeed(&quot;foo &amp;&quot;)</literal> to close
stdout. This can be done with a redirect such as
<literal>succeed(&quot;foo &gt;&amp;2 &amp;&quot;)</literal>.
This breaking change was necessitated by a race condition
causing tests to fail or hang. It applies to all methods that
invoke commands on the nodes, including
<literal>execute</literal>, <literal>succeed</literal>,
<literal>fail</literal>,
<literal>wait_until_succeeds</literal>,
<literal>wait_until_fails</literal>.
</para>
</listitem>
<listitem>
<para>
The <literal>services.wakeonlan</literal> option was removed,

View File

@ -128,6 +128,10 @@ In addition to numerous new and upgraded packages, this release has the followin
## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
- The NixOS VM test framework, `pkgs.nixosTest`/`make-test-python.nix`, now requires non-terminating commands such as `succeed("foo &")` to close stdout.
This can be done with a redirect such as `succeed("foo >&2 &")`. This breaking change was necessitated by a race condition causing tests to fail or hang.
It applies to all methods that invoke commands on the nodes, including `execute`, `succeed`, `fail`, `wait_until_succeeds`, `wait_until_fails`.
- The `services.wakeonlan` option was removed, and replaced with `networking.interfaces.<name>.wakeOnLan`.
- The `security.wrappers` option now requires to always specify an owner, group and whether the setuid/setgid bit should be set.

View File

@ -119,7 +119,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
with subtest("Stop a container early"):
machine.succeed(f"nixos-container stop {id1}")
machine.succeed(f"nixos-container start {id1} &")
machine.succeed(f"nixos-container start {id1} >&2 &")
machine.wait_for_console_text("Stage 2")
machine.succeed(f"nixos-container stop {id1}")
machine.wait_for_console_text(f"Container {id1} exited successfully")

View File

@ -38,7 +38,7 @@ in {
sender.execute("echo Hello World > testfile01.txt")
sender.execute("echo Hello Earth > testfile02.txt")
sender.execute(
"croc --pass ${pass} --relay relay send --code topSecret testfile01.txt testfile02.txt &"
"croc --pass ${pass} --relay relay send --code topSecret testfile01.txt testfile02.txt >&2 &"
)
# receive the testfiles and check them

View File

@ -33,7 +33,7 @@ import ./make-test-python.nix ({ pkgs, ...} : {
)
# connects to the daemon
machine.succeed("emacsclient --create-frame $EDITOR &")
machine.succeed("emacsclient --create-frame $EDITOR >&2 &")
# checks that Emacs shows the edited filename
machine.wait_for_text("emacseditor")

View File

@ -88,7 +88,7 @@ import ./make-test-python.nix ({ pkgs, ...} :
machine.screenshot("wizard12")
with subtest("Run Terminology"):
machine.succeed("terminology &")
machine.succeed("terminology >&2 &")
machine.sleep(5)
machine.send_chars("ls --color -alF\n")
machine.sleep(2)

View File

@ -13,7 +13,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
''
machine.wait_for_unit("multi-user.target")
machine.succeed("etesync-dav --version")
machine.execute("etesync-dav &")
machine.execute("etesync-dav >&2 &")
machine.wait_for_open_port(37358)
with subtest("Check that the web interface is accessible"):
assert "Add User" in machine.succeed("curl -s http://localhost:37358/.web/add/")

View File

@ -91,7 +91,7 @@ import ./make-test-python.nix ({ pkgs, firefoxPackage, ... }: {
with subtest("Wait until Firefox has finished loading the Valgrind docs page"):
machine.execute(
"xterm -e 'firefox file://${pkgs.valgrind.doc}/share/doc/valgrind/html/index.html' &"
"xterm -e 'firefox file://${pkgs.valgrind.doc}/share/doc/valgrind/html/index.html' >&2 &"
)
machine.wait_for_window("Valgrind")
machine.sleep(40)
@ -99,7 +99,7 @@ import ./make-test-python.nix ({ pkgs, firefoxPackage, ... }: {
with subtest("Check whether Firefox can play sound"):
with audio_recording(machine):
machine.succeed(
"firefox file://${pkgs.sound-theme-freedesktop}/share/sounds/freedesktop/stereo/phone-incoming-call.oga &"
"firefox file://${pkgs.sound-theme-freedesktop}/share/sounds/freedesktop/stereo/phone-incoming-call.oga >&2 &"
)
wait_for_sound(machine)
machine.copy_from_vm("/tmp/record.wav")

View File

@ -22,7 +22,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
# Add a dummy sound card, or the program won't start
machine.execute("modprobe snd-dummy")
machine.execute("ft2-clone &")
machine.execute("ft2-clone >&2 &")
machine.wait_for_window(r"Fasttracker")
machine.sleep(5)

View File

@ -110,7 +110,7 @@ in makeTest {
)
# Hibernate machine
hibernate.execute("systemctl hibernate &", check_return=False)
hibernate.execute("systemctl hibernate >&2 &", check_return=False)
hibernate.wait_for_shutdown()
# Restore machine from hibernation, validate our ramfs file is there.

View File

@ -26,7 +26,7 @@ import ./make-test-python.nix ({ pkgs, ...} :
machine.wait_for_x()
# start KeePassXC window
machine.execute("su - alice -c keepassxc &")
machine.execute("su - alice -c keepassxc >&2 &")
machine.wait_for_text("KeePassXC ${pkgs.keepassxc.version}")
machine.screenshot("KeePassXC")

View File

@ -18,7 +18,7 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : {
testScript =
''
machine.wait_for_unit("multi-user.target")
machine.execute("systemctl kexec &", check_return=False)
machine.execute("systemctl kexec >&2 &", check_return=False)
machine.connected = False
machine.wait_for_unit("multi-user.target")
'';

View File

@ -46,7 +46,7 @@ let
# set up process that expects all the keys to be entered
machine.succeed(
"{} {} {} {} &".format(
"{} {} {} {} >&2 &".format(
cmd,
"${testReader}",
len(inputs),

View File

@ -89,7 +89,7 @@ in
"""
Sends a message as Alice to Bob
"""
bob.execute("nc -lu ::0 1234 >/tmp/msg &")
bob.execute("nc -lu ::0 1234 >/tmp/msg >&2 &")
alice.sleep(1)
alice.succeed(f"echo '{msg}' | nc -uw 0 bob 1234")
bob.succeed(f"grep '{msg}' /tmp/msg")
@ -100,7 +100,7 @@ in
Starts eavesdropping on Alice and Bob
"""
match = "src host alice and dst host bob"
eve.execute(f"tcpdump -i br0 -c 1 -Avv {match} >/tmp/log &")
eve.execute(f"tcpdump -i br0 -c 1 -Avv {match} >/tmp/log >&2 &")
start_all()
@ -120,7 +120,7 @@ in
alice.succeed("ipsec verify 1>&2")
with subtest("Alice and Bob can start the tunnel"):
alice.execute("ipsec auto --start tunnel &")
alice.execute("ipsec auto --start tunnel >&2 &")
bob.succeed("ipsec auto --start tunnel")
# apparently this is needed to "wake" the tunnel
bob.execute("ping -c1 alice")

View File

@ -14,7 +14,7 @@ import ../make-test-python.nix {
)
# Start the daemon and wait until it is ready
machine.execute("lorri daemon > lorri.stdout 2> lorri.stderr &")
machine.execute("lorri daemon > lorri.stdout 2> lorri.stderr >&2 &")
machine.wait_until_succeeds("grep --fixed-strings 'ready' lorri.stdout")
# Ping the daemon

View File

@ -29,7 +29,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
# Create a secret file and send it to Bob
client_alice.succeed("echo mysecret > secretfile")
client_alice.succeed("wormhole --relay-url=ws://server:4000/v1 send -0 secretfile &")
client_alice.succeed("wormhole --relay-url=ws://server:4000/v1 send -0 secretfile >&2 &")
# Retrieve a secret file from Alice and check its content
client_bob.succeed("wormhole --relay-url=ws://server:4000/v1 receive -0 --accept-file")

View File

@ -20,7 +20,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
let user = nodes.client.config.users.users.alice;
in ''
client.wait_for_x()
client.execute("su - alice -c minecraft-launcher &")
client.execute("su - alice -c minecraft-launcher >&2 &")
client.wait_for_text("Create a new Microsoft account")
client.sleep(10)
client.screenshot("launcher")

View File

@ -21,7 +21,7 @@ in
};
testScript = ''
machine.execute("set -m; mpv --script-opts=webui-port=${port} --idle=yes &")
machine.execute("set -m; mpv --script-opts=webui-port=${port} --idle=yes >&2 &")
machine.wait_for_open_port(${port})
assert "<title>simple-mpv-webui" in machine.succeed("curl -s localhost:${port}")
'';

View File

@ -38,8 +38,8 @@ in
client1.wait_for_x()
client2.wait_for_x()
client1.execute("mumble mumble://client1:testpassword\@server/test &")
client2.execute("mumble mumble://client2:testpassword\@server/test &")
client1.execute("mumble mumble://client1:testpassword\@server/test >&2 &")
client2.execute("mumble mumble://client2:testpassword\@server/test >&2 &")
# cancel client audio configuration
client1.wait_for_window(r"Audio Tuning Wizard")

View File

@ -44,7 +44,7 @@ in
)
# Start MuseScore window
machine.execute("DISPLAY=:0.0 mscore &")
machine.execute("DISPLAY=:0.0 mscore >&2 &")
# Wait until MuseScore has launched
machine.wait_for_window("MuseScore")

View File

@ -66,7 +66,7 @@ in
client2.succeed("time flock -n -s /data/lock true")
with subtest("client 2 fails to acquire lock held by client 1"):
client1.succeed("flock -x /data/lock -c 'touch locked; sleep 100000' &")
client1.succeed("flock -x /data/lock -c 'touch locked; sleep 100000' >&2 &")
client1.wait_for_file("locked")
client2.fail("flock -n -s /data/lock true")

View File

@ -76,7 +76,7 @@ import ./make-test-python.nix {
server.wait_for_unit("nginx.service")
client.wait_for_unit("multi-user.target")
client.execute("test-runner &")
client.execute("test-runner >&2 &")
client.wait_for_file("/tmp/passed_stage1")
server.succeed(

View File

@ -78,7 +78,7 @@ let
# Put newlines on console, to flush the console reader's line buffer
# in case nixops' last output did not end in a newline, as is the case
# with a status line (if implemented?)
deployer.succeed("while sleep 60s; do echo [60s passed] >/dev/console; done &")
deployer.succeed("while sleep 60s; do echo [60s passed]; done >&2 &")
deployer_do("cd ~/unicorn; ssh -oStrictHostKeyChecking=accept-new root@server echo hi")

View File

@ -38,8 +38,8 @@ in {
client1.wait_for_x()
client2.wait_for_x()
client1.execute("openarena +set r_fullscreen 0 +set name Foo +connect server &")
client2.execute("openarena +set r_fullscreen 0 +set name Bar +connect server &")
client1.execute("openarena +set r_fullscreen 0 +set name Foo +connect server >&2 &")
client2.execute("openarena +set r_fullscreen 0 +set name Bar +connect server >&2 &")
server.wait_until_succeeds(
"journalctl -u openarena -e | grep -q 'Foo.*entered the game'"

View File

@ -14,7 +14,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
testScript = ''
machine.wait_for_x()
machine.succeed("gnome-calculator &")
machine.succeed("gnome-calculator >&2 &")
machine.wait_for_window("gnome-calculator")
machine.succeed(
"xdotool search --sync --onlyvisible --class gnome-calculator "

View File

@ -22,7 +22,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
# Add a dummy sound card, or the program won't start
machine.execute("modprobe snd-dummy")
machine.execute("pt2-clone &")
machine.execute("pt2-clone >&2 &")
machine.wait_for_window(r"ProTracker")
machine.sleep(5)

View File

@ -19,7 +19,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
testScript =
''
machine.wait_for_x()
machine.execute("shattered-pixel-dungeon &")
machine.execute("shattered-pixel-dungeon >&2 &")
machine.wait_for_window(r"Shattered Pixel Dungeon")
machine.sleep(5)
if "Enter" not in machine.get_screen_text():

View File

@ -41,7 +41,7 @@ in {
machine.wait_for_x()
# start signal desktop
machine.execute("su - alice -c signal-desktop &")
machine.execute("su - alice -c signal-desktop >&2 &")
# Wait for the Signal window to appear. Since usually the tests
# are run sandboxed and therfore with no internet, we can not wait

View File

@ -16,7 +16,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
testScript = ''
machine.wait_for_x()
machine.succeed("soapui &")
machine.succeed("soapui >&2 &")
machine.wait_for_window(r"SoapUI \d+\.\d+\.\d+")
machine.sleep(1)
machine.screenshot("soapui")

View File

@ -35,13 +35,13 @@ makeTest {
for host in [server, client]:
host.succeed("echo foobar | vncpasswd -f > vncpasswd")
server.succeed("Xvnc -geometry 720x576 :1 -PasswordFile vncpasswd &")
server.succeed("Xvnc -geometry 720x576 :1 -PasswordFile vncpasswd >&2 &")
server.wait_until_succeeds("nc -z localhost 5901", timeout=10)
server.succeed("DISPLAY=:1 xwininfo -root | grep 720x576")
server.execute("DISPLAY=:1 display -size 360x200 -font sans -gravity south label:'HELLO VNC WORLD' &")
server.execute("DISPLAY=:1 display -size 360x200 -font sans -gravity south label:'HELLO VNC WORLD' >&2 &")
client.wait_for_x()
client.execute("vncviewer server:1 -PasswordFile vncpasswd &")
client.execute("vncviewer server:1 -PasswordFile vncpasswd >&2 &")
client.wait_for_window(r"VNC")
client.screenshot("screenshot")
text = client.get_screen_text()

View File

@ -97,7 +97,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
)
machine.execute(
# Note trailing & for backgrounding.
f"({xvnc_command} | tee /tmp/Xvnc.stdout) 3>&1 1>&2 2>&3 | tee /tmp/Xvnc.stderr &",
f"({xvnc_command} | tee /tmp/Xvnc.stdout) 3>&1 1>&2 2>&3 | tee /tmp/Xvnc.stderr >&2 &",
)
@ -119,7 +119,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
def test_glxgears_failing_with_bad_driver_path():
machine.execute(
# Note trailing & for backgrounding.
"(env DISPLAY=:0 LIBGL_DRIVERS_PATH=/nonexistent glxgears -info | tee /tmp/glxgears-should-fail.stdout) 3>&1 1>&2 2>&3 | tee /tmp/glxgears-should-fail.stderr &"
"(env DISPLAY=:0 LIBGL_DRIVERS_PATH=/nonexistent glxgears -info | tee /tmp/glxgears-should-fail.stdout) 3>&1 1>&2 2>&3 | tee /tmp/glxgears-should-fail.stderr >&2 &"
)
machine.wait_until_succeeds("test -f /tmp/glxgears-should-fail.stderr")
wait_until_terminated_or_succeeds(
@ -136,7 +136,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
def test_glxgears_prints_renderer():
machine.execute(
# Note trailing & for backgrounding.
"(env DISPLAY=:0 glxgears -info | tee /tmp/glxgears.stdout) 3>&1 1>&2 2>&3 | tee /tmp/glxgears.stderr &"
"(env DISPLAY=:0 glxgears -info | tee /tmp/glxgears.stdout) 3>&1 1>&2 2>&3 | tee /tmp/glxgears.stderr >&2 &"
)
machine.wait_until_succeeds("test -f /tmp/glxgears.stderr")
wait_until_terminated_or_succeeds(

View File

@ -16,7 +16,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
testScript = ''
machine.wait_for_x()
machine.succeed("tuxguitar &")
machine.succeed("tuxguitar >&2 &")
machine.wait_for_window("TuxGuitar - Untitled.tg")
machine.sleep(1)
machine.screenshot("tuxguitar")

View File

@ -430,7 +430,7 @@ in mapAttrs (mkVBoxTest false vboxVMs) {
create_vm_simple()
machine.succeed(ru("VirtualBox &"))
machine.succeed(ru("VirtualBox >&2 &"))
machine.wait_until_succeeds(ru("xprop -name 'Oracle VM VirtualBox Manager'"))
machine.sleep(5)
machine.screenshot("gui_manager_started")

View File

@ -31,7 +31,7 @@ import ./make-test-python.nix ({ pkgs, ...} :
# Start VSCodium with a file that doesn't exist yet
machine.fail("ls /home/alice/foo.txt")
machine.succeed("su - alice -c 'codium foo.txt' &")
machine.succeed("su - alice -c 'codium foo.txt' >&2 &")
# Wait for the window to appear
machine.wait_for_text("VSCodium")

View File

@ -32,13 +32,13 @@ import ./make-test-python.nix ({ pkgs, ...} : {
client.sleep(5)
client.execute("xterm &")
client.execute("xterm >&2 &")
client.sleep(1)
client.send_chars("xfreerdp /cert-tofu /w:640 /h:480 /v:127.0.0.1 /u:${user.name} /p:${user.password}\n")
client.sleep(5)
client.screenshot("localrdp")
client.execute("xterm &")
client.execute("xterm >&2 &")
client.sleep(1)
client.send_chars("xfreerdp /cert-tofu /w:640 /h:480 /v:server /u:${user.name} /p:${user.password}\n")
client.sleep(5)

View File

@ -13,7 +13,7 @@ import ./make-test-python.nix ({ pkgs, ...} : {
testScript =
''
machine.wait_for_x()
machine.succeed("DISPLAY=:0 xterm -title testterm -class testterm -fullscreen &")
machine.succeed("DISPLAY=:0 xterm -title testterm -class testterm -fullscreen >&2 &")
machine.sleep(2)
machine.send_chars("echo $XTERM_VERSION >> /tmp/xterm_version\n")
machine.wait_for_file("/tmp/xterm_version")