consul.passthru.tests: Refactor into functions.

For better naming and commentary.

nixos/tests/consul.nix

@@ -107,13 +107,28 @@ in {
     for m in machines:
         m.wait_for_unit("consul.service")
 
+
+    def wait_for_healthy_servers():
+        for m in machines:
+            m.wait_until_succeeds("[ $(consul members | grep -o alive | wc -l) == 5 ]")
+
+
+    wait_for_healthy_servers()
+    # Also wait for clients to be alive.
     for m in machines:
         m.wait_until_succeeds("[ $(consul members | grep -o alive | wc -l) == 5 ]")
 
     client1.succeed("consul kv put testkey 42")
     client2.succeed("[ $(consul kv get testkey) == 42 ]")
 
-    # Test that the cluster can tolearate failures of any single server:
+
+    def rolling_reboot_test():
+        """
+        Tests that the cluster can tolearate failures of any single server,
+        following the recommended rolling upgrade procedure from
+        https://www.consul.io/docs/upgrading#standard-upgrades
+        """
+
         for server in servers:
             server.crash()
 
@@ -131,8 +146,7 @@ in {
             server.start()
 
             # Wait for recovery.
-        for m in machines:
+            wait_for_healthy_servers()
-            m.wait_until_succeeds("[ $(consul members | grep -o alive | wc -l) == 5 ]")
 
             # Wait for client connections.
             client1.wait_until_succeeds("consul kv get -recurse")
@@ -142,5 +156,9 @@ in {
             client1.succeed("consul kv put testkey 44")
             client2.succeed("[ $(consul kv get testkey) == 44 ]")
             client2.succeed("consul kv delete testkey")
+
+
+    # Run the tests.
+    rolling_reboot_test()
   '';
 })