[libvirt] [PATCH 4/4] libvirt-guests: Add parallel startup and shutdown of guests

[Peter Krempa]

With this patch, it's possible to shut down guests in parallel. Parallel
startup was possible before, but this functionality was not documented
properly.

To enable parallel startup set the START_DELAY to 0.

Parallel shutdown has a configurable parameter PARALLEL_SHUTDOWN that
defines the number of machines being shut down in parallel. Enabling
this feature changes the semantics of SHUTDOWN_TIMEOUT parameter that is
applied as a cumulative timeout to shutdown all guests on a URI.
---
 tools/libvirt-guests.init.sh |  140 +++++++++++++++++++++++++++++++++++++++--
 tools/libvirt-guests.sysconf |   10 +++-
 2 files changed, 141 insertions(+), 9 deletions(-)

diff --git a/tools/libvirt-guests.init.sh b/tools/libvirt-guests.init.sh
index 47914e3..dd933c7 100644
--- a/tools/libvirt-guests.init.sh
+++ b/tools/libvirt-guests.init.sh
@@ -273,6 +273,127 @@ shutdown_guest()
     fi
 }

+# shutdown_guest_async URI GUEST
+# Start a ACPI shutdown of GUEST on URI. This function returns after the command
+# was issued to libvirt to allow parallel shutdown.
+shutdown_guest_async()
+{
+    uri=$1
+    guest=$2
+
+    name=$(guest_name "$uri" "$guest")
+    eval_gettext "Starting shutdown on guest: \$name"
+    echo
+    retval run_virsh "$uri" shutdown "$guest" > /dev/null
+}
+
+# guest_count GUEST_LIST
+# Returns number of guests in GUEST_LIST
+guest_count()
+{
+    set -- $1
+    echo $#
+}
+
+# guest_first GUEST GUEST GUEST...
+# Returns the first guest in GUEST...
+guest_first()
+{
+    echo $1
+}
+
+# guest_remove GUEST GUEST_LIST
+# Remove GUEST from GUEST_LIST
+guest_remove()
+{
+    guest=$1
+    guests=$2
+
+    newguests=
+    for dom in $guests; do
+        if [ "$dom" != "$guest" ]; then
+            newguests="$newguests $dom"
+        fi
+    done
+
+    echo "$newguests"
+}
+
+# check_domains_shutdown URI GUESTS
+# check if shutdown is complete on guests in "GUESTS" and returns only
+# guests that are still shutting down
+check_domains_shutdown()
+{
+    uri=$1
+    guests=$2
+
+    guests_up=
+    for guest in $guests; do
+        guest_is_on "$uri" "$dom" 2>&1 > /dev/null || continue
+        if "$guest_running"; then
+            guests_up="$guests_up $guest"
+        fi
+    done
+    echo "$guests_up"
+}
+
+# print_domains_shutdown URI BEFORE AFTER
+# Checks for differences in the lists BEFORE and AFTER and prints
+# a shutdown complete notice for guests that have finished
+print_domains_shutdown()
+{
+    uri=$1
+    before=$2
+    after=$3
+
+    for guest in $before; do
+        found=false
+        for running in $after; do
+           if [ $guest = $running ]; then
+               found=true
+               break
+           fi
+        done
+
+        if ! "$found"; then
+            name=$(guest_name "$uri" "$guest")
+            eval_gettext "Shutdown of guest \$name complete."
+            echo
+        fi
+    done
+}
+
+# shutdown_guests_parallel URI GUESTS
+# Shutdown guests GUESTS on machine URI in parallel
+shutdown_guests_parallel()
+{
+    uri=$1
+    guests=$2
+
+    on_shutdown=
+    timeout=$SHUTDOWN_TIMEOUT
+    while [ -n "$on_shutdown" ] || [ -n "$guests" ]; do
+        while [ -n "$guests" ] &&
+              [ $(guest_count "$on_shutdown") -lt "$PARALLEL_SHUTDOWN" ]; do
+            guest=$(guest_first $guests)
+            guests=$(guest_remove "$guest" "$guests")
+            shutdown_guest_async "$uri" "$guest"
+            on_shutdown="$on_shutdown $guest"
+        done
+        sleep 1
+        timeout=$(($timeout - 1))
+        if [ $timeout -le 0 ]; then
+            eval_gettext "Timeout expired while shutting down domains"; echo
+            RETVAL=1
+            return
+        fi
+        on_shutdown_prev=$on_shutdown
+        on_shutdown=$(check_domains_shutdown "$uri" "$on_shutdown")
+        print_domains_shutdown "$uri" "$on_shutdown_prev" "$on_shutdown"
+    done
+
+}
+
 # stop
 # Shutdown or save guests on the configured uris
 stop() {
@@ -357,13 +478,18 @@ stop() {
             eval_gettext "Shutting down guests on \$uri URI..."; echo
         fi

-        for guest in $list; do
-            if "$suspending"; then
-                suspend_guest "$uri" "$guest"
-            else
-                shutdown_guest "$uri" "$guest"
-            fi
-        done
+        if [ "$PARALLEL_SHUTDOWN" -gt 1 ] &&
+           ! "$suspending"; then
+            shutdown_guests_parallel "$uri" "$list"
+        else
+            for guest in $list; do
+                if "$suspending"; then
+                    suspend_guest "$uri" "$guest"
+                else
+                    shutdown_guest "$uri" "$guest"
+                fi
+            done
+        fi
     done <"$LISTFILE"

     rm -f "$VAR_SUBSYS_LIBVIRT_GUESTS"
diff --git a/tools/libvirt-guests.sysconf b/tools/libvirt-guests.sysconf
index 9b8b64f..e16af4f 100644
--- a/tools/libvirt-guests.sysconf
+++ b/tools/libvirt-guests.sysconf
@@ -10,7 +10,8 @@
 #           libvirtd
 #ON_BOOT=start

-# number of seconds to wait between each guest start
+# number of seconds to wait between each guest start. Set to 0 to allow parallel
+# startup.
 #START_DELAY=0

 # action taken on host shutdown
@@ -23,7 +24,12 @@
 #             value suitable for your guests.
 #ON_SHUTDOWN=suspend

-# number of seconds we're willing to wait for a guest to shut down
+# If set to non-zero, shutdown will suspend domains concurently. Number of domains
+# on shutdown at any time will not exceed number set in this variable.
+#PARALLEL_SHUTDOWN=0
+
+# number of seconds we're willing to wait for a guest to shut down. If parallel
+# shutdown is enabled, this timeout applies as a timeout for shutting down all guests.
 #SHUTDOWN_TIMEOUT=0

 # If non-zero, try to bypass the file system cache when saving and
-- 
1.7.3.4