From 52c12cbad6cd0ca35f892a0b07519b41326ea91e Mon Sep 17 00:00:00 2001
From: Joe Lawrence <joe.lawrence@redhat.com>
Date: Wed, 15 Nov 2017 10:44:42 -0500
Subject: [PATCH] kpatch: wait for livepatch transitions, poke stragglers

When loading a livepatch, wait for the patching transition to complete
within a reasonable timeframe, then poke any stalled tasks with a
signal.  If the transition is still taking too long, reverse the patch
and unload the livepatch.

When re-enabling a livepatch, do the same wait and signaling.  If the
expected time expires, disable the livepatch.

When unloading a livepatch, perform the wait/signaling, but only emit an
error message if the transition exceeds the time limit.

Signed-off-by: Joe Lawrence <joe.lawrence@redhat.com>
---
 kpatch/kpatch | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/kpatch/kpatch b/kpatch/kpatch
index 303c67a..f61cd7e 100755
--- a/kpatch/kpatch
+++ b/kpatch/kpatch
@@ -26,6 +26,8 @@
 INSTALLDIR=/var/lib/kpatch
 SCRIPTDIR="$(readlink -f "$(dirname "$(type -p "$0")")")"
 VERSION="0.4.0"
+POST_ENABLE_WAIT=5	# seconds
+POST_SIGNAL_WAIT=60	# seconds
 
 usage_cmd() {
 	printf '   %-20s\n      %s\n' "$1" "$2" >&2
@@ -241,6 +243,36 @@ signal_stalled_processes() {
 	fi
 }
 
+wait_for_patch_transition() {
+	local module="$1"
+	local i
+
+	in_transition "$module" || return 0
+
+	echo "waiting (up to $POST_ENABLE_WAIT seconds) for patch transition to complete..."
+	for (( i=0; i<POST_ENABLE_WAIT; i++ )); do
+		if ! in_transition "$module" ; then
+			echo "transition complete ($i seconds)"
+			return 0
+		fi
+		sleep 1s
+	done
+
+	echo "patch transition has stalled, signaling stalled process(es):"
+	signal_stalled_processes
+
+	echo "waiting (up to $POST_SIGNAL_WAIT seconds) for patch transition to complete..."
+	for (( i=0; i<POST_SIGNAL_WAIT; i++ )); do
+		if ! in_transition "$module" ; then
+			echo "transition complete ($i seconds)"
+			return 0
+		fi
+		sleep 1s
+	done
+
+	return 1
+}
+
 load_module () {
 	local module="$1"
 
@@ -266,6 +298,12 @@ load_module () {
 			if verify_module_checksum "$module"; then # same checksum
 				echo "module already loaded, re-enabling"
 				echo 1 > "${moddir}/enabled" || die "failed to re-enable module $modname"
+				if ! wait_for_patch_transition "$modname" ; then
+					echo "module $modname did not complete its transition, disabling..."
+					echo 0 > "${moddir}/enabled" || die "failed to disable module $modname"
+					wait_for_patch_transition "$modname"
+					die "error: failed to re-enable module $modname (transition stalled), patch disabled"
+				fi
 				return
 			else
 				die "error: cannot re-enable patch module $modname, cannot verify checksum match"
@@ -296,6 +334,12 @@ load_module () {
 		fi
 	done
 
+	if ! wait_for_patch_transition "$modname" ; then
+		echo "module $modname did not complete its transition, unloading..."
+		unload_module "$modname"
+		die "error: failed to load module $modname (transition stalled)"
+	fi
+
 	return 0
 }
 
@@ -309,6 +353,10 @@ unload_module () {
 		echo 0 > "$ENABLED" || die "can't disable $PATCH"
 	fi
 
+	if ! wait_for_patch_transition "$PATCH" ; then
+		die "error: failed to unload module $PATCH (transition stalled)"
+	fi
+
 	echo "unloading patch module: $PATCH"
 	# ignore any error here because rmmod can fail if the module used
 	# KPATCH_FORCE_UNSAFE.