Restart client after heartbeat failures
This commit is contained in:
1 parent
b6ca819ec4
commit
69d6fdf968
1 file changed
+38
-1
@@ -634,12 +634,49 @@ send_heartbeat() {
|
|||||||
rm -f "$tmp"
|
rm -f "$tmp"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
heartbeat_mark_ok() {
|
||||||
|
rm -f "${TMPDIR:-/tmp}/owrt-remote-heartbeat.fail"
|
||||||
|
}
|
||||||
|
|
||||||
|
heartbeat_self_heal() {
|
||||||
|
local fail_file restart_file fails now last_restart threshold cooldown
|
||||||
|
fail_file="${TMPDIR:-/tmp}/owrt-remote-heartbeat.fail"
|
||||||
|
restart_file="${TMPDIR:-/tmp}/owrt-remote-heartbeat.restart"
|
||||||
|
threshold=3
|
||||||
|
cooldown=300
|
||||||
|
now="$(date +%s 2>/dev/null || echo 0)"
|
||||||
|
fails=0
|
||||||
|
last_restart=0
|
||||||
|
|
||||||
|
[ -f "$fail_file" ] && fails="$(cat "$fail_file" 2>/dev/null || echo 0)"
|
||||||
|
fails="$(safe_num "$fails" 0)"
|
||||||
|
fails=$((fails + 1))
|
||||||
|
printf '%s\n' "$fails" >"$fail_file"
|
||||||
|
|
||||||
|
[ "$fails" -lt "$threshold" ] && return 0
|
||||||
|
[ -f "$restart_file" ] && last_restart="$(cat "$restart_file" 2>/dev/null || echo 0)"
|
||||||
|
last_restart="$(safe_num "$last_restart" 0)"
|
||||||
|
[ $((now - last_restart)) -lt "$cooldown" ] && return 0
|
||||||
|
|
||||||
|
printf '%s\n' "$now" >"$restart_file"
|
||||||
|
rm -f "$fail_file"
|
||||||
|
log "heartbeat failed ${threshold} times in a row, restarting owrt-remote"
|
||||||
|
(
|
||||||
|
sleep 1
|
||||||
|
/etc/init.d/owrt-remote restart >/dev/null 2>&1
|
||||||
|
) &
|
||||||
|
}
|
||||||
|
|
||||||
heartbeat_loop() {
|
heartbeat_loop() {
|
||||||
local interval
|
local interval
|
||||||
interval="$(safe_num "$(uci_get heartbeat_interval 30)" 30)"
|
interval="$(safe_num "$(uci_get heartbeat_interval 30)" 30)"
|
||||||
[ "$interval" -lt 10 ] && interval=10
|
[ "$interval" -lt 10 ] && interval=10
|
||||||
while :; do
|
while :; do
|
||||||
send_heartbeat >/dev/null 2>&1 || true
|
if send_heartbeat >/dev/null 2>&1; then
|
||||||
|
heartbeat_mark_ok
|
||||||
|
else
|
||||||
|
heartbeat_self_heal
|
||||||
|
fi
|
||||||
sleep "$interval"
|
sleep "$interval"
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user