Fix persistent log collector liveness and BusyBox lock waits
This commit is contained in:
1 parent
b62db40daf
commit
e738d6fa69
8 files changed
+846
-37
No files matched your search
@@ -8,7 +8,8 @@ start_service() {
|
|||||||
/bin/sh /usr/lib/owrt-remote/logs.sh migrate || logger -t owrt-remote 'WARN: старый RAM-журнал ещё не перенесён; см. свободное место Flash.'
|
/bin/sh /usr/lib/owrt-remote/logs.sh migrate || logger -t owrt-remote 'WARN: старый RAM-журнал ещё не перенесён; см. свободное место Flash.'
|
||||||
procd_open_instance journal
|
procd_open_instance journal
|
||||||
procd_set_param command /bin/sh /usr/lib/owrt-remote/logs.sh collect
|
procd_set_param command /bin/sh /usr/lib/owrt-remote/logs.sh collect
|
||||||
procd_set_param respawn 3600 5 5
|
# Retry at a bounded rate even after repeated logd/storage outages.
|
||||||
|
procd_set_param respawn 3600 10 0
|
||||||
procd_set_param stdout 1
|
procd_set_param stdout 1
|
||||||
procd_set_param stderr 1
|
procd_set_param stderr 1
|
||||||
procd_close_instance
|
procd_close_instance
|
||||||
|
|||||||
@@ -7,10 +7,20 @@ LOG_PART_LIMIT=256
|
|||||||
LOG_BUFFER_LIMIT=16384
|
LOG_BUFFER_LIMIT=16384
|
||||||
LOG_BATCH_LIMIT=16384
|
LOG_BATCH_LIMIT=16384
|
||||||
LOG_FLUSH_SECONDS=60
|
LOG_FLUSH_SECONDS=60
|
||||||
|
LOG_HEARTBEAT_SECONDS=300
|
||||||
LOG_FREE_KB=2048
|
LOG_FREE_KB=2048
|
||||||
|
|
||||||
log_clock() { cut -d. -f1 /proc/uptime; }
|
log_clock() { cut -d. -f1 /proc/uptime; }
|
||||||
log_epoch() { date +%s; }
|
log_epoch() { date +%s; }
|
||||||
|
log_pause() {
|
||||||
|
# Some OpenWrt BusyBox builds omit fractional sleep. Keep lock/reap deadlines
|
||||||
|
# in tenths, including the one-second fallback, instead of spinning on failure.
|
||||||
|
if [ "${LOG_PAUSE_TENTHS:-0}" != 10 ]; then
|
||||||
|
if sleep 0.1 2>/dev/null; then LOG_PAUSE_TENTHS=1; return 0; fi
|
||||||
|
LOG_PAUSE_TENTHS=10
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
}
|
||||||
log_self_pid() {
|
log_self_pid() {
|
||||||
# $$ is the parent shell's PID inside ash subshells/pipelines. Reading proc
|
# $$ is the parent shell's PID inside ash subshells/pipelines. Reading proc
|
||||||
# with the shell builtin yields the actual lock holder, including the writer.
|
# with the shell builtin yields the actual lock holder, including the writer.
|
||||||
@@ -105,8 +115,8 @@ log_lock() {
|
|||||||
[ "$attempt" -lt 20 ] || log_reap_lock "$RUN_DIR/lock" "$attempt" ;;
|
[ "$attempt" -lt 20 ] || log_reap_lock "$RUN_DIR/lock" "$attempt" ;;
|
||||||
*) kill -0 "$owner" 2>/dev/null || log_reap_lock "$RUN_DIR/lock" "$attempt" ;;
|
*) kill -0 "$owner" 2>/dev/null || log_reap_lock "$RUN_DIR/lock" "$attempt" ;;
|
||||||
esac
|
esac
|
||||||
attempt=$((attempt + 1)); [ "$attempt" -lt 100 ] || return 1
|
log_pause
|
||||||
sleep 0.1
|
attempt=$((attempt + LOG_PAUSE_TENTHS)); [ "$attempt" -lt 100 ] || { log_diag lock-timeout; return 1; }
|
||||||
done
|
done
|
||||||
log_self_pid >"$RUN_DIR/lock/pid"
|
log_self_pid >"$RUN_DIR/lock/pid"
|
||||||
# Another first writer may have selected storage while we waited.
|
# Another first writer may have selected storage while we waited.
|
||||||
@@ -139,8 +149,24 @@ log_reap_lock() {
|
|||||||
owner="$(cat "$1/pid" 2>/dev/null)"
|
owner="$(cat "$1/pid" 2>/dev/null)"
|
||||||
case "$owner" in
|
case "$owner" in
|
||||||
''|*[!0-9]*) rm -f "$1/pid" ;;
|
''|*[!0-9]*) rm -f "$1/pid" ;;
|
||||||
*) if kill -0 "$owner" 2>/dev/null; then rm -f "$1/reap/pid"; rmdir "$1/reap"; return 0; else rm -f "$1/pid"; fi ;;
|
*)
|
||||||
|
if { [ "$1" = "$RUN_DIR/collector-lock" ] && log_process_info "$owner" >/dev/null; } ||
|
||||||
|
{ [ "$1" != "$RUN_DIR/collector-lock" ] && kill -0 "$owner" 2>/dev/null; }; then
|
||||||
|
rm -f "$1/reap/pid"; rmdir "$1/reap"; return 0
|
||||||
|
else rm -f "$1/pid"; fi ;;
|
||||||
esac
|
esac
|
||||||
|
if [ "$1" = "$RUN_DIR/collector-lock" ] && [ -s "$1/reader" ]; then
|
||||||
|
local old_reader old_birth
|
||||||
|
read -r old_reader old_birth <"$1/reader"
|
||||||
|
if [ -n "$old_birth" ] && [ "$(log_process_info "$old_reader")" = "$old_birth" ]; then
|
||||||
|
kill "$old_reader" 2>/dev/null || true
|
||||||
|
local remaining
|
||||||
|
remaining=0
|
||||||
|
while [ "$(log_process_info "$old_reader")" = "$old_birth" ] && [ "$remaining" -lt 10 ]; do log_pause; remaining=$((remaining + LOG_PAUSE_TENTHS)); done
|
||||||
|
if [ "$(log_process_info "$old_reader")" = "$old_birth" ]; then kill -KILL "$old_reader" 2>/dev/null || true; fi
|
||||||
|
fi
|
||||||
|
rm -f "$1/reader"
|
||||||
|
fi
|
||||||
rm -f "$1/reap/pid"; rmdir "$1/reap"; rmdir "$1" 2>/dev/null || true
|
rm -f "$1/reap/pid"; rmdir "$1/reap"; rmdir "$1" 2>/dev/null || true
|
||||||
}
|
}
|
||||||
log_fallback_locked() {
|
log_fallback_locked() {
|
||||||
@@ -397,6 +423,7 @@ log_flush_locked() {
|
|||||||
TOTAL=$((TOTAL + bytes))
|
TOTAL=$((TOTAL + bytes))
|
||||||
log_checkpoint_locked && log_commit_locked || { log_warn_locked write-warning 'Flash: ошибка фиксации состояния журнала.'; return 1; }
|
log_checkpoint_locked && log_commit_locked || { log_warn_locked write-warning 'Flash: ошибка фиксации состояния журнала.'; return 1; }
|
||||||
: >"$RUN_DIR/pending"
|
: >"$RUN_DIR/pending"
|
||||||
|
printf '%s\n' "$(log_epoch)" >"$RUN_DIR/commit-health"
|
||||||
rm -f "$RUN_DIR/inflight"
|
rm -f "$RUN_DIR/inflight"
|
||||||
rm -f "$RUN_DIR/write-warning" "$RUN_DIR/queue-warning" "$RUN_DIR/buffer-warning"
|
rm -f "$RUN_DIR/write-warning" "$RUN_DIR/queue-warning" "$RUN_DIR/buffer-warning"
|
||||||
dropped="$(cat "$RUN_DIR/dropped" 2>/dev/null)"; dropped="${dropped:-0}"
|
dropped="$(cat "$RUN_DIR/dropped" 2>/dev/null)"; dropped="${dropped:-0}"
|
||||||
@@ -406,6 +433,7 @@ log_flush_locked() {
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
log_clock >"$RUN_DIR/last-flush"
|
log_clock >"$RUN_DIR/last-flush"
|
||||||
|
printf '%s %s\n' "$(log_epoch)" "$(cat "$RUN_DIR/last-flush")" >"$RUN_DIR/flush-health.new" && mv -f "$RUN_DIR/flush-health.new" "$RUN_DIR/flush-health"
|
||||||
}
|
}
|
||||||
log_open() {
|
log_open() {
|
||||||
log_lock || return 1
|
log_lock || return 1
|
||||||
@@ -439,14 +467,22 @@ log_redact() {
|
|||||||
}'
|
}'
|
||||||
}
|
}
|
||||||
log_new_fact_locked() {
|
log_new_fact_locked() {
|
||||||
local key now heartbeat new_fact
|
local key now heartbeat new_fact last
|
||||||
# Compare event text without syslog timestamp, facility, PID or Xray time.
|
# Compare event text without syslog timestamp, facility, PID or Xray time.
|
||||||
key="$(printf '%s\n' "$1" | tr '\t' ' ' | sed -e 's/^\[[^]]*\] *//' -e 's/.*owrt-remote\(\[[0-9][0-9]*\]\)\{0,1\}: *//' -e 's/^[0-9][0-9]*\/[0-9/:. ]* *//' -e 's/ *(\([0-9][0-9]*\)\/[0-9][0-9]*)//')"
|
key="$(printf '%s\n' "$1" | tr '\t' ' ' | sed -e 's/^\[[^]]*\] *//' -e 's/.*owrt-remote\(\[[0-9][0-9]*\]\)\{0,1\}: *//' -e 's/^[0-9][0-9]*\/[0-9/:. ]* *//' -e 's/ *(\([0-9][0-9]*\)\/[0-9][0-9]*)//')"
|
||||||
heartbeat=0
|
heartbeat=0
|
||||||
case "$key" in
|
case "$key" in
|
||||||
'Heartbeat: OK;'*)
|
'Heartbeat: OK;'*)
|
||||||
heartbeat=1
|
heartbeat=1
|
||||||
if [ "$(cat "$RUN_DIR/heartbeat-ok" 2>/dev/null)" = "$key" ]; then return 1; fi
|
last=0
|
||||||
|
[ ! -f "$RUN_DIR/heartbeat-stored" ] || read -r last <"$RUN_DIR/heartbeat-stored"
|
||||||
|
case "$last" in ''|*[!0-9]*) last=0 ;; esac
|
||||||
|
now="$(log_clock)"
|
||||||
|
# Keep one liveness sample every five minutes, plus every change/recovery.
|
||||||
|
# Input freshness is tracked in RAM even when a duplicate is suppressed.
|
||||||
|
if [ "$(cat "$RUN_DIR/heartbeat-ok" 2>/dev/null)" = "$key" ] &&
|
||||||
|
[ "$now" -ge "$last" ] && [ "$((now - last))" -lt "$LOG_HEARTBEAT_SECONDS" ]; then return 1; fi
|
||||||
|
printf '%s\n' "$now" >"$RUN_DIR/heartbeat-stored"
|
||||||
printf '%s\n' "$key" >"$RUN_DIR/heartbeat-ok"
|
printf '%s\n' "$key" >"$RUN_DIR/heartbeat-ok"
|
||||||
# A recovery lets the same failure appear again; keys are fixed-size hashes.
|
# A recovery lets the same failure appear again; keys are fixed-size hashes.
|
||||||
: >"$RUN_DIR/facts" ;;
|
: >"$RUN_DIR/facts" ;;
|
||||||
@@ -479,24 +515,50 @@ log_new_fact_locked() {
|
|||||||
[ "$new_fact" = 0 ] || return 1
|
[ "$new_fact" = 0 ] || return 1
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
log_event_health_locked() {
|
||||||
|
local epoch uptime accepted_epoch accepted_up accepted suppressed failed extra
|
||||||
|
accepted_epoch=0; accepted_up=0; accepted=0; suppressed=0; failed=0
|
||||||
|
if [ -s "$RUN_DIR/event-health" ]; then
|
||||||
|
read -r epoch uptime accepted_epoch accepted_up accepted suppressed failed extra <"$RUN_DIR/event-health"
|
||||||
|
case "$accepted_epoch:$accepted_up:$accepted:$suppressed:$failed" in *[!0-9:]*|:*|*:) accepted_epoch=0; accepted_up=0; accepted=0; suppressed=0; failed=0 ;; esac
|
||||||
|
fi
|
||||||
|
epoch="$(log_epoch)"; uptime="$(log_clock)"
|
||||||
|
case "$1" in
|
||||||
|
written) accepted_epoch="$epoch"; accepted_up="$uptime"; [ "$accepted" -ge 2147483647 ] || accepted=$((accepted + 1)) ;;
|
||||||
|
suppressed) [ "$suppressed" -ge 2147483647 ] || suppressed=$((suppressed + 1)) ;;
|
||||||
|
failed) [ "$failed" -ge 2147483647 ] || failed=$((failed + 1)) ;;
|
||||||
|
esac
|
||||||
|
printf '%s %s %s %s %s %s %s\n' "$epoch" "$uptime" "$accepted_epoch" "$accepted_up" "$accepted" "$suppressed" "$failed" >"$RUN_DIR/event-health.new" &&
|
||||||
|
mv -f "$RUN_DIR/event-health.new" "$RUN_DIR/event-health"
|
||||||
|
}
|
||||||
log_append() {
|
log_append() {
|
||||||
local line size bytes dropped
|
local line size bytes dropped outcome
|
||||||
log_redact "${1:-OK}" | while IFS= read -r line; do
|
log_redact "${1:-OK}" | while IFS= read -r line; do
|
||||||
log_open || exit 1
|
log_open || { log_diag append-open-failed; exit 1; }
|
||||||
log_flush_locked || true
|
if ! log_flush_locked; then log_diag flush-failed; fi
|
||||||
[ -f "$RUN_DIR/facts" ] || : >"$RUN_DIR/facts"
|
[ -f "$RUN_DIR/facts" ] || : >"$RUN_DIR/facts"
|
||||||
|
outcome=suppressed
|
||||||
if log_new_fact_locked "$line"; then
|
if log_new_fact_locked "$line"; then
|
||||||
size="$(wc -c <"$RUN_DIR/pending")"; bytes="$(printf '%s\n' "$line" | wc -c)"
|
size="$(wc -c <"$RUN_DIR/pending")"; bytes="$(printf '%s\n' "$line" | wc -c)"
|
||||||
# Reserve room for a single overflow warning, too.
|
# Reserve room for a single overflow warning, too.
|
||||||
if [ "$((size + bytes))" -le "$((LOG_BUFFER_LIMIT - 1024))" ]; then
|
if [ "$((size + bytes))" -le "$((LOG_BUFFER_LIMIT - 1024))" ]; then
|
||||||
printf '%s\n' "$line" >>"$RUN_DIR/pending"
|
if ! printf '%s\n' "$line" >>"$RUN_DIR/pending"; then
|
||||||
|
# A failed append must be eligible for retry, not hidden by dedup.
|
||||||
|
rm -f "$RUN_DIR/heartbeat-ok" "$RUN_DIR/facts"
|
||||||
|
log_event_health_locked failed; log_diag append-write-failed
|
||||||
|
log_close; exit 1
|
||||||
|
fi
|
||||||
|
outcome=written
|
||||||
else
|
else
|
||||||
|
outcome=failed
|
||||||
|
rm -f "$RUN_DIR/heartbeat-ok" "$RUN_DIR/facts"
|
||||||
dropped="$(cat "$RUN_DIR/dropped" 2>/dev/null)"; dropped="${dropped:-0}"
|
dropped="$(cat "$RUN_DIR/dropped" 2>/dev/null)"; dropped="${dropped:-0}"
|
||||||
[ "$dropped" -ge 2147483647 ] || dropped=$((dropped + 1))
|
[ "$dropped" -ge 2147483647 ] || dropped=$((dropped + 1))
|
||||||
printf '%s\n' "$dropped" >"$RUN_DIR/dropped"
|
printf '%s\n' "$dropped" >"$RUN_DIR/dropped"
|
||||||
log_warn_locked buffer-warning 'Буфер логов заполнен: новые события сверх лимита пропускаются, неподтверждённый журнал сохранён.'
|
log_warn_locked buffer-warning 'Буфер логов заполнен: новые события сверх лимита пропускаются, неподтверждённый журнал сохранён.'
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
log_event_health_locked "$outcome" || { log_diag health-write-failed; log_close; exit 1; }
|
||||||
log_close
|
log_close
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
@@ -542,7 +604,7 @@ log_reset_locked() {
|
|||||||
log_checkpoint_locked && log_commit_locked || return 1
|
log_checkpoint_locked && log_commit_locked || return 1
|
||||||
rm -f "$LOG_DIR"/part.* "$LOG_DIR/damaged.log"
|
rm -f "$LOG_DIR"/part.* "$LOG_DIR/damaged.log"
|
||||||
: >"$RUN_DIR/pending"
|
: >"$RUN_DIR/pending"
|
||||||
rm -f "$RUN_DIR/ack" "$RUN_DIR/facts" "$RUN_DIR/heartbeat-ok" "$RUN_DIR/dropped" "$RUN_DIR/buffer-warning" "$RUN_DIR/inflight"
|
rm -f "$RUN_DIR/ack" "$RUN_DIR/facts" "$RUN_DIR/heartbeat-ok" "$RUN_DIR/heartbeat-stored" "$RUN_DIR/dropped" "$RUN_DIR/buffer-warning" "$RUN_DIR/inflight"
|
||||||
log_clock >"$RUN_DIR/last-flush"
|
log_clock >"$RUN_DIR/last-flush"
|
||||||
}
|
}
|
||||||
log_clear() { log_open || return 1; log_reset_locked; result=$?; log_close; return "$result"; }
|
log_clear() { log_open || return 1; log_reset_locked; result=$?; log_close; return "$result"; }
|
||||||
@@ -592,10 +654,16 @@ log_ack() {
|
|||||||
fi
|
fi
|
||||||
log_close
|
log_close
|
||||||
}
|
}
|
||||||
log_flush() { log_open || return 1; log_flush_locked "${1:-}"; result=$?; log_close; return "$result"; }
|
log_flush() {
|
||||||
|
log_open || return 1
|
||||||
|
log_flush_locked "${1:-}"; result=$?
|
||||||
|
[ "$result" = 0 ] || log_diag flush-failed
|
||||||
|
log_close; return "$result"
|
||||||
|
}
|
||||||
log_status() {
|
log_status() {
|
||||||
log_open || return 1
|
log_open || return 1
|
||||||
printf 'storage=%s\npersistent=%s\nruntime=%s\nlimit_bytes=%s\nmax_parts=%s\nbuffer_limit_bytes=%s\nflush_seconds=%s\njournal_id=%s\ncommitted_bytes=%s\nack_bytes=%s\npending_bytes=%s\n' "$LOG_DIR" "$PERSISTENT" "$RUN_DIR" "$LOG_LIMIT" "$LOG_PART_LIMIT" "$LOG_BUFFER_LIMIT" "$LOG_FLUSH_SECONDS" "$JOURNAL" "$TOTAL" "$ACK" "$(wc -c <"$RUN_DIR/pending")"
|
printf 'storage=%s\npersistent=%s\nruntime=%s\nlimit_bytes=%s\nmax_parts=%s\nbuffer_limit_bytes=%s\nflush_seconds=%s\njournal_id=%s\ncommitted_bytes=%s\nack_bytes=%s\npending_bytes=%s\n' "$LOG_DIR" "$PERSISTENT" "$RUN_DIR" "$LOG_LIMIT" "$LOG_PART_LIMIT" "$LOG_BUFFER_LIMIT" "$LOG_FLUSH_SECONDS" "$JOURNAL" "$TOTAL" "$ACK" "$(wc -c <"$RUN_DIR/pending")"
|
||||||
|
log_health_status
|
||||||
log_close
|
log_close
|
||||||
}
|
}
|
||||||
log_migrate() {
|
log_migrate() {
|
||||||
@@ -687,36 +755,183 @@ log_sync_loop() {
|
|||||||
sleep 3
|
sleep 3
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
# Diagnostics contain fixed reason codes only, never a line, URL or credentials.
|
||||||
|
# At most 4 KiB plus a 2 KiB compaction file, all on tmpfs. No syslog feedback.
|
||||||
|
log_diag() {
|
||||||
|
local code now last previous bytes
|
||||||
|
code="$1"; now="$(log_clock)"; last=0; previous=''
|
||||||
|
[ -d "$RUN_DIR" ] || return 0
|
||||||
|
mkdir "$RUN_DIR/diag-lock" 2>/dev/null || return 0
|
||||||
|
if [ -s "$RUN_DIR/diag-last" ]; then read -r last previous <"$RUN_DIR/diag-last"; fi
|
||||||
|
case "$last" in ''|*[!0-9]*) last=0 ;; esac
|
||||||
|
if [ "$code" != "$previous" ] || [ "$now" -lt "$last" ] || [ "$((now - last))" -ge 60 ]; then
|
||||||
|
bytes=0; [ ! -f "$RUN_DIR/diagnostics" ] || bytes="$(wc -c <"$RUN_DIR/diagnostics")"
|
||||||
|
if [ "$bytes" -gt 3900 ]; then
|
||||||
|
tail -c 2048 "$RUN_DIR/diagnostics" >"$RUN_DIR/diagnostics.new" && mv -f "$RUN_DIR/diagnostics.new" "$RUN_DIR/diagnostics"
|
||||||
|
fi
|
||||||
|
printf '%s %s\n' "$(log_epoch)" "$code" >>"$RUN_DIR/diagnostics"
|
||||||
|
printf '%s %s\n' "$now" "$code" >"$RUN_DIR/diag-last"
|
||||||
|
fi
|
||||||
|
rmdir "$RUN_DIR/diag-lock" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
log_process_info() {
|
||||||
|
local stat
|
||||||
|
case "$1" in ''|*[!0-9]*) return 1 ;; esac
|
||||||
|
kill -0 "$1" 2>/dev/null || return 1
|
||||||
|
if ! read -r stat <"/proc/$1/stat" 2>/dev/null; then return 1; fi
|
||||||
|
stat="${stat##*) }"
|
||||||
|
set -- $stat
|
||||||
|
[ "$#" -ge 20 ] || return 1
|
||||||
|
case "$1" in Z|X) return 1 ;; esac
|
||||||
|
shift 19
|
||||||
|
printf '%s\n' "$1"
|
||||||
|
}
|
||||||
|
log_collector_lock() {
|
||||||
|
local owner attempt
|
||||||
|
attempt=0
|
||||||
|
while ! mkdir "$RUN_DIR/collector-lock" 2>/dev/null; do
|
||||||
|
owner="$(cat "$RUN_DIR/collector-lock/pid" 2>/dev/null)"
|
||||||
|
case "$owner" in
|
||||||
|
''|*[!0-9]*) [ "$attempt" -lt 20 ] || log_reap_lock "$RUN_DIR/collector-lock" "$attempt" ;;
|
||||||
|
*)
|
||||||
|
if log_process_info "$owner" >/dev/null; then return 2; fi
|
||||||
|
# Reap a logread orphan only if its PID AND kernel start time match.
|
||||||
|
log_reap_lock "$RUN_DIR/collector-lock" "$attempt" ;;
|
||||||
|
esac
|
||||||
|
log_pause
|
||||||
|
attempt=$((attempt + LOG_PAUSE_TENTHS)); [ "$attempt" -lt 40 ] || { log_diag collector-lock-timeout; return 1; }
|
||||||
|
done
|
||||||
|
log_self_pid >"$RUN_DIR/collector-lock/pid"
|
||||||
|
}
|
||||||
|
log_collector_progress() {
|
||||||
|
printf '%s %s %s %s %s %s\n' "$collector" "$(log_epoch)" "$(log_clock)" "$received" "$append_errors" "$flush_errors" >"$RUN_DIR/collector-progress.new" &&
|
||||||
|
mv -f "$RUN_DIR/collector-progress.new" "$RUN_DIR/collector-progress"
|
||||||
|
}
|
||||||
|
log_collector_cleanup() {
|
||||||
|
local code owner attempt
|
||||||
|
code="$1"; trap - EXIT INT TERM
|
||||||
|
exec 3>&-
|
||||||
|
if [ -n "$reader" ]; then
|
||||||
|
kill "$reader" 2>/dev/null || true
|
||||||
|
# Never leave an uninterruptible wait in the procd shutdown path.
|
||||||
|
attempt=0
|
||||||
|
while log_process_info "$reader" >/dev/null && [ "$attempt" -lt 10 ]; do log_pause; attempt=$((attempt + LOG_PAUSE_TENTHS)); done
|
||||||
|
if log_process_info "$reader" >/dev/null; then kill -KILL "$reader" 2>/dev/null || true; fi
|
||||||
|
wait "$reader" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "$fifo"
|
||||||
|
owner="$(cat "$RUN_DIR/collector-lock/pid" 2>/dev/null)"
|
||||||
|
if [ "$owner" = "$collector" ]; then
|
||||||
|
# Flush while still owning the collector lock. The successor never overlaps.
|
||||||
|
(log_flush force) || true
|
||||||
|
rm -f "$RUN_DIR/collector-lock/reader" "$RUN_DIR/collector-lock/pid"
|
||||||
|
rmdir "$RUN_DIR/collector-lock" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
exit "$code"
|
||||||
|
}
|
||||||
|
log_health_status() {
|
||||||
|
local owner reader birth now state last_input input_up last_event event_up accepted suppressed failed extra flush_epoch flush_up received errors flush_errors progress_owner
|
||||||
|
owner=''; reader=''; birth=''; last_input=0; input_up=0; last_event=0; event_up=0; accepted=0; suppressed=0; failed=0; flush_epoch=0; flush_up=0; received=0; errors=0; flush_errors=0
|
||||||
|
[ ! -s "$RUN_DIR/collector-lock/pid" ] || read -r owner <"$RUN_DIR/collector-lock/pid"
|
||||||
|
[ ! -s "$RUN_DIR/collector-lock/reader" ] || read -r reader birth <"$RUN_DIR/collector-lock/reader"
|
||||||
|
[ ! -s "$RUN_DIR/event-health" ] || read -r last_input input_up last_event event_up accepted suppressed failed extra <"$RUN_DIR/event-health"
|
||||||
|
[ ! -s "$RUN_DIR/flush-health" ] || read -r flush_epoch flush_up <"$RUN_DIR/flush-health"
|
||||||
|
if [ -s "$RUN_DIR/collector-progress" ]; then
|
||||||
|
read -r progress_owner extra extra received errors flush_errors <"$RUN_DIR/collector-progress"
|
||||||
|
fi
|
||||||
|
now="$(log_clock)"; state=stopped
|
||||||
|
if log_process_info "$owner" >/dev/null; then
|
||||||
|
state=starting
|
||||||
|
if [ -n "$birth" ] && [ "$(log_process_info "$reader")" = "$birth" ]; then
|
||||||
|
state=waiting-for-events
|
||||||
|
if [ -s "$RUN_DIR/collector-progress" ]; then
|
||||||
|
read -r progress_owner extra extra received errors flush_errors <"$RUN_DIR/collector-progress"
|
||||||
|
if [ "$progress_owner" = "$owner" ] && [ "$received" -gt 0 ]; then
|
||||||
|
state=idle-unverified
|
||||||
|
[ "$input_up" -gt 0 ] && [ "$now" -ge "$input_up" ] && [ "$((now - input_up))" -le 120 ] && state=receiving
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
else state=reader-stopped; fi
|
||||||
|
fi
|
||||||
|
printf 'collector_state=%s\ncollector_pid=%s\nreader_pid=%s\nreceived_lines=%s\nlast_input_epoch=%s\nlast_event_epoch=%s\naccepted_events=%s\nsuppressed_events=%s\nappend_errors=%s\nflush_errors=%s\nlast_flush_epoch=%s\nheartbeat_sample_seconds=%s\n' "$state" "$owner" "$reader" "$received" "$last_input" "$last_event" "$accepted" "$suppressed" "$errors" "$flush_errors" "$flush_epoch" "$LOG_HEARTBEAT_SECONDS"
|
||||||
|
if [ "$input_up" -gt 0 ] && [ "$now" -ge "$input_up" ]; then printf 'last_input_age_seconds=%s\n' "$((now - input_up))"; fi
|
||||||
|
if [ "$event_up" -gt 0 ] && [ "$now" -ge "$event_up" ]; then printf 'last_event_age_seconds=%s\n' "$((now - event_up))"; fi
|
||||||
|
printf 'rejected_events=%s\nlast_commit_epoch=%s\n' "$failed" "$(cat "$RUN_DIR/commit-health" 2>/dev/null || printf 0)"
|
||||||
|
printf 'runtime_bytes=%s\n' "$(du -sk "$RUN_DIR" 2>/dev/null | awk '{print $1 * 1024}')"
|
||||||
|
for extra in "$owner" "$reader"; do
|
||||||
|
case "$extra" in ''|*[!0-9]*) continue ;; esac
|
||||||
|
if [ -r "/proc/$extra/status" ]; then
|
||||||
|
awk -v pid="$extra" '/^VmRSS:/ {print "process_" pid "_rss_kib=" $2}' "/proc/$extra/status"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
log_collect_line() {
|
||||||
|
# Explicit stdin is essential: awk/redaction must never inherit the live FIFO
|
||||||
|
# and drain subsequent syslog records behind the collector's read loop.
|
||||||
|
printf '%s\n' "$1" | log_append
|
||||||
|
}
|
||||||
log_collect() {
|
log_collect() {
|
||||||
local owner
|
# Lifecycle variables belong to this collect process: the EXIT trap runs after
|
||||||
|
# this function returns, when function-local variables have gone out of scope.
|
||||||
log_open || return 1
|
log_open || return 1
|
||||||
log_close
|
log_close
|
||||||
if ! mkdir "$RUN_DIR/collector-lock" 2>/dev/null; then
|
reader=''; birth=''
|
||||||
owner="$(cat "$RUN_DIR/collector-lock/pid" 2>/dev/null)"
|
log_collector_lock; rc=$?
|
||||||
case "$owner" in ''|*[!0-9]*) ;; *) kill -0 "$owner" 2>/dev/null && return 0 ;; esac
|
[ "$rc" != 2 ] || return 0
|
||||||
rm -f "$RUN_DIR/collector-lock/pid"; rmdir "$RUN_DIR/collector-lock" 2>/dev/null || return 1
|
[ "$rc" = 0 ] || return 1
|
||||||
mkdir "$RUN_DIR/collector-lock" || return 1
|
collector="$(cat "$RUN_DIR/collector-lock/pid")"
|
||||||
fi
|
fifo="$RUN_DIR/stream.$collector"
|
||||||
log_self_pid >"$RUN_DIR/collector-lock/pid"
|
received=0; append_errors=0; flush_errors=0; last_tick=0; fragment=''
|
||||||
rm -f "$RUN_DIR"/stream.*
|
trap 'log_collector_cleanup "$?"' EXIT
|
||||||
fifo="$RUN_DIR/stream.$$"; reader=''; writer=''; timer=''
|
|
||||||
trap 'trap - EXIT; for child in "$reader" "$writer" "$timer"; do [ -z "$child" ] || kill "$child" 2>/dev/null || true; done; wait; rm -f "$fifo" "$RUN_DIR/collector-lock/pid"; rmdir "$RUN_DIR/collector-lock" 2>/dev/null || true; (log_flush force) || true' EXIT
|
|
||||||
trap 'exit 0' INT TERM
|
trap 'exit 0' INT TERM
|
||||||
mkfifo "$fifo" || return 1
|
# Only the owner may remove stale runtime FIFOs. Persistent chunks are untouched.
|
||||||
logread -f -e 'owrt-remote\(\[[0-9][0-9]*\]\)\{0,1\}:' >"$fifo" 2>/dev/null & reader=$!
|
rm -f "$RUN_DIR"/stream.*
|
||||||
while IFS= read -r line; do printf '%s\n' "$line" | log_append; done <"$fifo" & writer=$!
|
rmdir "$RUN_DIR/diag-lock" 2>/dev/null || true
|
||||||
|
mkfifo "$fifo" || { log_diag fifo-create-failed; return 1; }
|
||||||
|
# A single parent reads and appends. No background writer can silently die or
|
||||||
|
# consume signals while procd still observes a live supervising shell.
|
||||||
|
exec 3<>"$fifo" || { log_diag fifo-open-failed; return 1; }
|
||||||
|
logread -f -l 64 -e 'owrt-remote\(\[[0-9][0-9]*\]\)\{0,1\}:' 3>&- >"$fifo" 2>/dev/null & reader=$!
|
||||||
|
birth="$(log_process_info "$reader")"
|
||||||
|
printf '%s %s\n' "$reader" "$birth" >"$RUN_DIR/collector-lock/reader"
|
||||||
|
log_collector_progress || return 1
|
||||||
while :; do
|
while :; do
|
||||||
# Lock-operation traps must not replace the collector's child/FIFO cleanup.
|
line=''; rc=0
|
||||||
(log_flush) || true
|
# BusyBox ash supports timed, bounded reads. The FIFO itself stays on tmpfs.
|
||||||
sleep 5 & timer=$!
|
IFS= read -r -t 5 -n 4096 line <&3 || rc=$?
|
||||||
wait "$timer" || return 0
|
if [ "$rc" = 0 ] || [ -n "$line" ]; then
|
||||||
kill -0 "$reader" 2>/dev/null && kill -0 "$writer" 2>/dev/null || return 1
|
# A logd line is <= its fixed line buffer. Drop an oversized fragment
|
||||||
|
# instead of growing an unbounded shell string on malformed input.
|
||||||
|
if [ "$rc" != 0 ] || [ "${#line}" -ge 4096 ]; then
|
||||||
|
fragment=1; log_diag fragmented-input
|
||||||
|
elif [ -n "$fragment" ]; then
|
||||||
|
fragment=''
|
||||||
|
else
|
||||||
|
received=$((received + 1))
|
||||||
|
log_collector_progress || return 1
|
||||||
|
if ! (log_collect_line "$line"); then
|
||||||
|
append_errors=$((append_errors + 1)); log_collector_progress || true
|
||||||
|
log_diag collector-append-failed
|
||||||
|
# Do not pretend to be healthy after an unhandled write failure.
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
now="$(log_clock)"
|
||||||
|
if [ "$now" -lt "$last_tick" ] || [ "$((now - last_tick))" -ge 5 ]; then
|
||||||
|
last_tick="$now"
|
||||||
|
if ! (log_flush); then flush_errors=$((flush_errors + 1)); log_diag collector-flush-failed; fi
|
||||||
|
log_collector_progress || return 1
|
||||||
|
fi
|
||||||
|
if ! log_process_info "$reader" >/dev/null; then log_diag reader-exited; return 1; fi
|
||||||
|
# A timeout with no input is normal idle, not a failure or proof of health.
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
check) log_open && log_close ;;
|
check) log_open && log_close ;;
|
||||||
status) log_status ;;
|
status) log_status ;;
|
||||||
|
diagnostics) log_init && { [ ! -f "$RUN_DIR/diagnostics" ] || cat "$RUN_DIR/diagnostics"; } ;;
|
||||||
migrate) log_migrate ;;
|
migrate) log_migrate ;;
|
||||||
flush) log_flush "${2:-}" ;;
|
flush) log_flush "${2:-}" ;;
|
||||||
collect) log_collect ;;
|
collect) log_collect ;;
|
||||||
@@ -729,5 +944,5 @@ case "${1:-}" in
|
|||||||
event)
|
event)
|
||||||
case "${3:-ok}" in bad|error) level=ERROR ;; warn) level=WARN ;; *) level=OK ;; esac
|
case "${3:-ok}" in bad|error) level=ERROR ;; warn) level=WARN ;; *) level=OK ;; esac
|
||||||
{ printf '%s owrt-remote: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$2"; cat; } | log_append "$level" ;;
|
{ printf '%s owrt-remote: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$2"; cat; } | log_append "$level" ;;
|
||||||
*) printf 'Usage: logs.sh collect|read [live]|clear|append|event MESSAGE|status|flush [force]\n' >&2; exit 2 ;;
|
*) printf 'Usage: logs.sh collect|read [live]|clear|append|event MESSAGE|status|diagnostics|flush [force]\n' >&2; exit 2 ;;
|
||||||
esac
|
esac
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
# Journal collector liveness fix (local v117)
|
||||||
|
|
||||||
|
## What the reported evidence does and does not prove
|
||||||
|
|
||||||
|
The previous `log_new_fact_locked()` normalized syslog timestamps and stored the
|
||||||
|
first successful `Heartbeat: OK; ...` in `heartbeat-ok`. The identical success
|
||||||
|
was then suppressed **without an expiry**, even after hours. A 136-byte chunk,
|
||||||
|
`ack=total`, empty pending and live logread therefore do not prove a stalled
|
||||||
|
collector. They also do not establish a memory leak. A shell syntax check and
|
||||||
|
`persistent=1` prove neither event delivery nor process health.
|
||||||
|
|
||||||
|
The old supervisor checked only `kill -0` for its background writer and reader;
|
||||||
|
it ignored `log_append` failure. This could conceal write/lock errors while the
|
||||||
|
supervisor remained alive. These failure paths are corrected independently of
|
||||||
|
the deduplication issue. No claim is made that they caused this particular
|
||||||
|
AX3000T report without measurements from that device.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- One supervising collect shell consumes its private FIFO with BusyBox ash
|
||||||
|
timed reads. There is one logread child, no separate persistent writer shell
|
||||||
|
and no five-second sleep child. At most one 4096-byte input fragment is kept
|
||||||
|
in a shell variable. Oversized/fragmented input is diagnosed and discarded;
|
||||||
|
it cannot grow a shell string without a bound.
|
||||||
|
- Each complete line is handed to `log_append` over an explicit short pipe,
|
||||||
|
isolated from the live FIFO. Append failure is checked, recorded and causes
|
||||||
|
a nonzero exit so procd can recover. Lock-operation traps run in subshells
|
||||||
|
and cannot replace the collector's cleanup trap.
|
||||||
|
- Normal stop terminates/reaps logread, closes/removes the FIFO, flushes a final
|
||||||
|
bounded batch and releases only its own collector lock. SIGKILL recovery
|
||||||
|
recognizes a stale owner and terminates its recorded orphan reader only if
|
||||||
|
PID **and kernel start time** match. Stale-lock reaping is serialized.
|
||||||
|
- procd's journal instance retries every 10 seconds without exhausting a
|
||||||
|
five-crash budget. The archive instance and networking services are unchanged.
|
||||||
|
- Builds without fractional BusyBox sleep use whole-second waits. Lock and
|
||||||
|
child cleanup deadlines account for that duration; unsupported `sleep 0.1`
|
||||||
|
cannot produce a fast loop, repeated errors or a 100-second lock wait.
|
||||||
|
- A successful heartbeat is saved at most once per 300 seconds when unchanged,
|
||||||
|
plus changes and recovery. Intervening duplicates update small RAM health
|
||||||
|
metadata only. Flash flush is still once per 60 seconds, <=16 KiB per batch,
|
||||||
|
with existing size/age/part limits. No per-heartbeat Flash write is introduced.
|
||||||
|
- Starting logread requests only a 64-message system-ring tail plus follow,
|
||||||
|
to recover a short interruption. Existing fact dedup and bounded pending
|
||||||
|
apply; this is not a guarantee of lossless collection during a long outage.
|
||||||
|
- `status` adds input and accepted-event times, counters, collector state,
|
||||||
|
append/flush errors, the last completed batch/commit, allocated runtime bytes
|
||||||
|
and RSS for the collector/reader when proc provides it. `receiving` means
|
||||||
|
recently processed input; `idle-unverified` explicitly does not assert health.
|
||||||
|
- `diagnostics` is a private RAM-only <=4 KiB reason-code history (plus <=2 KiB
|
||||||
|
compaction), with duplicate-rate limiting. It records FIFO creation/open,
|
||||||
|
fragmented input, lock timeout, append, flush and reader exit failures without
|
||||||
|
storing event text, keys or command lines. It is not sent back through logger,
|
||||||
|
avoiding diagnostic feedback loops.
|
||||||
|
- Existing validated persistent storage, immutable checksummed chunks, ack
|
||||||
|
protocol and limited RAM fallback remain. Runtime health is lost on reboot;
|
||||||
|
retained journal/ack data are not cleared by a collector restart.
|
||||||
|
|
||||||
|
## Local install and rollback (Windows PowerShell 5.1/11)
|
||||||
|
|
||||||
|
The dedicated installer replaces **only** `logs.sh` and `owrt-remote-logs`.
|
||||||
|
It backs up code/service state, preserves journal chunks and UCI configuration,
|
||||||
|
restarts only the log service, verifies a real logger marker through the FIFO
|
||||||
|
and performs one explicit deployment flush. Failure restores the previous code.
|
||||||
|
It does not remove `xray.zip`, update Xray, restart the tunnel, modify Podkop,
|
||||||
|
WAN/DNS or policy routing. VPS files and hosting URLs are untouched.
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "C:\Users\k.zolotarev95\Documents\openwrt overlay\vps\deploy-log-collector-fix.ps1" -Router "root@192.168.2.1" -Action Install
|
||||||
|
```
|
||||||
|
|
||||||
|
Retained backup: `/root/owrt-remote-backup-before-log-collector-v117`.
|
||||||
|
Rollback leaves journal data intact:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "C:\Users\k.zolotarev95\Documents\openwrt overlay\vps\deploy-log-collector-fix.ps1" -Router "root@192.168.2.1" -Action Rollback
|
||||||
|
```
|
||||||
|
|
||||||
|
After installation compare two samples a minute apart. `last_input_epoch` and
|
||||||
|
`suppressed_events` should advance on stable heartbeat. An unchanged heartbeat
|
||||||
|
becomes a new retained sample after five minutes. A distinct marker is visible
|
||||||
|
immediately in read, then committed by the next normal flush (up to ~65 seconds).
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
ssh root@192.168.2.1 'logger -t owrt-remote collector-user-check; sleep 3; sh /usr/lib/owrt-remote/logs.sh read | tail -n 12; sh /usr/lib/owrt-remote/logs.sh status; sh /usr/lib/owrt-remote/logs.sh diagnostics; free'
|
||||||
|
```
|
||||||
|
|
||||||
|
Measure memory using `MemAvailable`, tmpfs allocation and process RSS over time;
|
||||||
|
`free` memory alone includes reclaimable kernel/filesystem cache. Do not attribute
|
||||||
|
all process RSS or low available RAM to the log spool without these measurements.
|
||||||
|
No router or VPS is modified by automated tests.
|
||||||
|
|
||||||
|
## Automated coverage and limits
|
||||||
|
|
||||||
|
- Real FIFO tests using MSYS shell on Windows: live heartbeat sampling, distinct
|
||||||
|
events through pending/Flash, duplicate collector rejection, reader death,
|
||||||
|
append failure, owner SIGKILL and successor recovery, TERM cleanup/final flush.
|
||||||
|
- Real BusyBox ash/applets: spool bounds, atomic/checksum/ack recovery, sampling,
|
||||||
|
limited diagnostics, fallback and cleanup paths; Windows BusyBox lacks FIFOs.
|
||||||
|
- Sandboxed installer: install, first-backup preservation, rollback, bad syntax,
|
||||||
|
failed verification restoration, journal retention.
|
||||||
|
- Existing redaction/CGI/Hub transport regressions and shell syntax checks.
|
||||||
|
|
||||||
|
This is not a claim of physical AX3000T / OpenWrt 24.10.3 UBIFS testing, hardware
|
||||||
|
power-cut testing or a measured RAM reduction in MiB. The actual device still
|
||||||
|
needs event/flush/reboot and memory observations before release.
|
||||||
|
|
||||||
|
Implementation references checked:
|
||||||
|
[OpenWrt 24.10 ubox version](https://github.com/openwrt/openwrt/blob/openwrt-24.10/package/system/ubox/Makefile),
|
||||||
|
[its logread](https://github.com/openwrt/ubox/blob/85f1053019caf4cd333795760950235ee4529ba7/log/logread.c),
|
||||||
|
[BusyBox ash timed read](https://github.com/mirror/busybox/blob/1_36_stable/shell/ash.c),
|
||||||
|
[procd respawn](https://github.com/openwrt/procd/blob/master/service/instance.c).
|
||||||
@@ -108,7 +108,7 @@ class GlobalLogTests(unittest.TestCase):
|
|||||||
self.helper_call('append', good * 3)
|
self.helper_call('append', good * 3)
|
||||||
self.helper_call('append', 'Other action\n')
|
self.helper_call('append', 'Other action\n')
|
||||||
self.helper_call('append', good, env={**self.env, 'TEST_UPTIME': '2000'})
|
self.helper_call('append', good, env={**self.env, 'TEST_UPTIME': '2000'})
|
||||||
self.assertEqual(self.helper_call('read', env={**self.env, 'TEST_UPTIME': '2000'}).count('Heartbeat: OK;'), 1)
|
self.assertEqual(self.helper_call('read', env={**self.env, 'TEST_UPTIME': '2000'}).count('Heartbeat: OK;'), 2)
|
||||||
failed = 'Heartbeat: ошибка; endpoint=https://hub.example; rc=7; connection refused\n'
|
failed = 'Heartbeat: ошибка; endpoint=https://hub.example; rc=7; connection refused\n'
|
||||||
self.helper_call('append', failed * 3, env={**self.env, 'TEST_UPTIME': '2001'})
|
self.helper_call('append', failed * 3, env={**self.env, 'TEST_UPTIME': '2001'})
|
||||||
self.helper_call('append', good, env={**self.env, 'TEST_UPTIME': '2002'})
|
self.helper_call('append', good, env={**self.env, 'TEST_UPTIME': '2002'})
|
||||||
@@ -117,7 +117,7 @@ class GlobalLogTests(unittest.TestCase):
|
|||||||
env={**self.env, 'TEST_UPTIME': '2004'})
|
env={**self.env, 'TEST_UPTIME': '2004'})
|
||||||
stored = self.helper_call('read', env={**self.env, 'TEST_UPTIME': '2004'})
|
stored = self.helper_call('read', env={**self.env, 'TEST_UPTIME': '2004'})
|
||||||
self.assertEqual(stored.count('[ERROR] Heartbeat:'), 2, 'same failure after recovery must appear')
|
self.assertEqual(stored.count('[ERROR] Heartbeat:'), 2, 'same failure after recovery must appear')
|
||||||
self.assertEqual(stored.count('[OK] Heartbeat:'), 3, 'first, recovery and another WAN')
|
self.assertEqual(stored.count('[OK] Heartbeat:'), 4, 'first, periodic sample, recovery and another WAN')
|
||||||
|
|
||||||
def test_recurring_failure_is_recorded_again_after_a_quiet_period(self):
|
def test_recurring_failure_is_recorded_again_after_a_quiet_period(self):
|
||||||
failure = 'owrt-remote: cannot download Xray\n'
|
failure = 'owrt-remote: cannot download Xray\n'
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Real FIFO lifecycle tests; MSYS provides FIFOs on Windows, BusyBox tests the applets separately."""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import test_persistent_logs as spool
|
||||||
|
|
||||||
|
|
||||||
|
class CollectorTests(unittest.TestCase):
|
||||||
|
setUp = spool.PersistentLogsTests.setUp
|
||||||
|
call = spool.PersistentLogsTests.call
|
||||||
|
status = spool.PersistentLogsTests.status
|
||||||
|
flush = spool.PersistentLogsTests.flush
|
||||||
|
journal = spool.PersistentLogsTests.journal
|
||||||
|
# Reuse filesystem fixtures without inheriting the persistent spool test cases.
|
||||||
|
def start(self, extra=""):
|
||||||
|
self.input = self.root / 'input'
|
||||||
|
self.env['TEST_INPUT'] = self.input.as_posix()
|
||||||
|
self.env['TEST_CLOCK_FILE'] = (self.root / 'clock').as_posix()
|
||||||
|
(self.root / 'clock').write_text('1000\n')
|
||||||
|
mocks = r'''
|
||||||
|
log_clock() { cat "$TEST_CLOCK_FILE"; }
|
||||||
|
logread() {
|
||||||
|
exec 4<>"$TEST_INPUT"
|
||||||
|
while :; do if IFS= read -r -t 1 item <&4; then printf '%s\n' "$item"; fi; done
|
||||||
|
}
|
||||||
|
'''
|
||||||
|
source = self.source.replace('case "${1:-}" in', mocks + extra + '\ncase "${1:-}" in', 1)
|
||||||
|
self.helper.write_text(source, encoding='utf-8', newline='\n')
|
||||||
|
if not getattr(self, 'input_created', False):
|
||||||
|
result = subprocess.run([*self.shell, '-c', 'mkfifo "$TEST_INPUT"'], env=self.env,
|
||||||
|
capture_output=True, timeout=10)
|
||||||
|
if result.returncode:
|
||||||
|
self.skipTest('This shell has no FIFO support')
|
||||||
|
self.input_created = True
|
||||||
|
output = (self.root / ('process-%d.log' % len(getattr(self, 'processes', [])))).open('wb')
|
||||||
|
proc = subprocess.Popen([*self.shell, self.helper.as_posix(), 'collect'], env=self.env,
|
||||||
|
stdout=output, stderr=output)
|
||||||
|
if not hasattr(self, 'processes'):
|
||||||
|
self.processes = []
|
||||||
|
self.addCleanup(self.stop_processes)
|
||||||
|
self.processes.append((proc, output))
|
||||||
|
return proc
|
||||||
|
|
||||||
|
def wait_for(self, predicate, seconds=20):
|
||||||
|
deadline = time.monotonic() + seconds
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
value = predicate()
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
time.sleep(.15)
|
||||||
|
self.fail('Collector did not reach expected state')
|
||||||
|
|
||||||
|
def ready(self):
|
||||||
|
return self.wait_for(lambda: (self.run_dir / 'collector-lock/reader').exists())
|
||||||
|
|
||||||
|
def signal(self, pid, signal='TERM'):
|
||||||
|
return subprocess.run([*self.shell, '-c', 'kill -"$1" "$2"', '--', signal, str(pid)],
|
||||||
|
env=self.env, capture_output=True, timeout=10)
|
||||||
|
|
||||||
|
def stop_processes(self):
|
||||||
|
lock = self.run_dir / 'collector-lock/pid'
|
||||||
|
if lock.exists():
|
||||||
|
self.signal(lock.read_text().strip())
|
||||||
|
for proc, output in self.processes:
|
||||||
|
try:
|
||||||
|
proc.wait(timeout=15)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
proc.kill()
|
||||||
|
proc.wait(timeout=5)
|
||||||
|
output.close()
|
||||||
|
|
||||||
|
def send(self, *lines):
|
||||||
|
result = subprocess.run([*self.shell, '-c', 'cat >"$TEST_INPUT"'], input='\n'.join(lines) + '\n',
|
||||||
|
encoding='utf-8', env=self.env, capture_output=True, timeout=15)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
|
||||||
|
def test_actual_fifo_heartbeat_freshness_sample_and_batch_flush(self):
|
||||||
|
proc = self.start(); self.ready()
|
||||||
|
good = 'Thu Oct 15 22:52:36 2026 user.notice owrt-remote: Heartbeat: OK; Hub=https://hub.example; WAN_direct=1; WAN_active=wan'
|
||||||
|
self.send(good)
|
||||||
|
self.wait_for(lambda: self.status()['accepted_events'] == '1')
|
||||||
|
self.send(good.replace('22:52:36', '22:53:08'))
|
||||||
|
self.wait_for(lambda: self.status()['suppressed_events'] == '1')
|
||||||
|
state = self.status()
|
||||||
|
self.assertEqual(state['collector_state'], 'receiving')
|
||||||
|
self.assertEqual(state['received_lines'], '2')
|
||||||
|
self.assertGreater(int(state['last_input_epoch']), 0)
|
||||||
|
self.assertEqual(state['committed_bytes'], '0')
|
||||||
|
(self.root / 'clock').write_text('1301\n')
|
||||||
|
self.send(good.replace('22:52:36', '22:57:38'), 'owrt-remote: panel action after heartbeat')
|
||||||
|
self.wait_for(lambda: self.status()['accepted_events'] == '3')
|
||||||
|
read = self.call('read').stdout
|
||||||
|
self.assertEqual(read.count('Heartbeat: OK;'), 2)
|
||||||
|
self.assertIn('panel action after heartbeat', read)
|
||||||
|
self.flush()
|
||||||
|
self.assertEqual(self.status()['pending_bytes'], '0')
|
||||||
|
self.assertIn(b'panel action after heartbeat', b''.join(p.read_bytes() for p in self.journal.glob('part.*')))
|
||||||
|
self.assertIsNone(proc.poll())
|
||||||
|
|
||||||
|
def test_duplicate_collect_does_not_remove_owner_fifo(self):
|
||||||
|
proc = self.start(); self.ready()
|
||||||
|
owner = (self.run_dir / 'collector-lock/pid').read_text()
|
||||||
|
fifo = list(self.run_dir.glob('stream.*'))
|
||||||
|
other = self.start(); self.assertEqual(other.wait(timeout=15), 0)
|
||||||
|
self.assertEqual((self.run_dir / 'collector-lock/pid').read_text(), owner)
|
||||||
|
self.assertEqual(list(self.run_dir.glob('stream.*')), fifo)
|
||||||
|
self.send('owrt-remote: owner still receives')
|
||||||
|
self.wait_for(lambda: self.status()['accepted_events'] == '1')
|
||||||
|
self.assertIsNone(proc.poll())
|
||||||
|
|
||||||
|
def test_reader_death_exits_nonzero_and_retains_journal_for_restart(self):
|
||||||
|
proc = self.start(); self.ready()
|
||||||
|
self.send('owrt-remote: before logread failure')
|
||||||
|
self.wait_for(lambda: self.status()['accepted_events'] == '1')
|
||||||
|
reader = (self.run_dir / 'collector-lock/reader').read_text().split()[0]
|
||||||
|
self.signal(reader, 'KILL')
|
||||||
|
self.assertNotEqual(proc.wait(timeout=20), 0)
|
||||||
|
self.assertFalse((self.run_dir / 'collector-lock').exists())
|
||||||
|
self.assertFalse(list(self.run_dir.glob('stream.*')))
|
||||||
|
self.assertIn('reader-exited', self.call('diagnostics').stdout)
|
||||||
|
self.assertIn('before logread failure', self.call('read').stdout)
|
||||||
|
new = self.start(); self.ready()
|
||||||
|
self.send('owrt-remote: after reader recovery')
|
||||||
|
self.wait_for(lambda: self.status()['accepted_events'] == '2')
|
||||||
|
self.assertIsNone(new.poll())
|
||||||
|
|
||||||
|
def test_sigkill_owner_reaps_identified_reader_and_stale_fifo(self):
|
||||||
|
proc = self.start(); self.ready()
|
||||||
|
owner = (self.run_dir / 'collector-lock/pid').read_text().strip()
|
||||||
|
self.send('owrt-remote: retained before killed collector')
|
||||||
|
self.wait_for(lambda: self.status()['accepted_events'] == '1')
|
||||||
|
old_reader = (self.run_dir / 'collector-lock/reader').read_text().split()[0]
|
||||||
|
self.signal(owner, 'KILL'); proc.wait(timeout=10)
|
||||||
|
new = self.start()
|
||||||
|
self.wait_for(lambda: (self.run_dir / 'collector-lock/pid').exists() and
|
||||||
|
(self.run_dir / 'collector-lock/pid').read_text().strip() != owner)
|
||||||
|
self.wait_for(lambda: (self.run_dir / 'collector-lock/reader').exists() and
|
||||||
|
(self.run_dir / 'collector-lock/reader').read_text().split()[0] != old_reader)
|
||||||
|
self.assertEqual(len(list(self.run_dir.glob('stream.*'))), 1)
|
||||||
|
self.send('owrt-remote: recovered collector')
|
||||||
|
self.wait_for(lambda: self.status()['accepted_events'] == '2')
|
||||||
|
text = self.call('read').stdout
|
||||||
|
self.assertIn('retained before killed collector', text)
|
||||||
|
self.assertIn('recovered collector', text)
|
||||||
|
self.assertIsNone(new.poll())
|
||||||
|
|
||||||
|
def test_append_failure_is_not_silently_ignored(self):
|
||||||
|
proc = self.start('\nlog_collect_line() { return 1; }\n'); self.ready()
|
||||||
|
self.send('owrt-remote: rejected input')
|
||||||
|
self.assertNotEqual(proc.wait(timeout=20), 0)
|
||||||
|
self.assertIn('collector-append-failed', self.call('diagnostics').stdout)
|
||||||
|
self.assertEqual(self.status()['append_errors'], '1')
|
||||||
|
self.assertFalse((self.run_dir / 'collector-lock').exists())
|
||||||
|
|
||||||
|
def test_term_flushes_and_cleans_children_fifo_and_owner_lock(self):
|
||||||
|
proc = self.start(); self.ready()
|
||||||
|
self.send('owrt-remote: shutdown flush')
|
||||||
|
self.wait_for(lambda: self.status()['accepted_events'] == '1')
|
||||||
|
owner = (self.run_dir / 'collector-lock/pid').read_text().strip()
|
||||||
|
self.signal(owner)
|
||||||
|
self.assertEqual(proc.wait(timeout=15), 0)
|
||||||
|
self.assertEqual(self.status()['pending_bytes'], '0')
|
||||||
|
self.assertIn('shutdown flush', self.call('read').stdout)
|
||||||
|
self.assertFalse(list(self.run_dir.glob('stream.*')))
|
||||||
|
self.assertFalse((self.run_dir / 'collector-lock').exists())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
"""Deploy only the journal helper/service, preserving data and networking."""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
GIT_BIN = Path(os.environ.get('ProgramFiles', 'C:/Program Files')) / 'Git/usr/bin'
|
||||||
|
SH = str(GIT_BIN / 'sh.exe') if os.name == 'nt' else shutil.which('sh')
|
||||||
|
|
||||||
|
class CollectorDeploymentTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory(prefix='collector-deploy-')
|
||||||
|
self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
self.targets = {'logs.sh': '/usr/lib/owrt-remote/logs.sh', 'logs-init': '/etc/init.d/owrt-remote-logs'}
|
||||||
|
self.initial = {}
|
||||||
|
for item, target in self.targets.items():
|
||||||
|
path = self.root / target.lstrip('/')
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text('#!/bin/sh\necho old-' + item + '-$1\n', newline='\n')
|
||||||
|
self.initial[item] = path.read_bytes()
|
||||||
|
self.backup = self.root / 'backup'
|
||||||
|
self.journal = self.root / 'retained-journal'; self.journal.write_text('keep history')
|
||||||
|
text = (ROOT / 'vps/deploy-log-collector-fix.ps1').read_text(encoding='utf-8')
|
||||||
|
self.script = text.split("$RemoteScript = @'\n", 1)[1].split("\n'@", 1)[0]
|
||||||
|
mapping = {target: (self.root / target.lstrip('/')).as_posix() for target in self.targets.values()}
|
||||||
|
mapping['/root/owrt-remote-backup-before-log-collector-v117'] = self.backup.as_posix()
|
||||||
|
pattern = '|'.join(re.escape(path) for path in sorted(mapping, key=len, reverse=True))
|
||||||
|
self.script = re.sub(pattern, lambda m: mapping[m.group()], self.script)
|
||||||
|
mocks = r'''
|
||||||
|
ubus() { printf '{"running":true}\n'; }
|
||||||
|
jsonfilter() { printf true; }
|
||||||
|
logread() { return 0; }
|
||||||
|
mkfifo() { return 0; }
|
||||||
|
logger() { printf '%s\n' "$3" >"$TEST_MARKER"; }
|
||||||
|
sleep() { return 0; }
|
||||||
|
'''
|
||||||
|
self.script = self.script.replace('trap cleanup EXIT', mocks + '\ntrap cleanup EXIT')
|
||||||
|
self.env = {**os.environ, 'PATH': str(GIT_BIN) + os.pathsep + os.environ['PATH'],
|
||||||
|
'TEST_MARKER': (self.root / 'marker').as_posix()}
|
||||||
|
|
||||||
|
def deploy(self, action, bad=False, fail=False):
|
||||||
|
stage = self.root / 'stage'; stage.mkdir()
|
||||||
|
if action == 'Install':
|
||||||
|
helper = r'''#!/bin/sh
|
||||||
|
# log_collector_progress()
|
||||||
|
# LOG_HEARTBEAT_SECONDS=300
|
||||||
|
case "$1" in
|
||||||
|
read) cat "$TEST_MARKER" ;;
|
||||||
|
status) echo persistent=1 ;;
|
||||||
|
esac
|
||||||
|
'''
|
||||||
|
if bad: helper = '#!/bin/sh\nif true; then\n'
|
||||||
|
(stage / 'logs.sh').write_text(helper, newline='\n')
|
||||||
|
(stage / 'logs-init').write_text('#!/bin/sh\necho new-service-$1\n', newline='\n')
|
||||||
|
script = self.script.replace('stage=/tmp/__UPLOAD_NAME__', 'stage=' + shlex.quote(stage.as_posix()))
|
||||||
|
script = script.replace('__UPLOAD_NAME__', 'sandbox').replace('__ACTION__', action)
|
||||||
|
if fail: script = script.replace('cmp "$source_dir/$item" "$path"', 'false')
|
||||||
|
result = subprocess.run([SH], input=script, text=True, encoding='utf-8', capture_output=True, env=self.env, timeout=20)
|
||||||
|
self.assertFalse(stage.exists())
|
||||||
|
self.assertEqual(self.journal.read_text(), 'keep history')
|
||||||
|
return result
|
||||||
|
|
||||||
|
def test_install_and_rollback_keep_first_backup_and_journal(self):
|
||||||
|
for _ in range(2):
|
||||||
|
result = self.deploy('Install')
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertIn('LOG_COLLECTOR_ROUTER_Install_OK', result.stdout)
|
||||||
|
for item, initial in self.initial.items(): self.assertEqual((self.backup / item).read_bytes(), initial)
|
||||||
|
result = self.deploy('Rollback')
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
for item, target in self.targets.items(): self.assertEqual((self.root / target.lstrip('/')).read_bytes(), self.initial[item])
|
||||||
|
|
||||||
|
def test_missing_backup_is_noop(self):
|
||||||
|
result = self.deploy('Rollback')
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertIn('Nothing changed', result.stderr)
|
||||||
|
|
||||||
|
def test_invalid_shell_rejected_before_backup(self):
|
||||||
|
result = self.deploy('Install', bad=True)
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertFalse(self.backup.exists())
|
||||||
|
|
||||||
|
def test_failed_verify_restores_previous_code(self):
|
||||||
|
result = self.deploy('Install', fail=True)
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertIn('Restoring journal code', result.stderr)
|
||||||
|
for item, target in self.targets.items(): self.assertEqual((self.root / target.lstrip('/')).read_bytes(), self.initial[item])
|
||||||
|
|
||||||
|
if __name__ == '__main__': unittest.main()
|
||||||
@@ -128,7 +128,7 @@ class PersistentLogsTests(unittest.TestCase):
|
|||||||
(self.run_dir / 'facts').write_text(''.join('1000\ts256-%035d\n' % i for i in range(63)))
|
(self.run_dir / 'facts').write_text(''.join('1000\ts256-%035d\n' % i for i in range(63)))
|
||||||
self.append('distinct event 63\ndistinct event 64\n')
|
self.append('distinct event 63\ndistinct event 64\n')
|
||||||
self.assertEqual(len((self.run_dir / 'facts').read_text().splitlines()), 64)
|
self.assertEqual(len((self.run_dir / 'facts').read_text().splitlines()), 64)
|
||||||
self.assertLessEqual((self.run_dir / 'facts').stat().st_size, 4096)
|
self.assertLessEqual((self.run_dir / 'facts').stat().st_size if (self.run_dir / 'facts').exists() else 0, 4096)
|
||||||
|
|
||||||
def test_empty_digest_files_falsely_acked_by_old_hub_are_recovered(self):
|
def test_empty_digest_files_falsely_acked_by_old_hub_are_recovered(self):
|
||||||
self.append('retained restart event\n'); self.flush()
|
self.append('retained restart event\n'); self.flush()
|
||||||
@@ -242,7 +242,7 @@ class PersistentLogsTests(unittest.TestCase):
|
|||||||
self.flush()
|
self.flush()
|
||||||
self.assertEqual({p.name: p.read_bytes() for p in parts}, original)
|
self.assertEqual({p.name: p.read_bytes() for p in parts}, original)
|
||||||
self.assertLessEqual((self.run_dir / 'pending').stat().st_size, 16384)
|
self.assertLessEqual((self.run_dir / 'pending').stat().st_size, 16384)
|
||||||
self.assertLessEqual((self.run_dir / 'facts').stat().st_size, 4096)
|
self.assertLessEqual((self.run_dir / 'facts').stat().st_size if (self.run_dir / 'facts').exists() else 0, 4096)
|
||||||
first = self.export()
|
first = self.export()
|
||||||
self.acknowledge(first)
|
self.acknowledge(first)
|
||||||
self.flush()
|
self.flush()
|
||||||
@@ -353,6 +353,32 @@ class PersistentLogsTests(unittest.TestCase):
|
|||||||
self.assertIn('after killed reaper', self.call('read').stdout)
|
self.assertIn('after killed reaper', self.call('read').stdout)
|
||||||
self.assertFalse(lock.exists())
|
self.assertFalse(lock.exists())
|
||||||
|
|
||||||
|
def test_integer_only_sleep_keeps_lock_timeout_bounded_and_quiet(self):
|
||||||
|
self.call('check')
|
||||||
|
source = self.source.split('case "${1:-}" in', 1)[0]
|
||||||
|
commands = r'''
|
||||||
|
log_init
|
||||||
|
mkdir "$RUN_DIR/lock"
|
||||||
|
printf '12345\n' >"$RUN_DIR/lock/pid"
|
||||||
|
sleep() {
|
||||||
|
printf '%s\n' "$1" >>"$RUN_DIR/pauses"
|
||||||
|
if [ "$1" = 0.1 ]; then echo "sleep: invalid number '0.1'" >&2; return 1; fi
|
||||||
|
[ "$1" = 1 ]
|
||||||
|
}
|
||||||
|
kill() { return 0; }
|
||||||
|
if log_lock; then exit 17; fi
|
||||||
|
'''
|
||||||
|
script = self.root / 'integer-sleep.sh'
|
||||||
|
script.write_text(source + commands, encoding='utf-8', newline='\n')
|
||||||
|
result = subprocess.run([*self.shell, script.as_posix()], env=self.env,
|
||||||
|
capture_output=True, text=True, timeout=15)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertEqual(result.stderr, '')
|
||||||
|
pauses = (self.run_dir / 'pauses').read_text().splitlines()
|
||||||
|
self.assertEqual(pauses.count('0.1'), 1, 'probe once per process')
|
||||||
|
self.assertEqual(pauses.count('1'), 10, 'ten-second deadline, no busy spin or 100-second wait')
|
||||||
|
self.assertIn('lock-timeout', self.call('diagnostics').stdout)
|
||||||
|
|
||||||
def test_collector_failure_reaps_children_and_removes_fifo(self):
|
def test_collector_failure_reaps_children_and_removes_fifo(self):
|
||||||
# Actual FIFO integration is tested on POSIX; Windows BusyBox has no
|
# Actual FIFO integration is tested on POSIX; Windows BusyBox has no
|
||||||
# FIFO support. Exercise its traps with mocked process/FIFO operations.
|
# FIFO support. Exercise its traps with mocked process/FIFO operations.
|
||||||
@@ -373,7 +399,48 @@ wait() { return 0; }
|
|||||||
self.assertFalse(list(self.run_dir.glob('stream.*')))
|
self.assertFalse(list(self.run_dir.glob('stream.*')))
|
||||||
self.assertFalse((self.run_dir / 'collector-lock').exists())
|
self.assertFalse((self.run_dir / 'collector-lock').exists())
|
||||||
lines = (self.run_dir / 'kills').read_text().splitlines()
|
lines = (self.run_dir / 'kills').read_text().splitlines()
|
||||||
self.assertGreaterEqual(len([line for line in lines if not line.startswith('-0')]), 2)
|
self.assertGreaterEqual(len([line for line in lines if not line.startswith('-0')]), 1)
|
||||||
|
|
||||||
|
def test_heartbeat_sampling_and_freshness_are_independent_of_flash_writes(self):
|
||||||
|
good = 'Thu Oct 15 22:52:36 2026 user.notice owrt-remote: Heartbeat: OK; Hub=https://hub.example; WAN_direct=1; WAN_active=wan\n'
|
||||||
|
self.append(good)
|
||||||
|
self.append(good.replace('22:52:36', '22:53:08'), TEST_UPTIME='1032')
|
||||||
|
state = self.status()
|
||||||
|
self.assertEqual((state['accepted_events'], state['suppressed_events']), ('1', '1'))
|
||||||
|
self.assertEqual(state['committed_bytes'], '0')
|
||||||
|
self.assertGreater(int(state['last_input_epoch']), 0)
|
||||||
|
self.assertEqual(state['collector_state'], 'stopped', 'persistent storage does not imply a running collector')
|
||||||
|
self.append(good.replace('22:52:36', '22:57:38'), TEST_UPTIME='1302')
|
||||||
|
self.assertEqual(self.call('read').stdout.count('Heartbeat: OK;'), 2)
|
||||||
|
self.flush()
|
||||||
|
self.assertEqual(self.status()['pending_bytes'], '0')
|
||||||
|
self.assertGreater(int(self.status()['last_commit_epoch']), 0)
|
||||||
|
|
||||||
|
def test_private_diagnostics_are_bounded_rate_limited_and_no_syslog_feedback(self):
|
||||||
|
self.call('check')
|
||||||
|
source = self.source.split('case "${1:-}" in', 1)[0]
|
||||||
|
commands = r'''
|
||||||
|
log_init
|
||||||
|
logger() { echo forbidden-syslog-feedback >&2; return 1; }
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 140 ]; do
|
||||||
|
if [ "$((i % 2))" = 0 ]; then log_diag flush-failed; else log_diag lock-timeout; fi
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
log_diag lock-timeout
|
||||||
|
'''
|
||||||
|
script = self.root / 'diag-test.sh'
|
||||||
|
script.write_text(source + commands, encoding='utf-8', newline='\n')
|
||||||
|
result = subprocess.run([*self.shell, script.as_posix()], env=self.env,
|
||||||
|
capture_output=True, text=True, timeout=30)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertEqual(result.stderr, '')
|
||||||
|
diagnostics = self.run_dir / 'diagnostics'
|
||||||
|
self.assertLessEqual(diagnostics.stat().st_size, 4096)
|
||||||
|
self.assertFalse((self.run_dir / 'diagnostics.new').exists())
|
||||||
|
self.assertFalse((self.run_dir / 'diag-lock').exists())
|
||||||
|
self.assertEqual(self.call('diagnostics').stdout, diagnostics.read_text())
|
||||||
|
self.assertFalse((self.journal / 'diagnostics').exists())
|
||||||
|
|
||||||
def test_legacy_migration_preserves_identity_ack_and_only_undelivered_bytes(self):
|
def test_legacy_migration_preserves_identity_ack_and_only_undelivered_bytes(self):
|
||||||
legacy = self.root / 'legacy'
|
legacy = self.root / 'legacy'
|
||||||
|
|||||||
@@ -0,0 +1,148 @@
|
|||||||
|
param(
|
||||||
|
[ValidateNotNullOrEmpty()][string]$Router = 'root@192.168.2.1',
|
||||||
|
[ValidateSet('Install', 'Rollback')][string]$Action = 'Install',
|
||||||
|
[switch]$PrepareOnly
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = 'Stop'
|
||||||
|
$RepoRoot = Split-Path -Parent $PSScriptRoot
|
||||||
|
$UploadName = 'owrt-log-collector-' + [guid]::NewGuid().ToString('N')
|
||||||
|
$StageRoot = Join-Path ([System.IO.Path]::GetTempPath()) $UploadName
|
||||||
|
$Utf8 = [System.Text.UTF8Encoding]::new($false)
|
||||||
|
$Payloads = @{
|
||||||
|
'logs.sh' = 'files\usr\lib\owrt-remote\logs.sh'
|
||||||
|
'logs-init' = 'files\etc\init.d\owrt-remote-logs'
|
||||||
|
}
|
||||||
|
$RemoteScript = @'
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
stage=/tmp/__UPLOAD_NAME__
|
||||||
|
backup_dir=/root/owrt-remote-backup-before-log-collector-v117
|
||||||
|
changed=0
|
||||||
|
phase=preflight
|
||||||
|
items='logs.sh logs-init'
|
||||||
|
item_path() {
|
||||||
|
case "$1" in
|
||||||
|
logs.sh) printf /usr/lib/owrt-remote/logs.sh ;;
|
||||||
|
logs-init) printf /etc/init.d/owrt-remote-logs ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
complete_backup() {
|
||||||
|
[ -f "$1/complete" ] && [ -s "$1/logs.sh" ] && [ -s "$1/logs-init" ]
|
||||||
|
}
|
||||||
|
collector_running() {
|
||||||
|
ubus call service list '{"name":"owrt-remote-logs"}' | jsonfilter -e '@["owrt-remote-logs"].instances.journal.running' | grep -qx true
|
||||||
|
}
|
||||||
|
save_files() {
|
||||||
|
local item path
|
||||||
|
mkdir -p "$1"
|
||||||
|
chmod 700 "$1"
|
||||||
|
for item in $items; do path=$(item_path "$item"); cp -p "$path" "$1/$item" || return 1; done
|
||||||
|
if /etc/init.d/owrt-remote-logs enabled; then : >"$1/enabled"; fi
|
||||||
|
if collector_running; then : >"$1/running"; fi
|
||||||
|
: >"$1/complete"
|
||||||
|
}
|
||||||
|
restore_files() {
|
||||||
|
local item path
|
||||||
|
complete_backup "$1" || return 1
|
||||||
|
/etc/init.d/owrt-remote-logs stop || true
|
||||||
|
for item in $items; do
|
||||||
|
path=$(item_path "$item")
|
||||||
|
cp -p "$1/$item" "$path.new-__UPLOAD_NAME__" && mv -f "$path.new-__UPLOAD_NAME__" "$path" || return 1
|
||||||
|
done
|
||||||
|
if [ -f "$1/enabled" ]; then /etc/init.d/owrt-remote-logs enable; else /etc/init.d/owrt-remote-logs disable; fi
|
||||||
|
if [ -f "$1/running" ]; then /etc/init.d/owrt-remote-logs start; fi
|
||||||
|
}
|
||||||
|
cleanup() {
|
||||||
|
code=$?
|
||||||
|
trap - EXIT
|
||||||
|
if [ "$code" -ne 0 ]; then
|
||||||
|
printf 'Failed during %s (rc=%s).\n' "$phase" "$code" >&2
|
||||||
|
if [ "$changed" = 1 ]; then
|
||||||
|
echo 'Restoring journal code/service state from this attempt; journal data retained.' >&2
|
||||||
|
restore_files "$stage/previous" || echo 'Restore failed; retained backup is available.' >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
for item in $items; do path=$(item_path "$item"); rm -f "$path.new-__UPLOAD_NAME__"; done
|
||||||
|
rm -rf "$stage"
|
||||||
|
exit "$code"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
for item in $items; do path=$(item_path "$item"); test -s "$path"; done
|
||||||
|
command -v ubus >/dev/null
|
||||||
|
command -v jsonfilter >/dev/null
|
||||||
|
if [ '__ACTION__' = Install ]; then
|
||||||
|
for item in $items; do test -s "$stage/$item"; sh -n "$stage/$item"; done
|
||||||
|
command -v logread >/dev/null
|
||||||
|
command -v logger >/dev/null
|
||||||
|
command -v mkfifo >/dev/null
|
||||||
|
grep -Fq 'log_collector_progress()' "$stage/logs.sh"
|
||||||
|
grep -Fq 'LOG_HEARTBEAT_SECONDS=300' "$stage/logs.sh"
|
||||||
|
if [ ! -e "$backup_dir" ]; then save_files "$stage/retained"; mv "$stage/retained" "$backup_dir"; fi
|
||||||
|
complete_backup "$backup_dir" || { echo "Incomplete backup: $backup_dir. Nothing changed." >&2; exit 1; }
|
||||||
|
source_dir=$stage
|
||||||
|
else
|
||||||
|
complete_backup "$backup_dir" || { echo "No complete backup: $backup_dir. Nothing changed." >&2; exit 1; }
|
||||||
|
for item in $items; do sh -n "$backup_dir/$item"; done
|
||||||
|
source_dir=$backup_dir
|
||||||
|
fi
|
||||||
|
save_files "$stage/previous"
|
||||||
|
phase=install
|
||||||
|
changed=1
|
||||||
|
/etc/init.d/owrt-remote-logs stop
|
||||||
|
for item in $items; do
|
||||||
|
path=$(item_path "$item")
|
||||||
|
cp "$source_dir/$item" "$path.new-__UPLOAD_NAME__"
|
||||||
|
case "$item" in logs.sh) chmod 644 "$path.new-__UPLOAD_NAME__" ;; *) chmod 755 "$path.new-__UPLOAD_NAME__" ;; esac
|
||||||
|
mv -f "$path.new-__UPLOAD_NAME__" "$path"
|
||||||
|
done
|
||||||
|
if [ '__ACTION__' = Install ]; then
|
||||||
|
/etc/init.d/owrt-remote-logs enable
|
||||||
|
/etc/init.d/owrt-remote-logs start
|
||||||
|
phase=verify-delivery
|
||||||
|
marker=collector-check-__UPLOAD_NAME__
|
||||||
|
ready=0
|
||||||
|
for attempt in 1 2 3 4 5 6 7 8 9 10; do
|
||||||
|
logger -t owrt-remote "$marker"
|
||||||
|
sleep 1
|
||||||
|
if collector_running && sh /usr/lib/owrt-remote/logs.sh read | grep -Fq "$marker"; then ready=1; break; fi
|
||||||
|
done
|
||||||
|
[ "$ready" = 1 ] || { echo 'Collector did not capture the test event; restoring code.' >&2; exit 1; }
|
||||||
|
# A forced flush verifies the write path once at deployment, not each Heartbeat.
|
||||||
|
phase=verify-flush
|
||||||
|
sh /usr/lib/owrt-remote/logs.sh flush force
|
||||||
|
sh /usr/lib/owrt-remote/logs.sh read | grep -Fq "$marker"
|
||||||
|
sh /usr/lib/owrt-remote/logs.sh status
|
||||||
|
sh /usr/lib/owrt-remote/logs.sh diagnostics
|
||||||
|
else
|
||||||
|
if [ -f "$backup_dir/enabled" ]; then /etc/init.d/owrt-remote-logs enable; else /etc/init.d/owrt-remote-logs disable; fi
|
||||||
|
if [ -f "$backup_dir/running" ]; then /etc/init.d/owrt-remote-logs start; fi
|
||||||
|
fi
|
||||||
|
for item in $items; do path=$(item_path "$item"); cmp "$source_dir/$item" "$path"; done
|
||||||
|
printf 'Backup: %s\n' "$backup_dir"
|
||||||
|
printf '\nLOG_COLLECTOR_ROUTER_%s_OK\n' '__ACTION__'
|
||||||
|
'@
|
||||||
|
$RemoteScript = $RemoteScript.Replace('__UPLOAD_NAME__', $UploadName).Replace('__ACTION__', $Action)
|
||||||
|
try {
|
||||||
|
[System.IO.Directory]::CreateDirectory($StageRoot) | Out-Null
|
||||||
|
[System.IO.File]::WriteAllText((Join-Path $StageRoot 'run.sh'), $RemoteScript.Replace("`r`n", "`n") + "`n", $Utf8)
|
||||||
|
if ($Action -eq 'Install') {
|
||||||
|
foreach ($Item in $Payloads.GetEnumerator()) {
|
||||||
|
$Content = [System.IO.File]::ReadAllText((Join-Path $RepoRoot $Item.Value)).Replace("`r`n", "`n")
|
||||||
|
[System.IO.File]::WriteAllText((Join-Path $StageRoot $Item.Key), $Content, $Utf8)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($PrepareOnly) { Write-Host "Prepared only: $StageRoot"; return }
|
||||||
|
& scp -O -r -- $StageRoot "${Router}:/tmp/"
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw 'Upload failed. The installation has not started.' }
|
||||||
|
& ssh -T -o ConnectTimeout=10 $Router "sh /tmp/$UploadName/run.sh"
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw "$Action failed. See the output above; success was not confirmed." }
|
||||||
|
Write-Host "Log collector fix $Action finished successfully."
|
||||||
|
} finally {
|
||||||
|
if (-not $PrepareOnly) {
|
||||||
|
$Resolved = [System.IO.Path]::GetFullPath($StageRoot)
|
||||||
|
$Expected = [System.IO.Path]::GetFullPath((Join-Path ([System.IO.Path]::GetTempPath()) $UploadName))
|
||||||
|
if ($Resolved -ne $Expected -or (Split-Path -Leaf $Resolved) -ne $UploadName) { throw 'Unsafe staging cleanup path.' }
|
||||||
|
if (Test-Path -LiteralPath $Resolved) { Remove-Item -LiteralPath $Resolved -Recurse -Force }
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user