Fix persistent log collector liveness and BusyBox lock waits

This commit is contained in:
kzolotarev95 committed 2026-10-16 00:12:54 +04:00
1 parent b62db40daf
commit e738d6fa69
8 files changed
+846 -37

No files matched your search

+2 -1
View File
@@ -8,7 +8,8 @@ start_service() {
/bin/sh /usr/lib/owrt-remote/logs.sh migrate || logger -t owrt-remote 'WARN: старый RAM-журнал ещё не перенесён; см. свободное место Flash.' /bin/sh /usr/lib/owrt-remote/logs.sh migrate || logger -t owrt-remote 'WARN: старый RAM-журнал ещё не перенесён; см. свободное место Flash.'
procd_open_instance journal procd_open_instance journal
procd_set_param command /bin/sh /usr/lib/owrt-remote/logs.sh collect procd_set_param command /bin/sh /usr/lib/owrt-remote/logs.sh collect
procd_set_param respawn 3600 5 5 # Retry at a bounded rate even after repeated logd/storage outages.
procd_set_param respawn 3600 10 0
procd_set_param stdout 1 procd_set_param stdout 1
procd_set_param stderr 1 procd_set_param stderr 1
procd_close_instance procd_close_instance
+246 -31
View File
@@ -7,10 +7,20 @@ LOG_PART_LIMIT=256
LOG_BUFFER_LIMIT=16384 LOG_BUFFER_LIMIT=16384
LOG_BATCH_LIMIT=16384 LOG_BATCH_LIMIT=16384
LOG_FLUSH_SECONDS=60 LOG_FLUSH_SECONDS=60
LOG_HEARTBEAT_SECONDS=300
LOG_FREE_KB=2048 LOG_FREE_KB=2048
log_clock() { cut -d. -f1 /proc/uptime; } log_clock() { cut -d. -f1 /proc/uptime; }
log_epoch() { date +%s; } log_epoch() { date +%s; }
log_pause() {
# Some OpenWrt BusyBox builds omit fractional sleep. Keep lock/reap deadlines
# in tenths, including the one-second fallback, instead of spinning on failure.
if [ "${LOG_PAUSE_TENTHS:-0}" != 10 ]; then
if sleep 0.1 2>/dev/null; then LOG_PAUSE_TENTHS=1; return 0; fi
LOG_PAUSE_TENTHS=10
fi
sleep 1
}
log_self_pid() { log_self_pid() {
# $$ is the parent shell's PID inside ash subshells/pipelines. Reading proc # $$ is the parent shell's PID inside ash subshells/pipelines. Reading proc
# with the shell builtin yields the actual lock holder, including the writer. # with the shell builtin yields the actual lock holder, including the writer.
@@ -105,8 +115,8 @@ log_lock() {
[ "$attempt" -lt 20 ] || log_reap_lock "$RUN_DIR/lock" "$attempt" ;; [ "$attempt" -lt 20 ] || log_reap_lock "$RUN_DIR/lock" "$attempt" ;;
*) kill -0 "$owner" 2>/dev/null || log_reap_lock "$RUN_DIR/lock" "$attempt" ;; *) kill -0 "$owner" 2>/dev/null || log_reap_lock "$RUN_DIR/lock" "$attempt" ;;
esac esac
attempt=$((attempt + 1)); [ "$attempt" -lt 100 ] || return 1 log_pause
sleep 0.1 attempt=$((attempt + LOG_PAUSE_TENTHS)); [ "$attempt" -lt 100 ] || { log_diag lock-timeout; return 1; }
done done
log_self_pid >"$RUN_DIR/lock/pid" log_self_pid >"$RUN_DIR/lock/pid"
# Another first writer may have selected storage while we waited. # Another first writer may have selected storage while we waited.
@@ -139,8 +149,24 @@ log_reap_lock() {
owner="$(cat "$1/pid" 2>/dev/null)" owner="$(cat "$1/pid" 2>/dev/null)"
case "$owner" in case "$owner" in
''|*[!0-9]*) rm -f "$1/pid" ;; ''|*[!0-9]*) rm -f "$1/pid" ;;
*) if kill -0 "$owner" 2>/dev/null; then rm -f "$1/reap/pid"; rmdir "$1/reap"; return 0; else rm -f "$1/pid"; fi ;; *)
if { [ "$1" = "$RUN_DIR/collector-lock" ] && log_process_info "$owner" >/dev/null; } ||
{ [ "$1" != "$RUN_DIR/collector-lock" ] && kill -0 "$owner" 2>/dev/null; }; then
rm -f "$1/reap/pid"; rmdir "$1/reap"; return 0
else rm -f "$1/pid"; fi ;;
esac esac
if [ "$1" = "$RUN_DIR/collector-lock" ] && [ -s "$1/reader" ]; then
local old_reader old_birth
read -r old_reader old_birth <"$1/reader"
if [ -n "$old_birth" ] && [ "$(log_process_info "$old_reader")" = "$old_birth" ]; then
kill "$old_reader" 2>/dev/null || true
local remaining
remaining=0
while [ "$(log_process_info "$old_reader")" = "$old_birth" ] && [ "$remaining" -lt 10 ]; do log_pause; remaining=$((remaining + LOG_PAUSE_TENTHS)); done
if [ "$(log_process_info "$old_reader")" = "$old_birth" ]; then kill -KILL "$old_reader" 2>/dev/null || true; fi
fi
rm -f "$1/reader"
fi
rm -f "$1/reap/pid"; rmdir "$1/reap"; rmdir "$1" 2>/dev/null || true rm -f "$1/reap/pid"; rmdir "$1/reap"; rmdir "$1" 2>/dev/null || true
} }
log_fallback_locked() { log_fallback_locked() {
@@ -397,6 +423,7 @@ log_flush_locked() {
TOTAL=$((TOTAL + bytes)) TOTAL=$((TOTAL + bytes))
log_checkpoint_locked && log_commit_locked || { log_warn_locked write-warning 'Flash: ошибка фиксации состояния журнала.'; return 1; } log_checkpoint_locked && log_commit_locked || { log_warn_locked write-warning 'Flash: ошибка фиксации состояния журнала.'; return 1; }
: >"$RUN_DIR/pending" : >"$RUN_DIR/pending"
printf '%s\n' "$(log_epoch)" >"$RUN_DIR/commit-health"
rm -f "$RUN_DIR/inflight" rm -f "$RUN_DIR/inflight"
rm -f "$RUN_DIR/write-warning" "$RUN_DIR/queue-warning" "$RUN_DIR/buffer-warning" rm -f "$RUN_DIR/write-warning" "$RUN_DIR/queue-warning" "$RUN_DIR/buffer-warning"
dropped="$(cat "$RUN_DIR/dropped" 2>/dev/null)"; dropped="${dropped:-0}" dropped="$(cat "$RUN_DIR/dropped" 2>/dev/null)"; dropped="${dropped:-0}"
@@ -406,6 +433,7 @@ log_flush_locked() {
fi fi
fi fi
log_clock >"$RUN_DIR/last-flush" log_clock >"$RUN_DIR/last-flush"
printf '%s %s\n' "$(log_epoch)" "$(cat "$RUN_DIR/last-flush")" >"$RUN_DIR/flush-health.new" && mv -f "$RUN_DIR/flush-health.new" "$RUN_DIR/flush-health"
} }
log_open() { log_open() {
log_lock || return 1 log_lock || return 1
@@ -439,14 +467,22 @@ log_redact() {
}' }'
} }
log_new_fact_locked() { log_new_fact_locked() {
local key now heartbeat new_fact local key now heartbeat new_fact last
# Compare event text without syslog timestamp, facility, PID or Xray time. # Compare event text without syslog timestamp, facility, PID or Xray time.
key="$(printf '%s\n' "$1" | tr '\t' ' ' | sed -e 's/^\[[^]]*\] *//' -e 's/.*owrt-remote\(\[[0-9][0-9]*\]\)\{0,1\}: *//' -e 's/^[0-9][0-9]*\/[0-9/:. ]* *//' -e 's/ *(\([0-9][0-9]*\)\/[0-9][0-9]*)//')" key="$(printf '%s\n' "$1" | tr '\t' ' ' | sed -e 's/^\[[^]]*\] *//' -e 's/.*owrt-remote\(\[[0-9][0-9]*\]\)\{0,1\}: *//' -e 's/^[0-9][0-9]*\/[0-9/:. ]* *//' -e 's/ *(\([0-9][0-9]*\)\/[0-9][0-9]*)//')"
heartbeat=0 heartbeat=0
case "$key" in case "$key" in
'Heartbeat: OK;'*) 'Heartbeat: OK;'*)
heartbeat=1 heartbeat=1
if [ "$(cat "$RUN_DIR/heartbeat-ok" 2>/dev/null)" = "$key" ]; then return 1; fi last=0
[ ! -f "$RUN_DIR/heartbeat-stored" ] || read -r last <"$RUN_DIR/heartbeat-stored"
case "$last" in ''|*[!0-9]*) last=0 ;; esac
now="$(log_clock)"
# Keep one liveness sample every five minutes, plus every change/recovery.
# Input freshness is tracked in RAM even when a duplicate is suppressed.
if [ "$(cat "$RUN_DIR/heartbeat-ok" 2>/dev/null)" = "$key" ] &&
[ "$now" -ge "$last" ] && [ "$((now - last))" -lt "$LOG_HEARTBEAT_SECONDS" ]; then return 1; fi
printf '%s\n' "$now" >"$RUN_DIR/heartbeat-stored"
printf '%s\n' "$key" >"$RUN_DIR/heartbeat-ok" printf '%s\n' "$key" >"$RUN_DIR/heartbeat-ok"
# A recovery lets the same failure appear again; keys are fixed-size hashes. # A recovery lets the same failure appear again; keys are fixed-size hashes.
: >"$RUN_DIR/facts" ;; : >"$RUN_DIR/facts" ;;
@@ -479,24 +515,50 @@ log_new_fact_locked() {
[ "$new_fact" = 0 ] || return 1 [ "$new_fact" = 0 ] || return 1
return 0 return 0
} }
log_event_health_locked() {
local epoch uptime accepted_epoch accepted_up accepted suppressed failed extra
accepted_epoch=0; accepted_up=0; accepted=0; suppressed=0; failed=0
if [ -s "$RUN_DIR/event-health" ]; then
read -r epoch uptime accepted_epoch accepted_up accepted suppressed failed extra <"$RUN_DIR/event-health"
case "$accepted_epoch:$accepted_up:$accepted:$suppressed:$failed" in *[!0-9:]*|:*|*:) accepted_epoch=0; accepted_up=0; accepted=0; suppressed=0; failed=0 ;; esac
fi
epoch="$(log_epoch)"; uptime="$(log_clock)"
case "$1" in
written) accepted_epoch="$epoch"; accepted_up="$uptime"; [ "$accepted" -ge 2147483647 ] || accepted=$((accepted + 1)) ;;
suppressed) [ "$suppressed" -ge 2147483647 ] || suppressed=$((suppressed + 1)) ;;
failed) [ "$failed" -ge 2147483647 ] || failed=$((failed + 1)) ;;
esac
printf '%s %s %s %s %s %s %s\n' "$epoch" "$uptime" "$accepted_epoch" "$accepted_up" "$accepted" "$suppressed" "$failed" >"$RUN_DIR/event-health.new" &&
mv -f "$RUN_DIR/event-health.new" "$RUN_DIR/event-health"
}
log_append() { log_append() {
local line size bytes dropped local line size bytes dropped outcome
log_redact "${1:-OK}" | while IFS= read -r line; do log_redact "${1:-OK}" | while IFS= read -r line; do
log_open || exit 1 log_open || { log_diag append-open-failed; exit 1; }
log_flush_locked || true if ! log_flush_locked; then log_diag flush-failed; fi
[ -f "$RUN_DIR/facts" ] || : >"$RUN_DIR/facts" [ -f "$RUN_DIR/facts" ] || : >"$RUN_DIR/facts"
outcome=suppressed
if log_new_fact_locked "$line"; then if log_new_fact_locked "$line"; then
size="$(wc -c <"$RUN_DIR/pending")"; bytes="$(printf '%s\n' "$line" | wc -c)" size="$(wc -c <"$RUN_DIR/pending")"; bytes="$(printf '%s\n' "$line" | wc -c)"
# Reserve room for a single overflow warning, too. # Reserve room for a single overflow warning, too.
if [ "$((size + bytes))" -le "$((LOG_BUFFER_LIMIT - 1024))" ]; then if [ "$((size + bytes))" -le "$((LOG_BUFFER_LIMIT - 1024))" ]; then
printf '%s\n' "$line" >>"$RUN_DIR/pending" if ! printf '%s\n' "$line" >>"$RUN_DIR/pending"; then
# A failed append must be eligible for retry, not hidden by dedup.
rm -f "$RUN_DIR/heartbeat-ok" "$RUN_DIR/facts"
log_event_health_locked failed; log_diag append-write-failed
log_close; exit 1
fi
outcome=written
else else
outcome=failed
rm -f "$RUN_DIR/heartbeat-ok" "$RUN_DIR/facts"
dropped="$(cat "$RUN_DIR/dropped" 2>/dev/null)"; dropped="${dropped:-0}" dropped="$(cat "$RUN_DIR/dropped" 2>/dev/null)"; dropped="${dropped:-0}"
[ "$dropped" -ge 2147483647 ] || dropped=$((dropped + 1)) [ "$dropped" -ge 2147483647 ] || dropped=$((dropped + 1))
printf '%s\n' "$dropped" >"$RUN_DIR/dropped" printf '%s\n' "$dropped" >"$RUN_DIR/dropped"
log_warn_locked buffer-warning 'Буфер логов заполнен: новые события сверх лимита пропускаются, неподтверждённый журнал сохранён.' log_warn_locked buffer-warning 'Буфер логов заполнен: новые события сверх лимита пропускаются, неподтверждённый журнал сохранён.'
fi fi
fi fi
log_event_health_locked "$outcome" || { log_diag health-write-failed; log_close; exit 1; }
log_close log_close
done done
} }
@@ -542,7 +604,7 @@ log_reset_locked() {
log_checkpoint_locked && log_commit_locked || return 1 log_checkpoint_locked && log_commit_locked || return 1
rm -f "$LOG_DIR"/part.* "$LOG_DIR/damaged.log" rm -f "$LOG_DIR"/part.* "$LOG_DIR/damaged.log"
: >"$RUN_DIR/pending" : >"$RUN_DIR/pending"
rm -f "$RUN_DIR/ack" "$RUN_DIR/facts" "$RUN_DIR/heartbeat-ok" "$RUN_DIR/dropped" "$RUN_DIR/buffer-warning" "$RUN_DIR/inflight" rm -f "$RUN_DIR/ack" "$RUN_DIR/facts" "$RUN_DIR/heartbeat-ok" "$RUN_DIR/heartbeat-stored" "$RUN_DIR/dropped" "$RUN_DIR/buffer-warning" "$RUN_DIR/inflight"
log_clock >"$RUN_DIR/last-flush" log_clock >"$RUN_DIR/last-flush"
} }
log_clear() { log_open || return 1; log_reset_locked; result=$?; log_close; return "$result"; } log_clear() { log_open || return 1; log_reset_locked; result=$?; log_close; return "$result"; }
@@ -592,10 +654,16 @@ log_ack() {
fi fi
log_close log_close
} }
log_flush() { log_open || return 1; log_flush_locked "${1:-}"; result=$?; log_close; return "$result"; } log_flush() {
log_open || return 1
log_flush_locked "${1:-}"; result=$?
[ "$result" = 0 ] || log_diag flush-failed
log_close; return "$result"
}
log_status() { log_status() {
log_open || return 1 log_open || return 1
printf 'storage=%s\npersistent=%s\nruntime=%s\nlimit_bytes=%s\nmax_parts=%s\nbuffer_limit_bytes=%s\nflush_seconds=%s\njournal_id=%s\ncommitted_bytes=%s\nack_bytes=%s\npending_bytes=%s\n' "$LOG_DIR" "$PERSISTENT" "$RUN_DIR" "$LOG_LIMIT" "$LOG_PART_LIMIT" "$LOG_BUFFER_LIMIT" "$LOG_FLUSH_SECONDS" "$JOURNAL" "$TOTAL" "$ACK" "$(wc -c <"$RUN_DIR/pending")" printf 'storage=%s\npersistent=%s\nruntime=%s\nlimit_bytes=%s\nmax_parts=%s\nbuffer_limit_bytes=%s\nflush_seconds=%s\njournal_id=%s\ncommitted_bytes=%s\nack_bytes=%s\npending_bytes=%s\n' "$LOG_DIR" "$PERSISTENT" "$RUN_DIR" "$LOG_LIMIT" "$LOG_PART_LIMIT" "$LOG_BUFFER_LIMIT" "$LOG_FLUSH_SECONDS" "$JOURNAL" "$TOTAL" "$ACK" "$(wc -c <"$RUN_DIR/pending")"
log_health_status
log_close log_close
} }
log_migrate() { log_migrate() {
@@ -687,36 +755,183 @@ log_sync_loop() {
sleep 3 sleep 3
done done
} }
# Diagnostics contain fixed reason codes only, never a line, URL or credentials.
# At most 4 KiB plus a 2 KiB compaction file, all on tmpfs. No syslog feedback.
log_diag() {
local code now last previous bytes
code="$1"; now="$(log_clock)"; last=0; previous=''
[ -d "$RUN_DIR" ] || return 0
mkdir "$RUN_DIR/diag-lock" 2>/dev/null || return 0
if [ -s "$RUN_DIR/diag-last" ]; then read -r last previous <"$RUN_DIR/diag-last"; fi
case "$last" in ''|*[!0-9]*) last=0 ;; esac
if [ "$code" != "$previous" ] || [ "$now" -lt "$last" ] || [ "$((now - last))" -ge 60 ]; then
bytes=0; [ ! -f "$RUN_DIR/diagnostics" ] || bytes="$(wc -c <"$RUN_DIR/diagnostics")"
if [ "$bytes" -gt 3900 ]; then
tail -c 2048 "$RUN_DIR/diagnostics" >"$RUN_DIR/diagnostics.new" && mv -f "$RUN_DIR/diagnostics.new" "$RUN_DIR/diagnostics"
fi
printf '%s %s\n' "$(log_epoch)" "$code" >>"$RUN_DIR/diagnostics"
printf '%s %s\n' "$now" "$code" >"$RUN_DIR/diag-last"
fi
rmdir "$RUN_DIR/diag-lock" 2>/dev/null || true
}
log_process_info() {
local stat
case "$1" in ''|*[!0-9]*) return 1 ;; esac
kill -0 "$1" 2>/dev/null || return 1
if ! read -r stat <"/proc/$1/stat" 2>/dev/null; then return 1; fi
stat="${stat##*) }"
set -- $stat
[ "$#" -ge 20 ] || return 1
case "$1" in Z|X) return 1 ;; esac
shift 19
printf '%s\n' "$1"
}
log_collector_lock() {
local owner attempt
attempt=0
while ! mkdir "$RUN_DIR/collector-lock" 2>/dev/null; do
owner="$(cat "$RUN_DIR/collector-lock/pid" 2>/dev/null)"
case "$owner" in
''|*[!0-9]*) [ "$attempt" -lt 20 ] || log_reap_lock "$RUN_DIR/collector-lock" "$attempt" ;;
*)
if log_process_info "$owner" >/dev/null; then return 2; fi
# Reap a logread orphan only if its PID AND kernel start time match.
log_reap_lock "$RUN_DIR/collector-lock" "$attempt" ;;
esac
log_pause
attempt=$((attempt + LOG_PAUSE_TENTHS)); [ "$attempt" -lt 40 ] || { log_diag collector-lock-timeout; return 1; }
done
log_self_pid >"$RUN_DIR/collector-lock/pid"
}
log_collector_progress() {
printf '%s %s %s %s %s %s\n' "$collector" "$(log_epoch)" "$(log_clock)" "$received" "$append_errors" "$flush_errors" >"$RUN_DIR/collector-progress.new" &&
mv -f "$RUN_DIR/collector-progress.new" "$RUN_DIR/collector-progress"
}
log_collector_cleanup() {
local code owner attempt
code="$1"; trap - EXIT INT TERM
exec 3>&-
if [ -n "$reader" ]; then
kill "$reader" 2>/dev/null || true
# Never leave an uninterruptible wait in the procd shutdown path.
attempt=0
while log_process_info "$reader" >/dev/null && [ "$attempt" -lt 10 ]; do log_pause; attempt=$((attempt + LOG_PAUSE_TENTHS)); done
if log_process_info "$reader" >/dev/null; then kill -KILL "$reader" 2>/dev/null || true; fi
wait "$reader" 2>/dev/null || true
fi
rm -f "$fifo"
owner="$(cat "$RUN_DIR/collector-lock/pid" 2>/dev/null)"
if [ "$owner" = "$collector" ]; then
# Flush while still owning the collector lock. The successor never overlaps.
(log_flush force) || true
rm -f "$RUN_DIR/collector-lock/reader" "$RUN_DIR/collector-lock/pid"
rmdir "$RUN_DIR/collector-lock" 2>/dev/null || true
fi
exit "$code"
}
log_health_status() {
local owner reader birth now state last_input input_up last_event event_up accepted suppressed failed extra flush_epoch flush_up received errors flush_errors progress_owner
owner=''; reader=''; birth=''; last_input=0; input_up=0; last_event=0; event_up=0; accepted=0; suppressed=0; failed=0; flush_epoch=0; flush_up=0; received=0; errors=0; flush_errors=0
[ ! -s "$RUN_DIR/collector-lock/pid" ] || read -r owner <"$RUN_DIR/collector-lock/pid"
[ ! -s "$RUN_DIR/collector-lock/reader" ] || read -r reader birth <"$RUN_DIR/collector-lock/reader"
[ ! -s "$RUN_DIR/event-health" ] || read -r last_input input_up last_event event_up accepted suppressed failed extra <"$RUN_DIR/event-health"
[ ! -s "$RUN_DIR/flush-health" ] || read -r flush_epoch flush_up <"$RUN_DIR/flush-health"
if [ -s "$RUN_DIR/collector-progress" ]; then
read -r progress_owner extra extra received errors flush_errors <"$RUN_DIR/collector-progress"
fi
now="$(log_clock)"; state=stopped
if log_process_info "$owner" >/dev/null; then
state=starting
if [ -n "$birth" ] && [ "$(log_process_info "$reader")" = "$birth" ]; then
state=waiting-for-events
if [ -s "$RUN_DIR/collector-progress" ]; then
read -r progress_owner extra extra received errors flush_errors <"$RUN_DIR/collector-progress"
if [ "$progress_owner" = "$owner" ] && [ "$received" -gt 0 ]; then
state=idle-unverified
[ "$input_up" -gt 0 ] && [ "$now" -ge "$input_up" ] && [ "$((now - input_up))" -le 120 ] && state=receiving
fi
fi
else state=reader-stopped; fi
fi
printf 'collector_state=%s\ncollector_pid=%s\nreader_pid=%s\nreceived_lines=%s\nlast_input_epoch=%s\nlast_event_epoch=%s\naccepted_events=%s\nsuppressed_events=%s\nappend_errors=%s\nflush_errors=%s\nlast_flush_epoch=%s\nheartbeat_sample_seconds=%s\n' "$state" "$owner" "$reader" "$received" "$last_input" "$last_event" "$accepted" "$suppressed" "$errors" "$flush_errors" "$flush_epoch" "$LOG_HEARTBEAT_SECONDS"
if [ "$input_up" -gt 0 ] && [ "$now" -ge "$input_up" ]; then printf 'last_input_age_seconds=%s\n' "$((now - input_up))"; fi
if [ "$event_up" -gt 0 ] && [ "$now" -ge "$event_up" ]; then printf 'last_event_age_seconds=%s\n' "$((now - event_up))"; fi
printf 'rejected_events=%s\nlast_commit_epoch=%s\n' "$failed" "$(cat "$RUN_DIR/commit-health" 2>/dev/null || printf 0)"
printf 'runtime_bytes=%s\n' "$(du -sk "$RUN_DIR" 2>/dev/null | awk '{print $1 * 1024}')"
for extra in "$owner" "$reader"; do
case "$extra" in ''|*[!0-9]*) continue ;; esac
if [ -r "/proc/$extra/status" ]; then
awk -v pid="$extra" '/^VmRSS:/ {print "process_" pid "_rss_kib=" $2}' "/proc/$extra/status"
fi
done
}
log_collect_line() {
# Explicit stdin is essential: awk/redaction must never inherit the live FIFO
# and drain subsequent syslog records behind the collector's read loop.
printf '%s\n' "$1" | log_append
}
log_collect() { log_collect() {
local owner # Lifecycle variables belong to this collect process: the EXIT trap runs after
# this function returns, when function-local variables have gone out of scope.
log_open || return 1 log_open || return 1
log_close log_close
if ! mkdir "$RUN_DIR/collector-lock" 2>/dev/null; then reader=''; birth=''
owner="$(cat "$RUN_DIR/collector-lock/pid" 2>/dev/null)" log_collector_lock; rc=$?
case "$owner" in ''|*[!0-9]*) ;; *) kill -0 "$owner" 2>/dev/null && return 0 ;; esac [ "$rc" != 2 ] || return 0
rm -f "$RUN_DIR/collector-lock/pid"; rmdir "$RUN_DIR/collector-lock" 2>/dev/null || return 1 [ "$rc" = 0 ] || return 1
mkdir "$RUN_DIR/collector-lock" || return 1 collector="$(cat "$RUN_DIR/collector-lock/pid")"
fi fifo="$RUN_DIR/stream.$collector"
log_self_pid >"$RUN_DIR/collector-lock/pid" received=0; append_errors=0; flush_errors=0; last_tick=0; fragment=''
rm -f "$RUN_DIR"/stream.* trap 'log_collector_cleanup "$?"' EXIT
fifo="$RUN_DIR/stream.$$"; reader=''; writer=''; timer=''
trap 'trap - EXIT; for child in "$reader" "$writer" "$timer"; do [ -z "$child" ] || kill "$child" 2>/dev/null || true; done; wait; rm -f "$fifo" "$RUN_DIR/collector-lock/pid"; rmdir "$RUN_DIR/collector-lock" 2>/dev/null || true; (log_flush force) || true' EXIT
trap 'exit 0' INT TERM trap 'exit 0' INT TERM
mkfifo "$fifo" || return 1 # Only the owner may remove stale runtime FIFOs. Persistent chunks are untouched.
logread -f -e 'owrt-remote\(\[[0-9][0-9]*\]\)\{0,1\}:' >"$fifo" 2>/dev/null & reader=$! rm -f "$RUN_DIR"/stream.*
while IFS= read -r line; do printf '%s\n' "$line" | log_append; done <"$fifo" & writer=$! rmdir "$RUN_DIR/diag-lock" 2>/dev/null || true
mkfifo "$fifo" || { log_diag fifo-create-failed; return 1; }
# A single parent reads and appends. No background writer can silently die or
# consume signals while procd still observes a live supervising shell.
exec 3<>"$fifo" || { log_diag fifo-open-failed; return 1; }
logread -f -l 64 -e 'owrt-remote\(\[[0-9][0-9]*\]\)\{0,1\}:' 3>&- >"$fifo" 2>/dev/null & reader=$!
birth="$(log_process_info "$reader")"
printf '%s %s\n' "$reader" "$birth" >"$RUN_DIR/collector-lock/reader"
log_collector_progress || return 1
while :; do while :; do
# Lock-operation traps must not replace the collector's child/FIFO cleanup. line=''; rc=0
(log_flush) || true # BusyBox ash supports timed, bounded reads. The FIFO itself stays on tmpfs.
sleep 5 & timer=$! IFS= read -r -t 5 -n 4096 line <&3 || rc=$?
wait "$timer" || return 0 if [ "$rc" = 0 ] || [ -n "$line" ]; then
kill -0 "$reader" 2>/dev/null && kill -0 "$writer" 2>/dev/null || return 1 # A logd line is <= its fixed line buffer. Drop an oversized fragment
# instead of growing an unbounded shell string on malformed input.
if [ "$rc" != 0 ] || [ "${#line}" -ge 4096 ]; then
fragment=1; log_diag fragmented-input
elif [ -n "$fragment" ]; then
fragment=''
else
received=$((received + 1))
log_collector_progress || return 1
if ! (log_collect_line "$line"); then
append_errors=$((append_errors + 1)); log_collector_progress || true
log_diag collector-append-failed
# Do not pretend to be healthy after an unhandled write failure.
return 1
fi
fi
fi
now="$(log_clock)"
if [ "$now" -lt "$last_tick" ] || [ "$((now - last_tick))" -ge 5 ]; then
last_tick="$now"
if ! (log_flush); then flush_errors=$((flush_errors + 1)); log_diag collector-flush-failed; fi
log_collector_progress || return 1
fi
if ! log_process_info "$reader" >/dev/null; then log_diag reader-exited; return 1; fi
# A timeout with no input is normal idle, not a failure or proof of health.
done done
} }
case "${1:-}" in case "${1:-}" in
check) log_open && log_close ;; check) log_open && log_close ;;
status) log_status ;; status) log_status ;;
diagnostics) log_init && { [ ! -f "$RUN_DIR/diagnostics" ] || cat "$RUN_DIR/diagnostics"; } ;;
migrate) log_migrate ;; migrate) log_migrate ;;
flush) log_flush "${2:-}" ;; flush) log_flush "${2:-}" ;;
collect) log_collect ;; collect) log_collect ;;
@@ -729,5 +944,5 @@ case "${1:-}" in
event) event)
case "${3:-ok}" in bad|error) level=ERROR ;; warn) level=WARN ;; *) level=OK ;; esac case "${3:-ok}" in bad|error) level=ERROR ;; warn) level=WARN ;; *) level=OK ;; esac
{ printf '%s owrt-remote: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$2"; cat; } | log_append "$level" ;; { printf '%s owrt-remote: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$2"; cat; } | log_append "$level" ;;
*) printf 'Usage: logs.sh collect|read [live]|clear|append|event MESSAGE|status|flush [force]\n' >&2; exit 2 ;; *) printf 'Usage: logs.sh collect|read [live]|clear|append|event MESSAGE|status|diagnostics|flush [force]\n' >&2; exit 2 ;;
esac esac
+111
View File
@@ -0,0 +1,111 @@
# Journal collector liveness fix (local v117)
## What the reported evidence does and does not prove
The previous `log_new_fact_locked()` normalized syslog timestamps and stored the
first successful `Heartbeat: OK; ...` in `heartbeat-ok`. The identical success
was then suppressed **without an expiry**, even after hours. A 136-byte chunk,
`ack=total`, empty pending and live logread therefore do not prove a stalled
collector. They also do not establish a memory leak. A shell syntax check and
`persistent=1` prove neither event delivery nor process health.
The old supervisor checked only `kill -0` for its background writer and reader;
it ignored `log_append` failure. This could conceal write/lock errors while the
supervisor remained alive. These failure paths are corrected independently of
the deduplication issue. No claim is made that they caused this particular
AX3000T report without measurements from that device.
## Changes
- One supervising collect shell consumes its private FIFO with BusyBox ash
timed reads. There is one logread child, no separate persistent writer shell
and no five-second sleep child. At most one 4096-byte input fragment is kept
in a shell variable. Oversized/fragmented input is diagnosed and discarded;
it cannot grow a shell string without a bound.
- Each complete line is handed to `log_append` over an explicit short pipe,
isolated from the live FIFO. Append failure is checked, recorded and causes
a nonzero exit so procd can recover. Lock-operation traps run in subshells
and cannot replace the collector's cleanup trap.
- Normal stop terminates/reaps logread, closes/removes the FIFO, flushes a final
bounded batch and releases only its own collector lock. SIGKILL recovery
recognizes a stale owner and terminates its recorded orphan reader only if
PID **and kernel start time** match. Stale-lock reaping is serialized.
- procd's journal instance retries every 10 seconds without exhausting a
five-crash budget. The archive instance and networking services are unchanged.
- Builds without fractional BusyBox sleep use whole-second waits. Lock and
child cleanup deadlines account for that duration; unsupported `sleep 0.1`
cannot produce a fast loop, repeated errors or a 100-second lock wait.
- A successful heartbeat is saved at most once per 300 seconds when unchanged,
plus changes and recovery. Intervening duplicates update small RAM health
metadata only. Flash flush is still once per 60 seconds, <=16 KiB per batch,
with existing size/age/part limits. No per-heartbeat Flash write is introduced.
- Starting logread requests only a 64-message system-ring tail plus follow,
to recover a short interruption. Existing fact dedup and bounded pending
apply; this is not a guarantee of lossless collection during a long outage.
- `status` adds input and accepted-event times, counters, collector state,
append/flush errors, the last completed batch/commit, allocated runtime bytes
and RSS for the collector/reader when proc provides it. `receiving` means
recently processed input; `idle-unverified` explicitly does not assert health.
- `diagnostics` is a private RAM-only <=4 KiB reason-code history (plus <=2 KiB
compaction), with duplicate-rate limiting. It records FIFO creation/open,
fragmented input, lock timeout, append, flush and reader exit failures without
storing event text, keys or command lines. It is not sent back through logger,
avoiding diagnostic feedback loops.
- Existing validated persistent storage, immutable checksummed chunks, ack
protocol and limited RAM fallback remain. Runtime health is lost on reboot;
retained journal/ack data are not cleared by a collector restart.
## Local install and rollback (Windows PowerShell 5.1/11)
The dedicated installer replaces **only** `logs.sh` and `owrt-remote-logs`.
It backs up code/service state, preserves journal chunks and UCI configuration,
restarts only the log service, verifies a real logger marker through the FIFO
and performs one explicit deployment flush. Failure restores the previous code.
It does not remove `xray.zip`, update Xray, restart the tunnel, modify Podkop,
WAN/DNS or policy routing. VPS files and hosting URLs are untouched.
```powershell
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "C:\Users\k.zolotarev95\Documents\openwrt overlay\vps\deploy-log-collector-fix.ps1" -Router "root@192.168.2.1" -Action Install
```
Retained backup: `/root/owrt-remote-backup-before-log-collector-v117`.
Rollback leaves journal data intact:
```powershell
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "C:\Users\k.zolotarev95\Documents\openwrt overlay\vps\deploy-log-collector-fix.ps1" -Router "root@192.168.2.1" -Action Rollback
```
After installation compare two samples a minute apart. `last_input_epoch` and
`suppressed_events` should advance on stable heartbeat. An unchanged heartbeat
becomes a new retained sample after five minutes. A distinct marker is visible
immediately in read, then committed by the next normal flush (up to ~65 seconds).
```powershell
ssh root@192.168.2.1 'logger -t owrt-remote collector-user-check; sleep 3; sh /usr/lib/owrt-remote/logs.sh read | tail -n 12; sh /usr/lib/owrt-remote/logs.sh status; sh /usr/lib/owrt-remote/logs.sh diagnostics; free'
```
Measure memory using `MemAvailable`, tmpfs allocation and process RSS over time;
`free` memory alone includes reclaimable kernel/filesystem cache. Do not attribute
all process RSS or low available RAM to the log spool without these measurements.
No router or VPS is modified by automated tests.
## Automated coverage and limits
- Real FIFO tests using MSYS shell on Windows: live heartbeat sampling, distinct
events through pending/Flash, duplicate collector rejection, reader death,
append failure, owner SIGKILL and successor recovery, TERM cleanup/final flush.
- Real BusyBox ash/applets: spool bounds, atomic/checksum/ack recovery, sampling,
limited diagnostics, fallback and cleanup paths; Windows BusyBox lacks FIFOs.
- Sandboxed installer: install, first-backup preservation, rollback, bad syntax,
failed verification restoration, journal retention.
- Existing redaction/CGI/Hub transport regressions and shell syntax checks.
This is not a claim of physical AX3000T / OpenWrt 24.10.3 UBIFS testing, hardware
power-cut testing or a measured RAM reduction in MiB. The actual device still
needs event/flush/reboot and memory observations before release.
Implementation references checked:
[OpenWrt 24.10 ubox version](https://github.com/openwrt/openwrt/blob/openwrt-24.10/package/system/ubox/Makefile),
[its logread](https://github.com/openwrt/ubox/blob/85f1053019caf4cd333795760950235ee4529ba7/log/logread.c),
[BusyBox ash timed read](https://github.com/mirror/busybox/blob/1_36_stable/shell/ash.c),
[procd respawn](https://github.com/openwrt/procd/blob/master/service/instance.c).
+2 -2
View File
@@ -108,7 +108,7 @@ class GlobalLogTests(unittest.TestCase):
self.helper_call('append', good * 3) self.helper_call('append', good * 3)
self.helper_call('append', 'Other action\n') self.helper_call('append', 'Other action\n')
self.helper_call('append', good, env={**self.env, 'TEST_UPTIME': '2000'}) self.helper_call('append', good, env={**self.env, 'TEST_UPTIME': '2000'})
self.assertEqual(self.helper_call('read', env={**self.env, 'TEST_UPTIME': '2000'}).count('Heartbeat: OK;'), 1) self.assertEqual(self.helper_call('read', env={**self.env, 'TEST_UPTIME': '2000'}).count('Heartbeat: OK;'), 2)
failed = 'Heartbeat: ошибка; endpoint=https://hub.example; rc=7; connection refused\n' failed = 'Heartbeat: ошибка; endpoint=https://hub.example; rc=7; connection refused\n'
self.helper_call('append', failed * 3, env={**self.env, 'TEST_UPTIME': '2001'}) self.helper_call('append', failed * 3, env={**self.env, 'TEST_UPTIME': '2001'})
self.helper_call('append', good, env={**self.env, 'TEST_UPTIME': '2002'}) self.helper_call('append', good, env={**self.env, 'TEST_UPTIME': '2002'})
@@ -117,7 +117,7 @@ class GlobalLogTests(unittest.TestCase):
env={**self.env, 'TEST_UPTIME': '2004'}) env={**self.env, 'TEST_UPTIME': '2004'})
stored = self.helper_call('read', env={**self.env, 'TEST_UPTIME': '2004'}) stored = self.helper_call('read', env={**self.env, 'TEST_UPTIME': '2004'})
self.assertEqual(stored.count('[ERROR] Heartbeat:'), 2, 'same failure after recovery must appear') self.assertEqual(stored.count('[ERROR] Heartbeat:'), 2, 'same failure after recovery must appear')
self.assertEqual(stored.count('[OK] Heartbeat:'), 3, 'first, recovery and another WAN') self.assertEqual(stored.count('[OK] Heartbeat:'), 4, 'first, periodic sample, recovery and another WAN')
def test_recurring_failure_is_recorded_again_after_a_quiet_period(self): def test_recurring_failure_is_recorded_again_after_a_quiet_period(self):
failure = 'owrt-remote: cannot download Xray\n' failure = 'owrt-remote: cannot download Xray\n'
+172
View File
@@ -0,0 +1,172 @@
"""Real FIFO lifecycle tests; MSYS provides FIFOs on Windows, BusyBox tests the applets separately."""
import os
from pathlib import Path
import subprocess
import time
import unittest
import test_persistent_logs as spool
class CollectorTests(unittest.TestCase):
setUp = spool.PersistentLogsTests.setUp
call = spool.PersistentLogsTests.call
status = spool.PersistentLogsTests.status
flush = spool.PersistentLogsTests.flush
journal = spool.PersistentLogsTests.journal
# Reuse filesystem fixtures without inheriting the persistent spool test cases.
def start(self, extra=""):
self.input = self.root / 'input'
self.env['TEST_INPUT'] = self.input.as_posix()
self.env['TEST_CLOCK_FILE'] = (self.root / 'clock').as_posix()
(self.root / 'clock').write_text('1000\n')
mocks = r'''
log_clock() { cat "$TEST_CLOCK_FILE"; }
logread() {
exec 4<>"$TEST_INPUT"
while :; do if IFS= read -r -t 1 item <&4; then printf '%s\n' "$item"; fi; done
}
'''
source = self.source.replace('case "${1:-}" in', mocks + extra + '\ncase "${1:-}" in', 1)
self.helper.write_text(source, encoding='utf-8', newline='\n')
if not getattr(self, 'input_created', False):
result = subprocess.run([*self.shell, '-c', 'mkfifo "$TEST_INPUT"'], env=self.env,
capture_output=True, timeout=10)
if result.returncode:
self.skipTest('This shell has no FIFO support')
self.input_created = True
output = (self.root / ('process-%d.log' % len(getattr(self, 'processes', [])))).open('wb')
proc = subprocess.Popen([*self.shell, self.helper.as_posix(), 'collect'], env=self.env,
stdout=output, stderr=output)
if not hasattr(self, 'processes'):
self.processes = []
self.addCleanup(self.stop_processes)
self.processes.append((proc, output))
return proc
def wait_for(self, predicate, seconds=20):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(.15)
self.fail('Collector did not reach expected state')
def ready(self):
return self.wait_for(lambda: (self.run_dir / 'collector-lock/reader').exists())
def signal(self, pid, signal='TERM'):
return subprocess.run([*self.shell, '-c', 'kill -"$1" "$2"', '--', signal, str(pid)],
env=self.env, capture_output=True, timeout=10)
def stop_processes(self):
lock = self.run_dir / 'collector-lock/pid'
if lock.exists():
self.signal(lock.read_text().strip())
for proc, output in self.processes:
try:
proc.wait(timeout=15)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=5)
output.close()
def send(self, *lines):
result = subprocess.run([*self.shell, '-c', 'cat >"$TEST_INPUT"'], input='\n'.join(lines) + '\n',
encoding='utf-8', env=self.env, capture_output=True, timeout=15)
self.assertEqual(result.returncode, 0, result.stderr)
def test_actual_fifo_heartbeat_freshness_sample_and_batch_flush(self):
proc = self.start(); self.ready()
good = 'Thu Oct 15 22:52:36 2026 user.notice owrt-remote: Heartbeat: OK; Hub=https://hub.example; WAN_direct=1; WAN_active=wan'
self.send(good)
self.wait_for(lambda: self.status()['accepted_events'] == '1')
self.send(good.replace('22:52:36', '22:53:08'))
self.wait_for(lambda: self.status()['suppressed_events'] == '1')
state = self.status()
self.assertEqual(state['collector_state'], 'receiving')
self.assertEqual(state['received_lines'], '2')
self.assertGreater(int(state['last_input_epoch']), 0)
self.assertEqual(state['committed_bytes'], '0')
(self.root / 'clock').write_text('1301\n')
self.send(good.replace('22:52:36', '22:57:38'), 'owrt-remote: panel action after heartbeat')
self.wait_for(lambda: self.status()['accepted_events'] == '3')
read = self.call('read').stdout
self.assertEqual(read.count('Heartbeat: OK;'), 2)
self.assertIn('panel action after heartbeat', read)
self.flush()
self.assertEqual(self.status()['pending_bytes'], '0')
self.assertIn(b'panel action after heartbeat', b''.join(p.read_bytes() for p in self.journal.glob('part.*')))
self.assertIsNone(proc.poll())
def test_duplicate_collect_does_not_remove_owner_fifo(self):
proc = self.start(); self.ready()
owner = (self.run_dir / 'collector-lock/pid').read_text()
fifo = list(self.run_dir.glob('stream.*'))
other = self.start(); self.assertEqual(other.wait(timeout=15), 0)
self.assertEqual((self.run_dir / 'collector-lock/pid').read_text(), owner)
self.assertEqual(list(self.run_dir.glob('stream.*')), fifo)
self.send('owrt-remote: owner still receives')
self.wait_for(lambda: self.status()['accepted_events'] == '1')
self.assertIsNone(proc.poll())
def test_reader_death_exits_nonzero_and_retains_journal_for_restart(self):
proc = self.start(); self.ready()
self.send('owrt-remote: before logread failure')
self.wait_for(lambda: self.status()['accepted_events'] == '1')
reader = (self.run_dir / 'collector-lock/reader').read_text().split()[0]
self.signal(reader, 'KILL')
self.assertNotEqual(proc.wait(timeout=20), 0)
self.assertFalse((self.run_dir / 'collector-lock').exists())
self.assertFalse(list(self.run_dir.glob('stream.*')))
self.assertIn('reader-exited', self.call('diagnostics').stdout)
self.assertIn('before logread failure', self.call('read').stdout)
new = self.start(); self.ready()
self.send('owrt-remote: after reader recovery')
self.wait_for(lambda: self.status()['accepted_events'] == '2')
self.assertIsNone(new.poll())
def test_sigkill_owner_reaps_identified_reader_and_stale_fifo(self):
proc = self.start(); self.ready()
owner = (self.run_dir / 'collector-lock/pid').read_text().strip()
self.send('owrt-remote: retained before killed collector')
self.wait_for(lambda: self.status()['accepted_events'] == '1')
old_reader = (self.run_dir / 'collector-lock/reader').read_text().split()[0]
self.signal(owner, 'KILL'); proc.wait(timeout=10)
new = self.start()
self.wait_for(lambda: (self.run_dir / 'collector-lock/pid').exists() and
(self.run_dir / 'collector-lock/pid').read_text().strip() != owner)
self.wait_for(lambda: (self.run_dir / 'collector-lock/reader').exists() and
(self.run_dir / 'collector-lock/reader').read_text().split()[0] != old_reader)
self.assertEqual(len(list(self.run_dir.glob('stream.*'))), 1)
self.send('owrt-remote: recovered collector')
self.wait_for(lambda: self.status()['accepted_events'] == '2')
text = self.call('read').stdout
self.assertIn('retained before killed collector', text)
self.assertIn('recovered collector', text)
self.assertIsNone(new.poll())
def test_append_failure_is_not_silently_ignored(self):
proc = self.start('\nlog_collect_line() { return 1; }\n'); self.ready()
self.send('owrt-remote: rejected input')
self.assertNotEqual(proc.wait(timeout=20), 0)
self.assertIn('collector-append-failed', self.call('diagnostics').stdout)
self.assertEqual(self.status()['append_errors'], '1')
self.assertFalse((self.run_dir / 'collector-lock').exists())
def test_term_flushes_and_cleans_children_fifo_and_owner_lock(self):
proc = self.start(); self.ready()
self.send('owrt-remote: shutdown flush')
self.wait_for(lambda: self.status()['accepted_events'] == '1')
owner = (self.run_dir / 'collector-lock/pid').read_text().strip()
self.signal(owner)
self.assertEqual(proc.wait(timeout=15), 0)
self.assertEqual(self.status()['pending_bytes'], '0')
self.assertIn('shutdown flush', self.call('read').stdout)
self.assertFalse(list(self.run_dir.glob('stream.*')))
self.assertFalse((self.run_dir / 'collector-lock').exists())
if __name__ == '__main__':
unittest.main()
+95
View File
@@ -0,0 +1,95 @@
"""Deploy only the journal helper/service, preserving data and networking."""
import os
from pathlib import Path
import re
import shlex
import shutil
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
GIT_BIN = Path(os.environ.get('ProgramFiles', 'C:/Program Files')) / 'Git/usr/bin'
SH = str(GIT_BIN / 'sh.exe') if os.name == 'nt' else shutil.which('sh')
class CollectorDeploymentTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory(prefix='collector-deploy-')
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.targets = {'logs.sh': '/usr/lib/owrt-remote/logs.sh', 'logs-init': '/etc/init.d/owrt-remote-logs'}
self.initial = {}
for item, target in self.targets.items():
path = self.root / target.lstrip('/')
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text('#!/bin/sh\necho old-' + item + '-$1\n', newline='\n')
self.initial[item] = path.read_bytes()
self.backup = self.root / 'backup'
self.journal = self.root / 'retained-journal'; self.journal.write_text('keep history')
text = (ROOT / 'vps/deploy-log-collector-fix.ps1').read_text(encoding='utf-8')
self.script = text.split("$RemoteScript = @'\n", 1)[1].split("\n'@", 1)[0]
mapping = {target: (self.root / target.lstrip('/')).as_posix() for target in self.targets.values()}
mapping['/root/owrt-remote-backup-before-log-collector-v117'] = self.backup.as_posix()
pattern = '|'.join(re.escape(path) for path in sorted(mapping, key=len, reverse=True))
self.script = re.sub(pattern, lambda m: mapping[m.group()], self.script)
mocks = r'''
ubus() { printf '{"running":true}\n'; }
jsonfilter() { printf true; }
logread() { return 0; }
mkfifo() { return 0; }
logger() { printf '%s\n' "$3" >"$TEST_MARKER"; }
sleep() { return 0; }
'''
self.script = self.script.replace('trap cleanup EXIT', mocks + '\ntrap cleanup EXIT')
self.env = {**os.environ, 'PATH': str(GIT_BIN) + os.pathsep + os.environ['PATH'],
'TEST_MARKER': (self.root / 'marker').as_posix()}
def deploy(self, action, bad=False, fail=False):
stage = self.root / 'stage'; stage.mkdir()
if action == 'Install':
helper = r'''#!/bin/sh
# log_collector_progress()
# LOG_HEARTBEAT_SECONDS=300
case "$1" in
read) cat "$TEST_MARKER" ;;
status) echo persistent=1 ;;
esac
'''
if bad: helper = '#!/bin/sh\nif true; then\n'
(stage / 'logs.sh').write_text(helper, newline='\n')
(stage / 'logs-init').write_text('#!/bin/sh\necho new-service-$1\n', newline='\n')
script = self.script.replace('stage=/tmp/__UPLOAD_NAME__', 'stage=' + shlex.quote(stage.as_posix()))
script = script.replace('__UPLOAD_NAME__', 'sandbox').replace('__ACTION__', action)
if fail: script = script.replace('cmp "$source_dir/$item" "$path"', 'false')
result = subprocess.run([SH], input=script, text=True, encoding='utf-8', capture_output=True, env=self.env, timeout=20)
self.assertFalse(stage.exists())
self.assertEqual(self.journal.read_text(), 'keep history')
return result
def test_install_and_rollback_keep_first_backup_and_journal(self):
for _ in range(2):
result = self.deploy('Install')
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('LOG_COLLECTOR_ROUTER_Install_OK', result.stdout)
for item, initial in self.initial.items(): self.assertEqual((self.backup / item).read_bytes(), initial)
result = self.deploy('Rollback')
self.assertEqual(result.returncode, 0, result.stderr)
for item, target in self.targets.items(): self.assertEqual((self.root / target.lstrip('/')).read_bytes(), self.initial[item])
def test_missing_backup_is_noop(self):
result = self.deploy('Rollback')
self.assertNotEqual(result.returncode, 0)
self.assertIn('Nothing changed', result.stderr)
def test_invalid_shell_rejected_before_backup(self):
result = self.deploy('Install', bad=True)
self.assertNotEqual(result.returncode, 0)
self.assertFalse(self.backup.exists())
def test_failed_verify_restores_previous_code(self):
result = self.deploy('Install', fail=True)
self.assertNotEqual(result.returncode, 0)
self.assertIn('Restoring journal code', result.stderr)
for item, target in self.targets.items(): self.assertEqual((self.root / target.lstrip('/')).read_bytes(), self.initial[item])
if __name__ == '__main__': unittest.main()
+70 -3
View File
@@ -128,7 +128,7 @@ class PersistentLogsTests(unittest.TestCase):
(self.run_dir / 'facts').write_text(''.join('1000\ts256-%035d\n' % i for i in range(63))) (self.run_dir / 'facts').write_text(''.join('1000\ts256-%035d\n' % i for i in range(63)))
self.append('distinct event 63\ndistinct event 64\n') self.append('distinct event 63\ndistinct event 64\n')
self.assertEqual(len((self.run_dir / 'facts').read_text().splitlines()), 64) self.assertEqual(len((self.run_dir / 'facts').read_text().splitlines()), 64)
self.assertLessEqual((self.run_dir / 'facts').stat().st_size, 4096) self.assertLessEqual((self.run_dir / 'facts').stat().st_size if (self.run_dir / 'facts').exists() else 0, 4096)
def test_empty_digest_files_falsely_acked_by_old_hub_are_recovered(self): def test_empty_digest_files_falsely_acked_by_old_hub_are_recovered(self):
self.append('retained restart event\n'); self.flush() self.append('retained restart event\n'); self.flush()
@@ -242,7 +242,7 @@ class PersistentLogsTests(unittest.TestCase):
self.flush() self.flush()
self.assertEqual({p.name: p.read_bytes() for p in parts}, original) self.assertEqual({p.name: p.read_bytes() for p in parts}, original)
self.assertLessEqual((self.run_dir / 'pending').stat().st_size, 16384) self.assertLessEqual((self.run_dir / 'pending').stat().st_size, 16384)
self.assertLessEqual((self.run_dir / 'facts').stat().st_size, 4096) self.assertLessEqual((self.run_dir / 'facts').stat().st_size if (self.run_dir / 'facts').exists() else 0, 4096)
first = self.export() first = self.export()
self.acknowledge(first) self.acknowledge(first)
self.flush() self.flush()
@@ -353,6 +353,32 @@ class PersistentLogsTests(unittest.TestCase):
self.assertIn('after killed reaper', self.call('read').stdout) self.assertIn('after killed reaper', self.call('read').stdout)
self.assertFalse(lock.exists()) self.assertFalse(lock.exists())
def test_integer_only_sleep_keeps_lock_timeout_bounded_and_quiet(self):
self.call('check')
source = self.source.split('case "${1:-}" in', 1)[0]
commands = r'''
log_init
mkdir "$RUN_DIR/lock"
printf '12345\n' >"$RUN_DIR/lock/pid"
sleep() {
printf '%s\n' "$1" >>"$RUN_DIR/pauses"
if [ "$1" = 0.1 ]; then echo "sleep: invalid number '0.1'" >&2; return 1; fi
[ "$1" = 1 ]
}
kill() { return 0; }
if log_lock; then exit 17; fi
'''
script = self.root / 'integer-sleep.sh'
script.write_text(source + commands, encoding='utf-8', newline='\n')
result = subprocess.run([*self.shell, script.as_posix()], env=self.env,
capture_output=True, text=True, timeout=15)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stderr, '')
pauses = (self.run_dir / 'pauses').read_text().splitlines()
self.assertEqual(pauses.count('0.1'), 1, 'probe once per process')
self.assertEqual(pauses.count('1'), 10, 'ten-second deadline, no busy spin or 100-second wait')
self.assertIn('lock-timeout', self.call('diagnostics').stdout)
def test_collector_failure_reaps_children_and_removes_fifo(self): def test_collector_failure_reaps_children_and_removes_fifo(self):
# Actual FIFO integration is tested on POSIX; Windows BusyBox has no # Actual FIFO integration is tested on POSIX; Windows BusyBox has no
# FIFO support. Exercise its traps with mocked process/FIFO operations. # FIFO support. Exercise its traps with mocked process/FIFO operations.
@@ -373,7 +399,48 @@ wait() { return 0; }
self.assertFalse(list(self.run_dir.glob('stream.*'))) self.assertFalse(list(self.run_dir.glob('stream.*')))
self.assertFalse((self.run_dir / 'collector-lock').exists()) self.assertFalse((self.run_dir / 'collector-lock').exists())
lines = (self.run_dir / 'kills').read_text().splitlines() lines = (self.run_dir / 'kills').read_text().splitlines()
self.assertGreaterEqual(len([line for line in lines if not line.startswith('-0')]), 2) self.assertGreaterEqual(len([line for line in lines if not line.startswith('-0')]), 1)
def test_heartbeat_sampling_and_freshness_are_independent_of_flash_writes(self):
good = 'Thu Oct 15 22:52:36 2026 user.notice owrt-remote: Heartbeat: OK; Hub=https://hub.example; WAN_direct=1; WAN_active=wan\n'
self.append(good)
self.append(good.replace('22:52:36', '22:53:08'), TEST_UPTIME='1032')
state = self.status()
self.assertEqual((state['accepted_events'], state['suppressed_events']), ('1', '1'))
self.assertEqual(state['committed_bytes'], '0')
self.assertGreater(int(state['last_input_epoch']), 0)
self.assertEqual(state['collector_state'], 'stopped', 'persistent storage does not imply a running collector')
self.append(good.replace('22:52:36', '22:57:38'), TEST_UPTIME='1302')
self.assertEqual(self.call('read').stdout.count('Heartbeat: OK;'), 2)
self.flush()
self.assertEqual(self.status()['pending_bytes'], '0')
self.assertGreater(int(self.status()['last_commit_epoch']), 0)
def test_private_diagnostics_are_bounded_rate_limited_and_no_syslog_feedback(self):
self.call('check')
source = self.source.split('case "${1:-}" in', 1)[0]
commands = r'''
log_init
logger() { echo forbidden-syslog-feedback >&2; return 1; }
i=0
while [ "$i" -lt 140 ]; do
if [ "$((i % 2))" = 0 ]; then log_diag flush-failed; else log_diag lock-timeout; fi
i=$((i + 1))
done
log_diag lock-timeout
'''
script = self.root / 'diag-test.sh'
script.write_text(source + commands, encoding='utf-8', newline='\n')
result = subprocess.run([*self.shell, script.as_posix()], env=self.env,
capture_output=True, text=True, timeout=30)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stderr, '')
diagnostics = self.run_dir / 'diagnostics'
self.assertLessEqual(diagnostics.stat().st_size, 4096)
self.assertFalse((self.run_dir / 'diagnostics.new').exists())
self.assertFalse((self.run_dir / 'diag-lock').exists())
self.assertEqual(self.call('diagnostics').stdout, diagnostics.read_text())
self.assertFalse((self.journal / 'diagnostics').exists())
def test_legacy_migration_preserves_identity_ack_and_only_undelivered_bytes(self): def test_legacy_migration_preserves_identity_ack_and_only_undelivered_bytes(self):
legacy = self.root / 'legacy' legacy = self.root / 'legacy'
+148
View File
@@ -0,0 +1,148 @@
param(
[ValidateNotNullOrEmpty()][string]$Router = 'root@192.168.2.1',
[ValidateSet('Install', 'Rollback')][string]$Action = 'Install',
[switch]$PrepareOnly
)
$ErrorActionPreference = 'Stop'
$RepoRoot = Split-Path -Parent $PSScriptRoot
$UploadName = 'owrt-log-collector-' + [guid]::NewGuid().ToString('N')
$StageRoot = Join-Path ([System.IO.Path]::GetTempPath()) $UploadName
$Utf8 = [System.Text.UTF8Encoding]::new($false)
$Payloads = @{
'logs.sh' = 'files\usr\lib\owrt-remote\logs.sh'
'logs-init' = 'files\etc\init.d\owrt-remote-logs'
}
$RemoteScript = @'
#!/bin/sh
set -eu
stage=/tmp/__UPLOAD_NAME__
backup_dir=/root/owrt-remote-backup-before-log-collector-v117
changed=0
phase=preflight
items='logs.sh logs-init'
item_path() {
case "$1" in
logs.sh) printf /usr/lib/owrt-remote/logs.sh ;;
logs-init) printf /etc/init.d/owrt-remote-logs ;;
esac
}
complete_backup() {
[ -f "$1/complete" ] && [ -s "$1/logs.sh" ] && [ -s "$1/logs-init" ]
}
collector_running() {
ubus call service list '{"name":"owrt-remote-logs"}' | jsonfilter -e '@["owrt-remote-logs"].instances.journal.running' | grep -qx true
}
save_files() {
local item path
mkdir -p "$1"
chmod 700 "$1"
for item in $items; do path=$(item_path "$item"); cp -p "$path" "$1/$item" || return 1; done
if /etc/init.d/owrt-remote-logs enabled; then : >"$1/enabled"; fi
if collector_running; then : >"$1/running"; fi
: >"$1/complete"
}
restore_files() {
local item path
complete_backup "$1" || return 1
/etc/init.d/owrt-remote-logs stop || true
for item in $items; do
path=$(item_path "$item")
cp -p "$1/$item" "$path.new-__UPLOAD_NAME__" && mv -f "$path.new-__UPLOAD_NAME__" "$path" || return 1
done
if [ -f "$1/enabled" ]; then /etc/init.d/owrt-remote-logs enable; else /etc/init.d/owrt-remote-logs disable; fi
if [ -f "$1/running" ]; then /etc/init.d/owrt-remote-logs start; fi
}
cleanup() {
code=$?
trap - EXIT
if [ "$code" -ne 0 ]; then
printf 'Failed during %s (rc=%s).\n' "$phase" "$code" >&2
if [ "$changed" = 1 ]; then
echo 'Restoring journal code/service state from this attempt; journal data retained.' >&2
restore_files "$stage/previous" || echo 'Restore failed; retained backup is available.' >&2
fi
fi
for item in $items; do path=$(item_path "$item"); rm -f "$path.new-__UPLOAD_NAME__"; done
rm -rf "$stage"
exit "$code"
}
trap cleanup EXIT
for item in $items; do path=$(item_path "$item"); test -s "$path"; done
command -v ubus >/dev/null
command -v jsonfilter >/dev/null
if [ '__ACTION__' = Install ]; then
for item in $items; do test -s "$stage/$item"; sh -n "$stage/$item"; done
command -v logread >/dev/null
command -v logger >/dev/null
command -v mkfifo >/dev/null
grep -Fq 'log_collector_progress()' "$stage/logs.sh"
grep -Fq 'LOG_HEARTBEAT_SECONDS=300' "$stage/logs.sh"
if [ ! -e "$backup_dir" ]; then save_files "$stage/retained"; mv "$stage/retained" "$backup_dir"; fi
complete_backup "$backup_dir" || { echo "Incomplete backup: $backup_dir. Nothing changed." >&2; exit 1; }
source_dir=$stage
else
complete_backup "$backup_dir" || { echo "No complete backup: $backup_dir. Nothing changed." >&2; exit 1; }
for item in $items; do sh -n "$backup_dir/$item"; done
source_dir=$backup_dir
fi
save_files "$stage/previous"
phase=install
changed=1
/etc/init.d/owrt-remote-logs stop
for item in $items; do
path=$(item_path "$item")
cp "$source_dir/$item" "$path.new-__UPLOAD_NAME__"
case "$item" in logs.sh) chmod 644 "$path.new-__UPLOAD_NAME__" ;; *) chmod 755 "$path.new-__UPLOAD_NAME__" ;; esac
mv -f "$path.new-__UPLOAD_NAME__" "$path"
done
if [ '__ACTION__' = Install ]; then
/etc/init.d/owrt-remote-logs enable
/etc/init.d/owrt-remote-logs start
phase=verify-delivery
marker=collector-check-__UPLOAD_NAME__
ready=0
for attempt in 1 2 3 4 5 6 7 8 9 10; do
logger -t owrt-remote "$marker"
sleep 1
if collector_running && sh /usr/lib/owrt-remote/logs.sh read | grep -Fq "$marker"; then ready=1; break; fi
done
[ "$ready" = 1 ] || { echo 'Collector did not capture the test event; restoring code.' >&2; exit 1; }
# A forced flush verifies the write path once at deployment, not each Heartbeat.
phase=verify-flush
sh /usr/lib/owrt-remote/logs.sh flush force
sh /usr/lib/owrt-remote/logs.sh read | grep -Fq "$marker"
sh /usr/lib/owrt-remote/logs.sh status
sh /usr/lib/owrt-remote/logs.sh diagnostics
else
if [ -f "$backup_dir/enabled" ]; then /etc/init.d/owrt-remote-logs enable; else /etc/init.d/owrt-remote-logs disable; fi
if [ -f "$backup_dir/running" ]; then /etc/init.d/owrt-remote-logs start; fi
fi
for item in $items; do path=$(item_path "$item"); cmp "$source_dir/$item" "$path"; done
printf 'Backup: %s\n' "$backup_dir"
printf '\nLOG_COLLECTOR_ROUTER_%s_OK\n' '__ACTION__'
'@
$RemoteScript = $RemoteScript.Replace('__UPLOAD_NAME__', $UploadName).Replace('__ACTION__', $Action)
try {
[System.IO.Directory]::CreateDirectory($StageRoot) | Out-Null
[System.IO.File]::WriteAllText((Join-Path $StageRoot 'run.sh'), $RemoteScript.Replace("`r`n", "`n") + "`n", $Utf8)
if ($Action -eq 'Install') {
foreach ($Item in $Payloads.GetEnumerator()) {
$Content = [System.IO.File]::ReadAllText((Join-Path $RepoRoot $Item.Value)).Replace("`r`n", "`n")
[System.IO.File]::WriteAllText((Join-Path $StageRoot $Item.Key), $Content, $Utf8)
}
}
if ($PrepareOnly) { Write-Host "Prepared only: $StageRoot"; return }
& scp -O -r -- $StageRoot "${Router}:/tmp/"
if ($LASTEXITCODE -ne 0) { throw 'Upload failed. The installation has not started.' }
& ssh -T -o ConnectTimeout=10 $Router "sh /tmp/$UploadName/run.sh"
if ($LASTEXITCODE -ne 0) { throw "$Action failed. See the output above; success was not confirmed." }
Write-Host "Log collector fix $Action finished successfully."
} finally {
if (-not $PrepareOnly) {
$Resolved = [System.IO.Path]::GetFullPath($StageRoot)
$Expected = [System.IO.Path]::GetFullPath((Join-Path ([System.IO.Path]::GetTempPath()) $UploadName))
if ($Resolved -ne $Expected -or (Split-Path -Leaf $Resolved) -ne $UploadName) { throw 'Unsafe staging cleanup path.' }
if (Test-Path -LiteralPath $Resolved) { Remove-Item -LiteralPath $Resolved -Recurse -Force }
}
}