564 lines
24 KiB
Python
564 lines
24 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded VPS log maintenance. Never opens the Hub database or router archives.
|
|
|
|
Uses the distribution's logrotate rules and reopen hooks, with retention
|
|
overrides in a temporary configuration. One owner: system logrotate.timer.
|
|
"""
|
|
import argparse
|
|
import ast
|
|
import datetime as dt
|
|
import glob
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
MARKER = "# Managed by OpenWrt Remote log maintenance"
|
|
INSTALL_PATH = Path("/opt/owrt-remote/log-maintenance.py")
|
|
BACKUP = Path("/root/owrt-remote-backup-before-log-maintenance-v121")
|
|
STATE = Path("/var/lib/owrt-remote-log-maintenance")
|
|
RUN = Path("/run/owrt-remote-log-maintenance")
|
|
RESULT = Path("/var/log/owrt-remote-maintenance.log")
|
|
HUB = Path("/opt/owrt-remote/owrt-remote-hub.py")
|
|
LEGACY = Path("/etc/logrotate.d/owrt-remote")
|
|
POLICY = "\n daily\n maxsize 20M\n rotate 1\n maxage 1\n compress\n nodelaycompress\n"
|
|
CONFIGS = {
|
|
"/etc/systemd/journald.conf.d/owrt-remote-limits.conf": MARKER + """
|
|
[Journal]
|
|
SystemMaxUse=50M
|
|
SystemKeepFree=500M
|
|
SystemMaxFileSize=10M
|
|
RuntimeMaxUse=16M
|
|
RuntimeKeepFree=64M
|
|
MaxRetentionSec=7day
|
|
Compress=yes
|
|
""",
|
|
"/etc/systemd/system/logrotate.timer.d/owrt-remote.conf": MARKER + """
|
|
[Timer]
|
|
OnCalendar=
|
|
OnCalendar=*-*-* 03:20:00
|
|
AccuracySec=1min
|
|
RandomizedDelaySec=0
|
|
Persistent=true
|
|
""",
|
|
"/etc/systemd/system/logrotate.service.d/owrt-remote.conf": MARKER + """
|
|
[Service]
|
|
ExecStart=
|
|
ExecStart=/usr/bin/python3 /opt/owrt-remote/log-maintenance.py run
|
|
Nice=10
|
|
IOSchedulingClass=idle
|
|
TimeoutStartSec=30min
|
|
""",
|
|
"/etc/nginx/conf.d/owrt-remote-log-policy.conf": MARKER + """
|
|
# Only routine successful traffic is quiet; errors and other actions remain.
|
|
map "$uri:$status" $owrt_remote_access_log {
|
|
default 1;
|
|
~^/api/notifications/wait:[23][0-9][0-9]$ 0;
|
|
~^/api/ssh-session/[^/]+/read:[23][0-9][0-9]$ 0;
|
|
~^/api/heartbeat:[23][0-9][0-9]$ 0;
|
|
~^/api/router-logs:[23][0-9][0-9]$ 0;
|
|
}
|
|
""",
|
|
}
|
|
|
|
|
|
def atomic(path, data, mode=0o600):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if path.is_symlink():
|
|
raise RuntimeError("Refusing symlink: " + str(path))
|
|
fd, name = tempfile.mkstemp(prefix="." + path.name + ".", dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, "wb") as stream:
|
|
os.fchmod(stream.fileno(), mode)
|
|
stream.write(data if isinstance(data, bytes) else data.encode())
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(name, path)
|
|
finally:
|
|
if os.path.exists(name):
|
|
os.unlink(name)
|
|
|
|
|
|
def command(args, timeout=120, check=True):
|
|
result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
timeout=timeout, text=True, errors="replace")
|
|
if check and result.returncode:
|
|
raise RuntimeError(f"{shlex.join(args)}: rc={result.returncode}; {result.stdout[-6000:]}")
|
|
return result
|
|
|
|
|
|
def record(level, message):
|
|
"""One small result file, at most 64 KiB plus one 64 KiB previous file."""
|
|
line = (dt.datetime.now().astimezone().isoformat(timespec="seconds") +
|
|
f" [{level}] " + str(message).replace("\n", " ")[:4000] + "\n").encode()
|
|
print(line.decode().rstrip(), flush=True)
|
|
if RESULT.is_symlink() or RESULT.with_suffix(".log.1").is_symlink():
|
|
raise RuntimeError("Unsafe maintenance log path")
|
|
if RESULT.exists() and RESULT.stat().st_size + len(line) > 65536:
|
|
with RESULT.open("rb") as stream:
|
|
stream.seek(max(0, os.fstat(stream.fileno()).st_size - 65536))
|
|
previous = stream.read(65536)
|
|
atomic(RESULT.with_suffix(".log.1"), previous)
|
|
# Keep the active inode available to readers.
|
|
with RESULT.open("r+b") as stream:
|
|
stream.truncate(0)
|
|
fd = os.open(RESULT, os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, "ab") as stream:
|
|
stream.write(line)
|
|
|
|
|
|
def legacy_policy(path=LEGACY):
|
|
if not path.exists():
|
|
return False
|
|
if path.is_symlink():
|
|
return False
|
|
lines = [line.strip() for line in path.read_text().splitlines()
|
|
if line.strip() and not line.lstrip().startswith("#")]
|
|
expected = {"daily", "rotate 7", "size 50M", "missingok", "notifempty",
|
|
"compress", "delaycompress", "copytruncate"}
|
|
return (len(lines) == 10 and lines[0] == "/var/log/nginx/*.log {"
|
|
and lines[-1] == "}" and set(lines[1:-1]) == expected)
|
|
|
|
|
|
def nginx_policies():
|
|
"""Patch existing standard destinations; never add a second access logger.
|
|
|
|
sites-enabled entries are commonly symlinks: backup/edit their resolved
|
|
targets in /etc/nginx, leaving the symlinks themselves intact.
|
|
"""
|
|
rendered = command(["nginx", "-T"]).stdout
|
|
configs = {}
|
|
for filename in re.findall(r"^# configuration file ([^\n]+):$", rendered, re.M):
|
|
path = Path(filename).resolve()
|
|
if str(path) == "/etc/nginx/conf.d/owrt-remote-log-policy.conf":
|
|
continue
|
|
if not str(path).startswith("/etc/nginx/") or not path.is_file():
|
|
raise RuntimeError("Unexpected nginx configuration: " + str(path))
|
|
text = path.read_text()
|
|
def add_condition(match):
|
|
directive = match.group(0)
|
|
if "$owrt_remote_access_log" in directive:
|
|
return directive
|
|
if re.search(r"\bif=", directive):
|
|
raise RuntimeError("Custom nginx log condition requires manual review: " + str(path))
|
|
# nginx needs an explicit format before optional parameters. Ubuntu's
|
|
# default is simply `access_log /var/log/nginx/access.log;`.
|
|
prefix = directive[:-1]
|
|
if len(shlex.split(prefix)) == 2:
|
|
prefix += " combined"
|
|
return prefix + " if=$owrt_remote_access_log;"
|
|
updated = re.sub(r"(?m)^\s*access_log\s+/var/log/nginx/access\.log(?:\s+[^;\n]+)?;", add_condition, text)
|
|
if updated != text:
|
|
configs[str(path)] = updated
|
|
return configs
|
|
|
|
|
|
def flatten_config(path, seen=None):
|
|
"""Read trusted distribution configs; never execute hooks during parsing."""
|
|
path = Path(path)
|
|
seen = set() if seen is None else seen
|
|
if path in seen or len(seen) > 128:
|
|
raise RuntimeError("Recursive or oversized logrotate include: " + str(path))
|
|
info = path.lstat()
|
|
if path.is_symlink() or not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
|
raise RuntimeError("Untrusted logrotate config: " + str(path))
|
|
if info.st_size > 262144:
|
|
raise RuntimeError("Oversized logrotate config: " + str(path))
|
|
seen.add(path)
|
|
lines, script = [], False
|
|
for line in path.read_text().splitlines(keepends=True):
|
|
word = line.strip().split(" ", 1)[0]
|
|
if word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"):
|
|
script = True
|
|
elif word == "endscript":
|
|
script = False
|
|
if not script and re.match(r"^\s*include\s+", line):
|
|
parts = shlex.split(line, comments=True)
|
|
if len(parts) != 2 or not (parts[1] == "/etc/logrotate.conf" or parts[1] == "/etc/logrotate.d" or parts[1].startswith("/etc/logrotate.d/")):
|
|
raise RuntimeError("Unsupported logrotate include: " + line.strip())
|
|
include = Path(parts[1])
|
|
children = sorted(include.iterdir()) if include.is_dir() else [include]
|
|
for child in children:
|
|
if child.is_dir() or re.search(r"(?:~|\.(?:bak|old|disabled|dpkg-.+|rpmnew|rpmsave))$", child.name):
|
|
continue
|
|
if child == LEGACY and legacy_policy():
|
|
continue
|
|
lines.append(flatten_config(child, seen))
|
|
else:
|
|
lines.append(line)
|
|
seen.remove(path)
|
|
result = "".join(lines)
|
|
if len(result) > 1048576:
|
|
raise RuntimeError("Logrotate config exceeds 1 MiB")
|
|
return result + "\n"
|
|
|
|
|
|
def retention_config(source):
|
|
"""Keep native create/su/reopen hooks, override only retention per stanza.
|
|
|
|
Refuse ambiguous syntax and targets outside /var/log before any rotation.
|
|
In particular, no arbitrary service file in /var/lib can enter cleanup.
|
|
"""
|
|
output, patterns, header, block = [], [], [], []
|
|
script, inside = False, False
|
|
for line in source.splitlines(keepends=True):
|
|
stripped = line.strip()
|
|
if inside:
|
|
word = stripped.split(" ", 1)[0]
|
|
if not script and word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"):
|
|
script = True
|
|
elif script and word == "endscript":
|
|
script = False
|
|
if not script and stripped == "}":
|
|
# olddir may put archives outside /var/log: refuse to clean there.
|
|
check_script = False
|
|
for item in block:
|
|
word = item.strip().split(" ", 1)[0]
|
|
if word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"):
|
|
check_script = True
|
|
elif word == "endscript":
|
|
check_script = False
|
|
if check_script:
|
|
continue
|
|
if re.match(r"\s*(?:olddir|include)\s+", item):
|
|
raise RuntimeError("olddir/include inside log stanza requires administrator review")
|
|
output.extend(block)
|
|
output.append(POLICY + "}\n")
|
|
inside, block = False, []
|
|
elif not script and ("{" in stripped or stripped.startswith("}")):
|
|
raise RuntimeError("Unsupported logrotate block syntax: " + stripped)
|
|
else:
|
|
block.append(line)
|
|
continue
|
|
if header or stripped.startswith(("/", '"', "'")):
|
|
header.append(line)
|
|
if "{" in line:
|
|
combined = "".join(header)
|
|
if combined.count("{") != 1 or combined.split("{", 1)[1].strip():
|
|
raise RuntimeError("Unsupported inline logrotate stanza")
|
|
targets = shlex.split(combined.split("{", 1)[0], comments=True)
|
|
for target in targets:
|
|
if not target.startswith("/var/log/") or ".." in Path(target).parts:
|
|
raise RuntimeError("Refusing log target outside /var/log: " + target)
|
|
patterns.extend(targets)
|
|
output.extend(header)
|
|
header, inside = [], True
|
|
else:
|
|
output.append(line)
|
|
if header or inside or script:
|
|
raise RuntimeError("Incomplete logrotate configuration")
|
|
return "".join(output), patterns
|
|
|
|
|
|
def archive_name(active, candidate):
|
|
if not candidate.startswith(active):
|
|
return False
|
|
suffix = candidate[len(active):]
|
|
return bool(re.fullmatch(r"(?:\.\d+|\.old|[-.]\d{8}(?:[-.]\d{4,6})?)(?:\.(?:gz|xz|bz2|zst))?", suffix))
|
|
|
|
|
|
def open_inodes():
|
|
opened = set()
|
|
for directory in Path("/proc").iterdir():
|
|
if not directory.name.isdigit():
|
|
continue
|
|
try:
|
|
for fd in (directory / "fd").iterdir():
|
|
try:
|
|
info = fd.stat()
|
|
opened.add((info.st_dev, info.st_ino))
|
|
except FileNotFoundError:
|
|
pass
|
|
except FileNotFoundError:
|
|
pass
|
|
return opened
|
|
|
|
|
|
def prune_archives(active_paths, emergency=False, now=None, opened=None):
|
|
now = time.time() if now is None else now
|
|
opened = open_inodes() if opened is None else opened
|
|
removed, freed = 0, 0
|
|
for active in sorted(set(Path(p) for p in active_paths)):
|
|
if not active.is_file() or active.is_symlink() or active.resolve() != active:
|
|
continue
|
|
for candidate in active.parent.iterdir():
|
|
if not archive_name(active.name, candidate.name):
|
|
continue
|
|
info = candidate.lstat()
|
|
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
|
|
continue
|
|
if not emergency and now - info.st_mtime <= 86400:
|
|
continue
|
|
if (info.st_dev, info.st_ino) in opened:
|
|
continue
|
|
latest = candidate.lstat()
|
|
if (latest.st_dev, latest.st_ino, latest.st_mtime_ns) != (info.st_dev, info.st_ino, info.st_mtime_ns):
|
|
continue
|
|
candidate.unlink() # Only verified closed archives, never active logs.
|
|
removed += 1
|
|
freed += info.st_size
|
|
return removed, freed
|
|
|
|
|
|
def disk_levels():
|
|
result = []
|
|
for path in ("/", "/var/log"):
|
|
usage = shutil.disk_usage(path)
|
|
# Match df: blocks reserved for root are not available to log writers.
|
|
available_total = usage.used + usage.free
|
|
percent = 100 * usage.used / available_total if available_total else 100
|
|
level = "CRITICAL" if percent > 90 else "WARN" if percent > 75 else "OK"
|
|
result.append((path, percent, level))
|
|
return result
|
|
|
|
|
|
def service_status():
|
|
for name in ("owrt-remote", "nginx", "rsyslog", "systemd-journald"):
|
|
result = command(["systemctl", "is-active", name], check=False)
|
|
print(f"{name}: {result.stdout.strip()}")
|
|
|
|
|
|
def validate_journal_config():
|
|
effective, section = {}, ""
|
|
for line in command(["systemd-analyze", "cat-config", "systemd/journald.conf"]).stdout.splitlines():
|
|
text = line.strip()
|
|
if text.startswith("["):
|
|
section = text
|
|
elif section == "[Journal]" and not text.startswith(("#", ";")) and "=" in text:
|
|
key, value = text.split("=", 1)
|
|
effective[key.strip()] = value.strip()
|
|
for key, value in {"SystemMaxUse": "50M", "SystemKeepFree": "500M",
|
|
"MaxRetentionSec": "7day", "Compress": "yes"}.items():
|
|
if effective.get(key) != value:
|
|
raise RuntimeError("Another journald configuration overrides " + key + "; administrator review required")
|
|
|
|
|
|
def run(force=False):
|
|
import fcntl
|
|
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
RUN.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
with (RUN / "lock").open("a") as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
print("Maintenance already running; skipped")
|
|
return
|
|
today = dt.date.today().isoformat()
|
|
stamp = STATE / "last-attempt"
|
|
if not force and stamp.exists() and stamp.read_text().strip() == today:
|
|
print("Daily attempt already made; use run --force to retry an error")
|
|
return
|
|
# Write before work: even a process crash cannot cause a second daily run.
|
|
atomic(stamp, today + "\n")
|
|
try:
|
|
config, patterns = retention_config(flatten_config("/etc/logrotate.conf"))
|
|
active = [Path(p) for pattern in patterns for p in glob.glob(pattern)]
|
|
emergency = any(percent > 85 for _, percent, _ in disk_levels())
|
|
before = disk_levels()
|
|
for path, percent, level in before:
|
|
record(level, f"before {path}: {percent:.1f}% used")
|
|
atomic(RUN / "logrotate.conf", config)
|
|
command(["logrotate", "--debug", str(RUN / "logrotate.conf")])
|
|
removed, freed = prune_archives(active, emergency=emergency)
|
|
# Same state/lock as distro logrotate, not a private competing state.
|
|
command(["logrotate", "--wait-for-state-lock", "--force", str(RUN / "logrotate.conf")], timeout=1500)
|
|
command(["journalctl", "--rotate"], timeout=60)
|
|
vacuum = command(["journalctl", "--vacuum-time=1d", "--vacuum-size=" + ("10M" if emergency else "50M")])
|
|
more, size = prune_archives(active, emergency=emergency)
|
|
removed += more
|
|
freed += size
|
|
after = disk_levels()
|
|
result = {"day": today, "ok": True, "archives_removed": removed,
|
|
"bytes_removed": freed, "emergency": emergency,
|
|
"disk_after": after, "finished_at": time.time()}
|
|
atomic(STATE / "result.json", json.dumps(result) + "\n")
|
|
record("OK", f"completed; archives_removed={removed}; bytes_removed={freed}; emergency={emergency}; {vacuum.stdout[-500:]}")
|
|
for path, percent, level in after:
|
|
record(level, f"after {path}: {percent:.1f}% used; database and user data excluded")
|
|
print(command(["journalctl", "--disk-usage"]).stdout.strip())
|
|
except Exception as exc:
|
|
atomic(STATE / "result.json", json.dumps({"day": today, "ok": False, "error": str(exc)[-6000:]}) + "\n")
|
|
record("ERROR", exc)
|
|
raise
|
|
finally:
|
|
(RUN / "logrotate.conf").unlink(missing_ok=True)
|
|
|
|
|
|
def snapshot(paths, backup):
|
|
backup.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
manifest_path = backup / "manifest.json"
|
|
if manifest_path.exists():
|
|
manifest = json.loads(manifest_path.read_text())
|
|
else:
|
|
manifest = {"files": [], "timer_enabled": command(["systemctl", "is-enabled", "logrotate.timer"], check=False).stdout.strip(),
|
|
"timer_active": command(["systemctl", "is-active", "logrotate.timer"], check=False).stdout.strip()}
|
|
known = {item["path"] for item in manifest["files"]}
|
|
for path in paths:
|
|
path = Path(path)
|
|
if str(path) in known:
|
|
continue
|
|
if path.is_symlink():
|
|
raise RuntimeError("Refusing to replace symlink: " + str(path))
|
|
item = {"path": str(path), "exists": path.exists(), "copy": str(len(manifest["files"]))}
|
|
if path.exists():
|
|
info = path.stat()
|
|
item.update(mode=stat.S_IMODE(info.st_mode), uid=info.st_uid, gid=info.st_gid)
|
|
atomic(backup / item["copy"], path.read_bytes())
|
|
manifest["files"].append(item)
|
|
atomic(manifest_path, json.dumps(manifest))
|
|
|
|
|
|
def restore(backup):
|
|
manifest = json.loads((backup / "manifest.json").read_text())
|
|
command(["systemctl", "stop", "logrotate.timer"])
|
|
for item in manifest["files"]:
|
|
path = Path(item["path"])
|
|
if item["exists"]:
|
|
atomic(path, (backup / item["copy"]).read_bytes(), item["mode"])
|
|
os.chown(path, item["uid"], item["gid"])
|
|
else:
|
|
path.unlink(missing_ok=True)
|
|
command(["systemctl", "daemon-reload"])
|
|
command(["systemctl", "restart", "systemd-journald"])
|
|
if shutil.which("nginx"):
|
|
command(["nginx", "-t"])
|
|
if command(["systemctl", "is-active", "nginx"], check=False).returncode == 0:
|
|
command(["systemctl", "reload", "nginx"])
|
|
if manifest["timer_enabled"] == "enabled":
|
|
command(["systemctl", "enable", "logrotate.timer"])
|
|
else:
|
|
command(["systemctl", "disable", "logrotate.timer"])
|
|
if manifest["timer_active"] == "active":
|
|
command(["systemctl", "start", "logrotate.timer"])
|
|
if any(item["path"] == str(HUB) for item in manifest["files"]):
|
|
command(["systemctl", "restart", "owrt-remote"])
|
|
print("Configuration restored. Already rotated/deleted archives cannot be restored.")
|
|
|
|
|
|
def install(hub_candidate=None, backup=BACKUP):
|
|
for executable in ("logrotate", "journalctl", "systemctl", "systemd-analyze"):
|
|
if not shutil.which(executable):
|
|
raise RuntimeError("Missing prerequisite: " + executable)
|
|
if command(["systemctl", "is-active", "logrotate.service"], check=False).stdout.strip() in ("active", "activating"):
|
|
raise RuntimeError("Wait for the running system logrotate.service to finish")
|
|
if LEGACY.exists() and not legacy_policy():
|
|
raise RuntimeError("Custom /etc/logrotate.d/owrt-remote found; manual review required")
|
|
retention_config(flatten_config("/etc/logrotate.conf"))
|
|
if hub_candidate:
|
|
text = Path(hub_candidate).read_text()
|
|
ast.parse(text)
|
|
if "OWRT_REMOTE_HTTP_DEBUG" not in text:
|
|
raise RuntimeError("Hub candidate does not contain the HTTP log fix")
|
|
configs = dict(CONFIGS)
|
|
if not shutil.which("nginx"):
|
|
configs.pop("/etc/nginx/conf.d/owrt-remote-log-policy.conf")
|
|
else:
|
|
configs.update(nginx_policies())
|
|
paths = [*configs, str(INSTALL_PATH), str(LEGACY)] + ([str(HUB)] if hub_candidate else [])
|
|
snapshot(paths, backup)
|
|
try:
|
|
command(["systemctl", "stop", "logrotate.timer"])
|
|
for path, content in configs.items():
|
|
atomic(path, content, 0o644)
|
|
atomic(INSTALL_PATH, Path(__file__).read_bytes(), 0o755)
|
|
if legacy_policy():
|
|
LEGACY.unlink() # Old duplicate configuration, not a log file.
|
|
if hub_candidate:
|
|
atomic(HUB, Path(hub_candidate).read_bytes(), 0o755)
|
|
config, _ = retention_config(flatten_config("/etc/logrotate.conf"))
|
|
RUN.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
atomic(RUN / "validate.conf", config)
|
|
try:
|
|
command(["logrotate", "--debug", str(RUN / "validate.conf")])
|
|
finally:
|
|
(RUN / "validate.conf").unlink(missing_ok=True)
|
|
command(["systemctl", "daemon-reload"])
|
|
command(["systemd-analyze", "verify", "logrotate.service", "logrotate.timer"])
|
|
validate_journal_config()
|
|
if shutil.which("nginx"):
|
|
command(["nginx", "-t"])
|
|
if command(["systemctl", "is-active", "nginx"], check=False).returncode == 0:
|
|
command(["systemctl", "reload", "nginx"])
|
|
command(["systemctl", "restart", "systemd-journald"])
|
|
if hub_candidate:
|
|
command(["systemctl", "restart", "owrt-remote"])
|
|
check_health(hashlib.sha256(HUB.read_bytes()).hexdigest())
|
|
# Start the persistent timer only after main releases our lock. A missed
|
|
# calendar event can otherwise immediately run and be skipped as busy.
|
|
command(["systemctl", "enable", "logrotate.timer"])
|
|
print("Backup: " + str(backup))
|
|
print("LOG_MAINTENANCE_INSTALL_OK")
|
|
except Exception:
|
|
restore(backup)
|
|
raise
|
|
|
|
|
|
def check_health(revision):
|
|
import urllib.request
|
|
# No proxy environment / shell wildcard expansion. Retry startup quietly.
|
|
client = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
for attempt in range(15):
|
|
try:
|
|
with client.open("http://127.0.0.1:8088/health?panel-update=1", timeout=3) as response:
|
|
data = json.load(response)
|
|
if data.get("ok") and data.get("revision") == revision:
|
|
return
|
|
except (OSError, ValueError):
|
|
pass
|
|
time.sleep(1)
|
|
raise RuntimeError("Hub health check failed after restart")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("action", choices=("install", "rollback", "run", "status"))
|
|
parser.add_argument("--hub-candidate")
|
|
parser.add_argument("--force", action="store_true", help="Explicit manual retry; daily timer never uses this")
|
|
args = parser.parse_args()
|
|
if os.geteuid() != 0:
|
|
parser.error("Run as root")
|
|
if args.action in ("install", "rollback"):
|
|
import fcntl
|
|
RUN.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
with (RUN / "lock").open("a") as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise RuntimeError("Log maintenance/install already running; retry after completion")
|
|
if args.action == "install":
|
|
install(args.hub_candidate)
|
|
else:
|
|
restore(BACKUP)
|
|
if args.action == "install":
|
|
try:
|
|
command(["systemctl", "start", "logrotate.timer"])
|
|
except Exception:
|
|
restore(BACKUP)
|
|
raise
|
|
elif args.action == "run":
|
|
run(args.force)
|
|
else:
|
|
print(command(["systemctl", "list-timers", "logrotate.timer", "--no-pager"]).stdout)
|
|
print(command(["journalctl", "--disk-usage"]).stdout)
|
|
print((STATE / "result.json").read_text() if (STATE / "result.json").exists() else "No maintenance result yet")
|
|
for path, percent, level in disk_levels():
|
|
print(level, path, f"{percent:.1f}%")
|
|
service_status()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
print("ERROR: " + str(error), file=sys.stderr)
|
|
sys.exit(1)
|