102 lines
5.5 KiB
Python
102 lines
5.5 KiB
Python
"""Read-only Linux monitoring of owrt-remote; no credentials/DB writes needed.
|
|
|
|
Usage on VPS: python3 monitor_vps_resources.py 10
|
|
FD samples can briefly include active requests or SQLite transactions.
|
|
The report records trends; real router/proxy/browser actions should run alongside.
|
|
"""
|
|
import datetime
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
|
|
def main(minutes):
|
|
started = time.monotonic()
|
|
since = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S")
|
|
report_path = Path(f"/tmp/owrt-resource-monitor-{stamp}.jsonl")
|
|
ticks_per_second = os.sysconf("SC_CLK_TCK")
|
|
samples = []
|
|
previous = None
|
|
print(f"Read-only monitoring for {minutes} minute(s). Report: {report_path}", flush=True)
|
|
with report_path.open("x", encoding="utf-8") as report:
|
|
while True:
|
|
pid = int(subprocess.check_output(["systemctl", "show", "owrt-remote", "-p", "MainPID", "--value"], text=True, timeout=10).strip())
|
|
if pid <= 0:
|
|
raise RuntimeError("owrt-remote has no running MainPID")
|
|
proc = Path(f"/proc/{pid}")
|
|
links = []
|
|
for item in (proc / "fd").iterdir():
|
|
try:
|
|
links.append(os.readlink(item))
|
|
except FileNotFoundError:
|
|
pass # a live request can close a descriptor while sampled
|
|
fields = (proc / "stat").read_text().rsplit(")", 1)[1].split()
|
|
ticks = int(fields[11]) + int(fields[12])
|
|
sample_time = time.monotonic()
|
|
cpu = 0.0
|
|
if previous and previous[0] == pid:
|
|
cpu = 100 * (ticks - previous[1]) / ticks_per_second / (sample_time - previous[2])
|
|
previous = pid, ticks, sample_time
|
|
status = (proc / "status").read_text()
|
|
memory = re.search(r"^VmRSS:\s+(\d+)", status, re.MULTILINE)
|
|
threads = re.search(r"^Threads:\s+(\d+)", status, re.MULTILINE)
|
|
try:
|
|
with urllib.request.urlopen("http://127.0.0.1:8088/health", timeout=5) as response:
|
|
health = response.status == 200 and bool(json.load(response).get("ok"))
|
|
except Exception as exc:
|
|
health = f"error: {exc}"
|
|
sample = {"time": datetime.datetime.now().isoformat(timespec="seconds"),
|
|
"elapsed": round(sample_time - started, 1), "pid": pid, "fd": len(links),
|
|
"hub_db_fd": sum("/hub.db" in link for link in links),
|
|
"cpu_percent": round(cpu, 2), "rss_kib": int(memory[1]) if memory else None,
|
|
"threads": int(threads[1]) if threads else None, "health": health}
|
|
samples.append(sample)
|
|
line = json.dumps(sample, ensure_ascii=False)
|
|
print(line, flush=True)
|
|
report.write(line + "\n")
|
|
report.flush()
|
|
if time.monotonic() - started >= minutes * 60:
|
|
break
|
|
time.sleep(min(10, max(0, minutes * 60 - (time.monotonic() - started))))
|
|
|
|
journal = subprocess.check_output(["journalctl", "-u", "owrt-remote", "--since", since, "--no-pager"], text=True, errors="replace", timeout=30)
|
|
pattern = re.compile(r"unable to open database|database is locked|sqlite|traceback|exception|monitor.*error|connection reset|remote end closed", re.IGNORECASE)
|
|
errors = [line for line in journal.splitlines() if pattern.search(line)]
|
|
window = max(1, min(10, len(samples) // 3))
|
|
trend = {
|
|
key: {"first_median": statistics.median(s[key] for s in samples[:window]),
|
|
"last_median": statistics.median(s[key] for s in samples[-window:])}
|
|
for key in ("fd", "hub_db_fd", "rss_kib", "threads")
|
|
if all(s[key] is not None for s in samples)
|
|
}
|
|
# Concurrent requests can briefly raise FD counts. Compare windows and
|
|
# flag sustained growth; the report still needs workload-aware review.
|
|
fd_growth = trend["fd"]["last_median"] > trend["fd"]["first_median"] + 8
|
|
db_growth = trend["hub_db_fd"]["last_median"] > trend["hub_db_fd"]["first_median"] + 2
|
|
summary = {"type": "summary", "seconds": samples[-1]["elapsed"], "first": samples[0], "last": samples[-1],
|
|
"fd_min": min(s["fd"] for s in samples), "fd_max": max(s["fd"] for s in samples),
|
|
"hub_db_fd_max": max(s["hub_db_fd"] for s in samples),
|
|
"pid_changed": len({s["pid"] for s in samples}) != 1,
|
|
"health_failures": sum(s["health"] is not True for s in samples), "matching_log_lines": errors,
|
|
"trend": trend, "sustained_fd_growth": fd_growth, "sustained_db_growth": db_growth}
|
|
report.write(json.dumps(summary, ensure_ascii=False) + "\n")
|
|
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
|
print(f"Report saved: {report_path}", flush=True)
|
|
print("Check nginx and exercise the panel, a real router proxy, heartbeat and notifications alongside this report.", flush=True)
|
|
if summary["pid_changed"] or summary["health_failures"] or errors or fd_growth or db_growth:
|
|
raise RuntimeError("Monitoring found failures; inspect the saved report")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
minutes = int(sys.argv[1]) if len(sys.argv) > 1 else 10
|
|
if not 1 <= minutes <= 1440:
|
|
raise ValueError("minutes must be between 1 and 1440")
|
|
main(minutes)
|