Fix SQLite connection leaks and resource cleanup
This commit is contained in:
1 parent
7f809fd288
commit
5414eb6c4f
5 files changed
+1177
-173
No files matched your search
@@ -0,0 +1,101 @@
|
||||
"""Read-only Linux monitoring of owrt-remote; no credentials/DB writes needed.
|
||||
|
||||
Usage on VPS: python3 monitor_vps_resources.py 10
|
||||
FD samples can briefly include active requests or SQLite transactions.
|
||||
The report records trends; real router/proxy/browser actions should run alongside.
|
||||
"""
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
|
||||
def main(minutes):
|
||||
started = time.monotonic()
|
||||
since = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S")
|
||||
report_path = Path(f"/tmp/owrt-resource-monitor-{stamp}.jsonl")
|
||||
ticks_per_second = os.sysconf("SC_CLK_TCK")
|
||||
samples = []
|
||||
previous = None
|
||||
print(f"Read-only monitoring for {minutes} minute(s). Report: {report_path}", flush=True)
|
||||
with report_path.open("x", encoding="utf-8") as report:
|
||||
while True:
|
||||
pid = int(subprocess.check_output(["systemctl", "show", "owrt-remote", "-p", "MainPID", "--value"], text=True, timeout=10).strip())
|
||||
if pid <= 0:
|
||||
raise RuntimeError("owrt-remote has no running MainPID")
|
||||
proc = Path(f"/proc/{pid}")
|
||||
links = []
|
||||
for item in (proc / "fd").iterdir():
|
||||
try:
|
||||
links.append(os.readlink(item))
|
||||
except FileNotFoundError:
|
||||
pass # a live request can close a descriptor while sampled
|
||||
fields = (proc / "stat").read_text().rsplit(")", 1)[1].split()
|
||||
ticks = int(fields[11]) + int(fields[12])
|
||||
sample_time = time.monotonic()
|
||||
cpu = 0.0
|
||||
if previous and previous[0] == pid:
|
||||
cpu = 100 * (ticks - previous[1]) / ticks_per_second / (sample_time - previous[2])
|
||||
previous = pid, ticks, sample_time
|
||||
status = (proc / "status").read_text()
|
||||
memory = re.search(r"^VmRSS:\s+(\d+)", status, re.MULTILINE)
|
||||
threads = re.search(r"^Threads:\s+(\d+)", status, re.MULTILINE)
|
||||
try:
|
||||
with urllib.request.urlopen("http://127.0.0.1:8088/health", timeout=5) as response:
|
||||
health = response.status == 200 and bool(json.load(response).get("ok"))
|
||||
except Exception as exc:
|
||||
health = f"error: {exc}"
|
||||
sample = {"time": datetime.datetime.now().isoformat(timespec="seconds"),
|
||||
"elapsed": round(sample_time - started, 1), "pid": pid, "fd": len(links),
|
||||
"hub_db_fd": sum("/hub.db" in link for link in links),
|
||||
"cpu_percent": round(cpu, 2), "rss_kib": int(memory[1]) if memory else None,
|
||||
"threads": int(threads[1]) if threads else None, "health": health}
|
||||
samples.append(sample)
|
||||
line = json.dumps(sample, ensure_ascii=False)
|
||||
print(line, flush=True)
|
||||
report.write(line + "\n")
|
||||
report.flush()
|
||||
if time.monotonic() - started >= minutes * 60:
|
||||
break
|
||||
time.sleep(min(10, max(0, minutes * 60 - (time.monotonic() - started))))
|
||||
|
||||
journal = subprocess.check_output(["journalctl", "-u", "owrt-remote", "--since", since, "--no-pager"], text=True, errors="replace", timeout=30)
|
||||
pattern = re.compile(r"unable to open database|database is locked|sqlite|traceback|exception|monitor.*error|connection reset|remote end closed", re.IGNORECASE)
|
||||
errors = [line for line in journal.splitlines() if pattern.search(line)]
|
||||
window = max(1, min(10, len(samples) // 3))
|
||||
trend = {
|
||||
key: {"first_median": statistics.median(s[key] for s in samples[:window]),
|
||||
"last_median": statistics.median(s[key] for s in samples[-window:])}
|
||||
for key in ("fd", "hub_db_fd", "rss_kib", "threads")
|
||||
if all(s[key] is not None for s in samples)
|
||||
}
|
||||
# Concurrent requests can briefly raise FD counts. Compare windows and
|
||||
# flag sustained growth; the report still needs workload-aware review.
|
||||
fd_growth = trend["fd"]["last_median"] > trend["fd"]["first_median"] + 8
|
||||
db_growth = trend["hub_db_fd"]["last_median"] > trend["hub_db_fd"]["first_median"] + 2
|
||||
summary = {"type": "summary", "seconds": samples[-1]["elapsed"], "first": samples[0], "last": samples[-1],
|
||||
"fd_min": min(s["fd"] for s in samples), "fd_max": max(s["fd"] for s in samples),
|
||||
"hub_db_fd_max": max(s["hub_db_fd"] for s in samples),
|
||||
"pid_changed": len({s["pid"] for s in samples}) != 1,
|
||||
"health_failures": sum(s["health"] is not True for s in samples), "matching_log_lines": errors,
|
||||
"trend": trend, "sustained_fd_growth": fd_growth, "sustained_db_growth": db_growth}
|
||||
report.write(json.dumps(summary, ensure_ascii=False) + "\n")
|
||||
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
||||
print(f"Report saved: {report_path}", flush=True)
|
||||
print("Check nginx and exercise the panel, a real router proxy, heartbeat and notifications alongside this report.", flush=True)
|
||||
if summary["pid_changed"] or summary["health_failures"] or errors or fd_growth or db_growth:
|
||||
raise RuntimeError("Monitoring found failures; inspect the saved report")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
minutes = int(sys.argv[1]) if len(sys.argv) > 1 else 10
|
||||
if not 1 <= minutes <= 1440:
|
||||
raise ValueError("minutes must be between 1 and 1440")
|
||||
main(minutes)
|
||||
Reference in new issue
Block a user