diff --git a/tests/RESOURCE_AUDIT.md b/tests/RESOURCE_AUDIT.md new file mode 100644 index 0000000..70fdd12 --- /dev/null +++ b/tests/RESOURCE_AUDIT.md @@ -0,0 +1,136 @@ +# Аудит ресурсов OpenWrt Remote Hub v109 + +Исправления внесены поверх `7f809fd`. Версия панели остаётся v109. + +## SQLite + +В исходном коде нативный `sqlite3.Connection.__exit__` управлял транзакцией, +но не закрывал соединение. Это затрагивало мониторинг, API и CLI. + +Теперь `connect()` — единственный владелец создаваемого SQLite connection: + +```python +with connect(db_path) as conn: + # Работа с БД только внутри этого блока. + ... +``` + +Внутренний `with conn` сохраняет штатную семантику SQLite commit/rollback, +включая rollback после неудачного commit и `BaseException`. Внешний `finally` +всегда вызывает `close()`, в том числе при ошибке установки `row_factory`, +инициализации схемы, выходе через `return` и ошибке завершения транзакции. +Ошибки SQLite продолжают передаваться вызывающему коду. + +`App.conn()` инициализирует схему внутри того же защищённого контекста. +Аудит AST проверил один прямой `sqlite3.connect`, 13 вызовов общего `connect`, +30 вызовов `self.app.conn`, один `self.conn` и один `app.conn`. +Все потребители контекстных менеджеров используют `with`. + +Проверены мониторинг, heartbeat, группы, заметки, выдача конфигураций, +HTTP handlers, CLI, backup/restore и обновление Xray. Отдельных долгоживущих +cursor нет; результаты извлекаются внутри операции, connection закрывает +оставшиеся SQLite ресурсы. Backup закрывает source даже при ошибке открытия +destination. Возвращаемые `sqlite3.Row` остаются доступны после закрытия БД. + +## Другие ресурсы + +Исправлены найденные пути с неполной очисткой: + +* HTTPError OAuth закрывается после чтения тела ошибки. +* Ответы Web Push закрываются при успехе и при ответе с ошибкой. +* Ответ proxy закрывается контекстным менеджером, backend — через `finally`. +* HTTP сервер освобождает listening socket при ошибке TLS; главный и + дополнительные серверы закрываются при выходе или ошибке запуска. +* Монитор останавливается с ожиданием потока; неудачный запуск потока + не ломает cleanup. Ошибка первого снимка БД логируется, монитор повторяет + попытки с обычной паузой. +* HTTP terminal reader владеет PTY и сбором дочернего процесса. При + неудачном запуске reader или первой записи WebSocket ресурсы освобождаются. + Завершение ребёнка ожидается; при необходимости применяются HUP/TERM/KILL. +* Запись в HTTP terminal использует собственный временный дубликат FD, + который закрывается в `finally`, чтобы избежать гонки с reader. +* Закрытые terminal sessions удаляются спустя 60 секунд. HTTP sessions без + обращений браузера завершаются после 15 минут; живой polling продлевает их. + Буфер вывода ограничен 256 порциями по максимуму 4096 байт исходных данных. + При остановке сервера sessions помечаются закрытыми, reader threads ожидаются. +* Прерванный chunked request вызывает ошибку вместо бесконечного цикла на EOF. + Для сетевого чтения/записи установлен timeout 90 секунд. Долгий terminal + без ввода не истекает по этому timeout: его цикл ждёт данные через `select`. +* Атомарная запись и создание архива используют уникальные временные файлы + с очисткой при ошибке. SSH askpass очищается при неудачной подготовке + и после subprocess, включая timeout и ошибку подготовки аргументов. + +Проверенные прежние безопасные участки: `urlopen` успеха и TCP-проверки +используют `with`; `TemporaryDirectory` в подписи SSH, backup/restore и +upload/download очищаются контекстом; `subprocess.run` использует конечные +timeout и собирает child/PIPE после завершения или timeout. Web Push worker +имеет конечный список подписок и timeout HTTP; notification long polling +ждёт Condition с конечным deadline. Эти механизмы не отключались. +Лимит FD и конфигурация systemd не изменялись. + +## Автоматическая проверка + +```powershell +python -m unittest discover -s tests -v +python tests/test_resource_lifecycle.py --soak-seconds 600 +``` + +Тесты используют отдельные временные state/DB и loopback fixture роутера. +Они удерживают ссылки на connection, чтобы GC не маскировал утечку. +Проверяются 500 snapshots, commit, rollback, ранний return, init failure, +ошибка commit, setup/transaction exit failure, backup и остальные +исправленные пути очистки. AST-тест запрещает вызовы DB factory вне `with`. + +В Windows два POSIX-теста пропускаются: реальный PTY/child и сигналы. +В Linux установщик запускает их до замены рабочего файла на VPS. +CI в этом репозитории отсутствует; тесты подготовлены для его последующего +подключения без сторонних Python зависимостей. + +В исходном `7f809fd` тест с удержанными connection показал 101 незакрытое +соединение и рост Windows handles с 227 до 327 после 100 snapshots. +После фикса 500 snapshots не оставляют открытых connection. + +Первый локальный длительный прогон: 601.1 секунды, 116 циклов +панель/API/heartbeat/proxy/notifications/snapshot, handles 279 → 277, +максимум 281, CPU time 8.39 секунды. Это Windows handles, а не Linux FD. +Второй прогон после изменений сетевого cleanup: 601.2 секунды, 116 циклов, +handles 279 → 277, максимум 281, CPU time 8.73 секунды. +Локальный тест подтверждает fixture, а не реальные туннели/доставку push/nginx. + +## Установка и откат для проверки пользователем + +Из PowerShell Windows 11: + +```powershell +$Vps = "root@YOUR_VPS_IP" +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "vps\deploy-resource-fix.ps1" -Vps $Vps -Action Install +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "vps\deploy-resource-fix.ps1" -Vps $Vps -Action Monitor -Minutes 10 +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "vps\deploy-resource-fix.ps1" -Vps $Vps -Action Rollback +``` + +Первые две команды выполняются последовательно. Откат — отдельная команда +только при необходимости. Рабочая директория — корень репозитория. +Вместо `YOUR_VPS_IP` укажите адрес своего VPS. +Для длительного мониторинга можно использовать `-Minutes 120`. + +Установщик загружает локальный файл без Git, запускает изолированные тесты, +сохраняет исходный серверный файл: +`/opt/owrt-remote/owrt-remote-hub.py.bak-before-resource-fix-v109`, +атомарно заменяет Python файл, перезапускает `owrt-remote` и проверяет health. +Существующая резервная копия не перезаписывается. При ошибке установки +восстанавливается файл, работавший непосредственно перед текущей попыткой. +База и конфигурация роутеров этим скриптом не заменяются. + +Монитор читает `/proc/MainPID/fd`, hub.db FD, CPU за интервал, RSS, threads, +health и journal. Отчёт остаётся в `/tmp/owrt-resource-monitor-*.jsonl`. +Во время мониторинга нужно пользоваться реальной панелью и proxy роутера. +Значения FD могут кратковременно расти из-за активных запросов; важны +отсутствие накопления hub.db FD и устойчивого восходящего тренда между ними. + +## Что ещё требуется на реальном VPS + +Сам агент VPS не обновлял и длинный серверный тест не проводил. До финального +закрытия инцидента нужны минимум 10 минут Linux мониторинга, желательно 1–2 +часа с реальными роутерами, проверка online/offline уведомлений, Web Push, +heartbeat, SSH, API и nginx error log. Ошибки upstream/504 могут иметь +дополнительные причины; их отсутствие нельзя подтвердить Windows fixture. diff --git a/tests/monitor_vps_resources.py b/tests/monitor_vps_resources.py new file mode 100644 index 0000000..fb34154 --- /dev/null +++ b/tests/monitor_vps_resources.py @@ -0,0 +1,101 @@ +"""Read-only Linux monitoring of owrt-remote; no credentials/DB writes needed. + +Usage on VPS: python3 monitor_vps_resources.py 10 +FD samples can briefly include active requests or SQLite transactions. +The report records trends; real router/proxy/browser actions should run alongside. +""" +import datetime +import json +import os +from pathlib import Path +import re +import statistics +import subprocess +import sys +import time +import urllib.request + + +def main(minutes): + started = time.monotonic() + since = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S") + report_path = Path(f"/tmp/owrt-resource-monitor-{stamp}.jsonl") + ticks_per_second = os.sysconf("SC_CLK_TCK") + samples = [] + previous = None + print(f"Read-only monitoring for {minutes} minute(s). Report: {report_path}", flush=True) + with report_path.open("x", encoding="utf-8") as report: + while True: + pid = int(subprocess.check_output(["systemctl", "show", "owrt-remote", "-p", "MainPID", "--value"], text=True, timeout=10).strip()) + if pid <= 0: + raise RuntimeError("owrt-remote has no running MainPID") + proc = Path(f"/proc/{pid}") + links = [] + for item in (proc / "fd").iterdir(): + try: + links.append(os.readlink(item)) + except FileNotFoundError: + pass # a live request can close a descriptor while sampled + fields = (proc / "stat").read_text().rsplit(")", 1)[1].split() + ticks = int(fields[11]) + int(fields[12]) + sample_time = time.monotonic() + cpu = 0.0 + if previous and previous[0] == pid: + cpu = 100 * (ticks - previous[1]) / ticks_per_second / (sample_time - previous[2]) + previous = pid, ticks, sample_time + status = (proc / "status").read_text() + memory = re.search(r"^VmRSS:\s+(\d+)", status, re.MULTILINE) + threads = re.search(r"^Threads:\s+(\d+)", status, re.MULTILINE) + try: + with urllib.request.urlopen("http://127.0.0.1:8088/health", timeout=5) as response: + health = response.status == 200 and bool(json.load(response).get("ok")) + except Exception as exc: + health = f"error: {exc}" + sample = {"time": datetime.datetime.now().isoformat(timespec="seconds"), + "elapsed": round(sample_time - started, 1), "pid": pid, "fd": len(links), + "hub_db_fd": sum("/hub.db" in link for link in links), + "cpu_percent": round(cpu, 2), "rss_kib": int(memory[1]) if memory else None, + "threads": int(threads[1]) if threads else None, "health": health} + samples.append(sample) + line = json.dumps(sample, ensure_ascii=False) + print(line, flush=True) + report.write(line + "\n") + report.flush() + if time.monotonic() - started >= minutes * 60: + break + time.sleep(min(10, max(0, minutes * 60 - (time.monotonic() - started)))) + + journal = subprocess.check_output(["journalctl", "-u", "owrt-remote", "--since", since, "--no-pager"], text=True, errors="replace", timeout=30) + pattern = re.compile(r"unable to open database|database is locked|sqlite|traceback|exception|monitor.*error|connection reset|remote end closed", re.IGNORECASE) + errors = [line for line in journal.splitlines() if pattern.search(line)] + window = max(1, min(10, len(samples) // 3)) + trend = { + key: {"first_median": statistics.median(s[key] for s in samples[:window]), + "last_median": statistics.median(s[key] for s in samples[-window:])} + for key in ("fd", "hub_db_fd", "rss_kib", "threads") + if all(s[key] is not None for s in samples) + } + # Concurrent requests can briefly raise FD counts. Compare windows and + # flag sustained growth; the report still needs workload-aware review. + fd_growth = trend["fd"]["last_median"] > trend["fd"]["first_median"] + 8 + db_growth = trend["hub_db_fd"]["last_median"] > trend["hub_db_fd"]["first_median"] + 2 + summary = {"type": "summary", "seconds": samples[-1]["elapsed"], "first": samples[0], "last": samples[-1], + "fd_min": min(s["fd"] for s in samples), "fd_max": max(s["fd"] for s in samples), + "hub_db_fd_max": max(s["hub_db_fd"] for s in samples), + "pid_changed": len({s["pid"] for s in samples}) != 1, + "health_failures": sum(s["health"] is not True for s in samples), "matching_log_lines": errors, + "trend": trend, "sustained_fd_growth": fd_growth, "sustained_db_growth": db_growth} + report.write(json.dumps(summary, ensure_ascii=False) + "\n") + print(json.dumps(summary, ensure_ascii=False), flush=True) + print(f"Report saved: {report_path}", flush=True) + print("Check nginx and exercise the panel, a real router proxy, heartbeat and notifications alongside this report.", flush=True) + if summary["pid_changed"] or summary["health_failures"] or errors or fd_growth or db_growth: + raise RuntimeError("Monitoring found failures; inspect the saved report") + + +if __name__ == "__main__": + minutes = int(sys.argv[1]) if len(sys.argv) > 1 else 10 + if not 1 <= minutes <= 1440: + raise ValueError("minutes must be between 1 and 1440") + main(minutes) diff --git a/tests/test_resource_lifecycle.py b/tests/test_resource_lifecycle.py new file mode 100644 index 0000000..50a933c --- /dev/null +++ b/tests/test_resource_lifecycle.py @@ -0,0 +1,481 @@ +"""Resource regression tests; only temporary state and loopback servers are used. + +Run: python -m unittest discover -s tests -v +Soak: python tests/test_resource_lifecycle.py --soak-seconds 600 +Linux measures /proc/self/fd (including hub.db); Windows measures process handles. +Connections are deliberately retained so GC cannot conceal missing close(). +""" +import argparse +import ast +import contextlib +import ctypes +import importlib.util +import io +import json +import os +from pathlib import Path +import socket +import sqlite3 +import sys +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock +import urllib.error +import urllib.request + + +SOURCE = Path(os.environ.get("OWRT_RESOURCE_HUB_SOURCE", Path(__file__).resolve().parents[1] / "vps" / "owrt-remote-hub.py")) +STATE = tempfile.TemporaryDirectory(prefix="owrt-resource-tests-") +spec = importlib.util.spec_from_file_location("resource_test_hub", SOURCE) +hub = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = hub +with mock.patch.dict(os.environ, {"OWRT_REMOTE_STATE_DIR": STATE.name, "OWRT_REMOTE_ROUTER_NOTIFY_POLL": "3"}): + spec.loader.exec_module(hub) +REAL_CONNECT = sqlite3.connect + + +class TrackedConnection(sqlite3.Connection): + def close(self): + super().close() + self.closed = True + + +def track_connections(retained): + def factory(*args, **kwargs): + conn = REAL_CONNECT(*args, factory=TrackedConnection, **kwargs) + conn.closed = False + retained.append(conn) + return conn + return mock.patch.object(hub.sqlite3, "connect", side_effect=factory) + + +def resource_counts(): + if sys.platform.startswith("linux"): + links = [] + for path in Path("/proc/self/fd").iterdir(): + try: + links.append(os.readlink(path)) + except FileNotFoundError: + pass + return {"fd": len(links), "hub_db_fd": sum("hub.db" in item for item in links)} + if sys.platform == "win32": + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.GetCurrentProcess.restype = ctypes.c_void_p + kernel.GetProcessHandleCount.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_ulong)] + count = ctypes.c_ulong() + if not kernel.GetProcessHandleCount(kernel.GetCurrentProcess(), ctypes.byref(count)): + raise ctypes.WinError(ctypes.get_last_error()) + return {"handles": count.value} + return {} + + +class QuietHandler(hub.Handler): + def log_message(self, *args): + pass + + +class RouterBackend(BaseHTTPRequestHandler): + def do_GET(self): + body = b"router proxy fixture" + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +class HubFixture: + def __enter__(self): + self.tmp = tempfile.TemporaryDirectory(prefix="owrt-db-", dir=STATE.name) + self.app = hub.App(Path(self.tmp.name) / "hub.db", "test-session", "test-agent", "") + hub.write_json_private(hub.AUTH_FILE, {"username": "test-owner", "password_hash": "unused"}) + self.backend = ThreadingHTTPServer(("127.0.0.1", 0), RouterBackend) + with self.app.conn() as conn: + hub.upsert_router(conn, { + "id": "test-router", "name": "Test Router", "entry_port": self.backend.server_port, + "vps_host": "127.0.0.1", "public_url": "http://127.0.0.1", + }) + self.server = hub.HubHTTPServer(("127.0.0.1", 0), QuietHandler) + self.server.app = self.app + self.server.is_tls = False + self.threads = [] + self.push_patch = mock.patch.object(hub, "queue_web_push_payload") + self.push_patch.start() + for server in (self.backend, self.server): + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.threads.append(thread) + self.app.start_router_state_monitor() + return self + + def request(self, path, payload=None): + headers = {"Cookie": "owrt_remote_session=test-session", "Authorization": "Bearer test-agent"} + body = None + if payload is not None: + headers["Content-Type"] = "application/json" + body = json.dumps(payload).encode() + request = urllib.request.Request(f"http://127.0.0.1:{self.server.server_port}{path}", data=body, headers=headers) + with urllib.request.urlopen(request, timeout=5) as response: + return response.read() + + def exercise(self): + assert json.loads(self.request("/health"))["ok"] + assert json.loads(self.request("/api/heartbeat", {"id": "test-router", "hostname": "test-router"}))["ok"] + assert len(json.loads(self.request("/api/routers"))["routers"]) == 1 + assert "groups" in json.loads(self.request("/api/router-groups")) + assert "notifications" in json.loads(self.request("/api/notifications")) + assert self.request("/access/test-router/") == b"router proxy fixture" + assert b"v109" in self.request("/") + self.app.snapshot_router_states() + + def __exit__(self, *args): + self.app.stop_router_state_monitor() + for server in (self.server, self.backend): + server.shutdown() + server.server_close() + for thread in self.threads: + thread.join(timeout=5) + self.push_patch.stop() + self.tmp.cleanup() + + +class SQLiteLifecycleTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(dir=STATE.name) + self.addCleanup(self.tmp.cleanup) + self.db = Path(self.tmp.name) / "hub.db" + self.app = hub.App(self.db, "test-session", "test-agent", "") + self.retained = [] + self.patch = track_connections(self.retained) + self.patch.start() + self.addCleanup(self.patch.stop) + + def assert_closed(self): + self.assertTrue(self.retained) + for conn in self.retained: + self.assertTrue(conn.closed) + with self.assertRaises(sqlite3.ProgrammingError): + conn.execute("select 1") + + def test_success_return_and_commit(self): + def operation(): + with self.app.conn() as conn: + conn.execute("create table sample(value integer)") + conn.execute("insert into sample values (7)") + return 7 + self.assertEqual(operation(), 7) + with self.app.conn() as conn: + self.assertEqual(conn.execute("select value from sample").fetchone()[0], 7) + self.assert_closed() + + def test_exception_and_base_exception_rollback(self): + with self.app.conn() as conn: + conn.execute("create table sample(value integer)") + for error in (ValueError("body failed"), KeyboardInterrupt()): + with self.assertRaises(type(error)): + with self.app.conn() as conn: + conn.execute("insert into sample values (9)") + raise error + with self.app.conn() as conn: + self.assertEqual(conn.execute("select count(*) from sample").fetchone()[0], 0) + self.assert_closed() + + def test_init_failure_closes_and_propagates(self): + with mock.patch.object(hub, "init_db", side_effect=sqlite3.OperationalError("init failed")): + with self.assertRaisesRegex(sqlite3.OperationalError, "init failed"): + with self.app.conn(): + self.fail("must not enter body") + self.assert_closed() + + def test_commit_failure_rolls_back_and_closes(self): + with hub.connect(self.db) as conn: + conn.executescript("pragma foreign_keys=on; create table parent(id primary key); " + "create table child(id references parent(id) deferrable initially deferred);") + with self.assertRaises(sqlite3.IntegrityError): + with hub.connect(self.db) as conn: + conn.execute("pragma foreign_keys=on") + conn.execute("insert into child values (1)") + with hub.connect(self.db) as conn: + self.assertEqual(conn.execute("select count(*) from child").fetchone()[0], 0) + self.assert_closed() + + def test_setup_and_transaction_exit_failure_close(self): + for setup_error in (True, False): + connection = mock.MagicMock() + if setup_error: + type(connection).row_factory = mock.PropertyMock(side_effect=RuntimeError("setup failed")) + else: + connection.__exit__.side_effect = sqlite3.OperationalError("rollback failed") + with mock.patch.object(hub.sqlite3, "connect", return_value=connection): + with self.assertRaises((RuntimeError, sqlite3.OperationalError)): + with hub.connect(self.db): + pass + connection.close.assert_called_once() + + def test_backup_destination_open_failure_closes_source(self): + with self.app.conn(): + pass + with self.assertRaises(sqlite3.OperationalError): + hub.copy_sqlite_backup(self.db, self.tmp.name) # destination is a directory + self.assert_closed() + + def test_backup_success(self): + with self.app.conn() as conn: + hub.upsert_router(conn, {"id": "backup-router", "name": "backup", "entry_port": 0, "vps_host": "localhost"}) + copy = Path(self.tmp.name) / "copy.db" + hub.copy_sqlite_backup(self.db, copy) + with hub.connect(copy) as conn: + self.assertEqual(hub.list_router_rows(conn)[0]["id"], "backup-router") + self.assert_closed() + + def test_snapshot_fd_regression_without_gc(self): + with self.app.conn(): + pass + before = resource_counts() + for _ in range(500): + self.assertEqual(self.app.snapshot_router_states(), {}) + after = resource_counts() + self.assert_closed() + self.assertGreaterEqual(len(self.retained), 501) + for key, value in before.items(): + self.assertLessEqual(after[key], value + (2 if key != "hub_db_fd" else 0), (before, after)) + + def test_all_sqlite_creation_uses_managed_factory(self): + tree = ast.parse(SOURCE.read_text(encoding="utf-8")) + parents = {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} + direct = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == "sqlite3" and node.func.attr == "connect": + direct.append(node) + elif isinstance(node.func, ast.Name) and node.func.id == "connect" or isinstance(node.func, ast.Attribute) and node.func.attr == "conn": + self.assertIsInstance(parents[node], ast.withitem, f"unmanaged DB operation at line {node.lineno}") + self.assertEqual(len(direct), 1) + + +class OtherResourceTests(unittest.TestCase): + def handler(self, data=b"", headers=None): + handler = object.__new__(hub.Handler) + handler.rfile = io.BytesIO(data) + handler.headers = headers or {} + return handler + + def test_chunked_valid_and_truncated_bodies(self): + headers = {"Transfer-Encoding": "chunked"} + self.assertEqual(self.handler(b"3\r\nabc\r\n0\r\n\r\n", headers).read_body(), b"abc") + for data in (b"", b"3\r\nab", b"3\r\nabc", b"0\r\n", b"-1\r\n", b"\r\n"): + with self.subTest(data=data), self.assertRaises(ValueError): + self.handler(data, headers).read_body() + with self.assertRaises(ValueError): + self.handler(b"short", {"Content-Length": "100"}).read_body() + + def test_oauth_http_error_closes_response(self): + body = io.BytesIO(b'{"error":"denied"}') + error = urllib.error.HTTPError("http://test", 403, "denied", {}, body) + with mock.patch.object(hub.urllib.request, "urlopen", side_effect=error): + with self.assertRaisesRegex(ValueError, "denied"): + hub.oauth_fetch_json("http://test") + self.assertTrue(body.closed) + + def test_push_success_and_failure_close_response(self): + response = mock.Mock(status_code=410, text="gone") + error = RuntimeError("push failed") + error.response = response + with mock.patch.object(hub, "web_push_ready", return_value=True), mock.patch.object(hub, "webpush", return_value=response) as push: + self.assertEqual(hub.send_web_push({"endpoint": "https://test"}, {}), "ok") + response.close.assert_called_once() + response.reset_mock() + push.side_effect = error + self.assertEqual(hub.send_web_push({"endpoint": "https://test"}, {}), "gone") + response.close.assert_called_once() + + def test_atomic_failure_removes_temp_and_preserves_target(self): + with tempfile.TemporaryDirectory(dir=STATE.name) as name: + target = Path(name) / "original.txt" + target.write_text("original") + with mock.patch.object(hub.os, "replace", side_effect=OSError("replace failed")): + with self.assertRaises(OSError): + hub.atomic_write_text(target, "new") + self.assertEqual(target.read_text(), "original") + self.assertEqual(list(Path(name).iterdir()), [target]) + + def test_askpass_setup_failure_removes_directory(self): + with tempfile.TemporaryDirectory(dir=STATE.name) as name: + directory = Path(name) / "askpass" + directory.mkdir() + with mock.patch.object(hub.tempfile, "mkdtemp", return_value=str(directory)), mock.patch.object(hub.os, "chmod", side_effect=OSError("chmod failed")): + with self.assertRaises(OSError): + self.handler().prepare_ssh_askpass({}, "fake-password") + self.assertFalse(directory.exists()) + + def test_https_failure_releases_listening_port(self): + server = hub.make_http_server(None, "127.0.0.1", 0) + port = server.server_port + server.server_close() + with self.assertRaises(OSError): + hub.make_http_server(None, "127.0.0.1", port, "missing-cert.pem", "missing-key.pem") + replacement = hub.make_http_server(None, "127.0.0.1", port) + replacement.server_close() + + def test_terminal_reader_start_failure_cleans_process_and_map(self): + session = {"id": "failed-thread", "pid": 123, "fd": 456} + with mock.patch.object(hub.threading.Thread, "start", side_effect=RuntimeError("thread failed")), mock.patch.object(hub, "close_terminal_process") as close: + with self.assertRaises(RuntimeError): + self.handler().launch_ssh_http_reader(session) + close.assert_called_once_with(123, 456) + self.assertNotIn("failed-thread", hub.SSH_HTTP_SESSIONS) + + def test_initial_websocket_failure_cleans_terminal(self): + handler = self.handler() + handler.connection = object() + with mock.patch.object(handler, "spawn_terminal_pty", return_value=(123, 456)), mock.patch.object(hub, "ws_send_frame", side_effect=BrokenPipeError()), mock.patch.object(hub, "close_terminal_process") as close: + handler.run_terminal_session({}, ["sh"], "unavailable", "open", "sh", "terminal error") + close.assert_called_once_with(123, 456) + + def test_terminal_input_failure_closes_owned_descriptor(self): + handler = self.handler() + handler.send_json = mock.Mock() + session = {"alive": True, "fd": 456, "last_seen": 0, "lock": threading.Lock()} + with mock.patch.dict(hub.SSH_HTTP_SESSIONS, {"input-test": session}, clear=True), mock.patch.object(hub.os, "dup", return_value=789), mock.patch.object(hub.os, "close") as close, mock.patch.object(hub, "write_pty_all", side_effect=TimeoutError("write failed")): + handler.ssh_http_write("input-test", {"data": "hello"}) + close.assert_called_once_with(789) + self.assertEqual(handler.send_json.call_args.args[0], 500) + + @unittest.skipUnless(hasattr(hub.signal, "SIGHUP"), "POSIX terminal signals") + def test_terminal_shutdown_escalates_and_reaps(self): + # One unsuccessful poll per signal, followed by the final blocking reap. + with mock.patch.object(hub.os, "close") as close, mock.patch.object(hub.os, "waitpid", create=True, side_effect=[(0, 0)] * 3 + [(123, 0)]) as wait, mock.patch.object(hub.os, "kill") as kill, mock.patch.object(hub.time, "monotonic", side_effect=[0, 2, 0, 2, 0, 2]): + hub.close_terminal_process(123, 456) + close.assert_called_once_with(456) + self.assertEqual([c.args[1] for c in kill.call_args_list], [hub.signal.SIGHUP, hub.signal.SIGTERM, hub.signal.SIGKILL]) + self.assertEqual(wait.call_args.args, (123, 0)) + + def test_terminal_session_expiry_and_retained_close_notice(self): + now = hub.now_ts() + sessions = { + "expired": {"alive": True, "last_seen": now - 901, "lock": threading.Lock()}, + "closed": {"alive": False, "last_seen": now, "closed_at": now - 61, "lock": threading.Lock()}, + "recent": {"alive": False, "last_seen": now, "closed_at": now, "lock": threading.Lock()}, + "active": {"alive": True, "last_seen": now, "lock": threading.Lock()}, + } + with mock.patch.dict(hub.SSH_HTTP_SESSIONS, sessions, clear=True): + hub.expire_ssh_http_sessions() + self.assertEqual(set(hub.SSH_HTTP_SESSIONS), {"recent", "active"}) + + def test_monitor_recovers_from_initial_database_error(self): + app = hub.App("unused", "session", "agent", "") + with mock.patch.object(app, "prime_router_state_snapshot", side_effect=sqlite3.OperationalError("initial DB error")), mock.patch.object(app, "check_router_state_changes") as check, mock.patch.object(app.router_monitor_stop, "wait", side_effect=[False, True]), contextlib.redirect_stderr(io.StringIO()) as errors: + app.router_state_monitor_loop() + check.assert_called_once() + self.assertIn("initial DB error", errors.getvalue()) + + def test_monitor_start_failure_does_not_break_cleanup(self): + app = hub.App("unused", "session", "agent", "") + with mock.patch.object(hub.threading.Thread, "start", side_effect=RuntimeError("start failed")): + with self.assertRaises(RuntimeError): + app.start_router_state_monitor() + app.stop_router_state_monitor() + + def test_serve_start_failure_closes_main_server(self): + app = hub.App(Path(STATE.name) / "serve-failure.db", "session", "agent", "") + server = mock.Mock() + args = argparse.Namespace(db=app.db_path, public_url="", host="127.0.0.1", port=0) + with mock.patch.object(hub, "App", return_value=app), mock.patch.object(hub, "session_token", return_value="session"), mock.patch.object(hub, "agent_token", return_value="agent"), mock.patch.object(hub, "record_hub_start_event"), mock.patch.object(hub, "load_auth", return_value={}), mock.patch.object(hub, "make_http_server", return_value=server), mock.patch.object(app, "start_router_state_monitor", side_effect=RuntimeError("monitor failed")): + with self.assertRaises(RuntimeError): + hub.cmd_serve(args) + server.server_close.assert_called_once() + + def test_serve_stop_closes_main_and_extra_servers(self): + app = hub.App(Path(STATE.name) / "serve-stop.db", "session", "agent", "") + main = mock.Mock() + extra = mock.Mock() + main.serve_forever.side_effect = KeyboardInterrupt() + args = argparse.Namespace(db=app.db_path, public_url="", host="127.0.0.1", port=8088, + extra_ports="18088", tls_ports="", tls_cert="", tls_key="") + with mock.patch.object(hub, "App", return_value=app), mock.patch.object(hub, "session_token", return_value="session"), mock.patch.object(hub, "agent_token", return_value="agent"), mock.patch.object(hub, "record_hub_start_event"), mock.patch.object(hub, "load_auth", return_value={}), mock.patch.object(hub, "make_http_server", side_effect=[main, extra]), contextlib.redirect_stdout(io.StringIO()): + hub.cmd_serve(args) + main.server_close.assert_called_once() + extra.shutdown.assert_called_once() + extra.server_close.assert_called_once() + self.assertFalse(app.router_monitor_thread.is_alive()) + + def test_incoming_socket_has_finite_io_timeout(self): + server = hub.make_http_server(None, "127.0.0.1", 0) + self.addCleanup(server.server_close) + with socket.create_connection(server.server_address) as client: + incoming, _ = server.get_request() + try: + self.assertEqual(incoming.gettimeout(), 90) + finally: + incoming.close() + + @unittest.skipUnless(sys.platform.startswith("linux"), "real PTY/child reaping needs Linux") + def test_real_terminal_child_is_reaped_and_fd_closed(self): + handler = self.handler() + pid, fd = handler.spawn_terminal_pty(dict(os.environ), ["/bin/sh", "-c", "sleep 60"], "pty", "open", "sh") + hub.close_terminal_process(pid, fd) + with self.assertRaises(OSError): + os.fstat(fd) + with self.assertRaises(ChildProcessError): + os.waitpid(pid, os.WNOHANG) + + +class IntegrationTests(unittest.TestCase): + def test_panel_api_proxy_heartbeat_and_monitor(self): + with HubFixture() as fixture: + for _ in range(5): + fixture.exercise() + self.assertTrue(fixture.app.router_monitor_thread.is_alive()) + # Verify the monitor still emits the online -> offline transition. + fixture.app.prime_router_state_snapshot() + with fixture.app.conn() as conn: + conn.execute("update routers set last_seen = ?", (hub.now_ts() - 3600,)) + with mock.patch.object(hub, "notify_router_offline") as notify: + fixture.app.check_router_state_changes() + notify.assert_called_once() + self.assertFalse(fixture.app.router_monitor_thread.is_alive()) + + +def soak(seconds): + started = time.monotonic() + cpu_start = time.process_time() + samples = [] + operations = 0 + with HubFixture() as fixture: + fixture.exercise() # warm up auth, sessions, notifications and HTTP workers + while True: + fixture.exercise() + operations += 1 + # Measure between requests, after worker sockets have closed. + time.sleep(0.05) + counts = resource_counts() + sample = {"elapsed": round(time.monotonic() - started, 1), **counts} + samples.append(sample) + print(json.dumps(sample), flush=True) + if time.monotonic() - started >= seconds: + break + time.sleep(min(5, max(0, seconds - (time.monotonic() - started)))) + assert fixture.app.router_monitor_thread.is_alive(), "router monitor stopped" + for key in samples[0]: + if key == "elapsed": + continue + assert samples[-1][key] <= samples[0][key] + (4 if key != "hub_db_fd" else 0), samples + result = {"seconds": round(time.monotonic() - started, 1), "cpu_seconds": round(time.process_time() - cpu_start, 2), + "cycles": operations, "first": samples[0], "last": samples[-1], "peak": {key: max(s[key] for s in samples) for key in counts}} + print("SOAK_OK " + json.dumps(result), flush=True) + + +if __name__ == "__main__": + if "--soak-seconds" in sys.argv: + parser = argparse.ArgumentParser() + parser.add_argument("--soak-seconds", type=int, required=True) + soak(parser.parse_args().soak_seconds) + else: + unittest.main() diff --git a/vps/deploy-resource-fix.ps1 b/vps/deploy-resource-fix.ps1 new file mode 100644 index 0000000..8296949 --- /dev/null +++ b/vps/deploy-resource-fix.ps1 @@ -0,0 +1,177 @@ +param( + [ValidateSet('Install', 'Rollback', 'Monitor')][string]$Action = 'Install', + [ValidateRange(1, 1440)][int]$Minutes = 10, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string]$Vps, + [switch]$PrepareOnly +) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent $PSScriptRoot +$RemoteTarget = $Vps +$UploadName = 'owrt-resource-' + [guid]::NewGuid().ToString('N') +$Utf8 = [System.Text.UTF8Encoding]::new($false) +$UploadFiles = @() + +if ($Action -eq 'Install') { + $RemoteScript = @' +#!/bin/bash +set -Eeuo pipefail +stage=/tmp/__UPLOAD_NAME__ +target=/opt/owrt-remote/owrt-remote-hub.py +backup=/opt/owrt-remote/owrt-remote-hub.py.bak-before-resource-fix-v109 +candidate=$target.new-__UPLOAD_NAME__ +changed=0 +cleanup() { + code=$? + trap - EXIT + if [ "$code" -ne 0 ] && [ "$changed" = 1 ]; then + echo 'Installation failed; restoring the file from this attempt.' >&2 + systemctl --no-pager -l status owrt-remote >&2 || true + journalctl -u owrt-remote -n 60 --no-pager >&2 || true + cp -a "$stage/previous.py" "$candidate" + mv -f "$candidate" "$target" + systemctl restart owrt-remote || true + fi + rm -f "$candidate" + rm -rf -- "$stage" + exit "$code" +} +trap cleanup EXIT +test -s "$target" +test -s "$stage/hub.py" +python_bin=/opt/owrt-remote/venv/bin/python +if [ ! -x "$python_bin" ]; then python_bin=python3; fi +"$python_bin" - "$stage/hub.py" <<'PY' +import ast, pathlib, sys +source = pathlib.Path(sys.argv[1]).read_text(encoding='utf-8') +tree = ast.parse(source) +assert 'v109' in source +assert 'v108' not in source +factory = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == 'connect') +assert any(isinstance(item, ast.Name) and item.id == 'contextmanager' for item in factory.decorator_list) +assert any(isinstance(item, ast.Try) and item.finalbody for item in ast.walk(factory)) +PY +# The Linux regression suite uses its own temporary DB and loopback ports. +# It also exercises a real PTY and verifies child reaping. +OWRT_RESOURCE_HUB_SOURCE="$stage/hub.py" "$python_bin" "$stage/tests.py" -v + +if [ ! -e "$backup" ]; then cp -a "$target" "$backup"; fi +test -s "$backup" +printf 'Backup: %s\n' "$backup" +cp -a "$target" "$stage/previous.py" +install -m 0755 "$stage/hub.py" "$candidate" +changed=1 +mv -f "$candidate" "$target" +systemctl restart owrt-remote +healthy=0 +for attempt in {1..10}; do + if systemctl is-active --quiet owrt-remote && curl -fsS --max-time 5 http://127.0.0.1:8088/health; then + healthy=1 + break + fi + sleep 2 +done +test "$healthy" = 1 +cmp -s "$target" "$stage/hub.py" +printf '\n' +pid=$(systemctl show owrt-remote -p MainPID --value) +test "$pid" -gt 0 +printf 'PID=%s FD=%s\n' "$pid" "$(find /proc/$pid/fd -maxdepth 1 -type l | wc -l)" +systemctl --no-pager --full status owrt-remote | sed -n '1,16p' +echo RESOURCE_FIX_V109_OK +'@ +} elseif ($Action -eq 'Rollback') { + $RemoteScript = @' +#!/bin/bash +set -Eeuo pipefail +stage=/tmp/__UPLOAD_NAME__ +target=/opt/owrt-remote/owrt-remote-hub.py +backup=/opt/owrt-remote/owrt-remote-hub.py.bak-before-resource-fix-v109 +candidate=$target.rollback-__UPLOAD_NAME__ +changed=0 +cleanup() { + code=$? + trap - EXIT + if [ "$code" -ne 0 ] && [ "$changed" = 1 ]; then + echo 'Rollback failed; restoring the file from this attempt.' >&2 + cp -a "$stage/current.py" "$candidate" + mv -f "$candidate" "$target" + systemctl restart owrt-remote || true + fi + rm -f "$candidate" + rm -rf -- "$stage" + exit "$code" +} +trap cleanup EXIT +test -s "$target" +test -s "$backup" +python3 - "$backup" <<'PY' +import ast, pathlib, sys +ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding='utf-8')) +PY +cp -a "$target" "$stage/current.py" +cp -a "$backup" "$candidate" +changed=1 +mv -f "$candidate" "$target" +systemctl restart owrt-remote +healthy=0 +for attempt in {1..10}; do + if systemctl is-active --quiet owrt-remote && curl -fsS --max-time 5 http://127.0.0.1:8088/health; then + healthy=1 + break + fi + sleep 2 +done +test "$healthy" = 1 +printf '\n' +echo RESOURCE_FIX_ROLLBACK_OK +'@ +} else { + $RemoteScript = @' +#!/bin/bash +set -Eeuo pipefail +stage=/tmp/__UPLOAD_NAME__ +trap 'rm -rf -- "$stage"' EXIT +python3 "$stage/monitor.py" __MINUTES__ +'@ +} + +$RemoteScript = $RemoteScript.Replace('__UPLOAD_NAME__', $UploadName).Replace('__MINUTES__', [string]$Minutes) +try { + $StageRoot = Join-Path ([System.IO.Path]::GetTempPath()) $UploadName + [System.IO.Directory]::CreateDirectory($StageRoot) | Out-Null + $UploadFiles += $StageRoot + [System.IO.File]::WriteAllText((Join-Path $StageRoot 'run.sh'), $RemoteScript.Replace("`r`n", "`n") + "`n", $Utf8) + if ($Action -eq 'Install') { + $Payloads = @{ + 'hub.py' = Join-Path $RepoRoot 'vps\owrt-remote-hub.py' + 'tests.py' = Join-Path $RepoRoot 'tests\test_resource_lifecycle.py' + } + foreach ($Item in $Payloads.GetEnumerator()) { + $FileText = [System.IO.File]::ReadAllText($Item.Value) + [System.IO.File]::WriteAllText((Join-Path $StageRoot $Item.Key), $FileText.Replace("`r`n", "`n"), $Utf8) + } + } elseif ($Action -eq 'Monitor') { + $MonitorText = [System.IO.File]::ReadAllText((Join-Path $RepoRoot 'tests\monitor_vps_resources.py')) + [System.IO.File]::WriteAllText((Join-Path $StageRoot 'monitor.py'), $MonitorText.Replace("`r`n", "`n"), $Utf8) + } + if ($PrepareOnly) { + Write-Host "Prepared only: $StageRoot" + $UploadFiles = @() + return + } + & scp -r -- $StageRoot "${RemoteTarget}:/tmp/" + if ($LASTEXITCODE -ne 0) { throw 'Upload failed; installation has not started.' } + & ssh $RemoteTarget "bash /tmp/$UploadName/run.sh" + if ($LASTEXITCODE -ne 0) { throw "$Action failed. See the output above; success was not confirmed." } + Write-Host "$Action finished successfully." +} finally { + foreach ($UploadFile in $UploadFiles) { + $Resolved = [System.IO.Path]::GetFullPath($UploadFile) + $Expected = [System.IO.Path]::GetFullPath((Join-Path ([System.IO.Path]::GetTempPath()) $UploadName)) + if ($Resolved -ne $Expected -or (Split-Path -Leaf $Resolved) -ne $UploadName) { + throw 'Unsafe staging cleanup path.' + } + if (Test-Path -LiteralPath $Resolved) { Remove-Item -LiteralPath $Resolved -Recurse -Force } + } +} diff --git a/vps/owrt-remote-hub.py b/vps/owrt-remote-hub.py index f4f2ccb..6a99939 100644 --- a/vps/owrt-remote-hub.py +++ b/vps/owrt-remote-hub.py @@ -3,6 +3,8 @@ import argparse import base64 import datetime as dt import dataclasses +from contextlib import contextmanager +from collections import deque import enum from email.parser import BytesParser from email.policy import default as email_policy_default @@ -169,14 +171,82 @@ def now_ts(): return int(time.time()) -def atomic_write_text(path, text, mode=None): +def close_terminal_process(pid, fd): + """The terminal reader owns both the PTY descriptor and child reaping.""" + try: + os.close(fd) + except OSError: + pass + for sig in (signal.SIGHUP, signal.SIGTERM, signal.SIGKILL): + try: + # Reap first so an already exited child is never signalled again. + if os.waitpid(pid, os.WNOHANG)[0]: + return + os.kill(pid, sig) + except ProcessLookupError: + pass + except ChildProcessError: + return + deadline = time.monotonic() + 1 + while time.monotonic() < deadline: + try: + if os.waitpid(pid, os.WNOHANG)[0]: + return + except ChildProcessError: + return + time.sleep(0.05) + # SIGKILL has been sent; collect the child rather than leaving a zombie. + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + + +def expire_ssh_http_sessions(): + now = now_ts() + with SSH_HTTP_LOCK: + for sid, session in list(SSH_HTTP_SESSIONS.items()): + with session["lock"]: + expired = now - session["last_seen"] > 15 * 60 + closed = not session["alive"] and now - session.get("closed_at", now) > 60 + if expired or closed: + session["alive"] = False + SSH_HTTP_SESSIONS.pop(sid, None) + + +def stop_ssh_http_sessions(): + with SSH_HTTP_LOCK: + sessions = list(SSH_HTTP_SESSIONS.values()) + SSH_HTTP_SESSIONS.clear() + for session in sessions: + with session["lock"]: + session["alive"] = False + for session in sessions: + thread = session.get("thread") + if thread and thread.ident is not None and thread is not threading.current_thread(): + thread.join(timeout=5) + + +@contextmanager +def atomic_output_path(path): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") - tmp.write_text(text, encoding="utf-8") - if mode is not None: - os.chmod(tmp, mode) - os.replace(tmp, path) + tmp = None + try: + with tempfile.NamedTemporaryFile(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent, delete=False) as output: + tmp = Path(output.name) + yield tmp + os.replace(tmp, path) + finally: + if tmp is not None: + tmp.unlink(missing_ok=True) + + +def atomic_write_text(path, text, mode=None): + with atomic_output_path(path) as tmp: + tmp.write_text(text, encoding="utf-8") + if mode is not None: + os.chmod(tmp, mode) def iso_time(ts): @@ -1127,13 +1197,16 @@ def oauth_fetch_json(url, method="GET", headers=None, data=None): body = response.read().decode("utf-8", "replace") return json.loads(body or "{}") except urllib.error.HTTPError as exc: - body = exc.read().decode("utf-8", "replace") try: - detail = json.loads(body or "{}") - except Exception: - detail = {"error": body[:400] or exc.reason} - message = detail.get("error_description") or detail.get("error") or detail.get("message") or exc.reason - raise ValueError(str(message or "OAuth HTTP error")) + body = exc.read().decode("utf-8", "replace") + try: + detail = json.loads(body or "{}") + except Exception: + detail = {"error": body[:400] or exc.reason} + message = detail.get("error_description") or detail.get("error") or detail.get("message") or exc.reason + raise ValueError(str(message or "OAuth HTTP error")) + finally: + exc.close() except Exception as exc: raise ValueError(str(exc)) @@ -2310,8 +2383,9 @@ def push_payload_for_notification(item): def send_web_push(subscription, payload): if not web_push_ready(): return "unavailable" + response = None try: - webpush( + response = webpush( subscription_info={ "endpoint": subscription.get("endpoint"), "keys": subscription.get("keys") or {}, @@ -2339,6 +2413,9 @@ def send_web_push(subscription, payload): if detail: return f"error:{status_code or exc.__class__.__name__}:{detail}" return f"error:{status_code or exc.__class__.__name__}" + finally: + if response is not None: + response.close() def push_result_message(result, subscription=None): @@ -3027,11 +3104,21 @@ def encode_router_notes_entries(entries): ) +@contextmanager def connect(db_path=DB_PATH): + """Own the connection: commit/rollback the transaction, then always close. + + SQLite's native context manager handles transactions only; it does not close + the connection. Keep every DB operation inside this managed context. + """ ensure_state() conn = sqlite3.connect(str(db_path)) - conn.row_factory = sqlite3.Row - return conn + try: + conn.row_factory = sqlite3.Row + with conn: + yield conn + finally: + conn.close() def init_db(conn): @@ -3373,30 +3460,19 @@ def copy_sqlite_backup(src_path, dst_path): src_path.parent.mkdir(parents=True, exist_ok=True) dst_path.parent.mkdir(parents=True, exist_ok=True) if not src_path.exists(): - conn = connect(src_path) - try: + with connect(src_path) as conn: init_db(conn) - finally: - conn.close() - source = sqlite3.connect(str(src_path)) - dest = sqlite3.connect(str(dst_path)) - try: + with connect(src_path) as source, connect(dst_path) as dest: source.backup(dest) - finally: - dest.close() - source.close() def write_private_bytes(path, data, mode=0o600): - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") - tmp.write_bytes(data) - try: - os.chmod(tmp, mode) - except OSError: - pass - os.replace(tmp, path) + with atomic_output_path(path) as tmp: + tmp.write_bytes(data) + try: + os.chmod(tmp, mode) + except OSError: + pass def create_hub_backup(out_path, db_path=DB_PATH): @@ -3427,17 +3503,16 @@ def create_hub_backup(out_path, db_path=DB_PATH): "files": files, } (tmp_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - tmp_archive = out_path.with_name(f".{out_path.name}.{os.getpid()}.tmp") - with tarfile.open(tmp_archive, "w:gz") as tar: - tar.add(tmp_dir / "manifest.json", arcname="manifest.json") - for item in state_dir.iterdir(): - if item.is_file(): - tar.add(item, arcname=f"state/{item.name}") - try: - os.chmod(tmp_archive, 0o600) - except OSError: - pass - os.replace(tmp_archive, out_path) + with atomic_output_path(out_path) as tmp_archive: + with tarfile.open(tmp_archive, "w:gz") as tar: + tar.add(tmp_dir / "manifest.json", arcname="manifest.json") + for item in state_dir.iterdir(): + if item.is_file(): + tar.add(item, arcname=f"state/{item.name}") + try: + os.chmod(tmp_archive, 0o600) + except OSError: + pass return {"path": str(out_path), "filename": out_path.name, "files": files} @@ -3538,12 +3613,9 @@ def restore_hub_backup(archive_path, db_path=DB_PATH, vps_host="", public_url="" write_private_bytes(target, src.read_bytes()) restored_files.append(str(target)) rewritten = {} - conn = connect(db_path) - try: + with connect(db_path) as conn: init_db(conn) rewritten = rewrite_router_endpoints(conn, vps_host=vps_host, public_url=public_url) - finally: - conn.close() xray = None try: xray = reload_vps_xray(db_path) @@ -4044,12 +4116,9 @@ def make_router_xray_config(row): def reload_vps_xray(db_path=DB_PATH): out = Path(os.environ.get("OWRT_REMOTE_XRAY_CONFIG", "/etc/xray/owrt-remote.json")) service = os.environ.get("OWRT_REMOTE_XRAY_SERVICE", "owrt-remote-xray") - conn = connect(db_path) - try: + with connect(db_path) as conn: init_db(conn) rows = list_router_rows(conn) - finally: - conn.close() config = make_server_xray_config(rows) atomic_write_text(out, json.dumps(config, ensure_ascii=False, indent=2) + "\n", mode=0o600) try: @@ -13613,10 +13682,11 @@ class App: self.router_monitor_stop = threading.Event() self.router_monitor_thread = None + @contextmanager def conn(self): - conn = connect(self.db_path) - init_db(conn) - return conn + with connect(self.db_path) as conn: + init_db(conn) + yield conn def snapshot_router_states(self): with self.conn() as conn: @@ -13649,12 +13719,16 @@ class App: def router_state_monitor_loop(self): interval = max(3, int(os.environ.get("OWRT_REMOTE_ROUTER_NOTIFY_POLL", "5"))) - self.prime_router_state_snapshot() + try: + self.prime_router_state_snapshot() + except Exception as exc: + print(f"WARNING: router state monitor initialization error: {exc}", file=sys.stderr) while not self.router_monitor_stop.wait(interval): try: self.check_router_state_changes() except Exception as exc: print(f"WARNING: router state monitor error: {exc}", file=sys.stderr) + expire_ssh_http_sessions() def start_router_state_monitor(self): if self.router_monitor_thread and self.router_monitor_thread.is_alive(): @@ -13665,6 +13739,9 @@ class App: def stop_router_state_monitor(self): self.router_monitor_stop.set() + thread = self.router_monitor_thread + if thread and thread.ident is not None and thread is not threading.current_thread(): + thread.join(timeout=10) class Handler(BaseHTTPRequestHandler): @@ -13891,19 +13968,33 @@ class Handler(BaseHTTPRequestHandler): if self.headers.get("Transfer-Encoding", "").lower() == "chunked": chunks = [] while True: - line = self.rfile.readline().strip() - if not line: - continue + raw_line = self.rfile.readline() + if not raw_line: + raise ValueError("incomplete chunked request body") + line = raw_line.strip() size = int(line.split(b";", 1)[0], 16) + if size < 0: + raise ValueError("negative chunk size") if size == 0: - while self.rfile.readline().strip(): - pass + while True: + trailer = self.rfile.readline() + if not trailer: + raise ValueError("incomplete chunked request trailers") + if trailer == b"\r\n": + break break - chunks.append(self.rfile.read(size)) - self.rfile.read(2) + chunk = self.rfile.read(size) + if len(chunk) != size or self.rfile.read(2) != b"\r\n": + raise ValueError("incomplete chunked request data") + chunks.append(chunk) return b"".join(chunks) length = int(self.headers.get("Content-Length", "0") or "0") - return self.rfile.read(length) if length else b"" + if length < 0: + raise ValueError("negative content length") + body = self.rfile.read(length) if length else b"" + if len(body) != length: + raise ValueError("incomplete request body") + return body def read_payload(self): body = self.read_body() @@ -15358,15 +15449,19 @@ a{{display:inline-flex;margin-top:18px;color:#93c5fd}} return None askpass_dir = Path(tempfile.mkdtemp(prefix="owrt-askpass-")) askpass_script = askpass_dir / "askpass.sh" - askpass_script.write_text( - "#!/bin/sh\nprintf '%s\\n' \"$OWRT_REMOTE_SSH_PASSWORD\"\n", - encoding="utf-8", - ) - os.chmod(askpass_script, 0o700) - env["SSH_ASKPASS"] = str(askpass_script) - env["SSH_ASKPASS_REQUIRE"] = "force" - env["DISPLAY"] = env.get("DISPLAY") or "owrt-remote:0" - env["OWRT_REMOTE_SSH_PASSWORD"] = password + try: + askpass_script.write_text( + "#!/bin/sh\nprintf '%s\\n' \"$OWRT_REMOTE_SSH_PASSWORD\"\n", + encoding="utf-8", + ) + os.chmod(askpass_script, 0o700) + env["SSH_ASKPASS"] = str(askpass_script) + env["SSH_ASKPASS_REQUIRE"] = "force" + env["DISPLAY"] = env.get("DISPLAY") or "owrt-remote:0" + env["OWRT_REMOTE_SSH_PASSWORD"] = password + except BaseException: + shutil.rmtree(askpass_dir) + raise return askpass_dir def normalize_wol_mac(self, value): @@ -15389,8 +15484,8 @@ a{{display:inline-flex;margin-top:18px;color:#93c5fd}} password = str(ssh_password or "") env, args = self.ssh_exec_args(port, allow_password=bool(password)) askpass_dir = self.prepare_ssh_askpass(env, password) - argv = args + ["sh", "-s", "--"] + [str(item) for item in (script_args or [])] try: + argv = args + ["sh", "-s", "--"] + [str(item) for item in (script_args or [])] result = subprocess.run( argv, input=str(script), @@ -16148,13 +16243,19 @@ exit 127 os._exit(127) return pid, fd - def ssh_http_reader(self, sid): - session = SSH_HTTP_SESSIONS.get(sid) + def ssh_http_reader(self, sid, session=None): + # Pass ownership directly: the map may be cleared before the thread runs. + if session is None: + with SSH_HTTP_LOCK: + session = SSH_HTTP_SESSIONS.get(sid) if not session: return fd = session["fd"] try: while True: + with session["lock"]: + if not session["alive"] or now_ts() - session["last_seen"] > 15 * 60: + break ready, _, _ = select.select([fd], [], [], 0.5) if fd not in ready: with session["lock"]: @@ -16171,31 +16272,35 @@ exit 127 break with session["lock"]: session["buffer"].append(data.decode("utf-8", errors="replace")) - session["last_seen"] = now_ts() finally: with session["lock"]: session["alive"] = False + session["closed_at"] = now_ts() session["buffer"].append("\r\n[" + session.get("close_notice", "SSH соединение закрыто") + "]\r\n") - try: - os.close(fd) - except OSError: - pass - try: - os.waitpid(session["pid"], os.WNOHANG) - except OSError: - pass + close_terminal_process(session["pid"], fd) + + def launch_ssh_http_reader(self, session): + sid = session["id"] + try: + thread = threading.Thread(target=self.ssh_http_reader, args=(sid, session), daemon=True) + session["thread"] = thread + with SSH_HTTP_LOCK: + SSH_HTTP_SESSIONS[sid] = session + thread.start() + except BaseException: + with SSH_HTTP_LOCK: + SSH_HTTP_SESSIONS.pop(sid, None) + close_terminal_process(session["pid"], session["fd"]) + raise def start_ssh_http_session(self, router_id, port): env, args = self.ssh_args(port) - pid, fd = self.spawn_terminal_pty(env, args, "pty недоступен на VPS", "не удалось открыть SSH pty", "ssh") sid = secrets.token_urlsafe(24) session = { "id": sid, "router_id": router_id, "port": port, - "pid": pid, - "fd": fd, - "buffer": [], + "buffer": deque(maxlen=256), "alive": True, "created": now_ts(), "last_seen": now_ts(), @@ -16203,22 +16308,18 @@ exit 127 "label": "SSH", "close_notice": "SSH соединение закрыто", } - with SSH_HTTP_LOCK: - SSH_HTTP_SESSIONS[sid] = session - threading.Thread(target=self.ssh_http_reader, args=(sid,), daemon=True).start() + session["pid"], session["fd"] = self.spawn_terminal_pty(env, args, "pty недоступен на VPS", "не удалось открыть SSH pty", "ssh") + self.launch_ssh_http_reader(session) return session def start_vps_http_session(self): env, args = self.vps_shell_args() - pid, fd = self.spawn_terminal_pty(env, args, "pty недоступен на VPS", "не удалось открыть VPS pty", "vps-shell") sid = secrets.token_urlsafe(24) session = { "id": sid, "router_id": VPS_TERMINAL_ID, "port": 0, - "pid": pid, - "fd": fd, - "buffer": [], + "buffer": deque(maxlen=256), "alive": True, "created": now_ts(), "last_seen": now_ts(), @@ -16226,9 +16327,8 @@ exit 127 "label": "VPS terminal", "close_notice": "VPS terminal закрыт", } - with SSH_HTTP_LOCK: - SSH_HTTP_SESSIONS[sid] = session - threading.Thread(target=self.ssh_http_reader, args=(sid,), daemon=True).start() + session["pid"], session["fd"] = self.spawn_terminal_pty(env, args, "pty недоступен на VPS", "не удалось открыть VPS pty", "vps-shell") + self.launch_ssh_http_reader(session) return session def ssh_http_session(self): @@ -16292,17 +16392,24 @@ exit 127 data = payload.get("data", "") if not isinstance(data, str): data = str(data) - with session["lock"]: - alive = bool(session["alive"]) - fd = session["fd"] - if not alive: - self.send_json(409, {"ok": False, "error": "terminal session closed"}) - return + fd = None try: + with session["lock"]: + if session["alive"]: + session["last_seen"] = now_ts() + # Own a temporary descriptor while writing, so reader cleanup + # cannot close/reuse it and terminal output remains unblocked. + fd = os.dup(session["fd"]) + if fd is None: + self.send_json(409, {"ok": False, "error": "terminal session closed"}) + return write_pty_all(fd, data) self.send_json(200, {"ok": True}) except Exception as exc: self.send_json(500, {"ok": False, "error": str(exc)}) + finally: + if fd is not None: + os.close(fd) def ssh_http_resize(self, sid, payload=None): with SSH_HTTP_LOCK: @@ -16312,7 +16419,12 @@ exit 127 return if payload is None: payload = self.read_payload() - ok = set_pty_size(session["fd"], payload.get("rows", 24), payload.get("cols", 80)) + with session["lock"]: + if not session["alive"]: + self.send_json(409, {"ok": False, "error": "terminal session closed"}) + return + session["last_seen"] = now_ts() + ok = set_pty_size(session["fd"], payload.get("rows", 24), payload.get("cols", 80)) self.send_json(200, {"ok": bool(ok)}) def ssh_http_write_short(self): @@ -16333,10 +16445,6 @@ exit 127 return with session["lock"]: session["alive"] = False - try: - os.kill(session["pid"], signal.SIGHUP) - except OSError: - pass self.send_json(200, {"ok": True}) def ssh_session_action(self, path): @@ -16380,8 +16488,8 @@ exit 127 except Exception as exc: ws_send_frame(self.connection, f"{exc}\r\n") return - ws_send_frame(self.connection, "") try: + ws_send_frame(self.connection, "") while True: ready, _, _ = select.select([self.connection, fd], [], [], 0.25) if fd in ready: @@ -16415,18 +16523,7 @@ exit 127 except Exception: pass finally: - try: - os.kill(pid, signal.SIGHUP) - except OSError: - pass - try: - os.close(fd) - except OSError: - pass - try: - os.waitpid(pid, os.WNOHANG) - except OSError: - pass + close_terminal_process(pid, fd) def run_ssh_session(self, router_id, port): env, args = self.ssh_args(port) @@ -17483,11 +17580,11 @@ exit 127 try: backend = http.client.HTTPConnection("127.0.0.1", port, timeout=PROXY_TIMEOUT) backend.request(self.command, target, body=body, headers=headers) - resp = backend.getresponse() - resp_status = resp.status - resp_raw_headers = resp.getheaders() - resp_body = resp.read() - content_type = resp.getheader("Content-Type", "") + with backend.getresponse() as resp: + resp_status = resp.status + resp_raw_headers = resp.getheaders() + resp_body = resp.read() + content_type = resp.getheader("Content-Type", "") last_exc = None break except Exception as exc: @@ -18029,6 +18126,9 @@ class HubHTTPServer(ThreadingHTTPServer): def get_request(self): request, client_address = super().get_request() try: + # Bound incomplete HTTP/TLS/WebSocket reads without expiring an idle + # terminal (the terminal's select loop only reads when data arrives). + request.settimeout(90) request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except OSError: pass @@ -18037,19 +18137,23 @@ class HubHTTPServer(ThreadingHTTPServer): def make_http_server(app, host, port, tls_cert="", tls_key=""): server = HubHTTPServer((host, port), Handler) - server.app = app - server.is_tls = False - if tls_cert and tls_key: - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - context.set_alpn_protocols(["http/1.1"]) - context.load_cert_chain(tls_cert, tls_key) - server.socket = context.wrap_socket( - server.socket, - server_side=True, - do_handshake_on_connect=False, - ) - server.is_tls = True - return server + try: + server.app = app + server.is_tls = False + if tls_cert and tls_key: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.set_alpn_protocols(["http/1.1"]) + context.load_cert_chain(tls_cert, tls_key) + server.socket = context.wrap_socket( + server.socket, + server_side=True, + do_handshake_on_connect=False, + ) + server.is_tls = True + return server + except BaseException: + server.server_close() + raise def cmd_serve(args): @@ -18057,52 +18161,57 @@ def cmd_serve(args): with app.conn(): pass record_hub_start_event() - app.start_router_state_monitor() auth = load_auth() server = make_http_server(app, args.host, args.port) extra_servers = [] - tls_ports = parse_extra_ports(args.tls_ports) if args.tls_cert and args.tls_key else [] - tls_port_set = set(tls_ports) - for port in parse_extra_ports(args.extra_ports): - if port == args.port: - continue - if port in tls_port_set: - print(f"WARNING: port {port} skipped for plain HTTP because TLS is enabled on it", file=sys.stderr) - continue + + def start_extra(port, tls=False): + extra = make_http_server(app, args.host, port, args.tls_cert if tls else "", args.tls_key if tls else "") try: - extra_server = make_http_server(app, args.host, port) - except OSError as exc: - print(f"WARNING: extra port {port} not started: {exc}", file=sys.stderr) - continue - extra_servers.append(extra_server) - thread = threading.Thread(target=extra_server.serve_forever, daemon=True) - thread.start() - print(f"{APP_NAME} also listening on http://{args.host}:{port}") - if args.tls_cert and args.tls_key: + thread = threading.Thread(target=extra.serve_forever, daemon=True) + thread.start() + except BaseException: + extra.server_close() + raise + extra_servers.append((extra, thread)) + + try: + app.start_router_state_monitor() + tls_ports = parse_extra_ports(args.tls_ports) if args.tls_cert and args.tls_key else [] + tls_port_set = set(tls_ports) + for port in parse_extra_ports(args.extra_ports): + if port == args.port: + continue + if port in tls_port_set: + print(f"WARNING: port {port} skipped for plain HTTP because TLS is enabled on it", file=sys.stderr) + continue + try: + start_extra(port) + except OSError as exc: + print(f"WARNING: extra port {port} not started: {exc}", file=sys.stderr) + continue + print(f"{APP_NAME} also listening on http://{args.host}:{port}") for port in tls_ports: try: - tls_server = make_http_server(app, args.host, port, args.tls_cert, args.tls_key) - except OSError as exc: + start_extra(port, tls=True) + except (OSError, ssl.SSLError) as exc: print(f"WARNING: HTTPS port {port} not started: {exc}", file=sys.stderr) continue - except ssl.SSLError as exc: - print(f"WARNING: HTTPS cert/key error: {exc}", file=sys.stderr) - continue - extra_servers.append(tls_server) - thread = threading.Thread(target=tls_server.serve_forever, daemon=True) - thread.start() print(f"{APP_NAME} also listening on https://{args.host}:{port}") - print(f"{APP_NAME} listening on http://{args.host}:{args.port}") - print(f"HUB_LOGIN: {auth.get('username', 'admin')}") - print(f"AGENT_TOKEN: {app.agent_token}") - try: + print(f"{APP_NAME} listening on http://{args.host}:{args.port}") + print(f"HUB_LOGIN: {auth.get('username', 'admin')}") + print(f"AGENT_TOKEN: {app.agent_token}") server.serve_forever() except KeyboardInterrupt: print("") finally: app.stop_router_state_monitor() - for extra_server in extra_servers: + stop_ssh_http_sessions() + for extra_server, thread in extra_servers: extra_server.shutdown() + extra_server.server_close() + thread.join(timeout=5) + server.server_close() def parser():