"""Resource regression tests; only temporary state and loopback servers are used. Run: python -m unittest discover -s tests -v Soak: python tests/test_resource_lifecycle.py --soak-seconds 600 Linux measures /proc/self/fd (including hub.db); Windows measures process handles. Connections are deliberately retained so GC cannot conceal missing close(). """ import argparse import ast import contextlib import ctypes import importlib.util import io import json import os from pathlib import Path import socket import sqlite3 import sys import tempfile import threading import time import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest import mock import urllib.error import urllib.request SOURCE = Path(os.environ.get("OWRT_RESOURCE_HUB_SOURCE", Path(__file__).resolve().parents[1] / "vps" / "owrt-remote-hub.py")) STATE = tempfile.TemporaryDirectory(prefix="owrt-resource-tests-") spec = importlib.util.spec_from_file_location("resource_test_hub", SOURCE) hub = importlib.util.module_from_spec(spec) sys.modules[spec.name] = hub with mock.patch.dict(os.environ, {"OWRT_REMOTE_STATE_DIR": STATE.name, "OWRT_REMOTE_ROUTER_NOTIFY_POLL": "3"}): spec.loader.exec_module(hub) REAL_CONNECT = sqlite3.connect class TrackedConnection(sqlite3.Connection): def close(self): super().close() self.closed = True def track_connections(retained): def factory(*args, **kwargs): conn = REAL_CONNECT(*args, factory=TrackedConnection, **kwargs) conn.closed = False retained.append(conn) return conn return mock.patch.object(hub.sqlite3, "connect", side_effect=factory) def resource_counts(): if sys.platform.startswith("linux"): links = [] for path in Path("/proc/self/fd").iterdir(): try: links.append(os.readlink(path)) except FileNotFoundError: pass return {"fd": len(links), "hub_db_fd": sum("hub.db" in item for item in links)} if sys.platform == "win32": kernel = ctypes.WinDLL("kernel32", use_last_error=True) kernel.GetCurrentProcess.restype = ctypes.c_void_p kernel.GetProcessHandleCount.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_ulong)] count = ctypes.c_ulong() if not kernel.GetProcessHandleCount(kernel.GetCurrentProcess(), ctypes.byref(count)): raise ctypes.WinError(ctypes.get_last_error()) return {"handles": count.value} return {} class QuietHandler(hub.Handler): def log_message(self, *args): pass class RouterBackend(BaseHTTPRequestHandler): def do_GET(self): body = b"router proxy fixture" self.send_response(200) self.send_header("Content-Type", "text/plain") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *args): pass class HubFixture: def __enter__(self): self.tmp = tempfile.TemporaryDirectory(prefix="owrt-db-", dir=STATE.name) self.app = hub.App(Path(self.tmp.name) / "hub.db", "test-session", "test-agent", "") hub.write_json_private(hub.AUTH_FILE, {"username": "test-owner", "password_hash": "unused"}) self.backend = ThreadingHTTPServer(("127.0.0.1", 0), RouterBackend) with self.app.conn() as conn: hub.upsert_router(conn, { "id": "test-router", "name": "Test Router", "entry_port": self.backend.server_port, "vps_host": "127.0.0.1", "public_url": "http://127.0.0.1", }) self.server = hub.HubHTTPServer(("127.0.0.1", 0), QuietHandler) self.server.app = self.app self.server.is_tls = False self.threads = [] self.push_patch = mock.patch.object(hub, "queue_web_push_payload") self.push_patch.start() for server in (self.backend, self.server): thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() self.threads.append(thread) self.app.start_router_state_monitor() return self def request(self, path, payload=None): headers = {"Cookie": "owrt_remote_session=test-session", "Authorization": "Bearer test-agent"} body = None if payload is not None: headers["Content-Type"] = "application/json" body = json.dumps(payload).encode() request = urllib.request.Request(f"http://127.0.0.1:{self.server.server_port}{path}", data=body, headers=headers) with urllib.request.urlopen(request, timeout=5) as response: return response.read() def exercise(self): assert json.loads(self.request("/health"))["ok"] assert json.loads(self.request("/api/heartbeat", {"id": "test-router", "hostname": "test-router"}))["ok"] assert len(json.loads(self.request("/api/routers"))["routers"]) == 1 assert "groups" in json.loads(self.request("/api/router-groups")) assert "notifications" in json.loads(self.request("/api/notifications")) assert self.request("/access/test-router/") == b"router proxy fixture" assert b"v118" in self.request("/") self.app.snapshot_router_states() def __exit__(self, *args): self.app.stop_router_state_monitor() for server in (self.server, self.backend): server.shutdown() server.server_close() for thread in self.threads: thread.join(timeout=5) self.push_patch.stop() self.tmp.cleanup() class SQLiteLifecycleTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory(dir=STATE.name) self.addCleanup(self.tmp.cleanup) self.db = Path(self.tmp.name) / "hub.db" self.app = hub.App(self.db, "test-session", "test-agent", "") self.retained = [] self.patch = track_connections(self.retained) self.patch.start() self.addCleanup(self.patch.stop) def assert_closed(self): self.assertTrue(self.retained) for conn in self.retained: self.assertTrue(conn.closed) with self.assertRaises(sqlite3.ProgrammingError): conn.execute("select 1") def test_success_return_and_commit(self): def operation(): with self.app.conn() as conn: conn.execute("create table sample(value integer)") conn.execute("insert into sample values (7)") return 7 self.assertEqual(operation(), 7) with self.app.conn() as conn: self.assertEqual(conn.execute("select value from sample").fetchone()[0], 7) self.assert_closed() def test_exception_and_base_exception_rollback(self): with self.app.conn() as conn: conn.execute("create table sample(value integer)") for error in (ValueError("body failed"), KeyboardInterrupt()): with self.assertRaises(type(error)): with self.app.conn() as conn: conn.execute("insert into sample values (9)") raise error with self.app.conn() as conn: self.assertEqual(conn.execute("select count(*) from sample").fetchone()[0], 0) self.assert_closed() def test_init_failure_closes_and_propagates(self): with mock.patch.object(hub, "init_db", side_effect=sqlite3.OperationalError("init failed")): with self.assertRaisesRegex(sqlite3.OperationalError, "init failed"): with self.app.conn(): self.fail("must not enter body") self.assert_closed() def test_commit_failure_rolls_back_and_closes(self): with hub.connect(self.db) as conn: conn.executescript("pragma foreign_keys=on; create table parent(id primary key); " "create table child(id references parent(id) deferrable initially deferred);") with self.assertRaises(sqlite3.IntegrityError): with hub.connect(self.db) as conn: conn.execute("pragma foreign_keys=on") conn.execute("insert into child values (1)") with hub.connect(self.db) as conn: self.assertEqual(conn.execute("select count(*) from child").fetchone()[0], 0) self.assert_closed() def test_setup_and_transaction_exit_failure_close(self): for setup_error in (True, False): connection = mock.MagicMock() if setup_error: type(connection).row_factory = mock.PropertyMock(side_effect=RuntimeError("setup failed")) else: connection.__exit__.side_effect = sqlite3.OperationalError("rollback failed") with mock.patch.object(hub.sqlite3, "connect", return_value=connection): with self.assertRaises((RuntimeError, sqlite3.OperationalError)): with hub.connect(self.db): pass connection.close.assert_called_once() def test_backup_destination_open_failure_closes_source(self): with self.app.conn(): pass with self.assertRaises(sqlite3.OperationalError): hub.copy_sqlite_backup(self.db, self.tmp.name) # destination is a directory self.assert_closed() def test_backup_success(self): with self.app.conn() as conn: hub.upsert_router(conn, {"id": "backup-router", "name": "backup", "entry_port": 0, "vps_host": "localhost"}) copy = Path(self.tmp.name) / "copy.db" hub.copy_sqlite_backup(self.db, copy) with hub.connect(copy) as conn: self.assertEqual(hub.list_router_rows(conn)[0]["id"], "backup-router") self.assert_closed() def test_snapshot_fd_regression_without_gc(self): with self.app.conn(): pass before = resource_counts() for _ in range(500): self.assertEqual(self.app.snapshot_router_states(), {}) after = resource_counts() self.assert_closed() self.assertGreaterEqual(len(self.retained), 501) for key, value in before.items(): self.assertLessEqual(after[key], value + (2 if key != "hub_db_fd" else 0), (before, after)) def test_all_sqlite_creation_uses_managed_factory(self): tree = ast.parse(SOURCE.read_text(encoding="utf-8")) parents = {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} direct = [] for node in ast.walk(tree): if not isinstance(node, ast.Call): continue if isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == "sqlite3" and node.func.attr == "connect": direct.append(node) elif isinstance(node.func, ast.Name) and node.func.id == "connect" or isinstance(node.func, ast.Attribute) and node.func.attr == "conn": self.assertIsInstance(parents[node], ast.withitem, f"unmanaged DB operation at line {node.lineno}") self.assertEqual(len(direct), 1) class OtherResourceTests(unittest.TestCase): def handler(self, data=b"", headers=None): handler = object.__new__(hub.Handler) handler.rfile = io.BytesIO(data) handler.headers = headers or {} return handler def test_chunked_valid_and_truncated_bodies(self): headers = {"Transfer-Encoding": "chunked"} self.assertEqual(self.handler(b"3\r\nabc\r\n0\r\n\r\n", headers).read_body(), b"abc") for data in (b"", b"3\r\nab", b"3\r\nabc", b"0\r\n", b"-1\r\n", b"\r\n"): with self.subTest(data=data), self.assertRaises(ValueError): self.handler(data, headers).read_body() with self.assertRaises(ValueError): self.handler(b"short", {"Content-Length": "100"}).read_body() def test_oauth_http_error_closes_response(self): body = io.BytesIO(b'{"error":"denied"}') error = urllib.error.HTTPError("http://test", 403, "denied", {}, body) with mock.patch.object(hub.urllib.request, "urlopen", side_effect=error): with self.assertRaisesRegex(ValueError, "denied"): hub.oauth_fetch_json("http://test") self.assertTrue(body.closed) def test_push_success_and_failure_close_response(self): response = mock.Mock(status_code=410, text="gone") error = RuntimeError("push failed") error.response = response with mock.patch.object(hub, "web_push_ready", return_value=True), mock.patch.object(hub, "webpush", return_value=response) as push: self.assertEqual(hub.send_web_push({"endpoint": "https://test"}, {}), "ok") response.close.assert_called_once() response.reset_mock() push.side_effect = error self.assertEqual(hub.send_web_push({"endpoint": "https://test"}, {}), "gone") response.close.assert_called_once() def test_web_push_reuses_existing_registration_before_subscribing(self): page = hub.dashboard_html([], "test-owner", auth_bootstrap={"is_owner": True}) self.assertIn("subscription = await pushManager.getSubscription();", page) self.assertIn("Update the existing registration in place", page) self.assertIn("forceFresh: false", page) self.assertNotIn("directSubscribeFirst: isIOSDevice()", page) self.assertIn("pushsubscriptionchange", hub.service_worker_js()) self.assertIn("push_test: false", hub.service_worker_js()) self.assertIn('if not bool(payload.get("push_test", True))', SOURCE.read_text(encoding="utf-8")) def test_router_cards_hide_traffic_and_keep_wol_devices_readable(self): page = hub.dashboard_html([], "test-owner", auth_bootstrap={"is_owner": True}) self.assertNotIn("Клиенты Traffic", page) self.assertNotIn("data-traffic-toggle", page) self.assertIn(".wolPickerList{display:grid;grid-template-columns:minmax(0,1fr)", page) self.assertIn("grid-auto-rows:max-content", page) self.assertIn("overflow-x:hidden", page) self.assertIn("overflow-wrap:anywhere", page) self.assertIn('data-wol-picker-list="${{escapeAttr(router.id)}}"', SOURCE.read_text(encoding="utf-8")) self.assertIn("hasOpenWolPanel()", page) self.assertIn("loadRouters({automatic: true})", page) def test_vps_terminal_contains_sqlite_fd_leak_command(self): quick = hub.vps_quick_commands_html("193.233.82.38") self.assertIn("Сделать тест на утечку соединений SQLite панели Hub.", quick) self.assertIn("PID=$(pgrep -f '/opt/owrt-remote/owrt-remote-hub.py' | head -1)", quick) self.assertIn("HUBDB=$(ls -l /proc/$PID/fd", quick) self.assertIn("sleep 10", quick) def test_atomic_failure_removes_temp_and_preserves_target(self): with tempfile.TemporaryDirectory(dir=STATE.name) as name: target = Path(name) / "original.txt" target.write_text("original") with mock.patch.object(hub.os, "replace", side_effect=OSError("replace failed")): with self.assertRaises(OSError): hub.atomic_write_text(target, "new") self.assertEqual(target.read_text(), "original") self.assertEqual(list(Path(name).iterdir()), [target]) def test_askpass_setup_failure_removes_directory(self): with tempfile.TemporaryDirectory(dir=STATE.name) as name: directory = Path(name) / "askpass" directory.mkdir() with mock.patch.object(hub.tempfile, "mkdtemp", return_value=str(directory)), mock.patch.object(hub.os, "chmod", side_effect=OSError("chmod failed")): with self.assertRaises(OSError): self.handler().prepare_ssh_askpass({}, "fake-password") self.assertFalse(directory.exists()) def test_https_failure_releases_listening_port(self): server = hub.make_http_server(None, "127.0.0.1", 0) port = server.server_port server.server_close() with self.assertRaises(OSError): hub.make_http_server(None, "127.0.0.1", port, "missing-cert.pem", "missing-key.pem") replacement = hub.make_http_server(None, "127.0.0.1", port) replacement.server_close() def test_terminal_reader_start_failure_cleans_process_and_map(self): session = {"id": "failed-thread", "pid": 123, "fd": 456} with mock.patch.object(hub.threading.Thread, "start", side_effect=RuntimeError("thread failed")), mock.patch.object(hub, "close_terminal_process") as close: with self.assertRaises(RuntimeError): self.handler().launch_ssh_http_reader(session) close.assert_called_once_with(123, 456) self.assertNotIn("failed-thread", hub.SSH_HTTP_SESSIONS) def test_initial_websocket_failure_cleans_terminal(self): handler = self.handler() handler.connection = object() with mock.patch.object(handler, "spawn_terminal_pty", return_value=(123, 456)), mock.patch.object(hub, "ws_send_frame", side_effect=BrokenPipeError()), mock.patch.object(hub, "close_terminal_process") as close: handler.run_terminal_session({}, ["sh"], "unavailable", "open", "sh", "terminal error") close.assert_called_once_with(123, 456) def test_terminal_input_failure_closes_owned_descriptor(self): handler = self.handler() handler.send_json = mock.Mock() session = {"alive": True, "fd": 456, "last_seen": 0, "lock": threading.Lock()} with mock.patch.dict(hub.SSH_HTTP_SESSIONS, {"input-test": session}, clear=True), mock.patch.object(hub.os, "dup", return_value=789), mock.patch.object(hub.os, "close") as close, mock.patch.object(hub, "write_pty_all", side_effect=TimeoutError("write failed")): handler.ssh_http_write("input-test", {"data": "hello"}) close.assert_called_once_with(789) self.assertEqual(handler.send_json.call_args.args[0], 500) @unittest.skipUnless(hasattr(hub.signal, "SIGHUP"), "POSIX terminal signals") def test_terminal_shutdown_escalates_and_reaps(self): # One unsuccessful poll per signal, followed by the final blocking reap. with mock.patch.object(hub.os, "close") as close, mock.patch.object(hub.os, "waitpid", create=True, side_effect=[(0, 0)] * 3 + [(123, 0)]) as wait, mock.patch.object(hub.os, "kill") as kill, mock.patch.object(hub.time, "monotonic", side_effect=[0, 2, 0, 2, 0, 2]): hub.close_terminal_process(123, 456) close.assert_called_once_with(456) self.assertEqual([c.args[1] for c in kill.call_args_list], [hub.signal.SIGHUP, hub.signal.SIGTERM, hub.signal.SIGKILL]) self.assertEqual(wait.call_args.args, (123, 0)) def test_terminal_session_expiry_and_retained_close_notice(self): now = hub.now_ts() sessions = { "expired": {"alive": True, "last_seen": now - 901, "lock": threading.Lock()}, "closed": {"alive": False, "last_seen": now, "closed_at": now - 61, "lock": threading.Lock()}, "recent": {"alive": False, "last_seen": now, "closed_at": now, "lock": threading.Lock()}, "active": {"alive": True, "last_seen": now, "lock": threading.Lock()}, } with mock.patch.dict(hub.SSH_HTTP_SESSIONS, sessions, clear=True): hub.expire_ssh_http_sessions() self.assertEqual(set(hub.SSH_HTTP_SESSIONS), {"recent", "active"}) def test_monitor_recovers_from_initial_database_error(self): app = hub.App("unused", "session", "agent", "") with mock.patch.object(app, "prime_router_state_snapshot", side_effect=sqlite3.OperationalError("initial DB error")), mock.patch.object(app, "check_router_state_changes") as check, mock.patch.object(app.router_monitor_stop, "wait", side_effect=[False, True]), contextlib.redirect_stderr(io.StringIO()) as errors: app.router_state_monitor_loop() check.assert_called_once() self.assertIn("initial DB error", errors.getvalue()) def test_monitor_start_failure_does_not_break_cleanup(self): app = hub.App("unused", "session", "agent", "") with mock.patch.object(hub.threading.Thread, "start", side_effect=RuntimeError("start failed")): with self.assertRaises(RuntimeError): app.start_router_state_monitor() app.stop_router_state_monitor() def test_serve_start_failure_closes_main_server(self): app = hub.App(Path(STATE.name) / "serve-failure.db", "session", "agent", "") server = mock.Mock() args = argparse.Namespace(db=app.db_path, public_url="", host="127.0.0.1", port=0) with mock.patch.object(hub, "App", return_value=app), mock.patch.object(hub, "session_token", return_value="session"), mock.patch.object(hub, "agent_token", return_value="agent"), mock.patch.object(hub, "record_hub_start_event"), mock.patch.object(hub, "load_auth", return_value={}), mock.patch.object(hub, "make_http_server", return_value=server), mock.patch.object(app, "start_router_state_monitor", side_effect=RuntimeError("monitor failed")): with self.assertRaises(RuntimeError): hub.cmd_serve(args) server.server_close.assert_called_once() def test_serve_stop_closes_main_and_extra_servers(self): app = hub.App(Path(STATE.name) / "serve-stop.db", "session", "agent", "") main = mock.Mock() extra = mock.Mock() main.serve_forever.side_effect = KeyboardInterrupt() args = argparse.Namespace(db=app.db_path, public_url="", host="127.0.0.1", port=8088, extra_ports="18088", tls_ports="", tls_cert="", tls_key="") with mock.patch.object(hub, "App", return_value=app), mock.patch.object(hub, "session_token", return_value="session"), mock.patch.object(hub, "agent_token", return_value="agent"), mock.patch.object(hub, "record_hub_start_event"), mock.patch.object(hub, "load_auth", return_value={}), mock.patch.object(hub, "make_http_server", side_effect=[main, extra]), contextlib.redirect_stdout(io.StringIO()): hub.cmd_serve(args) main.server_close.assert_called_once() extra.shutdown.assert_called_once() extra.server_close.assert_called_once() self.assertFalse(app.router_monitor_thread.is_alive()) def test_incoming_socket_has_finite_io_timeout(self): server = hub.make_http_server(None, "127.0.0.1", 0) self.addCleanup(server.server_close) with socket.create_connection(server.server_address) as client: incoming, _ = server.get_request() try: self.assertEqual(incoming.gettimeout(), 90) finally: incoming.close() @unittest.skipUnless(sys.platform.startswith("linux"), "real PTY/child reaping needs Linux") def test_real_terminal_child_is_reaped_and_fd_closed(self): handler = self.handler() pid, fd = handler.spawn_terminal_pty(dict(os.environ), ["/bin/sh", "-c", "sleep 60"], "pty", "open", "sh") hub.close_terminal_process(pid, fd) with self.assertRaises(OSError): os.fstat(fd) with self.assertRaises(ChildProcessError): os.waitpid(pid, os.WNOHANG) class IntegrationTests(unittest.TestCase): def test_push_subscription_sync_does_not_send_duplicate_test(self): with HubFixture() as fixture: payload = { "endpoint": "https://push.example/subscription-sync", "keys": {"p256dh": "p256dh", "auth": "auth"}, "client_hint": "hub", "push_test": False, } with mock.patch.object(hub, "send_web_push") as send: response = json.loads(fixture.request("/api/push/subscribe", payload)) self.assertTrue(response["ok"]) self.assertEqual(response["delivery"], "registered") send.assert_not_called() def test_heartbeat_response_recovers_https_from_reported_router_settings(self): with HubFixture() as fixture: payload = {"id": "test-router", "hub_url": "https://hub.example:9443", "public_url": "http://hub.example:8088", "vps_host_mode": "manual"} response = json.loads(fixture.request("/api/heartbeat", payload)) self.assertTrue(response["ok"]) self.assertEqual(response["hub_update"]["hub_url"], "https://hub.example:9443") self.assertEqual(response["hub_update"]["public_url"], "https://hub.example:9443") self.assertNotIn("vps_host", response["hub_update"]) def test_router_online_grace_tolerates_delayed_heartbeat_and_custom_interval(self): now = hub.now_ts() base = { "id": "delayed-router", "name": "Delayed Router", "custom_name": "", "notes": "", "created_at": now, "updated_at": now, "status_json": json.dumps({"heartbeat_interval": 30}), } router = hub.row_to_router({**base, "last_seen": now - 120}) self.assertTrue(router["online"], "a short WAN/Xray rebind must not mark the router offline") router = hub.row_to_router({**base, "last_seen": now - 181}) self.assertFalse(router["online"]) custom = hub.row_to_router({ **base, "status_json": json.dumps({"heartbeat_interval": 120}), "last_seen": now - 350, }) self.assertTrue(custom["online"], "the grace period must follow a slower configured heartbeat") def test_panel_api_proxy_heartbeat_and_monitor(self): with HubFixture() as fixture: for _ in range(5): fixture.exercise() self.assertTrue(fixture.app.router_monitor_thread.is_alive()) # Verify the monitor still emits the online -> offline transition. fixture.app.prime_router_state_snapshot() with fixture.app.conn() as conn: conn.execute("update routers set last_seen = ?", (hub.now_ts() - 3600,)) with mock.patch.object(hub, "notify_router_offline") as notify: fixture.app.check_router_state_changes() notify.assert_called_once() self.assertFalse(fixture.app.router_monitor_thread.is_alive()) def soak(seconds): started = time.monotonic() cpu_start = time.process_time() samples = [] operations = 0 with HubFixture() as fixture: fixture.exercise() # warm up auth, sessions, notifications and HTTP workers while True: fixture.exercise() operations += 1 # Measure between requests, after worker sockets have closed. time.sleep(0.05) counts = resource_counts() sample = {"elapsed": round(time.monotonic() - started, 1), **counts} samples.append(sample) print(json.dumps(sample), flush=True) if time.monotonic() - started >= seconds: break time.sleep(min(5, max(0, seconds - (time.monotonic() - started)))) assert fixture.app.router_monitor_thread.is_alive(), "router monitor stopped" for key in samples[0]: if key == "elapsed": continue assert samples[-1][key] <= samples[0][key] + (4 if key != "hub_db_fd" else 0), samples result = {"seconds": round(time.monotonic() - started, 1), "cpu_seconds": round(time.process_time() - cpu_start, 2), "cycles": operations, "first": samples[0], "last": samples[-1], "peak": {key: max(s[key] for s in samples) for key in counts}} print("SOAK_OK " + json.dumps(result), flush=True) if __name__ == "__main__": if "--soak-seconds" in sys.argv: parser = argparse.ArgumentParser() parser.add_argument("--soak-seconds", type=int, required=True) soak(parser.parse_args().soak_seconds) else: unittest.main()