diff --git a/tests/REQUEST_LOAD.md b/tests/REQUEST_LOAD.md new file mode 100644 index 0000000..27cf909 --- /dev/null +++ b/tests/REQUEST_LOAD.md @@ -0,0 +1,17 @@ +# Notification long-poll and VPS log limits + +The dashboard keeps one notification long-poll for the current browser profile. +Tabs share a short lease in `localStorage`; the server also admits only one +`/api/notifications/wait` request for a session and client id. A stale cursor +after clearing or pruning the notification file returns an empty response with +the current serial instead of replaying the whole list in a tight loop. + +Successful long-poll and HTTP-terminal reads are omitted from the Hub stderr +request log. HTTP errors remain logged. The nginx policy excludes only 2xx +responses for these endpoints from `access.log`. + +`install-vps.sh` and `deploy-request-load-fix.ps1` install the same bounded +logging policy: journald is capped at 256 MiB with a seven-day retention limit, +and nginx logs rotate daily with seven compressed archives and a 50 MiB trigger. +The deploy script keeps a backup at +`/root/owrt-remote-backup-before-request-load-v119` and supports Rollback. diff --git a/tests/test_notification_load.py b/tests/test_notification_load.py new file mode 100644 index 0000000..911258d --- /dev/null +++ b/tests/test_notification_load.py @@ -0,0 +1,99 @@ +"""Regression tests for notification long-poll admission and stale cursors.""" +import importlib.util +import io +import os +from pathlib import Path +import tempfile +import threading +import unittest +from unittest import mock +import contextlib + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_hub(): + state = tempfile.TemporaryDirectory(prefix="notification-load-state-") + os.environ["OWRT_REMOTE_STATE_DIR"] = state.name + os.environ["OWRT_REMOTE_DB"] = str(Path(state.name) / "hub.db") + spec = importlib.util.spec_from_file_location("notification_load_hub", ROOT / "vps" / "owrt-remote-hub.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module._notification_test_state = state + return module + + +class NotificationLoadTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.hub = load_hub() + + @classmethod + def tearDownClass(cls): + state = getattr(cls.hub, "_notification_test_state", None) + if state: + state.cleanup() + os.environ.pop("OWRT_REMOTE_STATE_DIR", None) + os.environ.pop("OWRT_REMOTE_DB", None) + + def test_stale_cursor_does_not_return_old_notifications(self): + rows = [{"id": "one", "serial": 4, "ts": 1, "title": "old"}] + with mock.patch.object(self.hub, "load_notifications", return_value=rows): + self.assertEqual(self.hub.list_notifications(after_serial=99), []) + + def test_same_client_can_have_only_one_active_long_poll(self): + key = self.hub.notification_wait_context("session", "client-1234") + self.assertTrue(self.hub.claim_notification_wait(key)) + self.assertFalse(self.hub.claim_notification_wait(key)) + self.hub.release_notification_wait(key) + self.assertTrue(self.hub.claim_notification_wait(key)) + self.hub.release_notification_wait(key) + + def test_metrics_are_bounded_and_released(self): + key = self.hub.notification_wait_context("session", "client-5678") + before = self.hub.notification_wait_metrics()["active"] + self.assertTrue(self.hub.claim_notification_wait(key)) + self.assertEqual(self.hub.notification_wait_metrics()["active"], before + 1) + self.hub.release_notification_wait(key) + self.assertEqual(self.hub.notification_wait_metrics()["active"], before) + + def test_concurrent_claims_have_one_winner(self): + key = self.hub.notification_wait_context("session", "client-concurrent") + winners = [] + barrier = threading.Barrier(8) + + def claim(): + barrier.wait() + if self.hub.claim_notification_wait(key): + winners.append(True) + + threads = [threading.Thread(target=claim) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=2) + self.assertEqual(len(winners), 1) + self.hub.release_notification_wait(key) + + def test_successful_poll_logs_are_quiet_but_errors_are_kept(self): + class FakeHandler: + path = "/api/notifications/wait?client_id=test" + + def address_string(self): + return "127.0.0.1" + + def log_date_time_string(self): + return "now" + + output = io.StringIO() + with contextlib.redirect_stderr(output): + with mock.patch.object(self.hub.sys, "stderr", output): + self.hub.Handler.log_message(FakeHandler(), '"%s" %s %s', "GET", "200", "12") + self.hub.Handler.log_message(FakeHandler(), '"%s" %s %s', "GET", "500", "12") + self.assertIn("500", output.getvalue()) + self.assertNotIn("200", output.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_request_load_deploy.py b/tests/test_request_load_deploy.py new file mode 100644 index 0000000..3106f53 --- /dev/null +++ b/tests/test_request_load_deploy.py @@ -0,0 +1,39 @@ +"""Static validation for the local VPS request-load deployment script.""" +import os +from pathlib import Path +import shutil +import subprocess +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class RequestLoadDeployTests(unittest.TestCase): + def test_embedded_remote_script_is_valid_posix_shell(self): + source = (ROOT / "vps" / "deploy-request-load-fix.ps1").read_text(encoding="utf-8") + script = source.split("$RemoteScript = @'\n", 1)[1].split("\n'@", 1)[0] + script = script.replace("__UPLOAD_NAME__", "test-stage").replace("__ACTION__", "Install") + shell = shutil.which("sh") + if os.name == "nt": + shell = str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/usr/bin/sh.exe") + if not shell or not Path(shell).exists(): + self.skipTest("POSIX shell required") + result = subprocess.run([shell, "-n"], input=script, text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_policy_and_rollback_markers_are_present(self): + source = (ROOT / "vps" / "deploy-request-load-fix.ps1").read_text(encoding="utf-8") + for marker in ( + "owrt-remote-log-policy.conf", + "owrt-remote-limits.conf", + "MaxRetentionSec=7day", + "rotate 7", + "REQUEST_LOAD_FIX_V119_", + "restore_optional", + ): + self.assertIn(marker, source) + + +if __name__ == "__main__": + unittest.main() diff --git a/vps/deploy-request-load-fix.ps1 b/vps/deploy-request-load-fix.ps1 new file mode 100644 index 0000000..844bf5d --- /dev/null +++ b/vps/deploy-request-load-fix.ps1 @@ -0,0 +1,210 @@ +param( + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string]$Vps, + [ValidateSet('Install', 'Rollback')][string]$Action = 'Install' +) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent $PSScriptRoot +$UploadName = 'owrt-request-load-' + [guid]::NewGuid().ToString('N') +$StageRoot = Join-Path ([System.IO.Path]::GetTempPath()) $UploadName +$Utf8 = [System.Text.UTF8Encoding]::new($false) + +$RemoteScript = @' +#!/bin/sh +set -eu + +stage=/tmp/__UPLOAD_NAME__ +target=/opt/owrt-remote/owrt-remote-hub.py +backup=/root/owrt-remote-backup-before-request-load-v119 +policy=/etc/nginx/conf.d/owrt-remote-log-policy.conf +journal=/etc/systemd/journald.conf.d/owrt-remote-limits.conf +logrotate=/etc/logrotate.d/owrt-remote +candidate="$target.new-__UPLOAD_NAME__" +changed=0 + +show_failure() { + echo '--- owrt-remote status ---' >&2 + systemctl --no-pager -l status owrt-remote >&2 || true + echo '--- owrt-remote journal ---' >&2 + journalctl -u owrt-remote -n 80 --no-pager -l >&2 || true +} + +backup_optional() { + file="$1" + name="$2" + if [ -e "$file" ]; then + cp -p "$file" "$backup/$name" + else + : > "$backup/$name.absent" + fi +} + +restore_optional() { + file="$1" + name="$2" + if [ -e "$backup/$name" ]; then + cp -p "$backup/$name" "$file" + elif [ -e "$backup/$name.absent" ]; then + rm -f "$file" + fi +} + +cleanup() { + code=$? + trap - EXIT + if [ "$code" -ne 0 ]; then + echo "Request-load fix failed with rc=$code" >&2 + if [ "$changed" = 1 ] && [ -s "$backup/owrt-remote-hub.py" ]; then + cp -p "$backup/owrt-remote-hub.py" "$candidate" + mv -f "$candidate" "$target" + restore_optional "$policy" nginx-policy + restore_optional "$journal" journald-limits + restore_optional "$logrotate" logrotate + nginx -t >/dev/null 2>&1 && systemctl reload nginx >/dev/null 2>&1 || true + systemctl daemon-reload >/dev/null 2>&1 || true + systemctl restart owrt-remote >/dev/null 2>&1 || true + fi + show_failure + fi + rm -f "$candidate" + rm -rf "$stage" + exit "$code" +} +trap cleanup EXIT INT TERM + +test -s "$target" +python_bin=/opt/owrt-remote/venv/bin/python +[ -x "$python_bin" ] || python_bin=python3 + +if [ '__ACTION__' = Install ]; then + test -s "$stage/hub.py" + "$python_bin" -m py_compile "$stage/hub.py" + "$python_bin" - "$stage/hub.py" <<'PY' +from pathlib import Path +import sys +source = Path(sys.argv[1]).read_text(encoding='utf-8') +required = ( + 'def notification_wait_context(', + 'def claim_notification_wait(', + 'if after_serial > latest_serial:', + 'notification_wait_metrics()', + 'client_id: notificationClient', + 'notificationWaitController', +) +missing = [marker for marker in required if marker not in source] +if missing: + for marker in missing: + print('Missing marker:', repr(marker), file=sys.stderr) + raise SystemExit(1) +print('Candidate validation OK') +PY + mkdir -p "$backup" + if [ ! -e "$backup/owrt-remote-hub.py" ]; then + cp -p "$target" "$backup/owrt-remote-hub.py" + backup_optional "$policy" nginx-policy + backup_optional "$journal" journald-limits + backup_optional "$logrotate" logrotate + fi + changed=1 + mkdir -p "$(dirname "$policy")" "$(dirname "$journal")" "$(dirname "$logrotate")" + cat > "$policy" <<'EOF' +# OpenWrt Remote Hub: keep successful long-poll traffic out of access.log. +# 4xx/5xx responses remain logged for diagnostics. +map "$request_uri:$status" $owrt_remote_access_log { + default 1; + ~^/api/notifications/wait[^:]*:2[0-9][0-9]$ 0; + ~^/api/ssh-session/[^/]+/read[^:]*:2[0-9][0-9]$ 0; +} +access_log /var/log/nginx/access.log combined if=$owrt_remote_access_log; +EOF + cat > "$journal" <<'EOF' +[Journal] +SystemMaxUse=256M +SystemKeepFree=512M +MaxRetentionSec=7day +RateLimitIntervalSec=30s +RateLimitBurst=2000 +Compress=yes +EOF + cat > "$logrotate" <<'EOF' +/var/log/nginx/*.log { + daily + rotate 7 + size 50M + missingok + notifempty + compress + delaycompress + copytruncate +} +EOF + cp -p "$stage/hub.py" "$candidate" + chmod 0755 "$candidate" + mv -f "$candidate" "$target" + if command -v nginx >/dev/null 2>&1; then + nginx -t + systemctl reload nginx >/dev/null 2>&1 || systemctl restart nginx + fi + systemctl daemon-reload >/dev/null 2>&1 || true + systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true + journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true +else + test -s "$backup/owrt-remote-hub.py" + "$python_bin" -m py_compile "$backup/owrt-remote-hub.py" + cp -p "$backup/owrt-remote-hub.py" "$candidate" + mv -f "$candidate" "$target" + restore_optional "$policy" nginx-policy + restore_optional "$journal" journald-limits + restore_optional "$logrotate" logrotate + if command -v nginx >/dev/null 2>&1; then + nginx -t + systemctl reload nginx >/dev/null 2>&1 || systemctl restart nginx + fi + systemctl daemon-reload >/dev/null 2>&1 || true + systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true + journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true +fi + +systemctl restart owrt-remote +healthy=0 +for attempt in 1 2 3 4 5 6 7 8; do + if systemctl is-active --quiet owrt-remote && "$python_bin" - <<'PY' +import json +import urllib.request +with urllib.request.urlopen('http://127.0.0.1:8088/health?request-load-fix=1', timeout=3) as response: + data = json.loads(response.read(4096)) +assert data.get('ok') and 'notification_wait' in data +print('Running Hub health verified: notification_wait=' + json.dumps(data['notification_wait'])) +PY + then + healthy=1 + break + fi + sleep 2 +done +[ "$healthy" = 1 ] + +printf '\nBackup: %s\nREQUEST_LOAD_FIX_V119_%s_OK\n' "$backup" '__ACTION__' +'@ + +$RemoteScript = $RemoteScript.Replace('__UPLOAD_NAME__', $UploadName).Replace('__ACTION__', $Action) + +try { + [System.IO.Directory]::CreateDirectory($StageRoot) | Out-Null + [System.IO.File]::WriteAllText((Join-Path $StageRoot 'run.sh'), $RemoteScript.Replace("`r`n", "`n") + "`n", $Utf8) + if ($Action -eq 'Install') { + $Content = [System.IO.File]::ReadAllText((Join-Path $RepoRoot 'vps\owrt-remote-hub.py')).Replace("`r`n", "`n") + [System.IO.File]::WriteAllText((Join-Path $StageRoot 'hub.py'), $Content, $Utf8) + } + & scp -O -r -- $StageRoot "${Vps}:/tmp/" + if ($LASTEXITCODE -ne 0) { throw 'Upload failed. VPS was not changed.' } + & ssh -T -o ConnectTimeout=10 $Vps "sh /tmp/$UploadName/run.sh" + if ($LASTEXITCODE -ne 0) { throw "$Action failed. See the output above; success was not confirmed." } + Write-Host "Request-load fix $Action finished successfully." +} +finally { + $Resolved = [System.IO.Path]::GetFullPath($StageRoot) + $Expected = [System.IO.Path]::GetFullPath((Join-Path ([System.IO.Path]::GetTempPath()) $UploadName)) + if ($Resolved -ne $Expected -or (Split-Path -Leaf $Resolved) -ne $UploadName) { throw 'Unsafe staging cleanup path.' } + if (Test-Path -LiteralPath $Resolved) { Remove-Item -LiteralPath $Resolved -Recurse -Force } +} diff --git a/vps/install-vps.sh b/vps/install-vps.sh index 6063005..c1f1408 100755 --- a/vps/install-vps.sh +++ b/vps/install-vps.sh @@ -152,6 +152,44 @@ PY fi } +install_log_limits() { + $SUDO mkdir -p /etc/systemd/journald.conf.d /etc/nginx/conf.d + $SUDO tee /etc/systemd/journald.conf.d/owrt-remote-limits.conf >/dev/null <<'EOF' +[Journal] +SystemMaxUse=256M +SystemKeepFree=512M +MaxRetentionSec=7day +RateLimitIntervalSec=30s +RateLimitBurst=2000 +Compress=yes +EOF + $SUDO tee /etc/nginx/conf.d/owrt-remote-log-policy.conf >/dev/null <<'EOF' +# Successful long-poll and terminal reads are expected traffic. +# Keep 4xx/5xx responses in access.log for diagnostics. +map "$request_uri:$status" $owrt_remote_access_log { + default 1; + ~^/api/notifications/wait[^:]*:2[0-9][0-9]$ 0; + ~^/api/ssh-session/[^/]+/read[^:]*:2[0-9][0-9]$ 0; +} +access_log /var/log/nginx/access.log combined if=$owrt_remote_access_log; +EOF + $SUDO tee /etc/logrotate.d/owrt-remote >/dev/null <<'EOF' +/var/log/nginx/*.log { + daily + rotate 7 + size 50M + missingok + notifempty + compress + delaycompress + copytruncate +} +EOF + $SUDO systemctl daemon-reload >/dev/null 2>&1 || true + $SUDO systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true + $SUDO journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true +} + install_xray_service() { xray_bin="$(command -v xray || command -v /usr/local/bin/xray || command -v /usr/bin/xray || true)" [ -n "$xray_bin" ] || return 0 @@ -291,6 +329,7 @@ main() { install_xray_binary install_files install_python_deps_v2 + install_log_limits install_xray_service open_firewall start_hub diff --git a/vps/owrt-remote-hub.py b/vps/owrt-remote-hub.py index 8ea1447..3f294f7 100644 --- a/vps/owrt-remote-hub.py +++ b/vps/owrt-remote-hub.py @@ -154,6 +154,9 @@ ROUTER_PROXY_LIMITERS = {} STATIC_CACHE_LOCK = threading.Lock() NOTIFICATIONS_LOCK = threading.Lock() NOTIFICATIONS_COND = threading.Condition() +NOTIFICATION_WAIT_LOCK = threading.Lock() +NOTIFICATION_WAIT_ACTIVE = {} +NOTIFICATION_WAIT_MAX = max(1, int(os.environ.get("OWRT_REMOTE_NOTIFICATION_WAIT_MAX", "64"))) PUSH_LOCK = threading.Lock() TRAFFIC_COUNTERS_LOCK = threading.Lock() AUTH_STATE_LOCK = threading.Lock() @@ -2183,7 +2186,11 @@ def list_notifications(after=0, limit=60, after_serial=0): items = load_notifications() latest_serial = max((int(item.get("serial") or 0) for item in items), default=0) if after_serial > latest_serial: - after_serial = 0 + # A cleared/pruned notification file must not turn a stale browser + # cursor into an immediate full-list response. Returning no items + # lets the caller advance from the current server serial and keep the + # long-poll blocked until a genuinely new notification arrives. + return [] if after_serial > 0: items = [item for item in items if int(item.get("serial") or 0) > after_serial] elif after > 0: @@ -2204,6 +2211,10 @@ def wait_for_notifications(after_serial=0, timeout=25, limit=60): timeout = max(1, min(60, int(timeout or 25))) except (TypeError, ValueError): timeout = 25 + # A clear/prune can make a browser cursor newer than the current file. + # Clamp it before entering the wait so the next newly-created serial is + # delivered instead of being hidden behind the stale cursor. + after_serial = min(after_serial, latest_notification_serial()) deadline = time.time() + timeout while True: items = list_notifications(limit=limit, after_serial=after_serial) @@ -2216,6 +2227,40 @@ def wait_for_notifications(after_serial=0, timeout=25, limit=60): NOTIFICATIONS_COND.wait(timeout=remaining) +def notification_wait_context(session_token="", client_id="", client_ip=""): + """Return a bounded per-browser context key for long-poll admission.""" + token = str(session_token or "")[:160] + client_id = str(client_id or "")[:96] + if not re.fullmatch(r"[A-Za-z0-9_-]{8,96}", client_id): + client_id = "ip:" + str(client_ip or "unknown")[:96] + return hashlib.sha256((token + "\0" + client_id).encode("utf-8", "replace")).hexdigest() + + +def claim_notification_wait(key): + now = time.monotonic() + with NOTIFICATION_WAIT_LOCK: + for active_key, started in list(NOTIFICATION_WAIT_ACTIVE.items()): + if now - started > 90: + NOTIFICATION_WAIT_ACTIVE.pop(active_key, None) + if key in NOTIFICATION_WAIT_ACTIVE or len(NOTIFICATION_WAIT_ACTIVE) >= NOTIFICATION_WAIT_MAX: + return False + NOTIFICATION_WAIT_ACTIVE[key] = now + return True + + +def release_notification_wait(key): + with NOTIFICATION_WAIT_LOCK: + NOTIFICATION_WAIT_ACTIVE.pop(key, None) + + +def notification_wait_metrics(): + with NOTIFICATION_WAIT_LOCK: + return { + "active": len(NOTIFICATION_WAIT_ACTIVE), + "limit": NOTIFICATION_WAIT_MAX, + } + + def clear_notifications(): with NOTIFICATIONS_LOCK: save_notifications([]) @@ -11123,6 +11168,66 @@ let lastNotificationTs = initialNotificationTs(); let lastNotificationSerial = initialNotificationSerial(); let notificationWaitAbort = false; let notificationWaitRunning = false; +let notificationWaitController = null; +let notificationLeaderTimer = 0; +let notificationLeaderTab = ''; +const NOTIFICATION_CLIENT_KEY = 'owrtNotificationClientId'; +const NOTIFICATION_LEASE_KEY = 'owrtNotificationLease'; +const NOTIFICATION_LEASE_MS = 90000; + +function notificationClientId() {{ + try {{ + let value = localStorage.getItem(NOTIFICATION_CLIENT_KEY); + if (!/^[A-Za-z0-9_-]{{8,96}}$/.test(value || '')) {{ + value = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + Math.random().toString(36).slice(2)) + .replace(/[^A-Za-z0-9_-]/g, '').slice(0, 96); + localStorage.setItem(NOTIFICATION_CLIENT_KEY, value); + }} + return value; + }} catch (e) {{ + // Let the server fall back to the client IP when storage is unavailable; + // random per-tab ids would defeat the single-poll admission guard. + return ''; + }} +}} + +const notificationClient = notificationClientId(); +const notificationTab = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + Math.random()).replace(/[^A-Za-z0-9_-]/g, '').slice(0, 96); + +function notificationLeaderState() {{ + try {{ + const raw = localStorage.getItem(NOTIFICATION_LEASE_KEY); + const state = raw ? JSON.parse(raw) : null; + return state && typeof state === 'object' ? state : null; + }} catch (e) {{ + return null; + }} +}} + +function notificationIsLeader() {{ + const now = Date.now(); + const current = notificationLeaderState(); + if (!current || Number(current.expires || 0) <= now || current.tab === notificationTab) {{ + try {{ + localStorage.setItem(NOTIFICATION_LEASE_KEY, JSON.stringify({{tab: notificationTab, expires: now + NOTIFICATION_LEASE_MS}})); + notificationLeaderTab = notificationTab; + return true; + }} catch (e) {{ + notificationLeaderTab = notificationTab; + return true; + }} + }} + notificationLeaderTab = String(current.tab || ''); + return notificationLeaderTab === notificationTab; +}} + +function releaseNotificationLeader() {{ + try {{ + const current = notificationLeaderState(); + if (current && current.tab === notificationTab) localStorage.removeItem(NOTIFICATION_LEASE_KEY); + }} catch (e) {{}} + notificationLeaderTab = ''; +}} function storeNotificationCursor() {{ localStorage.setItem('owrtLastNotificationTs', String(lastNotificationTs)); @@ -11167,7 +11272,9 @@ async function loadNotifications({{initial = false}} = {{}}) {{ const data = await res.json().catch(() => ({{}})); const items = Array.isArray(data.notifications) ? data.notifications : []; if (!items.length && !initial) {{ - lastNotificationSerial = Math.max(lastNotificationSerial, Number(data.serial || 0)); + const serverSerial = Number(data.serial || 0); + lastNotificationSerial = serverSerial < lastNotificationSerial + ? serverSerial : Math.max(lastNotificationSerial, serverSerial); storeNotificationCursor(); return; }} @@ -11177,20 +11284,38 @@ async function loadNotifications({{initial = false}} = {{}}) {{ async function waitNotificationsLoop() {{ if (!authIsOwner()) return; if (notificationWaitRunning || notificationWaitAbort) return; + if (!notificationIsLeader()) {{ + if (!notificationLeaderTimer) notificationLeaderTimer = setTimeout(() => {{ + notificationLeaderTimer = 0; + waitNotificationsLoop(); + }}, 5000); + return; + }} notificationWaitRunning = true; try {{ while (!notificationWaitAbort) {{ + if (!notificationIsLeader()) break; const params = new URLSearchParams({{ after_serial: String(lastNotificationSerial || 0), timeout: '25', - limit: '60' + limit: '60', + client_id: notificationClient }}); let res; try {{ - res = await fetch('/api/notifications/wait?' + params.toString(), {{cache: 'no-store'}}); + notificationWaitController = new AbortController(); + res = await fetch('/api/notifications/wait?' + params.toString(), {{cache: 'no-store', signal: notificationWaitController.signal}}); }} catch (e) {{ + if (notificationWaitAbort) break; await new Promise(resolve => setTimeout(resolve, 2000)); continue; + }} finally {{ + notificationWaitController = null; + }} + if (res.status === 429) {{ + const retry = Math.max(2000, Math.min(15000, Number(res.headers.get('Retry-After') || 5) * 1000)); + await new Promise(resolve => setTimeout(resolve, retry)); + continue; }} if (!res.ok) {{ await new Promise(resolve => setTimeout(resolve, 2000)); @@ -11202,7 +11327,9 @@ async function waitNotificationsLoop() {{ applyNotificationItems(items); continue; }} - lastNotificationSerial = Math.max(lastNotificationSerial, Number(data.serial || 0)); + const serverSerial = Number(data.serial || 0); + lastNotificationSerial = serverSerial < lastNotificationSerial + ? serverSerial : Math.max(lastNotificationSerial, serverSerial); storeNotificationCursor(); }} }} finally {{ @@ -11210,6 +11337,19 @@ async function waitNotificationsLoop() {{ }} }} +function stopNotificationWait() {{ + notificationWaitAbort = true; + if (notificationWaitController) notificationWaitController.abort(); + if (notificationLeaderTimer) {{ clearTimeout(notificationLeaderTimer); notificationLeaderTimer = 0; }} + releaseNotificationLeader(); +}} + +window.addEventListener('pagehide', stopNotificationWait); +window.addEventListener('pageshow', () => {{ + notificationWaitAbort = false; + waitNotificationsLoop(); +}}); + function bytesToBase64(bytes) {{ let binary = ''; const chunk = 0x8000; @@ -15704,6 +15844,23 @@ class Handler(BaseHTTPRequestHandler): return self.server.app def log_message(self, fmt, *args): + # Successful long-poll and terminal-read responses are expected to be + # frequent. Nginx and journald already retain their failures; avoid + # writing a line for every normal 200 response while keeping 4xx/5xx + # diagnostics visible. + path = urllib.parse.urlsplit(str(self.path or "")).path + status = 0 + if len(args) > 1: + try: + status = int(str(args[1])) + except (TypeError, ValueError): + status = 0 + quiet = ( + path == "/api/notifications/wait" + or re.fullmatch(r"/api/ssh-session/[^/]+/read", path or "") + ) + if quiet and 200 <= status < 400: + return sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), fmt % args)) def parsed(self): @@ -15905,9 +16062,9 @@ class Handler(BaseHTTPRequestHandler): return self.send_bytes(200, body, content_type) - def send_json(self, status, payload): + def send_json(self, status, payload, extra_headers=None): body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - self.send_bytes(status, body, "application/json; charset=utf-8") + self.send_bytes(status, body, "application/json; charset=utf-8", extra_headers=extra_headers) def require_admin(self): if self.admin_ok(): @@ -18737,6 +18894,8 @@ exit 127 result = {"ok": True} if self.query().get('panel-update') == ['1']: result['revision'] = HUB_LOADED_REVISION + if self.query().get("request-load-fix") == ["1"]: + result["notification_wait"] = notification_wait_metrics() self.send_json(200, result) return if path == '/api/hub-update': @@ -18915,17 +19074,38 @@ exit 127 if not self.require_owner(json_mode=True): return query = self.query() - self.send_json( - 200, - { - "notifications": wait_for_notifications( - query.get("after_serial", ["0"])[0], - query.get("timeout", ["25"])[0], - query.get("limit", ["60"])[0], - ), - "serial": latest_notification_serial(), - }, + key = notification_wait_context( + self.current_session_token(), + query.get("client_id", [""])[0], + self.client_ip(), ) + if not claim_notification_wait(key): + self.send_json( + 429, + { + "ok": False, + "busy": True, + "notifications": [], + "serial": latest_notification_serial(), + "error": "для этого клиента уже выполняется long-poll", + }, + extra_headers=[("Retry-After", "5")], + ) + return + try: + self.send_json( + 200, + { + "notifications": wait_for_notifications( + query.get("after_serial", ["0"])[0], + query.get("timeout", ["25"])[0], + query.get("limit", ["60"])[0], + ), + "serial": latest_notification_serial(), + }, + ) + finally: + release_notification_wait(key) return if path == "/api/push/vapid-public-key": if not self.require_owner(json_mode=True):