Limit Hub polling and rotate VPS logs

This commit is contained in:
kzolotarev95 committed 2026-10-29 22:02:27 +04:00
1 parent fcde23bfc4
commit 927181dd54
6 files changed
+601 -17

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
# Notification long-poll and VPS log limits
The dashboard keeps one notification long-poll for the current browser profile.
Tabs share a short lease in `localStorage`; the server also admits only one
`/api/notifications/wait` request for a session and client id. A stale cursor
after clearing or pruning the notification file returns an empty response with
the current serial instead of replaying the whole list in a tight loop.
Successful long-poll and HTTP-terminal reads are omitted from the Hub stderr
request log. HTTP errors remain logged. The nginx policy excludes only 2xx
responses for these endpoints from `access.log`.
`install-vps.sh` and `deploy-request-load-fix.ps1` install the same bounded
logging policy: journald is capped at 256 MiB with a seven-day retention limit,
and nginx logs rotate daily with seven compressed archives and a 50 MiB trigger.
The deploy script keeps a backup at
`/root/owrt-remote-backup-before-request-load-v119` and supports Rollback.
+99
View File
@@ -0,0 +1,99 @@
"""Regression tests for notification long-poll admission and stale cursors."""
import importlib.util
import io
import os
from pathlib import Path
import tempfile
import threading
import unittest
from unittest import mock
import contextlib
ROOT = Path(__file__).resolve().parents[1]
def load_hub():
state = tempfile.TemporaryDirectory(prefix="notification-load-state-")
os.environ["OWRT_REMOTE_STATE_DIR"] = state.name
os.environ["OWRT_REMOTE_DB"] = str(Path(state.name) / "hub.db")
spec = importlib.util.spec_from_file_location("notification_load_hub", ROOT / "vps" / "owrt-remote-hub.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module._notification_test_state = state
return module
class NotificationLoadTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.hub = load_hub()
@classmethod
def tearDownClass(cls):
state = getattr(cls.hub, "_notification_test_state", None)
if state:
state.cleanup()
os.environ.pop("OWRT_REMOTE_STATE_DIR", None)
os.environ.pop("OWRT_REMOTE_DB", None)
def test_stale_cursor_does_not_return_old_notifications(self):
rows = [{"id": "one", "serial": 4, "ts": 1, "title": "old"}]
with mock.patch.object(self.hub, "load_notifications", return_value=rows):
self.assertEqual(self.hub.list_notifications(after_serial=99), [])
def test_same_client_can_have_only_one_active_long_poll(self):
key = self.hub.notification_wait_context("session", "client-1234")
self.assertTrue(self.hub.claim_notification_wait(key))
self.assertFalse(self.hub.claim_notification_wait(key))
self.hub.release_notification_wait(key)
self.assertTrue(self.hub.claim_notification_wait(key))
self.hub.release_notification_wait(key)
def test_metrics_are_bounded_and_released(self):
key = self.hub.notification_wait_context("session", "client-5678")
before = self.hub.notification_wait_metrics()["active"]
self.assertTrue(self.hub.claim_notification_wait(key))
self.assertEqual(self.hub.notification_wait_metrics()["active"], before + 1)
self.hub.release_notification_wait(key)
self.assertEqual(self.hub.notification_wait_metrics()["active"], before)
def test_concurrent_claims_have_one_winner(self):
key = self.hub.notification_wait_context("session", "client-concurrent")
winners = []
barrier = threading.Barrier(8)
def claim():
barrier.wait()
if self.hub.claim_notification_wait(key):
winners.append(True)
threads = [threading.Thread(target=claim) for _ in range(8)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=2)
self.assertEqual(len(winners), 1)
self.hub.release_notification_wait(key)
def test_successful_poll_logs_are_quiet_but_errors_are_kept(self):
class FakeHandler:
path = "/api/notifications/wait?client_id=test"
def address_string(self):
return "127.0.0.1"
def log_date_time_string(self):
return "now"
output = io.StringIO()
with contextlib.redirect_stderr(output):
with mock.patch.object(self.hub.sys, "stderr", output):
self.hub.Handler.log_message(FakeHandler(), '"%s" %s %s', "GET", "200", "12")
self.hub.Handler.log_message(FakeHandler(), '"%s" %s %s', "GET", "500", "12")
self.assertIn("500", output.getvalue())
self.assertNotIn("200", output.getvalue())
if __name__ == "__main__":
unittest.main()
+39
View File
@@ -0,0 +1,39 @@
"""Static validation for the local VPS request-load deployment script."""
import os
from pathlib import Path
import shutil
import subprocess
import unittest
ROOT = Path(__file__).resolve().parents[1]
class RequestLoadDeployTests(unittest.TestCase):
def test_embedded_remote_script_is_valid_posix_shell(self):
source = (ROOT / "vps" / "deploy-request-load-fix.ps1").read_text(encoding="utf-8")
script = source.split("$RemoteScript = @'\n", 1)[1].split("\n'@", 1)[0]
script = script.replace("__UPLOAD_NAME__", "test-stage").replace("__ACTION__", "Install")
shell = shutil.which("sh")
if os.name == "nt":
shell = str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/usr/bin/sh.exe")
if not shell or not Path(shell).exists():
self.skipTest("POSIX shell required")
result = subprocess.run([shell, "-n"], input=script, text=True, capture_output=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
def test_policy_and_rollback_markers_are_present(self):
source = (ROOT / "vps" / "deploy-request-load-fix.ps1").read_text(encoding="utf-8")
for marker in (
"owrt-remote-log-policy.conf",
"owrt-remote-limits.conf",
"MaxRetentionSec=7day",
"rotate 7",
"REQUEST_LOAD_FIX_V119_",
"restore_optional",
):
self.assertIn(marker, source)
if __name__ == "__main__":
unittest.main()
+210
View File
@@ -0,0 +1,210 @@
param(
[Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string]$Vps,
[ValidateSet('Install', 'Rollback')][string]$Action = 'Install'
)
$ErrorActionPreference = 'Stop'
$RepoRoot = Split-Path -Parent $PSScriptRoot
$UploadName = 'owrt-request-load-' + [guid]::NewGuid().ToString('N')
$StageRoot = Join-Path ([System.IO.Path]::GetTempPath()) $UploadName
$Utf8 = [System.Text.UTF8Encoding]::new($false)
$RemoteScript = @'
#!/bin/sh
set -eu
stage=/tmp/__UPLOAD_NAME__
target=/opt/owrt-remote/owrt-remote-hub.py
backup=/root/owrt-remote-backup-before-request-load-v119
policy=/etc/nginx/conf.d/owrt-remote-log-policy.conf
journal=/etc/systemd/journald.conf.d/owrt-remote-limits.conf
logrotate=/etc/logrotate.d/owrt-remote
candidate="$target.new-__UPLOAD_NAME__"
changed=0
show_failure() {
echo '--- owrt-remote status ---' >&2
systemctl --no-pager -l status owrt-remote >&2 || true
echo '--- owrt-remote journal ---' >&2
journalctl -u owrt-remote -n 80 --no-pager -l >&2 || true
}
backup_optional() {
file="$1"
name="$2"
if [ -e "$file" ]; then
cp -p "$file" "$backup/$name"
else
: > "$backup/$name.absent"
fi
}
restore_optional() {
file="$1"
name="$2"
if [ -e "$backup/$name" ]; then
cp -p "$backup/$name" "$file"
elif [ -e "$backup/$name.absent" ]; then
rm -f "$file"
fi
}
cleanup() {
code=$?
trap - EXIT
if [ "$code" -ne 0 ]; then
echo "Request-load fix failed with rc=$code" >&2
if [ "$changed" = 1 ] && [ -s "$backup/owrt-remote-hub.py" ]; then
cp -p "$backup/owrt-remote-hub.py" "$candidate"
mv -f "$candidate" "$target"
restore_optional "$policy" nginx-policy
restore_optional "$journal" journald-limits
restore_optional "$logrotate" logrotate
nginx -t >/dev/null 2>&1 && systemctl reload nginx >/dev/null 2>&1 || true
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl restart owrt-remote >/dev/null 2>&1 || true
fi
show_failure
fi
rm -f "$candidate"
rm -rf "$stage"
exit "$code"
}
trap cleanup EXIT INT TERM
test -s "$target"
python_bin=/opt/owrt-remote/venv/bin/python
[ -x "$python_bin" ] || python_bin=python3
if [ '__ACTION__' = Install ]; then
test -s "$stage/hub.py"
"$python_bin" -m py_compile "$stage/hub.py"
"$python_bin" - "$stage/hub.py" <<'PY'
from pathlib import Path
import sys
source = Path(sys.argv[1]).read_text(encoding='utf-8')
required = (
'def notification_wait_context(',
'def claim_notification_wait(',
'if after_serial > latest_serial:',
'notification_wait_metrics()',
'client_id: notificationClient',
'notificationWaitController',
)
missing = [marker for marker in required if marker not in source]
if missing:
for marker in missing:
print('Missing marker:', repr(marker), file=sys.stderr)
raise SystemExit(1)
print('Candidate validation OK')
PY
mkdir -p "$backup"
if [ ! -e "$backup/owrt-remote-hub.py" ]; then
cp -p "$target" "$backup/owrt-remote-hub.py"
backup_optional "$policy" nginx-policy
backup_optional "$journal" journald-limits
backup_optional "$logrotate" logrotate
fi
changed=1
mkdir -p "$(dirname "$policy")" "$(dirname "$journal")" "$(dirname "$logrotate")"
cat > "$policy" <<'EOF'
# OpenWrt Remote Hub: keep successful long-poll traffic out of access.log.
# 4xx/5xx responses remain logged for diagnostics.
map "$request_uri:$status" $owrt_remote_access_log {
default 1;
~^/api/notifications/wait[^:]*:2[0-9][0-9]$ 0;
~^/api/ssh-session/[^/]+/read[^:]*:2[0-9][0-9]$ 0;
}
access_log /var/log/nginx/access.log combined if=$owrt_remote_access_log;
EOF
cat > "$journal" <<'EOF'
[Journal]
SystemMaxUse=256M
SystemKeepFree=512M
MaxRetentionSec=7day
RateLimitIntervalSec=30s
RateLimitBurst=2000
Compress=yes
EOF
cat > "$logrotate" <<'EOF'
/var/log/nginx/*.log {
daily
rotate 7
size 50M
missingok
notifempty
compress
delaycompress
copytruncate
}
EOF
cp -p "$stage/hub.py" "$candidate"
chmod 0755 "$candidate"
mv -f "$candidate" "$target"
if command -v nginx >/dev/null 2>&1; then
nginx -t
systemctl reload nginx >/dev/null 2>&1 || systemctl restart nginx
fi
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true
journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true
else
test -s "$backup/owrt-remote-hub.py"
"$python_bin" -m py_compile "$backup/owrt-remote-hub.py"
cp -p "$backup/owrt-remote-hub.py" "$candidate"
mv -f "$candidate" "$target"
restore_optional "$policy" nginx-policy
restore_optional "$journal" journald-limits
restore_optional "$logrotate" logrotate
if command -v nginx >/dev/null 2>&1; then
nginx -t
systemctl reload nginx >/dev/null 2>&1 || systemctl restart nginx
fi
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true
journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true
fi
systemctl restart owrt-remote
healthy=0
for attempt in 1 2 3 4 5 6 7 8; do
if systemctl is-active --quiet owrt-remote && "$python_bin" - <<'PY'
import json
import urllib.request
with urllib.request.urlopen('http://127.0.0.1:8088/health?request-load-fix=1', timeout=3) as response:
data = json.loads(response.read(4096))
assert data.get('ok') and 'notification_wait' in data
print('Running Hub health verified: notification_wait=' + json.dumps(data['notification_wait']))
PY
then
healthy=1
break
fi
sleep 2
done
[ "$healthy" = 1 ]
printf '\nBackup: %s\nREQUEST_LOAD_FIX_V119_%s_OK\n' "$backup" '__ACTION__'
'@
$RemoteScript = $RemoteScript.Replace('__UPLOAD_NAME__', $UploadName).Replace('__ACTION__', $Action)
try {
[System.IO.Directory]::CreateDirectory($StageRoot) | Out-Null
[System.IO.File]::WriteAllText((Join-Path $StageRoot 'run.sh'), $RemoteScript.Replace("`r`n", "`n") + "`n", $Utf8)
if ($Action -eq 'Install') {
$Content = [System.IO.File]::ReadAllText((Join-Path $RepoRoot 'vps\owrt-remote-hub.py')).Replace("`r`n", "`n")
[System.IO.File]::WriteAllText((Join-Path $StageRoot 'hub.py'), $Content, $Utf8)
}
& scp -O -r -- $StageRoot "${Vps}:/tmp/"
if ($LASTEXITCODE -ne 0) { throw 'Upload failed. VPS was not changed.' }
& ssh -T -o ConnectTimeout=10 $Vps "sh /tmp/$UploadName/run.sh"
if ($LASTEXITCODE -ne 0) { throw "$Action failed. See the output above; success was not confirmed." }
Write-Host "Request-load fix $Action finished successfully."
}
finally {
$Resolved = [System.IO.Path]::GetFullPath($StageRoot)
$Expected = [System.IO.Path]::GetFullPath((Join-Path ([System.IO.Path]::GetTempPath()) $UploadName))
if ($Resolved -ne $Expected -or (Split-Path -Leaf $Resolved) -ne $UploadName) { throw 'Unsafe staging cleanup path.' }
if (Test-Path -LiteralPath $Resolved) { Remove-Item -LiteralPath $Resolved -Recurse -Force }
}
+39
View File
@@ -152,6 +152,44 @@ PY
fi fi
} }
install_log_limits() {
$SUDO mkdir -p /etc/systemd/journald.conf.d /etc/nginx/conf.d
$SUDO tee /etc/systemd/journald.conf.d/owrt-remote-limits.conf >/dev/null <<'EOF'
[Journal]
SystemMaxUse=256M
SystemKeepFree=512M
MaxRetentionSec=7day
RateLimitIntervalSec=30s
RateLimitBurst=2000
Compress=yes
EOF
$SUDO tee /etc/nginx/conf.d/owrt-remote-log-policy.conf >/dev/null <<'EOF'
# Successful long-poll and terminal reads are expected traffic.
# Keep 4xx/5xx responses in access.log for diagnostics.
map "$request_uri:$status" $owrt_remote_access_log {
default 1;
~^/api/notifications/wait[^:]*:2[0-9][0-9]$ 0;
~^/api/ssh-session/[^/]+/read[^:]*:2[0-9][0-9]$ 0;
}
access_log /var/log/nginx/access.log combined if=$owrt_remote_access_log;
EOF
$SUDO tee /etc/logrotate.d/owrt-remote >/dev/null <<'EOF'
/var/log/nginx/*.log {
daily
rotate 7
size 50M
missingok
notifempty
compress
delaycompress
copytruncate
}
EOF
$SUDO systemctl daemon-reload >/dev/null 2>&1 || true
$SUDO systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true
$SUDO journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true
}
install_xray_service() { install_xray_service() {
xray_bin="$(command -v xray || command -v /usr/local/bin/xray || command -v /usr/bin/xray || true)" xray_bin="$(command -v xray || command -v /usr/local/bin/xray || command -v /usr/bin/xray || true)"
[ -n "$xray_bin" ] || return 0 [ -n "$xray_bin" ] || return 0
@@ -291,6 +329,7 @@ main() {
install_xray_binary install_xray_binary
install_files install_files
install_python_deps_v2 install_python_deps_v2
install_log_limits
install_xray_service install_xray_service
open_firewall open_firewall
start_hub start_hub
+197 -17
View File
@@ -154,6 +154,9 @@ ROUTER_PROXY_LIMITERS = {}
STATIC_CACHE_LOCK = threading.Lock() STATIC_CACHE_LOCK = threading.Lock()
NOTIFICATIONS_LOCK = threading.Lock() NOTIFICATIONS_LOCK = threading.Lock()
NOTIFICATIONS_COND = threading.Condition() NOTIFICATIONS_COND = threading.Condition()
NOTIFICATION_WAIT_LOCK = threading.Lock()
NOTIFICATION_WAIT_ACTIVE = {}
NOTIFICATION_WAIT_MAX = max(1, int(os.environ.get("OWRT_REMOTE_NOTIFICATION_WAIT_MAX", "64")))
PUSH_LOCK = threading.Lock() PUSH_LOCK = threading.Lock()
TRAFFIC_COUNTERS_LOCK = threading.Lock() TRAFFIC_COUNTERS_LOCK = threading.Lock()
AUTH_STATE_LOCK = threading.Lock() AUTH_STATE_LOCK = threading.Lock()
@@ -2183,7 +2186,11 @@ def list_notifications(after=0, limit=60, after_serial=0):
items = load_notifications() items = load_notifications()
latest_serial = max((int(item.get("serial") or 0) for item in items), default=0) latest_serial = max((int(item.get("serial") or 0) for item in items), default=0)
if after_serial > latest_serial: if after_serial > latest_serial:
after_serial = 0 # A cleared/pruned notification file must not turn a stale browser
# cursor into an immediate full-list response. Returning no items
# lets the caller advance from the current server serial and keep the
# long-poll blocked until a genuinely new notification arrives.
return []
if after_serial > 0: if after_serial > 0:
items = [item for item in items if int(item.get("serial") or 0) > after_serial] items = [item for item in items if int(item.get("serial") or 0) > after_serial]
elif after > 0: elif after > 0:
@@ -2204,6 +2211,10 @@ def wait_for_notifications(after_serial=0, timeout=25, limit=60):
timeout = max(1, min(60, int(timeout or 25))) timeout = max(1, min(60, int(timeout or 25)))
except (TypeError, ValueError): except (TypeError, ValueError):
timeout = 25 timeout = 25
# A clear/prune can make a browser cursor newer than the current file.
# Clamp it before entering the wait so the next newly-created serial is
# delivered instead of being hidden behind the stale cursor.
after_serial = min(after_serial, latest_notification_serial())
deadline = time.time() + timeout deadline = time.time() + timeout
while True: while True:
items = list_notifications(limit=limit, after_serial=after_serial) items = list_notifications(limit=limit, after_serial=after_serial)
@@ -2216,6 +2227,40 @@ def wait_for_notifications(after_serial=0, timeout=25, limit=60):
NOTIFICATIONS_COND.wait(timeout=remaining) NOTIFICATIONS_COND.wait(timeout=remaining)
def notification_wait_context(session_token="", client_id="", client_ip=""):
"""Return a bounded per-browser context key for long-poll admission."""
token = str(session_token or "")[:160]
client_id = str(client_id or "")[:96]
if not re.fullmatch(r"[A-Za-z0-9_-]{8,96}", client_id):
client_id = "ip:" + str(client_ip or "unknown")[:96]
return hashlib.sha256((token + "\0" + client_id).encode("utf-8", "replace")).hexdigest()
def claim_notification_wait(key):
now = time.monotonic()
with NOTIFICATION_WAIT_LOCK:
for active_key, started in list(NOTIFICATION_WAIT_ACTIVE.items()):
if now - started > 90:
NOTIFICATION_WAIT_ACTIVE.pop(active_key, None)
if key in NOTIFICATION_WAIT_ACTIVE or len(NOTIFICATION_WAIT_ACTIVE) >= NOTIFICATION_WAIT_MAX:
return False
NOTIFICATION_WAIT_ACTIVE[key] = now
return True
def release_notification_wait(key):
with NOTIFICATION_WAIT_LOCK:
NOTIFICATION_WAIT_ACTIVE.pop(key, None)
def notification_wait_metrics():
with NOTIFICATION_WAIT_LOCK:
return {
"active": len(NOTIFICATION_WAIT_ACTIVE),
"limit": NOTIFICATION_WAIT_MAX,
}
def clear_notifications(): def clear_notifications():
with NOTIFICATIONS_LOCK: with NOTIFICATIONS_LOCK:
save_notifications([]) save_notifications([])
@@ -11123,6 +11168,66 @@ let lastNotificationTs = initialNotificationTs();
let lastNotificationSerial = initialNotificationSerial(); let lastNotificationSerial = initialNotificationSerial();
let notificationWaitAbort = false; let notificationWaitAbort = false;
let notificationWaitRunning = false; let notificationWaitRunning = false;
let notificationWaitController = null;
let notificationLeaderTimer = 0;
let notificationLeaderTab = '';
const NOTIFICATION_CLIENT_KEY = 'owrtNotificationClientId';
const NOTIFICATION_LEASE_KEY = 'owrtNotificationLease';
const NOTIFICATION_LEASE_MS = 90000;
function notificationClientId() {{
try {{
let value = localStorage.getItem(NOTIFICATION_CLIENT_KEY);
if (!/^[A-Za-z0-9_-]{{8,96}}$/.test(value || '')) {{
value = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + Math.random().toString(36).slice(2))
.replace(/[^A-Za-z0-9_-]/g, '').slice(0, 96);
localStorage.setItem(NOTIFICATION_CLIENT_KEY, value);
}}
return value;
}} catch (e) {{
// Let the server fall back to the client IP when storage is unavailable;
// random per-tab ids would defeat the single-poll admission guard.
return '';
}}
}}
const notificationClient = notificationClientId();
const notificationTab = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + Math.random()).replace(/[^A-Za-z0-9_-]/g, '').slice(0, 96);
function notificationLeaderState() {{
try {{
const raw = localStorage.getItem(NOTIFICATION_LEASE_KEY);
const state = raw ? JSON.parse(raw) : null;
return state && typeof state === 'object' ? state : null;
}} catch (e) {{
return null;
}}
}}
function notificationIsLeader() {{
const now = Date.now();
const current = notificationLeaderState();
if (!current || Number(current.expires || 0) <= now || current.tab === notificationTab) {{
try {{
localStorage.setItem(NOTIFICATION_LEASE_KEY, JSON.stringify({{tab: notificationTab, expires: now + NOTIFICATION_LEASE_MS}}));
notificationLeaderTab = notificationTab;
return true;
}} catch (e) {{
notificationLeaderTab = notificationTab;
return true;
}}
}}
notificationLeaderTab = String(current.tab || '');
return notificationLeaderTab === notificationTab;
}}
function releaseNotificationLeader() {{
try {{
const current = notificationLeaderState();
if (current && current.tab === notificationTab) localStorage.removeItem(NOTIFICATION_LEASE_KEY);
}} catch (e) {{}}
notificationLeaderTab = '';
}}
function storeNotificationCursor() {{ function storeNotificationCursor() {{
localStorage.setItem('owrtLastNotificationTs', String(lastNotificationTs)); localStorage.setItem('owrtLastNotificationTs', String(lastNotificationTs));
@@ -11167,7 +11272,9 @@ async function loadNotifications({{initial = false}} = {{}}) {{
const data = await res.json().catch(() => ({{}})); const data = await res.json().catch(() => ({{}}));
const items = Array.isArray(data.notifications) ? data.notifications : []; const items = Array.isArray(data.notifications) ? data.notifications : [];
if (!items.length && !initial) {{ if (!items.length && !initial) {{
lastNotificationSerial = Math.max(lastNotificationSerial, Number(data.serial || 0)); const serverSerial = Number(data.serial || 0);
lastNotificationSerial = serverSerial < lastNotificationSerial
? serverSerial : Math.max(lastNotificationSerial, serverSerial);
storeNotificationCursor(); storeNotificationCursor();
return; return;
}} }}
@@ -11177,20 +11284,38 @@ async function loadNotifications({{initial = false}} = {{}}) {{
async function waitNotificationsLoop() {{ async function waitNotificationsLoop() {{
if (!authIsOwner()) return; if (!authIsOwner()) return;
if (notificationWaitRunning || notificationWaitAbort) return; if (notificationWaitRunning || notificationWaitAbort) return;
if (!notificationIsLeader()) {{
if (!notificationLeaderTimer) notificationLeaderTimer = setTimeout(() => {{
notificationLeaderTimer = 0;
waitNotificationsLoop();
}}, 5000);
return;
}}
notificationWaitRunning = true; notificationWaitRunning = true;
try {{ try {{
while (!notificationWaitAbort) {{ while (!notificationWaitAbort) {{
if (!notificationIsLeader()) break;
const params = new URLSearchParams({{ const params = new URLSearchParams({{
after_serial: String(lastNotificationSerial || 0), after_serial: String(lastNotificationSerial || 0),
timeout: '25', timeout: '25',
limit: '60' limit: '60',
client_id: notificationClient
}}); }});
let res; let res;
try {{ try {{
res = await fetch('/api/notifications/wait?' + params.toString(), {{cache: 'no-store'}}); notificationWaitController = new AbortController();
res = await fetch('/api/notifications/wait?' + params.toString(), {{cache: 'no-store', signal: notificationWaitController.signal}});
}} catch (e) {{ }} catch (e) {{
if (notificationWaitAbort) break;
await new Promise(resolve => setTimeout(resolve, 2000)); await new Promise(resolve => setTimeout(resolve, 2000));
continue; continue;
}} finally {{
notificationWaitController = null;
}}
if (res.status === 429) {{
const retry = Math.max(2000, Math.min(15000, Number(res.headers.get('Retry-After') || 5) * 1000));
await new Promise(resolve => setTimeout(resolve, retry));
continue;
}} }}
if (!res.ok) {{ if (!res.ok) {{
await new Promise(resolve => setTimeout(resolve, 2000)); await new Promise(resolve => setTimeout(resolve, 2000));
@@ -11202,7 +11327,9 @@ async function waitNotificationsLoop() {{
applyNotificationItems(items); applyNotificationItems(items);
continue; continue;
}} }}
lastNotificationSerial = Math.max(lastNotificationSerial, Number(data.serial || 0)); const serverSerial = Number(data.serial || 0);
lastNotificationSerial = serverSerial < lastNotificationSerial
? serverSerial : Math.max(lastNotificationSerial, serverSerial);
storeNotificationCursor(); storeNotificationCursor();
}} }}
}} finally {{ }} finally {{
@@ -11210,6 +11337,19 @@ async function waitNotificationsLoop() {{
}} }}
}} }}
function stopNotificationWait() {{
notificationWaitAbort = true;
if (notificationWaitController) notificationWaitController.abort();
if (notificationLeaderTimer) {{ clearTimeout(notificationLeaderTimer); notificationLeaderTimer = 0; }}
releaseNotificationLeader();
}}
window.addEventListener('pagehide', stopNotificationWait);
window.addEventListener('pageshow', () => {{
notificationWaitAbort = false;
waitNotificationsLoop();
}});
function bytesToBase64(bytes) {{ function bytesToBase64(bytes) {{
let binary = ''; let binary = '';
const chunk = 0x8000; const chunk = 0x8000;
@@ -15704,6 +15844,23 @@ class Handler(BaseHTTPRequestHandler):
return self.server.app return self.server.app
def log_message(self, fmt, *args): def log_message(self, fmt, *args):
# Successful long-poll and terminal-read responses are expected to be
# frequent. Nginx and journald already retain their failures; avoid
# writing a line for every normal 200 response while keeping 4xx/5xx
# diagnostics visible.
path = urllib.parse.urlsplit(str(self.path or "")).path
status = 0
if len(args) > 1:
try:
status = int(str(args[1]))
except (TypeError, ValueError):
status = 0
quiet = (
path == "/api/notifications/wait"
or re.fullmatch(r"/api/ssh-session/[^/]+/read", path or "")
)
if quiet and 200 <= status < 400:
return
sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), fmt % args)) sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), fmt % args))
def parsed(self): def parsed(self):
@@ -15905,9 +16062,9 @@ class Handler(BaseHTTPRequestHandler):
return return
self.send_bytes(200, body, content_type) self.send_bytes(200, body, content_type)
def send_json(self, status, payload): def send_json(self, status, payload, extra_headers=None):
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
self.send_bytes(status, body, "application/json; charset=utf-8") self.send_bytes(status, body, "application/json; charset=utf-8", extra_headers=extra_headers)
def require_admin(self): def require_admin(self):
if self.admin_ok(): if self.admin_ok():
@@ -18737,6 +18894,8 @@ exit 127
result = {"ok": True} result = {"ok": True}
if self.query().get('panel-update') == ['1']: if self.query().get('panel-update') == ['1']:
result['revision'] = HUB_LOADED_REVISION result['revision'] = HUB_LOADED_REVISION
if self.query().get("request-load-fix") == ["1"]:
result["notification_wait"] = notification_wait_metrics()
self.send_json(200, result) self.send_json(200, result)
return return
if path == '/api/hub-update': if path == '/api/hub-update':
@@ -18915,17 +19074,38 @@ exit 127
if not self.require_owner(json_mode=True): if not self.require_owner(json_mode=True):
return return
query = self.query() query = self.query()
self.send_json( key = notification_wait_context(
200, self.current_session_token(),
{ query.get("client_id", [""])[0],
"notifications": wait_for_notifications( self.client_ip(),
query.get("after_serial", ["0"])[0],
query.get("timeout", ["25"])[0],
query.get("limit", ["60"])[0],
),
"serial": latest_notification_serial(),
},
) )
if not claim_notification_wait(key):
self.send_json(
429,
{
"ok": False,
"busy": True,
"notifications": [],
"serial": latest_notification_serial(),
"error": "для этого клиента уже выполняется long-poll",
},
extra_headers=[("Retry-After", "5")],
)
return
try:
self.send_json(
200,
{
"notifications": wait_for_notifications(
query.get("after_serial", ["0"])[0],
query.get("timeout", ["25"])[0],
query.get("limit", ["60"])[0],
),
"serial": latest_notification_serial(),
},
)
finally:
release_notification_wait(key)
return return
if path == "/api/push/vapid-public-key": if path == "/api/push/vapid-public-key":
if not self.require_owner(json_mode=True): if not self.require_owner(json_mode=True):