Limit Hub polling and rotate VPS logs
This commit is contained in:
1 parent
fcde23bfc4
commit
927181dd54
6 files changed
+591
-7
No files matched your search
@@ -0,0 +1,17 @@
|
||||
# Notification long-poll and VPS log limits
|
||||
|
||||
The dashboard keeps one notification long-poll for the current browser profile.
|
||||
Tabs share a short lease in `localStorage`; the server also admits only one
|
||||
`/api/notifications/wait` request for a session and client id. A stale cursor
|
||||
after clearing or pruning the notification file returns an empty response with
|
||||
the current serial instead of replaying the whole list in a tight loop.
|
||||
|
||||
Successful long-poll and HTTP-terminal reads are omitted from the Hub stderr
|
||||
request log. HTTP errors remain logged. The nginx policy excludes only 2xx
|
||||
responses for these endpoints from `access.log`.
|
||||
|
||||
`install-vps.sh` and `deploy-request-load-fix.ps1` install the same bounded
|
||||
logging policy: journald is capped at 256 MiB with a seven-day retention limit,
|
||||
and nginx logs rotate daily with seven compressed archives and a 50 MiB trigger.
|
||||
The deploy script keeps a backup at
|
||||
`/root/owrt-remote-backup-before-request-load-v119` and supports Rollback.
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Regression tests for notification long-poll admission and stale cursors."""
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import contextlib
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_hub():
|
||||
state = tempfile.TemporaryDirectory(prefix="notification-load-state-")
|
||||
os.environ["OWRT_REMOTE_STATE_DIR"] = state.name
|
||||
os.environ["OWRT_REMOTE_DB"] = str(Path(state.name) / "hub.db")
|
||||
spec = importlib.util.spec_from_file_location("notification_load_hub", ROOT / "vps" / "owrt-remote-hub.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
module._notification_test_state = state
|
||||
return module
|
||||
|
||||
|
||||
class NotificationLoadTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.hub = load_hub()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
state = getattr(cls.hub, "_notification_test_state", None)
|
||||
if state:
|
||||
state.cleanup()
|
||||
os.environ.pop("OWRT_REMOTE_STATE_DIR", None)
|
||||
os.environ.pop("OWRT_REMOTE_DB", None)
|
||||
|
||||
def test_stale_cursor_does_not_return_old_notifications(self):
|
||||
rows = [{"id": "one", "serial": 4, "ts": 1, "title": "old"}]
|
||||
with mock.patch.object(self.hub, "load_notifications", return_value=rows):
|
||||
self.assertEqual(self.hub.list_notifications(after_serial=99), [])
|
||||
|
||||
def test_same_client_can_have_only_one_active_long_poll(self):
|
||||
key = self.hub.notification_wait_context("session", "client-1234")
|
||||
self.assertTrue(self.hub.claim_notification_wait(key))
|
||||
self.assertFalse(self.hub.claim_notification_wait(key))
|
||||
self.hub.release_notification_wait(key)
|
||||
self.assertTrue(self.hub.claim_notification_wait(key))
|
||||
self.hub.release_notification_wait(key)
|
||||
|
||||
def test_metrics_are_bounded_and_released(self):
|
||||
key = self.hub.notification_wait_context("session", "client-5678")
|
||||
before = self.hub.notification_wait_metrics()["active"]
|
||||
self.assertTrue(self.hub.claim_notification_wait(key))
|
||||
self.assertEqual(self.hub.notification_wait_metrics()["active"], before + 1)
|
||||
self.hub.release_notification_wait(key)
|
||||
self.assertEqual(self.hub.notification_wait_metrics()["active"], before)
|
||||
|
||||
def test_concurrent_claims_have_one_winner(self):
|
||||
key = self.hub.notification_wait_context("session", "client-concurrent")
|
||||
winners = []
|
||||
barrier = threading.Barrier(8)
|
||||
|
||||
def claim():
|
||||
barrier.wait()
|
||||
if self.hub.claim_notification_wait(key):
|
||||
winners.append(True)
|
||||
|
||||
threads = [threading.Thread(target=claim) for _ in range(8)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(timeout=2)
|
||||
self.assertEqual(len(winners), 1)
|
||||
self.hub.release_notification_wait(key)
|
||||
|
||||
def test_successful_poll_logs_are_quiet_but_errors_are_kept(self):
|
||||
class FakeHandler:
|
||||
path = "/api/notifications/wait?client_id=test"
|
||||
|
||||
def address_string(self):
|
||||
return "127.0.0.1"
|
||||
|
||||
def log_date_time_string(self):
|
||||
return "now"
|
||||
|
||||
output = io.StringIO()
|
||||
with contextlib.redirect_stderr(output):
|
||||
with mock.patch.object(self.hub.sys, "stderr", output):
|
||||
self.hub.Handler.log_message(FakeHandler(), '"%s" %s %s', "GET", "200", "12")
|
||||
self.hub.Handler.log_message(FakeHandler(), '"%s" %s %s', "GET", "500", "12")
|
||||
self.assertIn("500", output.getvalue())
|
||||
self.assertNotIn("200", output.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Static validation for the local VPS request-load deployment script."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class RequestLoadDeployTests(unittest.TestCase):
|
||||
def test_embedded_remote_script_is_valid_posix_shell(self):
|
||||
source = (ROOT / "vps" / "deploy-request-load-fix.ps1").read_text(encoding="utf-8")
|
||||
script = source.split("$RemoteScript = @'\n", 1)[1].split("\n'@", 1)[0]
|
||||
script = script.replace("__UPLOAD_NAME__", "test-stage").replace("__ACTION__", "Install")
|
||||
shell = shutil.which("sh")
|
||||
if os.name == "nt":
|
||||
shell = str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/usr/bin/sh.exe")
|
||||
if not shell or not Path(shell).exists():
|
||||
self.skipTest("POSIX shell required")
|
||||
result = subprocess.run([shell, "-n"], input=script, text=True, capture_output=True, timeout=10)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_policy_and_rollback_markers_are_present(self):
|
||||
source = (ROOT / "vps" / "deploy-request-load-fix.ps1").read_text(encoding="utf-8")
|
||||
for marker in (
|
||||
"owrt-remote-log-policy.conf",
|
||||
"owrt-remote-limits.conf",
|
||||
"MaxRetentionSec=7day",
|
||||
"rotate 7",
|
||||
"REQUEST_LOAD_FIX_V119_",
|
||||
"restore_optional",
|
||||
):
|
||||
self.assertIn(marker, source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,210 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string]$Vps,
|
||||
[ValidateSet('Install', 'Rollback')][string]$Action = 'Install'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$RepoRoot = Split-Path -Parent $PSScriptRoot
|
||||
$UploadName = 'owrt-request-load-' + [guid]::NewGuid().ToString('N')
|
||||
$StageRoot = Join-Path ([System.IO.Path]::GetTempPath()) $UploadName
|
||||
$Utf8 = [System.Text.UTF8Encoding]::new($false)
|
||||
|
||||
$RemoteScript = @'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
stage=/tmp/__UPLOAD_NAME__
|
||||
target=/opt/owrt-remote/owrt-remote-hub.py
|
||||
backup=/root/owrt-remote-backup-before-request-load-v119
|
||||
policy=/etc/nginx/conf.d/owrt-remote-log-policy.conf
|
||||
journal=/etc/systemd/journald.conf.d/owrt-remote-limits.conf
|
||||
logrotate=/etc/logrotate.d/owrt-remote
|
||||
candidate="$target.new-__UPLOAD_NAME__"
|
||||
changed=0
|
||||
|
||||
show_failure() {
|
||||
echo '--- owrt-remote status ---' >&2
|
||||
systemctl --no-pager -l status owrt-remote >&2 || true
|
||||
echo '--- owrt-remote journal ---' >&2
|
||||
journalctl -u owrt-remote -n 80 --no-pager -l >&2 || true
|
||||
}
|
||||
|
||||
backup_optional() {
|
||||
file="$1"
|
||||
name="$2"
|
||||
if [ -e "$file" ]; then
|
||||
cp -p "$file" "$backup/$name"
|
||||
else
|
||||
: > "$backup/$name.absent"
|
||||
fi
|
||||
}
|
||||
|
||||
restore_optional() {
|
||||
file="$1"
|
||||
name="$2"
|
||||
if [ -e "$backup/$name" ]; then
|
||||
cp -p "$backup/$name" "$file"
|
||||
elif [ -e "$backup/$name.absent" ]; then
|
||||
rm -f "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
code=$?
|
||||
trap - EXIT
|
||||
if [ "$code" -ne 0 ]; then
|
||||
echo "Request-load fix failed with rc=$code" >&2
|
||||
if [ "$changed" = 1 ] && [ -s "$backup/owrt-remote-hub.py" ]; then
|
||||
cp -p "$backup/owrt-remote-hub.py" "$candidate"
|
||||
mv -f "$candidate" "$target"
|
||||
restore_optional "$policy" nginx-policy
|
||||
restore_optional "$journal" journald-limits
|
||||
restore_optional "$logrotate" logrotate
|
||||
nginx -t >/dev/null 2>&1 && systemctl reload nginx >/dev/null 2>&1 || true
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
systemctl restart owrt-remote >/dev/null 2>&1 || true
|
||||
fi
|
||||
show_failure
|
||||
fi
|
||||
rm -f "$candidate"
|
||||
rm -rf "$stage"
|
||||
exit "$code"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
test -s "$target"
|
||||
python_bin=/opt/owrt-remote/venv/bin/python
|
||||
[ -x "$python_bin" ] || python_bin=python3
|
||||
|
||||
if [ '__ACTION__' = Install ]; then
|
||||
test -s "$stage/hub.py"
|
||||
"$python_bin" -m py_compile "$stage/hub.py"
|
||||
"$python_bin" - "$stage/hub.py" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
source = Path(sys.argv[1]).read_text(encoding='utf-8')
|
||||
required = (
|
||||
'def notification_wait_context(',
|
||||
'def claim_notification_wait(',
|
||||
'if after_serial > latest_serial:',
|
||||
'notification_wait_metrics()',
|
||||
'client_id: notificationClient',
|
||||
'notificationWaitController',
|
||||
)
|
||||
missing = [marker for marker in required if marker not in source]
|
||||
if missing:
|
||||
for marker in missing:
|
||||
print('Missing marker:', repr(marker), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print('Candidate validation OK')
|
||||
PY
|
||||
mkdir -p "$backup"
|
||||
if [ ! -e "$backup/owrt-remote-hub.py" ]; then
|
||||
cp -p "$target" "$backup/owrt-remote-hub.py"
|
||||
backup_optional "$policy" nginx-policy
|
||||
backup_optional "$journal" journald-limits
|
||||
backup_optional "$logrotate" logrotate
|
||||
fi
|
||||
changed=1
|
||||
mkdir -p "$(dirname "$policy")" "$(dirname "$journal")" "$(dirname "$logrotate")"
|
||||
cat > "$policy" <<'EOF'
|
||||
# OpenWrt Remote Hub: keep successful long-poll traffic out of access.log.
|
||||
# 4xx/5xx responses remain logged for diagnostics.
|
||||
map "$request_uri:$status" $owrt_remote_access_log {
|
||||
default 1;
|
||||
~^/api/notifications/wait[^:]*:2[0-9][0-9]$ 0;
|
||||
~^/api/ssh-session/[^/]+/read[^:]*:2[0-9][0-9]$ 0;
|
||||
}
|
||||
access_log /var/log/nginx/access.log combined if=$owrt_remote_access_log;
|
||||
EOF
|
||||
cat > "$journal" <<'EOF'
|
||||
[Journal]
|
||||
SystemMaxUse=256M
|
||||
SystemKeepFree=512M
|
||||
MaxRetentionSec=7day
|
||||
RateLimitIntervalSec=30s
|
||||
RateLimitBurst=2000
|
||||
Compress=yes
|
||||
EOF
|
||||
cat > "$logrotate" <<'EOF'
|
||||
/var/log/nginx/*.log {
|
||||
daily
|
||||
rotate 7
|
||||
size 50M
|
||||
missingok
|
||||
notifempty
|
||||
compress
|
||||
delaycompress
|
||||
copytruncate
|
||||
}
|
||||
EOF
|
||||
cp -p "$stage/hub.py" "$candidate"
|
||||
chmod 0755 "$candidate"
|
||||
mv -f "$candidate" "$target"
|
||||
if command -v nginx >/dev/null 2>&1; then
|
||||
nginx -t
|
||||
systemctl reload nginx >/dev/null 2>&1 || systemctl restart nginx
|
||||
fi
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true
|
||||
journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true
|
||||
else
|
||||
test -s "$backup/owrt-remote-hub.py"
|
||||
"$python_bin" -m py_compile "$backup/owrt-remote-hub.py"
|
||||
cp -p "$backup/owrt-remote-hub.py" "$candidate"
|
||||
mv -f "$candidate" "$target"
|
||||
restore_optional "$policy" nginx-policy
|
||||
restore_optional "$journal" journald-limits
|
||||
restore_optional "$logrotate" logrotate
|
||||
if command -v nginx >/dev/null 2>&1; then
|
||||
nginx -t
|
||||
systemctl reload nginx >/dev/null 2>&1 || systemctl restart nginx
|
||||
fi
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true
|
||||
journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
systemctl restart owrt-remote
|
||||
healthy=0
|
||||
for attempt in 1 2 3 4 5 6 7 8; do
|
||||
if systemctl is-active --quiet owrt-remote && "$python_bin" - <<'PY'
|
||||
import json
|
||||
import urllib.request
|
||||
with urllib.request.urlopen('http://127.0.0.1:8088/health?request-load-fix=1', timeout=3) as response:
|
||||
data = json.loads(response.read(4096))
|
||||
assert data.get('ok') and 'notification_wait' in data
|
||||
print('Running Hub health verified: notification_wait=' + json.dumps(data['notification_wait']))
|
||||
PY
|
||||
then
|
||||
healthy=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
[ "$healthy" = 1 ]
|
||||
|
||||
printf '\nBackup: %s\nREQUEST_LOAD_FIX_V119_%s_OK\n' "$backup" '__ACTION__'
|
||||
'@
|
||||
|
||||
$RemoteScript = $RemoteScript.Replace('__UPLOAD_NAME__', $UploadName).Replace('__ACTION__', $Action)
|
||||
|
||||
try {
|
||||
[System.IO.Directory]::CreateDirectory($StageRoot) | Out-Null
|
||||
[System.IO.File]::WriteAllText((Join-Path $StageRoot 'run.sh'), $RemoteScript.Replace("`r`n", "`n") + "`n", $Utf8)
|
||||
if ($Action -eq 'Install') {
|
||||
$Content = [System.IO.File]::ReadAllText((Join-Path $RepoRoot 'vps\owrt-remote-hub.py')).Replace("`r`n", "`n")
|
||||
[System.IO.File]::WriteAllText((Join-Path $StageRoot 'hub.py'), $Content, $Utf8)
|
||||
}
|
||||
& scp -O -r -- $StageRoot "${Vps}:/tmp/"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Upload failed. VPS was not changed.' }
|
||||
& ssh -T -o ConnectTimeout=10 $Vps "sh /tmp/$UploadName/run.sh"
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Action failed. See the output above; success was not confirmed." }
|
||||
Write-Host "Request-load fix $Action finished successfully."
|
||||
}
|
||||
finally {
|
||||
$Resolved = [System.IO.Path]::GetFullPath($StageRoot)
|
||||
$Expected = [System.IO.Path]::GetFullPath((Join-Path ([System.IO.Path]::GetTempPath()) $UploadName))
|
||||
if ($Resolved -ne $Expected -or (Split-Path -Leaf $Resolved) -ne $UploadName) { throw 'Unsafe staging cleanup path.' }
|
||||
if (Test-Path -LiteralPath $Resolved) { Remove-Item -LiteralPath $Resolved -Recurse -Force }
|
||||
}
|
||||
@@ -152,6 +152,44 @@ PY
|
||||
fi
|
||||
}
|
||||
|
||||
install_log_limits() {
|
||||
$SUDO mkdir -p /etc/systemd/journald.conf.d /etc/nginx/conf.d
|
||||
$SUDO tee /etc/systemd/journald.conf.d/owrt-remote-limits.conf >/dev/null <<'EOF'
|
||||
[Journal]
|
||||
SystemMaxUse=256M
|
||||
SystemKeepFree=512M
|
||||
MaxRetentionSec=7day
|
||||
RateLimitIntervalSec=30s
|
||||
RateLimitBurst=2000
|
||||
Compress=yes
|
||||
EOF
|
||||
$SUDO tee /etc/nginx/conf.d/owrt-remote-log-policy.conf >/dev/null <<'EOF'
|
||||
# Successful long-poll and terminal reads are expected traffic.
|
||||
# Keep 4xx/5xx responses in access.log for diagnostics.
|
||||
map "$request_uri:$status" $owrt_remote_access_log {
|
||||
default 1;
|
||||
~^/api/notifications/wait[^:]*:2[0-9][0-9]$ 0;
|
||||
~^/api/ssh-session/[^/]+/read[^:]*:2[0-9][0-9]$ 0;
|
||||
}
|
||||
access_log /var/log/nginx/access.log combined if=$owrt_remote_access_log;
|
||||
EOF
|
||||
$SUDO tee /etc/logrotate.d/owrt-remote >/dev/null <<'EOF'
|
||||
/var/log/nginx/*.log {
|
||||
daily
|
||||
rotate 7
|
||||
size 50M
|
||||
missingok
|
||||
notifempty
|
||||
compress
|
||||
delaycompress
|
||||
copytruncate
|
||||
}
|
||||
EOF
|
||||
$SUDO systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
$SUDO systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true
|
||||
$SUDO journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
install_xray_service() {
|
||||
xray_bin="$(command -v xray || command -v /usr/local/bin/xray || command -v /usr/bin/xray || true)"
|
||||
[ -n "$xray_bin" ] || return 0
|
||||
@@ -291,6 +329,7 @@ main() {
|
||||
install_xray_binary
|
||||
install_files
|
||||
install_python_deps_v2
|
||||
install_log_limits
|
||||
install_xray_service
|
||||
open_firewall
|
||||
start_hub
|
||||
|
||||
+187
-7
@@ -154,6 +154,9 @@ ROUTER_PROXY_LIMITERS = {}
|
||||
STATIC_CACHE_LOCK = threading.Lock()
|
||||
NOTIFICATIONS_LOCK = threading.Lock()
|
||||
NOTIFICATIONS_COND = threading.Condition()
|
||||
NOTIFICATION_WAIT_LOCK = threading.Lock()
|
||||
NOTIFICATION_WAIT_ACTIVE = {}
|
||||
NOTIFICATION_WAIT_MAX = max(1, int(os.environ.get("OWRT_REMOTE_NOTIFICATION_WAIT_MAX", "64")))
|
||||
PUSH_LOCK = threading.Lock()
|
||||
TRAFFIC_COUNTERS_LOCK = threading.Lock()
|
||||
AUTH_STATE_LOCK = threading.Lock()
|
||||
@@ -2183,7 +2186,11 @@ def list_notifications(after=0, limit=60, after_serial=0):
|
||||
items = load_notifications()
|
||||
latest_serial = max((int(item.get("serial") or 0) for item in items), default=0)
|
||||
if after_serial > latest_serial:
|
||||
after_serial = 0
|
||||
# A cleared/pruned notification file must not turn a stale browser
|
||||
# cursor into an immediate full-list response. Returning no items
|
||||
# lets the caller advance from the current server serial and keep the
|
||||
# long-poll blocked until a genuinely new notification arrives.
|
||||
return []
|
||||
if after_serial > 0:
|
||||
items = [item for item in items if int(item.get("serial") or 0) > after_serial]
|
||||
elif after > 0:
|
||||
@@ -2204,6 +2211,10 @@ def wait_for_notifications(after_serial=0, timeout=25, limit=60):
|
||||
timeout = max(1, min(60, int(timeout or 25)))
|
||||
except (TypeError, ValueError):
|
||||
timeout = 25
|
||||
# A clear/prune can make a browser cursor newer than the current file.
|
||||
# Clamp it before entering the wait so the next newly-created serial is
|
||||
# delivered instead of being hidden behind the stale cursor.
|
||||
after_serial = min(after_serial, latest_notification_serial())
|
||||
deadline = time.time() + timeout
|
||||
while True:
|
||||
items = list_notifications(limit=limit, after_serial=after_serial)
|
||||
@@ -2216,6 +2227,40 @@ def wait_for_notifications(after_serial=0, timeout=25, limit=60):
|
||||
NOTIFICATIONS_COND.wait(timeout=remaining)
|
||||
|
||||
|
||||
def notification_wait_context(session_token="", client_id="", client_ip=""):
|
||||
"""Return a bounded per-browser context key for long-poll admission."""
|
||||
token = str(session_token or "")[:160]
|
||||
client_id = str(client_id or "")[:96]
|
||||
if not re.fullmatch(r"[A-Za-z0-9_-]{8,96}", client_id):
|
||||
client_id = "ip:" + str(client_ip or "unknown")[:96]
|
||||
return hashlib.sha256((token + "\0" + client_id).encode("utf-8", "replace")).hexdigest()
|
||||
|
||||
|
||||
def claim_notification_wait(key):
|
||||
now = time.monotonic()
|
||||
with NOTIFICATION_WAIT_LOCK:
|
||||
for active_key, started in list(NOTIFICATION_WAIT_ACTIVE.items()):
|
||||
if now - started > 90:
|
||||
NOTIFICATION_WAIT_ACTIVE.pop(active_key, None)
|
||||
if key in NOTIFICATION_WAIT_ACTIVE or len(NOTIFICATION_WAIT_ACTIVE) >= NOTIFICATION_WAIT_MAX:
|
||||
return False
|
||||
NOTIFICATION_WAIT_ACTIVE[key] = now
|
||||
return True
|
||||
|
||||
|
||||
def release_notification_wait(key):
|
||||
with NOTIFICATION_WAIT_LOCK:
|
||||
NOTIFICATION_WAIT_ACTIVE.pop(key, None)
|
||||
|
||||
|
||||
def notification_wait_metrics():
|
||||
with NOTIFICATION_WAIT_LOCK:
|
||||
return {
|
||||
"active": len(NOTIFICATION_WAIT_ACTIVE),
|
||||
"limit": NOTIFICATION_WAIT_MAX,
|
||||
}
|
||||
|
||||
|
||||
def clear_notifications():
|
||||
with NOTIFICATIONS_LOCK:
|
||||
save_notifications([])
|
||||
@@ -11123,6 +11168,66 @@ let lastNotificationTs = initialNotificationTs();
|
||||
let lastNotificationSerial = initialNotificationSerial();
|
||||
let notificationWaitAbort = false;
|
||||
let notificationWaitRunning = false;
|
||||
let notificationWaitController = null;
|
||||
let notificationLeaderTimer = 0;
|
||||
let notificationLeaderTab = '';
|
||||
const NOTIFICATION_CLIENT_KEY = 'owrtNotificationClientId';
|
||||
const NOTIFICATION_LEASE_KEY = 'owrtNotificationLease';
|
||||
const NOTIFICATION_LEASE_MS = 90000;
|
||||
|
||||
function notificationClientId() {{
|
||||
try {{
|
||||
let value = localStorage.getItem(NOTIFICATION_CLIENT_KEY);
|
||||
if (!/^[A-Za-z0-9_-]{{8,96}}$/.test(value || '')) {{
|
||||
value = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + Math.random().toString(36).slice(2))
|
||||
.replace(/[^A-Za-z0-9_-]/g, '').slice(0, 96);
|
||||
localStorage.setItem(NOTIFICATION_CLIENT_KEY, value);
|
||||
}}
|
||||
return value;
|
||||
}} catch (e) {{
|
||||
// Let the server fall back to the client IP when storage is unavailable;
|
||||
// random per-tab ids would defeat the single-poll admission guard.
|
||||
return '';
|
||||
}}
|
||||
}}
|
||||
|
||||
const notificationClient = notificationClientId();
|
||||
const notificationTab = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + Math.random()).replace(/[^A-Za-z0-9_-]/g, '').slice(0, 96);
|
||||
|
||||
function notificationLeaderState() {{
|
||||
try {{
|
||||
const raw = localStorage.getItem(NOTIFICATION_LEASE_KEY);
|
||||
const state = raw ? JSON.parse(raw) : null;
|
||||
return state && typeof state === 'object' ? state : null;
|
||||
}} catch (e) {{
|
||||
return null;
|
||||
}}
|
||||
}}
|
||||
|
||||
function notificationIsLeader() {{
|
||||
const now = Date.now();
|
||||
const current = notificationLeaderState();
|
||||
if (!current || Number(current.expires || 0) <= now || current.tab === notificationTab) {{
|
||||
try {{
|
||||
localStorage.setItem(NOTIFICATION_LEASE_KEY, JSON.stringify({{tab: notificationTab, expires: now + NOTIFICATION_LEASE_MS}}));
|
||||
notificationLeaderTab = notificationTab;
|
||||
return true;
|
||||
}} catch (e) {{
|
||||
notificationLeaderTab = notificationTab;
|
||||
return true;
|
||||
}}
|
||||
}}
|
||||
notificationLeaderTab = String(current.tab || '');
|
||||
return notificationLeaderTab === notificationTab;
|
||||
}}
|
||||
|
||||
function releaseNotificationLeader() {{
|
||||
try {{
|
||||
const current = notificationLeaderState();
|
||||
if (current && current.tab === notificationTab) localStorage.removeItem(NOTIFICATION_LEASE_KEY);
|
||||
}} catch (e) {{}}
|
||||
notificationLeaderTab = '';
|
||||
}}
|
||||
|
||||
function storeNotificationCursor() {{
|
||||
localStorage.setItem('owrtLastNotificationTs', String(lastNotificationTs));
|
||||
@@ -11167,7 +11272,9 @@ async function loadNotifications({{initial = false}} = {{}}) {{
|
||||
const data = await res.json().catch(() => ({{}}));
|
||||
const items = Array.isArray(data.notifications) ? data.notifications : [];
|
||||
if (!items.length && !initial) {{
|
||||
lastNotificationSerial = Math.max(lastNotificationSerial, Number(data.serial || 0));
|
||||
const serverSerial = Number(data.serial || 0);
|
||||
lastNotificationSerial = serverSerial < lastNotificationSerial
|
||||
? serverSerial : Math.max(lastNotificationSerial, serverSerial);
|
||||
storeNotificationCursor();
|
||||
return;
|
||||
}}
|
||||
@@ -11177,20 +11284,38 @@ async function loadNotifications({{initial = false}} = {{}}) {{
|
||||
async function waitNotificationsLoop() {{
|
||||
if (!authIsOwner()) return;
|
||||
if (notificationWaitRunning || notificationWaitAbort) return;
|
||||
if (!notificationIsLeader()) {{
|
||||
if (!notificationLeaderTimer) notificationLeaderTimer = setTimeout(() => {{
|
||||
notificationLeaderTimer = 0;
|
||||
waitNotificationsLoop();
|
||||
}}, 5000);
|
||||
return;
|
||||
}}
|
||||
notificationWaitRunning = true;
|
||||
try {{
|
||||
while (!notificationWaitAbort) {{
|
||||
if (!notificationIsLeader()) break;
|
||||
const params = new URLSearchParams({{
|
||||
after_serial: String(lastNotificationSerial || 0),
|
||||
timeout: '25',
|
||||
limit: '60'
|
||||
limit: '60',
|
||||
client_id: notificationClient
|
||||
}});
|
||||
let res;
|
||||
try {{
|
||||
res = await fetch('/api/notifications/wait?' + params.toString(), {{cache: 'no-store'}});
|
||||
notificationWaitController = new AbortController();
|
||||
res = await fetch('/api/notifications/wait?' + params.toString(), {{cache: 'no-store', signal: notificationWaitController.signal}});
|
||||
}} catch (e) {{
|
||||
if (notificationWaitAbort) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
continue;
|
||||
}} finally {{
|
||||
notificationWaitController = null;
|
||||
}}
|
||||
if (res.status === 429) {{
|
||||
const retry = Math.max(2000, Math.min(15000, Number(res.headers.get('Retry-After') || 5) * 1000));
|
||||
await new Promise(resolve => setTimeout(resolve, retry));
|
||||
continue;
|
||||
}}
|
||||
if (!res.ok) {{
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
@@ -11202,7 +11327,9 @@ async function waitNotificationsLoop() {{
|
||||
applyNotificationItems(items);
|
||||
continue;
|
||||
}}
|
||||
lastNotificationSerial = Math.max(lastNotificationSerial, Number(data.serial || 0));
|
||||
const serverSerial = Number(data.serial || 0);
|
||||
lastNotificationSerial = serverSerial < lastNotificationSerial
|
||||
? serverSerial : Math.max(lastNotificationSerial, serverSerial);
|
||||
storeNotificationCursor();
|
||||
}}
|
||||
}} finally {{
|
||||
@@ -11210,6 +11337,19 @@ async function waitNotificationsLoop() {{
|
||||
}}
|
||||
}}
|
||||
|
||||
function stopNotificationWait() {{
|
||||
notificationWaitAbort = true;
|
||||
if (notificationWaitController) notificationWaitController.abort();
|
||||
if (notificationLeaderTimer) {{ clearTimeout(notificationLeaderTimer); notificationLeaderTimer = 0; }}
|
||||
releaseNotificationLeader();
|
||||
}}
|
||||
|
||||
window.addEventListener('pagehide', stopNotificationWait);
|
||||
window.addEventListener('pageshow', () => {{
|
||||
notificationWaitAbort = false;
|
||||
waitNotificationsLoop();
|
||||
}});
|
||||
|
||||
function bytesToBase64(bytes) {{
|
||||
let binary = '';
|
||||
const chunk = 0x8000;
|
||||
@@ -15704,6 +15844,23 @@ class Handler(BaseHTTPRequestHandler):
|
||||
return self.server.app
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
# Successful long-poll and terminal-read responses are expected to be
|
||||
# frequent. Nginx and journald already retain their failures; avoid
|
||||
# writing a line for every normal 200 response while keeping 4xx/5xx
|
||||
# diagnostics visible.
|
||||
path = urllib.parse.urlsplit(str(self.path or "")).path
|
||||
status = 0
|
||||
if len(args) > 1:
|
||||
try:
|
||||
status = int(str(args[1]))
|
||||
except (TypeError, ValueError):
|
||||
status = 0
|
||||
quiet = (
|
||||
path == "/api/notifications/wait"
|
||||
or re.fullmatch(r"/api/ssh-session/[^/]+/read", path or "")
|
||||
)
|
||||
if quiet and 200 <= status < 400:
|
||||
return
|
||||
sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), fmt % args))
|
||||
|
||||
def parsed(self):
|
||||
@@ -15905,9 +16062,9 @@ class Handler(BaseHTTPRequestHandler):
|
||||
return
|
||||
self.send_bytes(200, body, content_type)
|
||||
|
||||
def send_json(self, status, payload):
|
||||
def send_json(self, status, payload, extra_headers=None):
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
self.send_bytes(status, body, "application/json; charset=utf-8")
|
||||
self.send_bytes(status, body, "application/json; charset=utf-8", extra_headers=extra_headers)
|
||||
|
||||
def require_admin(self):
|
||||
if self.admin_ok():
|
||||
@@ -18737,6 +18894,8 @@ exit 127
|
||||
result = {"ok": True}
|
||||
if self.query().get('panel-update') == ['1']:
|
||||
result['revision'] = HUB_LOADED_REVISION
|
||||
if self.query().get("request-load-fix") == ["1"]:
|
||||
result["notification_wait"] = notification_wait_metrics()
|
||||
self.send_json(200, result)
|
||||
return
|
||||
if path == '/api/hub-update':
|
||||
@@ -18915,6 +19074,25 @@ exit 127
|
||||
if not self.require_owner(json_mode=True):
|
||||
return
|
||||
query = self.query()
|
||||
key = notification_wait_context(
|
||||
self.current_session_token(),
|
||||
query.get("client_id", [""])[0],
|
||||
self.client_ip(),
|
||||
)
|
||||
if not claim_notification_wait(key):
|
||||
self.send_json(
|
||||
429,
|
||||
{
|
||||
"ok": False,
|
||||
"busy": True,
|
||||
"notifications": [],
|
||||
"serial": latest_notification_serial(),
|
||||
"error": "для этого клиента уже выполняется long-poll",
|
||||
},
|
||||
extra_headers=[("Retry-After", "5")],
|
||||
)
|
||||
return
|
||||
try:
|
||||
self.send_json(
|
||||
200,
|
||||
{
|
||||
@@ -18926,6 +19104,8 @@ exit 127
|
||||
"serial": latest_notification_serial(),
|
||||
},
|
||||
)
|
||||
finally:
|
||||
release_notification_wait(key)
|
||||
return
|
||||
if path == "/api/push/vapid-public-key":
|
||||
if not self.require_owner(json_mode=True):
|
||||
|
||||
Reference in new issue
Block a user