From 799ad8aad6478c6bc3bd7adb8e7466cb73da2b0e Mon Sep 17 00:00:00 2001
From: kzolotarev95
Date: Fri, 20 Nov 2026 00:31:35 +0400
Subject: [PATCH] Bound VPS log retention and quiet routine Hub traffic
---
README.md | 6 +
docs/VPS_LOG_MAINTENANCE.md | 195 +++++++++++
tests/test_log_maintenance.py | 189 +++++++++++
tests/test_notification_load.py | 34 ++
vps/deploy-log-maintenance.ps1 | 54 +++
vps/install-vps.sh | 51 +--
vps/log-maintenance.py | 563 ++++++++++++++++++++++++++++++++
vps/owrt-remote-hub.py | 14 +-
8 files changed, 1063 insertions(+), 43 deletions(-)
create mode 100644 docs/VPS_LOG_MAINTENANCE.md
create mode 100644 tests/test_log_maintenance.py
create mode 100644 vps/deploy-log-maintenance.ps1
create mode 100644 vps/log-maintenance.py
diff --git a/README.md b/README.md
index e9f18ee..099db42 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,12 @@ Xray и недоступном SSH-туннеле, если агент прод
Нужны обновлённые Hub и агент. Описание и проверка.
+
+Журналы VPS: ежедневная ротация в 03:20, ограничение journald,
+фильтрация успешных частых HTTP-запросов и проверка заполнения диска.
+Установка, проверка и откат.
+
+
diff --git a/docs/VPS_LOG_MAINTENANCE.md b/docs/VPS_LOG_MAINTENANCE.md
new file mode 100644
index 0000000..cc384ae
--- /dev/null
+++ b/docs/VPS_LOG_MAINTENANCE.md
@@ -0,0 +1,195 @@
+# Обслуживание журналов VPS
+
+Исправление для v121. Оно меняет только HTTP-логирование Hub,
+ротацию журналов и обслуживание диска. API, авторизация, Xray, heartbeat,
+база данных и архивы журналов роутеров остаются в прежнем формате.
+
+## Причина
+
+`Handler.log_message()` фильтровал успешные long-poll и чтение SSH-терминала,
+но записывал каждый успешный heartbeat и пакет `/api/router-logs`.
+При большом количестве роутеров эти записи попадали в journald/syslog и Nginx.
+Старая настройка Nginx могла добавлять второй `access_log`, сохраняя первый
+без фильтра. Установщик также создавал правило `/etc/logrotate.d/owrt-remote`
+для тех же файлов Nginx, которые уже обслуживает пакет Nginx.
+
+Частота SSH-перебора на рабочем VPS этим локальным тестом не установлена:
+доступ к серверу и чтение его журналов не выполнялись.
+
+## Что установлено
+
+- Hub и стандартный `/var/log/nginx/access.log` пропускают успешные 2xx/3xx
+ для `/api/heartbeat`, `/api/router-logs`, `/api/notifications/wait` и
+ `/api/ssh-session/*/read`. HTTP 4xx/5xx, исключения и остальные действия
+ продолжают записываться. Содержимое доставленных журналов не фильтруется.
+- Единственный планировщик — штатный `logrotate.timer`, ежедневно в **03:20
+ по часовому поясу VPS**. `Persistent=true` запускает пропущенную задачу после
+ включения сервера. Отдельного cron или второго таймера нет.
+- Штатные правила `/etc/logrotate.conf` и `/etc/logrotate.d/` сохраняют права,
+ владельцев и команды повторного открытия файлов Nginx/rsyslog. Временная
+ конфигурация добавляет `daily`, `maxsize 20M`, `rotate 1`, `maxage 1`,
+ `compress`, `nodelaycompress`. Активные журналы через `rm` не удаляются.
+- Старые распознанные архивы настроенных журналов удаляются после 24 часов.
+ Очистка не представляет собой удаление всех `*.gz` на сервере: проверяются
+ имена архивов конкретных логов, тип файла, ссылки и открытые дескрипторы.
+- journald: `SystemMaxUse=50M`, `SystemKeepFree=500M`, `MaxRetentionSec=7day`,
+ `Compress=yes`, файлы до 10 МиБ. Для непостоянного журнала задано 16 МиБ.
+ Ежедневно выполняются `journalctl --rotate` и очистка архивов старше суток
+ с ограничением 50 МиБ. Поэтому ежедневная политика строже предела 7 дней.
+- Проверяются `/` и `/var/log`: предупреждение выше 75%, усиленная очистка
+ **закрытых архивов** выше 85%, критическое сообщение выше 90%. При усиленной
+ очистке допустимо удалить также свежие архивы, journald сокращается до 10 МиБ.
+- Результат: `/var/lib/owrt-remote-log-maintenance/result.json`; собственный
+ журнал `/var/log/owrt-remote-maintenance.log` — до 64 КиБ плюс один архив
+ до 64 КиБ. Права состояния и резервных копий ограничены root.
+- `flock` исключает параллельные обслуживание/установку/откат. Используется
+ также штатная блокировка состояния logrotate. Маркер попытки записывается
+ до работы; автоматический повтор в тот же день пропускается даже после
+ ошибки. Явный `Run` нужен для повторной попытки после исправления причины.
+
+`maxsize` проверяется при запуске logrotate, а не непрерывно. Ежедневная
+задача не гарантирует, что сторонний сервис не заполнит диск между запусками.
+Лимиты journald не ограничивают размер базы Hub или пользовательских файлов.
+
+## Установка подготовленной версии
+
+PowerShell Windows 11:
+
+```powershell
+$Fix = "C:\Users\k.zolotarev95\Documents\openwrt overlay\vps\deploy-log-maintenance.ps1"
+powershell.exe -NoProfile -ExecutionPolicy Bypass -File $Fix -Vps "root@132.243.19.232" -Action Install
+```
+
+Нужны Python 3, systemd, journald и установленный `logrotate`. Если последнего
+нет, скрипт сообщит об этом до изменения конфигурации:
+
+```powershell
+ssh root@132.243.19.232 'apt-get update && apt-get install -y logrotate'
+```
+
+Установка сохраняет исходные файлы, проверяет Python, Nginx, systemd и
+`logrotate --debug`. При ошибке применения возвращает конфигурацию из копии.
+Hub перезапускается; проверяется SHA256 именно запущенной сборки через `/health`.
+Затем выполняется первое обслуживание. Если оно завершилось ошибкой,
+изучить её и выполнить `Run` либо `Rollback`; уже удалённые архивы не вернуть.
+
+Повторный `Install` обновляет исправление, сохраняя первую резервную копию.
+Основной `install-vps.sh` также подключает этот механизм после настройки HTTPS,
+когда конфигурация Nginx уже существует. Для полного обновления существующего
+VPS через GitHub с сохранением текущего логина выполнить на самом VPS:
+
+```sh
+curl -fsSL "https://raw.githubusercontent.com/kzolotarev95/luci-app-owrt-remote/main/vps/install-vps.sh?v=$(date +%s)" | sudo env RESET_LOGIN=0 sh
+```
+
+Кнопка обновления панели заменяет только файл Hub. Для установки новых правил
+обслуживания журналов нужен полный установщик выше или PowerShell-скрипт.
+
+Нестандартные правила с целями вне `/var/log`, `olddir`, неоднозначным синтаксисом
+или пользовательским условием `access_log if=...` требуют ручного рассмотрения.
+Установка остановится с причиной, не выполняя опасную очистку. Иные назначения
+access_log, например отдельные пользовательские файлы, автоматически не меняются.
+
+## Проверка и ручной запуск
+
+```powershell
+powershell.exe -NoProfile -ExecutionPolicy Bypass -File $Fix -Vps "root@132.243.19.232" -Action Status
+powershell.exe -NoProfile -ExecutionPolicy Bypass -File $Fix -Vps "root@132.243.19.232" -Action Run
+ssh root@132.243.19.232 'systemctl list-timers logrotate.timer --no-pager; journalctl -u logrotate.service -n 50 --no-pager; tail -n 20 /var/log/owrt-remote-maintenance.log; journalctl --disk-usage; nginx -t; systemctl is-active owrt-remote nginx rsyslog systemd-journald'
+```
+
+`Run` принудительно запускает ротацию повторно; для обычной проверки достаточно
+`Status`. Не применять `Run` многократно без необходимости.
+
+Временно включить полный HTTP-лог Hub для диагностики можно через
+`systemctl edit owrt-remote`, добавив:
+
+```ini
+[Service]
+Environment=OWRT_REMOTE_HTTP_DEBUG=1
+```
+
+После изменения нужен `systemctl restart owrt-remote`. Для выключения удалить
+эту строку из созданного drop-in и перезапустить Hub. На фильтр Nginx эта
+переменная не влияет; ошибки Nginx всегда сохраняются.
+
+## SSH
+
+Проверить попытки входа без изменения доступа:
+
+```powershell
+ssh root@132.243.19.232 'journalctl -u ssh -u sshd --since yesterday --no-pager | tail -n 100'
+```
+
+Возможные меры: ограничение SSH по доверенным адресам в firewall, Fail2ban,
+`MaxAuthTries 3`, `LoginGraceTime 30`, `MaxStartups 10:30:30`. Подбирать после
+просмотра реальной нагрузки и наличия резервного доступа. Перед применением
+проверять `sshd -t`, сохранять текущий SSH-сеанс и проверять второй вход.
+Исправление не меняет `PasswordAuthentication`, `PermitRootLogin`, SSH или firewall.
+
+## Откат
+
+```powershell
+powershell.exe -NoProfile -ExecutionPolicy Bypass -File $Fix -Vps "root@132.243.19.232" -Action Rollback
+```
+
+Копия: `/root/owrt-remote-backup-before-log-maintenance-v121/manifest.json`
+и файлы рядом. Откат восстанавливает исходный Hub, Nginx, настройки journald,
+таймера и старое правило logrotate; возвращает прежнее состояние таймера.
+База, сертификаты, SSH-ключи и данные пользователей не открываются обработчиком
+очистки. Удалённые или уже ротированные журналы откат не восстанавливает.
+
+## Изменённые файлы и проверки
+
+- `vps/owrt-remote-hub.py` — фильтрация успешных HTTP-записей и DEBUG.
+- `vps/log-maintenance.py` — настройка, обслуживание, проверки и откат.
+- `vps/install-vps.sh` — подключение обслуживания вместо старых правил.
+- `vps/deploy-log-maintenance.ps1` — установка локальной сборки и откат.
+- `tests/test_log_maintenance.py`, `tests/test_notification_load.py` — регрессии.
+- Этот документ — эксплуатация и ограничения.
+
+Тесты Python можно запустить отдельно:
+
+```powershell
+python -m unittest discover -s tests -p test_log_maintenance.py -v
+python -m unittest discover -s tests -p test_notification_load.py -v
+```
+
+Проверки POSIX-блокировок, ссылок и прав запускаются на Linux; на Windows
+они явно помечены как пропущенные. Проверка на рабочем VPS выполняется после
+установки пользователем; локальная Ubuntu не подтверждает состояние его диска
+или фактическую интенсивность SSH-перебора.
+
+### Результаты локальной проверки
+
+На отдельной виртуальной машине Ubuntu 24.04 с настоящими systemd, logrotate,
+Nginx, rsyslog и работающим Hub проверены:
+
+- 13 тестов обслуживания и 7 тестов HTTP-логирования — успешно на Linux.
+- Установка, расписание 03:20, фактические настройки journald, ручное
+ обслуживание, повтор в тот же день и блокировка параллельного запуска.
+- Успешные запросы к четырём частым API не попадают в access.log;
+ HTTP 404/503 и обычное действие остаются. Настоящий Hub возвращает и
+ записывает HTTP 401 при heartbeat без авторизации.
+- Удаление старых архивов, сохранение постороннего `.gz`, запись новых событий
+ Nginx и rsyslog после ротации, ограничение собственного журнала.
+- Переход со старого правила проекта и дублирующего access_log без сохранения
+ второго регистратора; исходные файлы возвращаются при откате.
+- Повторная установка, ручной откат и автоматический откат при намеренно
+ ошибочной конфигурации Nginx.
+- SQLite `integrity_check=ok`, тестовая запись в базе и SHA256 тестовых настроек,
+ токенов, сертификата, SSH-ключей и пользовательских файлов сохранены.
+- Дополнительно 19 регрессий архива журналов роутеров и 31 проверка жизненного
+ цикла ресурсов прошли на Windows; 2 проверки POSIX в последнем наборе пропущены.
+- Синтаксис Python, PowerShell, shell-установщика и `git diff --check` проверены.
+
+Пороги 75/85/90% проверяются с подменёнными показаниями диска в тестах:
+заполнение настоящего раздела до 100% не выполнялось. Существующее предупреждение
+Python `invalid escape sequence '\s'` в другом HTML-шаблоне Hub не относится к
+этому исправлению и не мешает запуску; шаблон в рамках этой задачи не менялся.
+
+После установки пользователем на рабочий VPS получен успешный результат:
+`ok=true`, удалены 263 архива, освобождены 2 365 938 байт, journald занимает
+18,8 МиБ. Hub, Nginx, rsyslog и journald подтвердили состояние `active`.
+Эта проверка выполнена по предоставленному выводу; независимое подключение
+к рабочему серверу не выполнялось.
diff --git a/tests/test_log_maintenance.py b/tests/test_log_maintenance.py
new file mode 100644
index 0000000..e944cdb
--- /dev/null
+++ b/tests/test_log_maintenance.py
@@ -0,0 +1,189 @@
+"""Retention policy, safe archive selection, disk levels and HTTP diagnostics."""
+import importlib.util
+import io
+import os
+from pathlib import Path
+import tempfile
+import unittest
+import subprocess
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location("log_maintenance", ROOT / "vps/log-maintenance.py")
+m = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(m)
+
+
+class MaintenanceTests(unittest.TestCase):
+ def test_only_exact_old_project_rule_can_be_removed(self):
+ with tempfile.TemporaryDirectory() as temp:
+ path = Path(temp) / 'owrt-remote'
+ rule = '/var/log/nginx/*.log {\n daily\n rotate 7\n size 50M\n missingok\n notifempty\n compress\n delaycompress\n copytruncate\n}\n'
+ path.write_text(rule)
+ self.assertTrue(m.legacy_policy(path))
+ path.write_text(rule.replace('rotate 7', 'rotate 30'))
+ self.assertFalse(m.legacy_policy(path))
+ path.write_text(rule.replace('copytruncate', 'postrotate\n echo custom hook\n endscript'))
+ self.assertFalse(m.legacy_policy(path))
+
+ def test_retention_preserves_native_reopen_hooks(self):
+ text = '''weekly
+/var/log/syslog
+/var/log/auth.log
+{
+ rotate 7
+ weekly
+ delaycompress
+ postrotate
+ if true; then
+ echo "{ brace inside shell }"
+ fi
+ endscript
+}
+'''
+ result, patterns = m.retention_config(text)
+ self.assertEqual(patterns, ["/var/log/syslog", "/var/log/auth.log"])
+ self.assertIn('echo "{ brace inside shell }"', result)
+ self.assertGreater(result.index("daily"), result.index("endscript"))
+ self.assertIn("maxsize 20M", result)
+ self.assertIn("nodelaycompress", result)
+
+ def test_non_log_paths_and_ambiguous_rules_fail_before_rotation(self):
+ for source in ("/var/lib/owrt-remote/hub.db {\n}\n", "/var/log/../lib/data {\n}\n",
+ "/var/log/a { rotate 1 }\n", "/var/log/a {\nolddir /home/user\n}\n"):
+ with self.subTest(source=source), self.assertRaises(RuntimeError):
+ m.retention_config(source)
+
+ def test_archive_names_do_not_match_active_or_user_files(self):
+ for suffix in (".1", ".20.gz", ".old", "-20261119.gz", ".20261119-032000.xz"):
+ self.assertTrue(m.archive_name("syslog", "syslog" + suffix), suffix)
+ for name in ("syslog", "syslog.db", "syslog.backup.tar.gz", "user.gz", "syslog.log", "syslog.old.key"):
+ self.assertFalse(m.archive_name("syslog", name), name)
+
+ def test_prune_only_old_closed_archives_and_keep_active_inode(self):
+ with tempfile.TemporaryDirectory() as temp:
+ root = Path(temp).resolve()
+ active = root / "access.log"
+ active.write_bytes(b"active")
+ old, fresh, opened, unrelated = [root / x for x in ("access.log.2.gz", "access.log.1", "access.log.old", "notes.gz")]
+ for p in (old, fresh, opened, unrelated):
+ p.write_bytes(b"archive")
+ for p in (old, opened, unrelated):
+ os.utime(p, (100, 100))
+ st = opened.stat()
+ inode = active.stat().st_ino
+ removed, size = m.prune_archives([active], now=100000, opened={(st.st_dev, st.st_ino)})
+ self.assertEqual((removed, size), (1, 7))
+ self.assertFalse(old.exists())
+ self.assertTrue(all(p.exists() for p in (active, fresh, opened, unrelated)))
+ self.assertEqual(active.stat().st_ino, inode)
+ self.assertEqual(m.prune_archives([active], emergency=True, opened={(st.st_dev, st.st_ino)})[0], 1)
+ self.assertTrue(opened.exists())
+
+ @unittest.skipIf(os.name == "nt", "Requires Unix link semantics")
+ def test_symlinks_and_hardlinks_are_not_deleted(self):
+ with tempfile.TemporaryDirectory() as temp:
+ root = Path(temp).resolve(); active = root / "a.log"; active.write_text("active")
+ user = root / "user"; user.write_text("private")
+ (root / "a.log.1").symlink_to(user)
+ os.link(user, root / "a.log.2")
+ self.assertEqual(m.prune_archives([active], emergency=True, opened=set()), (0, 0))
+ self.assertEqual(user.read_text(), "private")
+
+ def test_disk_checks_root_and_separate_log_mount(self):
+ usage = type("Usage", (), {"total": 110, "used": 91, "free": 9})()
+ with mock.patch.object(m.shutil, "disk_usage", return_value=usage) as get:
+ levels = m.disk_levels()
+ self.assertEqual([x[0] for x in levels], ["/", "/var/log"])
+ self.assertTrue(all(x[2] == "CRITICAL" for x in levels))
+ self.assertTrue(all(x[1] == 91 for x in levels))
+ self.assertEqual(get.call_count, 2)
+ for used, expected in ((75, 'OK'), (75.1, 'WARN'), (85, 'WARN'), (90, 'WARN'), (90.1, 'CRITICAL')):
+ usage = type('Usage', (), {'used': used, 'free': 100 - used})()
+ with self.subTest(used=used), mock.patch.object(m.shutil, 'disk_usage', return_value=usage):
+ self.assertEqual(m.disk_levels()[0][2], expected)
+
+ @unittest.skipIf(os.name == 'nt', 'Requires fcntl lock')
+ def test_disk_pressure_selects_aggressive_archives_and_smaller_journal_budget(self):
+ for percent, emergency in ((84.9, False), (85, False), (85.1, True)):
+ with self.subTest(percent=percent), tempfile.TemporaryDirectory() as temp:
+ with mock.patch.object(m, 'STATE', Path(temp) / 'state'), \
+ mock.patch.object(m, 'RUN', Path(temp) / 'run'), \
+ mock.patch.object(m, 'flatten_config', return_value=''), \
+ mock.patch.object(m, 'disk_levels', return_value=[('/', percent, 'WARN'), ('/var/log', 20, 'OK')]), \
+ mock.patch.object(m, 'prune_archives', return_value=(0, 0)) as prune, \
+ mock.patch.object(m, 'record'), mock.patch('sys.stdout', new=io.StringIO()), \
+ mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, '')) as command:
+ m.run()
+ self.assertTrue(all(call.kwargs['emergency'] == emergency for call in prune.call_args_list))
+ self.assertTrue(any('--vacuum-size=' + ('10M' if emergency else '50M') in call.args[0] for call in command.call_args_list))
+
+ def test_later_journald_override_cannot_silently_disable_limit(self):
+ text = '[Journal]\nSystemMaxUse=50M\nSystemKeepFree=500M\nMaxRetentionSec=7day\nCompress=yes\n'
+ with mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, text)):
+ m.validate_journal_config()
+ with mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, text + '\n[Journal]\nSystemMaxUse=1G\n')):
+ with self.assertRaisesRegex(RuntimeError, 'overrides SystemMaxUse'):
+ m.validate_journal_config()
+
+ @unittest.skipIf(os.name == 'nt', 'Requires POSIX nginx paths')
+ def test_nginx_modifies_existing_logger_without_adding_duplicate(self):
+ text = 'http {\n access_log /var/log/nginx/access.log;\n access_log /var/log/nginx/custom.log;\n}\n'
+ path = Path('/etc/nginx/nginx.conf')
+ with mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, '# configuration file /etc/nginx/nginx.conf:\n')), \
+ mock.patch.object(Path, 'resolve', return_value=path), \
+ mock.patch.object(Path, 'is_file', return_value=True), \
+ mock.patch.object(Path, 'read_text', return_value=text):
+ configs = m.nginx_policies()
+ changed = configs[str(path)]
+ self.assertEqual(changed.count('access_log '), 2)
+ self.assertIn('access.log combined if=$owrt_remote_access_log;', changed)
+ self.assertIn('access_log /var/log/nginx/custom.log;', changed)
+
+ @unittest.skipIf(os.name == 'nt', 'Requires POSIX nginx paths')
+ def test_existing_custom_nginx_condition_is_not_silently_overwritten(self):
+ with mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, '# configuration file /etc/nginx/nginx.conf:\n')), \
+ mock.patch.object(Path, 'resolve', return_value=Path('/etc/nginx/nginx.conf')), \
+ mock.patch.object(Path, 'is_file', return_value=True), \
+ mock.patch.object(Path, 'read_text', return_value='access_log /var/log/nginx/access.log combined if=$custom;'):
+ with self.assertRaisesRegex(RuntimeError, 'manual review'):
+ m.nginx_policies()
+
+ @unittest.skipIf(os.name == 'nt', 'Requires POSIX file permissions')
+ def test_result_log_rollover_is_bounded(self):
+ with tempfile.TemporaryDirectory() as temp:
+ path = Path(temp) / 'maintenance.log'
+ path.write_bytes(b'x' * 100000)
+ with mock.patch.object(m, 'RESULT', path), mock.patch('sys.stdout', new=io.StringIO()):
+ m.record('OK', 'done')
+ self.assertLess(path.stat().st_size, 200)
+ self.assertEqual(path.with_suffix('.log.1').stat().st_size, 65536)
+
+ @unittest.skipIf(os.name == 'nt', 'Requires fcntl lock')
+ def test_daily_guard_parallel_lock_and_failed_run_are_explicit(self):
+ import datetime
+ import fcntl
+ with tempfile.TemporaryDirectory() as temp:
+ state, runtime = Path(temp) / 'state', Path(temp) / 'run'
+ state.mkdir(); runtime.mkdir()
+ with mock.patch.object(m, 'STATE', state), mock.patch.object(m, 'RUN', runtime), \
+ mock.patch.object(m, 'record'), mock.patch('sys.stdout', new=io.StringIO()) as output, \
+ mock.patch.object(m, 'flatten_config', side_effect=RuntimeError('invalid policy')) as parse:
+ with (runtime / 'lock').open('a') as lock:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ m.run()
+ self.assertFalse(parse.called)
+ with self.assertRaisesRegex(RuntimeError, 'invalid policy'):
+ m.run()
+ self.assertIn('"ok": false', (state / 'result.json').read_text())
+ self.assertEqual((state / 'last-attempt').read_text().strip(), datetime.date.today().isoformat())
+ parse.reset_mock()
+ m.run()
+ self.assertFalse(parse.called)
+ self.assertIn('already', output.getvalue())
+ with self.assertRaisesRegex(RuntimeError, 'invalid policy'):
+ m.run(force=True)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_notification_load.py b/tests/test_notification_load.py
index 911258d..e3a2806 100644
--- a/tests/test_notification_load.py
+++ b/tests/test_notification_load.py
@@ -94,6 +94,40 @@ class NotificationLoadTests(unittest.TestCase):
self.assertIn("500", output.getvalue())
self.assertNotIn("200", output.getvalue())
+ def test_heartbeat_and_log_upload_keep_errors_debug_and_application_messages(self):
+ class Handler:
+ path = "/api/heartbeat?unused=1"
+ def address_string(self): return "127.0.0.1"
+ def log_date_time_string(self): return "now"
+ handler = Handler()
+ for path in ("/api/heartbeat", "/api/router-logs", "/api/ssh-session/test/read"):
+ handler.path = path + "?test=1"
+ with self.subTest(path=path), mock.patch.dict(os.environ, {"OWRT_REMOTE_HTTP_DEBUG": "0"}):
+ output = io.StringIO()
+ with mock.patch.object(self.hub.sys, "stderr", output):
+ for status in (200, 201, 204, 301, 304):
+ self.hub.Handler.log_message(handler, '"%s" %s %s', "POST /test HTTP/1.1", str(status), "0")
+ for status in (400, 401, 403, 429, 500, 503, 599):
+ self.hub.Handler.log_message(handler, '"%s" %s %s', "POST /test HTTP/1.1", str(status), "0")
+ self.hub.Handler.log_message(handler, "database failed: %s", "disk full")
+ self.hub.Handler.log_message(handler, "custom numeric error: %s %s", "bad", 200)
+ lines = output.getvalue().splitlines()
+ self.assertEqual(len(lines), 9)
+ self.assertIn("database failed", output.getvalue())
+ with mock.patch.dict(os.environ, {"OWRT_REMOTE_HTTP_DEBUG": "1"}), mock.patch.object(self.hub.sys, "stderr", output):
+ self.hub.Handler.log_message(handler, '"%s" %s %s', "POST /test HTTP/1.1", "200", "0")
+ self.assertEqual(len(output.getvalue().splitlines()), 10)
+
+ def test_malformed_http_request_without_path_is_still_logged(self):
+ class Handler:
+ def address_string(self): return '127.0.0.1'
+ def log_date_time_string(self): return 'now'
+ output = io.StringIO()
+ with mock.patch.object(self.hub.sys, 'stderr', output):
+ self.hub.Handler.log_message(Handler(), 'code %d, message %s', 400, 'Bad request syntax')
+ self.assertIn('400', output.getvalue())
+ self.assertIn('Bad request syntax', output.getvalue())
+
if __name__ == "__main__":
unittest.main()
diff --git a/vps/deploy-log-maintenance.ps1 b/vps/deploy-log-maintenance.ps1
new file mode 100644
index 0000000..16ba3f9
--- /dev/null
+++ b/vps/deploy-log-maintenance.ps1
@@ -0,0 +1,54 @@
+param(
+ [string]$Vps = 'root@132.243.19.232',
+ [ValidateSet('Install', 'Rollback', 'Status', 'Run')][string]$Action = 'Install'
+)
+
+$ErrorActionPreference = 'Stop'
+$RepoRoot = Split-Path -Parent $PSScriptRoot
+$UploadName = 'owrt-log-maintenance-' + [guid]::NewGuid().ToString('N')
+$StageRoot = Join-Path ([System.IO.Path]::GetTempPath()) $UploadName
+$Utf8 = [System.Text.UTF8Encoding]::new($false)
+$RemoteScript = @'
+#!/bin/sh
+set -eu
+umask 077
+stage=/tmp/__UPLOAD_NAME__
+trap 'rm -rf "$stage"' EXIT
+trap 'exit 1' INT TERM
+case '__ACTION__' in
+ Install)
+ command -v logrotate >/dev/null || { echo 'Install prerequisite first: apt-get install logrotate' >&2; exit 1; }
+ python3 "$stage/log-maintenance.py" install --hub-candidate "$stage/hub.py"
+ if ! systemctl start logrotate.service; then
+ journalctl -u logrotate.service -n 40 --no-pager >&2
+ exit 1
+ fi
+ python3 /opt/owrt-remote/log-maintenance.py status
+ ;;
+ Rollback) python3 "$stage/log-maintenance.py" rollback ;;
+ Status) python3 /opt/owrt-remote/log-maintenance.py status ;;
+ Run) python3 /opt/owrt-remote/log-maintenance.py run --force ;;
+esac
+echo 'LOG_MAINTENANCE___ACTION___OK'
+'@
+$RemoteScript = $RemoteScript.Replace('__UPLOAD_NAME__', $UploadName).Replace('__ACTION__', $Action)
+try {
+ [System.IO.Directory]::CreateDirectory($StageRoot) | Out-Null
+ [System.IO.File]::WriteAllText((Join-Path $StageRoot 'run.sh'), $RemoteScript.Replace("`r`n", "`n") + "`n", $Utf8)
+ foreach ($Pair in @(@('vps\log-maintenance.py', 'log-maintenance.py'), @('vps\owrt-remote-hub.py', 'hub.py'))) {
+ if ($Pair[1] -eq 'hub.py' -and $Action -ne 'Install') { continue }
+ $Content = [System.IO.File]::ReadAllText((Join-Path $RepoRoot $Pair[0])).Replace("`r`n", "`n")
+ [System.IO.File]::WriteAllText((Join-Path $StageRoot $Pair[1]), $Content, $Utf8)
+ }
+ & scp -O -r -- $StageRoot "${Vps}:/tmp/"
+ if ($LASTEXITCODE -ne 0) { throw 'Upload failed. VPS was not changed.' }
+ & ssh -T -o ConnectTimeout=15 $Vps "sh /tmp/$UploadName/run.sh"
+ if ($LASTEXITCODE -ne 0) { throw "$Action failed; success was not confirmed. See the error above." }
+ Write-Host "Log maintenance $Action finished successfully."
+}
+finally {
+ $Resolved = [System.IO.Path]::GetFullPath($StageRoot)
+ $Expected = [System.IO.Path]::GetFullPath((Join-Path ([System.IO.Path]::GetTempPath()) $UploadName))
+ if ($Resolved -ne $Expected -or (Split-Path -Leaf $Resolved) -ne $UploadName) { throw 'Unsafe staging cleanup path.' }
+ if (Test-Path -LiteralPath $Resolved) { Remove-Item -LiteralPath $Resolved -Recurse -Force }
+}
diff --git a/vps/install-vps.sh b/vps/install-vps.sh
index c1f1408..03cfe41 100755
--- a/vps/install-vps.sh
+++ b/vps/install-vps.sh
@@ -153,41 +153,20 @@ PY
}
install_log_limits() {
- $SUDO mkdir -p /etc/systemd/journald.conf.d /etc/nginx/conf.d
- $SUDO tee /etc/systemd/journald.conf.d/owrt-remote-limits.conf >/dev/null <<'EOF'
-[Journal]
-SystemMaxUse=256M
-SystemKeepFree=512M
-MaxRetentionSec=7day
-RateLimitIntervalSec=30s
-RateLimitBurst=2000
-Compress=yes
-EOF
- $SUDO tee /etc/nginx/conf.d/owrt-remote-log-policy.conf >/dev/null <<'EOF'
-# Successful long-poll and terminal reads are expected traffic.
-# Keep 4xx/5xx responses in access.log for diagnostics.
-map "$request_uri:$status" $owrt_remote_access_log {
- default 1;
- ~^/api/notifications/wait[^:]*:2[0-9][0-9]$ 0;
- ~^/api/ssh-session/[^/]+/read[^:]*:2[0-9][0-9]$ 0;
-}
-access_log /var/log/nginx/access.log combined if=$owrt_remote_access_log;
-EOF
- $SUDO tee /etc/logrotate.d/owrt-remote >/dev/null <<'EOF'
-/var/log/nginx/*.log {
- daily
- rotate 7
- size 50M
- missingok
- notifempty
- compress
- delaycompress
- copytruncate
-}
-EOF
- $SUDO systemctl daemon-reload >/dev/null 2>&1 || true
- $SUDO systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true
- $SUDO journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true
+ info "Настраиваю ежедневную ротацию журналов и защиту диска..."
+ if ! command -v logrotate >/dev/null 2>&1; then
+ apt_get install -y logrotate || die "не удалось установить logrotate"
+ fi
+ maintenance_candidate="$(mktemp /tmp/owrt-log-maintenance.XXXXXX.py)" || die "не удалось создать временный файл"
+ if ! curl -fsSL --max-time 120 -o "$maintenance_candidate" "$RAW_BASE/vps/log-maintenance.py?v=$(date +%s)"; then
+ rm -f "$maintenance_candidate"
+ die "не удалось загрузить обслуживание журналов"
+ fi
+ if ! $SUDO python3 "$maintenance_candidate" install; then
+ rm -f "$maintenance_candidate"
+ die "не удалось настроить обслуживание журналов; смотри причину выше"
+ fi
+ rm -f "$maintenance_candidate"
}
install_xray_service() {
@@ -329,12 +308,12 @@ main() {
install_xray_binary
install_files
install_python_deps_v2
- install_log_limits
install_xray_service
open_firewall
start_hub
check_hub || die "Hub установлен, но сервис не поднялся. Лог выше."
enable_https "$host"
+ install_log_limits
print_result "$host"
}
diff --git a/vps/log-maintenance.py b/vps/log-maintenance.py
new file mode 100644
index 0000000..f4d7003
--- /dev/null
+++ b/vps/log-maintenance.py
@@ -0,0 +1,563 @@
+#!/usr/bin/env python3
+"""Bounded VPS log maintenance. Never opens the Hub database or router archives.
+
+Uses the distribution's logrotate rules and reopen hooks, with retention
+overrides in a temporary configuration. One owner: system logrotate.timer.
+"""
+import argparse
+import ast
+import datetime as dt
+import glob
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shlex
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+
+MARKER = "# Managed by OpenWrt Remote log maintenance"
+INSTALL_PATH = Path("/opt/owrt-remote/log-maintenance.py")
+BACKUP = Path("/root/owrt-remote-backup-before-log-maintenance-v121")
+STATE = Path("/var/lib/owrt-remote-log-maintenance")
+RUN = Path("/run/owrt-remote-log-maintenance")
+RESULT = Path("/var/log/owrt-remote-maintenance.log")
+HUB = Path("/opt/owrt-remote/owrt-remote-hub.py")
+LEGACY = Path("/etc/logrotate.d/owrt-remote")
+POLICY = "\n daily\n maxsize 20M\n rotate 1\n maxage 1\n compress\n nodelaycompress\n"
+CONFIGS = {
+ "/etc/systemd/journald.conf.d/owrt-remote-limits.conf": MARKER + """
+[Journal]
+SystemMaxUse=50M
+SystemKeepFree=500M
+SystemMaxFileSize=10M
+RuntimeMaxUse=16M
+RuntimeKeepFree=64M
+MaxRetentionSec=7day
+Compress=yes
+""",
+ "/etc/systemd/system/logrotate.timer.d/owrt-remote.conf": MARKER + """
+[Timer]
+OnCalendar=
+OnCalendar=*-*-* 03:20:00
+AccuracySec=1min
+RandomizedDelaySec=0
+Persistent=true
+""",
+ "/etc/systemd/system/logrotate.service.d/owrt-remote.conf": MARKER + """
+[Service]
+ExecStart=
+ExecStart=/usr/bin/python3 /opt/owrt-remote/log-maintenance.py run
+Nice=10
+IOSchedulingClass=idle
+TimeoutStartSec=30min
+""",
+ "/etc/nginx/conf.d/owrt-remote-log-policy.conf": MARKER + """
+# Only routine successful traffic is quiet; errors and other actions remain.
+map "$uri:$status" $owrt_remote_access_log {
+ default 1;
+ ~^/api/notifications/wait:[23][0-9][0-9]$ 0;
+ ~^/api/ssh-session/[^/]+/read:[23][0-9][0-9]$ 0;
+ ~^/api/heartbeat:[23][0-9][0-9]$ 0;
+ ~^/api/router-logs:[23][0-9][0-9]$ 0;
+}
+""",
+}
+
+
+def atomic(path, data, mode=0o600):
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ if path.is_symlink():
+ raise RuntimeError("Refusing symlink: " + str(path))
+ fd, name = tempfile.mkstemp(prefix="." + path.name + ".", dir=path.parent)
+ try:
+ with os.fdopen(fd, "wb") as stream:
+ os.fchmod(stream.fileno(), mode)
+ stream.write(data if isinstance(data, bytes) else data.encode())
+ stream.flush()
+ os.fsync(stream.fileno())
+ os.replace(name, path)
+ finally:
+ if os.path.exists(name):
+ os.unlink(name)
+
+
+def command(args, timeout=120, check=True):
+ result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ timeout=timeout, text=True, errors="replace")
+ if check and result.returncode:
+ raise RuntimeError(f"{shlex.join(args)}: rc={result.returncode}; {result.stdout[-6000:]}")
+ return result
+
+
+def record(level, message):
+ """One small result file, at most 64 KiB plus one 64 KiB previous file."""
+ line = (dt.datetime.now().astimezone().isoformat(timespec="seconds") +
+ f" [{level}] " + str(message).replace("\n", " ")[:4000] + "\n").encode()
+ print(line.decode().rstrip(), flush=True)
+ if RESULT.is_symlink() or RESULT.with_suffix(".log.1").is_symlink():
+ raise RuntimeError("Unsafe maintenance log path")
+ if RESULT.exists() and RESULT.stat().st_size + len(line) > 65536:
+ with RESULT.open("rb") as stream:
+ stream.seek(max(0, os.fstat(stream.fileno()).st_size - 65536))
+ previous = stream.read(65536)
+ atomic(RESULT.with_suffix(".log.1"), previous)
+ # Keep the active inode available to readers.
+ with RESULT.open("r+b") as stream:
+ stream.truncate(0)
+ fd = os.open(RESULT, os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW, 0o600)
+ with os.fdopen(fd, "ab") as stream:
+ stream.write(line)
+
+
+def legacy_policy(path=LEGACY):
+ if not path.exists():
+ return False
+ if path.is_symlink():
+ return False
+ lines = [line.strip() for line in path.read_text().splitlines()
+ if line.strip() and not line.lstrip().startswith("#")]
+ expected = {"daily", "rotate 7", "size 50M", "missingok", "notifempty",
+ "compress", "delaycompress", "copytruncate"}
+ return (len(lines) == 10 and lines[0] == "/var/log/nginx/*.log {"
+ and lines[-1] == "}" and set(lines[1:-1]) == expected)
+
+
+def nginx_policies():
+ """Patch existing standard destinations; never add a second access logger.
+
+ sites-enabled entries are commonly symlinks: backup/edit their resolved
+ targets in /etc/nginx, leaving the symlinks themselves intact.
+ """
+ rendered = command(["nginx", "-T"]).stdout
+ configs = {}
+ for filename in re.findall(r"^# configuration file ([^\n]+):$", rendered, re.M):
+ path = Path(filename).resolve()
+ if str(path) == "/etc/nginx/conf.d/owrt-remote-log-policy.conf":
+ continue
+ if not str(path).startswith("/etc/nginx/") or not path.is_file():
+ raise RuntimeError("Unexpected nginx configuration: " + str(path))
+ text = path.read_text()
+ def add_condition(match):
+ directive = match.group(0)
+ if "$owrt_remote_access_log" in directive:
+ return directive
+ if re.search(r"\bif=", directive):
+ raise RuntimeError("Custom nginx log condition requires manual review: " + str(path))
+ # nginx needs an explicit format before optional parameters. Ubuntu's
+ # default is simply `access_log /var/log/nginx/access.log;`.
+ prefix = directive[:-1]
+ if len(shlex.split(prefix)) == 2:
+ prefix += " combined"
+ return prefix + " if=$owrt_remote_access_log;"
+ updated = re.sub(r"(?m)^\s*access_log\s+/var/log/nginx/access\.log(?:\s+[^;\n]+)?;", add_condition, text)
+ if updated != text:
+ configs[str(path)] = updated
+ return configs
+
+
+def flatten_config(path, seen=None):
+ """Read trusted distribution configs; never execute hooks during parsing."""
+ path = Path(path)
+ seen = set() if seen is None else seen
+ if path in seen or len(seen) > 128:
+ raise RuntimeError("Recursive or oversized logrotate include: " + str(path))
+ info = path.lstat()
+ if path.is_symlink() or not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+ raise RuntimeError("Untrusted logrotate config: " + str(path))
+ if info.st_size > 262144:
+ raise RuntimeError("Oversized logrotate config: " + str(path))
+ seen.add(path)
+ lines, script = [], False
+ for line in path.read_text().splitlines(keepends=True):
+ word = line.strip().split(" ", 1)[0]
+ if word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"):
+ script = True
+ elif word == "endscript":
+ script = False
+ if not script and re.match(r"^\s*include\s+", line):
+ parts = shlex.split(line, comments=True)
+ if len(parts) != 2 or not (parts[1] == "/etc/logrotate.conf" or parts[1] == "/etc/logrotate.d" or parts[1].startswith("/etc/logrotate.d/")):
+ raise RuntimeError("Unsupported logrotate include: " + line.strip())
+ include = Path(parts[1])
+ children = sorted(include.iterdir()) if include.is_dir() else [include]
+ for child in children:
+ if child.is_dir() or re.search(r"(?:~|\.(?:bak|old|disabled|dpkg-.+|rpmnew|rpmsave))$", child.name):
+ continue
+ if child == LEGACY and legacy_policy():
+ continue
+ lines.append(flatten_config(child, seen))
+ else:
+ lines.append(line)
+ seen.remove(path)
+ result = "".join(lines)
+ if len(result) > 1048576:
+ raise RuntimeError("Logrotate config exceeds 1 MiB")
+ return result + "\n"
+
+
+def retention_config(source):
+ """Keep native create/su/reopen hooks, override only retention per stanza.
+
+ Refuse ambiguous syntax and targets outside /var/log before any rotation.
+ In particular, no arbitrary service file in /var/lib can enter cleanup.
+ """
+ output, patterns, header, block = [], [], [], []
+ script, inside = False, False
+ for line in source.splitlines(keepends=True):
+ stripped = line.strip()
+ if inside:
+ word = stripped.split(" ", 1)[0]
+ if not script and word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"):
+ script = True
+ elif script and word == "endscript":
+ script = False
+ if not script and stripped == "}":
+ # olddir may put archives outside /var/log: refuse to clean there.
+ check_script = False
+ for item in block:
+ word = item.strip().split(" ", 1)[0]
+ if word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"):
+ check_script = True
+ elif word == "endscript":
+ check_script = False
+ if check_script:
+ continue
+ if re.match(r"\s*(?:olddir|include)\s+", item):
+ raise RuntimeError("olddir/include inside log stanza requires administrator review")
+ output.extend(block)
+ output.append(POLICY + "}\n")
+ inside, block = False, []
+ elif not script and ("{" in stripped or stripped.startswith("}")):
+ raise RuntimeError("Unsupported logrotate block syntax: " + stripped)
+ else:
+ block.append(line)
+ continue
+ if header or stripped.startswith(("/", '"', "'")):
+ header.append(line)
+ if "{" in line:
+ combined = "".join(header)
+ if combined.count("{") != 1 or combined.split("{", 1)[1].strip():
+ raise RuntimeError("Unsupported inline logrotate stanza")
+ targets = shlex.split(combined.split("{", 1)[0], comments=True)
+ for target in targets:
+ if not target.startswith("/var/log/") or ".." in Path(target).parts:
+ raise RuntimeError("Refusing log target outside /var/log: " + target)
+ patterns.extend(targets)
+ output.extend(header)
+ header, inside = [], True
+ else:
+ output.append(line)
+ if header or inside or script:
+ raise RuntimeError("Incomplete logrotate configuration")
+ return "".join(output), patterns
+
+
+def archive_name(active, candidate):
+ if not candidate.startswith(active):
+ return False
+ suffix = candidate[len(active):]
+ return bool(re.fullmatch(r"(?:\.\d+|\.old|[-.]\d{8}(?:[-.]\d{4,6})?)(?:\.(?:gz|xz|bz2|zst))?", suffix))
+
+
+def open_inodes():
+ opened = set()
+ for directory in Path("/proc").iterdir():
+ if not directory.name.isdigit():
+ continue
+ try:
+ for fd in (directory / "fd").iterdir():
+ try:
+ info = fd.stat()
+ opened.add((info.st_dev, info.st_ino))
+ except FileNotFoundError:
+ pass
+ except FileNotFoundError:
+ pass
+ return opened
+
+
+def prune_archives(active_paths, emergency=False, now=None, opened=None):
+ now = time.time() if now is None else now
+ opened = open_inodes() if opened is None else opened
+ removed, freed = 0, 0
+ for active in sorted(set(Path(p) for p in active_paths)):
+ if not active.is_file() or active.is_symlink() or active.resolve() != active:
+ continue
+ for candidate in active.parent.iterdir():
+ if not archive_name(active.name, candidate.name):
+ continue
+ info = candidate.lstat()
+ if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
+ continue
+ if not emergency and now - info.st_mtime <= 86400:
+ continue
+ if (info.st_dev, info.st_ino) in opened:
+ continue
+ latest = candidate.lstat()
+ if (latest.st_dev, latest.st_ino, latest.st_mtime_ns) != (info.st_dev, info.st_ino, info.st_mtime_ns):
+ continue
+ candidate.unlink() # Only verified closed archives, never active logs.
+ removed += 1
+ freed += info.st_size
+ return removed, freed
+
+
+def disk_levels():
+ result = []
+ for path in ("/", "/var/log"):
+ usage = shutil.disk_usage(path)
+ # Match df: blocks reserved for root are not available to log writers.
+ available_total = usage.used + usage.free
+ percent = 100 * usage.used / available_total if available_total else 100
+ level = "CRITICAL" if percent > 90 else "WARN" if percent > 75 else "OK"
+ result.append((path, percent, level))
+ return result
+
+
+def service_status():
+ for name in ("owrt-remote", "nginx", "rsyslog", "systemd-journald"):
+ result = command(["systemctl", "is-active", name], check=False)
+ print(f"{name}: {result.stdout.strip()}")
+
+
+def validate_journal_config():
+ effective, section = {}, ""
+ for line in command(["systemd-analyze", "cat-config", "systemd/journald.conf"]).stdout.splitlines():
+ text = line.strip()
+ if text.startswith("["):
+ section = text
+ elif section == "[Journal]" and not text.startswith(("#", ";")) and "=" in text:
+ key, value = text.split("=", 1)
+ effective[key.strip()] = value.strip()
+ for key, value in {"SystemMaxUse": "50M", "SystemKeepFree": "500M",
+ "MaxRetentionSec": "7day", "Compress": "yes"}.items():
+ if effective.get(key) != value:
+ raise RuntimeError("Another journald configuration overrides " + key + "; administrator review required")
+
+
+def run(force=False):
+ import fcntl
+ STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
+ RUN.mkdir(mode=0o700, parents=True, exist_ok=True)
+ with (RUN / "lock").open("a") as lock:
+ try:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except BlockingIOError:
+ print("Maintenance already running; skipped")
+ return
+ today = dt.date.today().isoformat()
+ stamp = STATE / "last-attempt"
+ if not force and stamp.exists() and stamp.read_text().strip() == today:
+ print("Daily attempt already made; use run --force to retry an error")
+ return
+ # Write before work: even a process crash cannot cause a second daily run.
+ atomic(stamp, today + "\n")
+ try:
+ config, patterns = retention_config(flatten_config("/etc/logrotate.conf"))
+ active = [Path(p) for pattern in patterns for p in glob.glob(pattern)]
+ emergency = any(percent > 85 for _, percent, _ in disk_levels())
+ before = disk_levels()
+ for path, percent, level in before:
+ record(level, f"before {path}: {percent:.1f}% used")
+ atomic(RUN / "logrotate.conf", config)
+ command(["logrotate", "--debug", str(RUN / "logrotate.conf")])
+ removed, freed = prune_archives(active, emergency=emergency)
+ # Same state/lock as distro logrotate, not a private competing state.
+ command(["logrotate", "--wait-for-state-lock", "--force", str(RUN / "logrotate.conf")], timeout=1500)
+ command(["journalctl", "--rotate"], timeout=60)
+ vacuum = command(["journalctl", "--vacuum-time=1d", "--vacuum-size=" + ("10M" if emergency else "50M")])
+ more, size = prune_archives(active, emergency=emergency)
+ removed += more
+ freed += size
+ after = disk_levels()
+ result = {"day": today, "ok": True, "archives_removed": removed,
+ "bytes_removed": freed, "emergency": emergency,
+ "disk_after": after, "finished_at": time.time()}
+ atomic(STATE / "result.json", json.dumps(result) + "\n")
+ record("OK", f"completed; archives_removed={removed}; bytes_removed={freed}; emergency={emergency}; {vacuum.stdout[-500:]}")
+ for path, percent, level in after:
+ record(level, f"after {path}: {percent:.1f}% used; database and user data excluded")
+ print(command(["journalctl", "--disk-usage"]).stdout.strip())
+ except Exception as exc:
+ atomic(STATE / "result.json", json.dumps({"day": today, "ok": False, "error": str(exc)[-6000:]}) + "\n")
+ record("ERROR", exc)
+ raise
+ finally:
+ (RUN / "logrotate.conf").unlink(missing_ok=True)
+
+
+def snapshot(paths, backup):
+ backup.mkdir(mode=0o700, parents=True, exist_ok=True)
+ manifest_path = backup / "manifest.json"
+ if manifest_path.exists():
+ manifest = json.loads(manifest_path.read_text())
+ else:
+ manifest = {"files": [], "timer_enabled": command(["systemctl", "is-enabled", "logrotate.timer"], check=False).stdout.strip(),
+ "timer_active": command(["systemctl", "is-active", "logrotate.timer"], check=False).stdout.strip()}
+ known = {item["path"] for item in manifest["files"]}
+ for path in paths:
+ path = Path(path)
+ if str(path) in known:
+ continue
+ if path.is_symlink():
+ raise RuntimeError("Refusing to replace symlink: " + str(path))
+ item = {"path": str(path), "exists": path.exists(), "copy": str(len(manifest["files"]))}
+ if path.exists():
+ info = path.stat()
+ item.update(mode=stat.S_IMODE(info.st_mode), uid=info.st_uid, gid=info.st_gid)
+ atomic(backup / item["copy"], path.read_bytes())
+ manifest["files"].append(item)
+ atomic(manifest_path, json.dumps(manifest))
+
+
+def restore(backup):
+ manifest = json.loads((backup / "manifest.json").read_text())
+ command(["systemctl", "stop", "logrotate.timer"])
+ for item in manifest["files"]:
+ path = Path(item["path"])
+ if item["exists"]:
+ atomic(path, (backup / item["copy"]).read_bytes(), item["mode"])
+ os.chown(path, item["uid"], item["gid"])
+ else:
+ path.unlink(missing_ok=True)
+ command(["systemctl", "daemon-reload"])
+ command(["systemctl", "restart", "systemd-journald"])
+ if shutil.which("nginx"):
+ command(["nginx", "-t"])
+ if command(["systemctl", "is-active", "nginx"], check=False).returncode == 0:
+ command(["systemctl", "reload", "nginx"])
+ if manifest["timer_enabled"] == "enabled":
+ command(["systemctl", "enable", "logrotate.timer"])
+ else:
+ command(["systemctl", "disable", "logrotate.timer"])
+ if manifest["timer_active"] == "active":
+ command(["systemctl", "start", "logrotate.timer"])
+ if any(item["path"] == str(HUB) for item in manifest["files"]):
+ command(["systemctl", "restart", "owrt-remote"])
+ print("Configuration restored. Already rotated/deleted archives cannot be restored.")
+
+
+def install(hub_candidate=None, backup=BACKUP):
+ for executable in ("logrotate", "journalctl", "systemctl", "systemd-analyze"):
+ if not shutil.which(executable):
+ raise RuntimeError("Missing prerequisite: " + executable)
+ if command(["systemctl", "is-active", "logrotate.service"], check=False).stdout.strip() in ("active", "activating"):
+ raise RuntimeError("Wait for the running system logrotate.service to finish")
+ if LEGACY.exists() and not legacy_policy():
+ raise RuntimeError("Custom /etc/logrotate.d/owrt-remote found; manual review required")
+ retention_config(flatten_config("/etc/logrotate.conf"))
+ if hub_candidate:
+ text = Path(hub_candidate).read_text()
+ ast.parse(text)
+ if "OWRT_REMOTE_HTTP_DEBUG" not in text:
+ raise RuntimeError("Hub candidate does not contain the HTTP log fix")
+ configs = dict(CONFIGS)
+ if not shutil.which("nginx"):
+ configs.pop("/etc/nginx/conf.d/owrt-remote-log-policy.conf")
+ else:
+ configs.update(nginx_policies())
+ paths = [*configs, str(INSTALL_PATH), str(LEGACY)] + ([str(HUB)] if hub_candidate else [])
+ snapshot(paths, backup)
+ try:
+ command(["systemctl", "stop", "logrotate.timer"])
+ for path, content in configs.items():
+ atomic(path, content, 0o644)
+ atomic(INSTALL_PATH, Path(__file__).read_bytes(), 0o755)
+ if legacy_policy():
+ LEGACY.unlink() # Old duplicate configuration, not a log file.
+ if hub_candidate:
+ atomic(HUB, Path(hub_candidate).read_bytes(), 0o755)
+ config, _ = retention_config(flatten_config("/etc/logrotate.conf"))
+ RUN.mkdir(mode=0o700, parents=True, exist_ok=True)
+ atomic(RUN / "validate.conf", config)
+ try:
+ command(["logrotate", "--debug", str(RUN / "validate.conf")])
+ finally:
+ (RUN / "validate.conf").unlink(missing_ok=True)
+ command(["systemctl", "daemon-reload"])
+ command(["systemd-analyze", "verify", "logrotate.service", "logrotate.timer"])
+ validate_journal_config()
+ if shutil.which("nginx"):
+ command(["nginx", "-t"])
+ if command(["systemctl", "is-active", "nginx"], check=False).returncode == 0:
+ command(["systemctl", "reload", "nginx"])
+ command(["systemctl", "restart", "systemd-journald"])
+ if hub_candidate:
+ command(["systemctl", "restart", "owrt-remote"])
+ check_health(hashlib.sha256(HUB.read_bytes()).hexdigest())
+ # Start the persistent timer only after main releases our lock. A missed
+ # calendar event can otherwise immediately run and be skipped as busy.
+ command(["systemctl", "enable", "logrotate.timer"])
+ print("Backup: " + str(backup))
+ print("LOG_MAINTENANCE_INSTALL_OK")
+ except Exception:
+ restore(backup)
+ raise
+
+
+def check_health(revision):
+ import urllib.request
+ # No proxy environment / shell wildcard expansion. Retry startup quietly.
+ client = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+ for attempt in range(15):
+ try:
+ with client.open("http://127.0.0.1:8088/health?panel-update=1", timeout=3) as response:
+ data = json.load(response)
+ if data.get("ok") and data.get("revision") == revision:
+ return
+ except (OSError, ValueError):
+ pass
+ time.sleep(1)
+ raise RuntimeError("Hub health check failed after restart")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("action", choices=("install", "rollback", "run", "status"))
+ parser.add_argument("--hub-candidate")
+ parser.add_argument("--force", action="store_true", help="Explicit manual retry; daily timer never uses this")
+ args = parser.parse_args()
+ if os.geteuid() != 0:
+ parser.error("Run as root")
+ if args.action in ("install", "rollback"):
+ import fcntl
+ RUN.mkdir(mode=0o700, parents=True, exist_ok=True)
+ with (RUN / "lock").open("a") as lock:
+ try:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except BlockingIOError:
+ raise RuntimeError("Log maintenance/install already running; retry after completion")
+ if args.action == "install":
+ install(args.hub_candidate)
+ else:
+ restore(BACKUP)
+ if args.action == "install":
+ try:
+ command(["systemctl", "start", "logrotate.timer"])
+ except Exception:
+ restore(BACKUP)
+ raise
+ elif args.action == "run":
+ run(args.force)
+ else:
+ print(command(["systemctl", "list-timers", "logrotate.timer", "--no-pager"]).stdout)
+ print(command(["journalctl", "--disk-usage"]).stdout)
+ print((STATE / "result.json").read_text() if (STATE / "result.json").exists() else "No maintenance result yet")
+ for path, percent, level in disk_levels():
+ print(level, path, f"{percent:.1f}%")
+ service_status()
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except Exception as error:
+ print("ERROR: " + str(error), file=sys.stderr)
+ sys.exit(1)
diff --git a/vps/owrt-remote-hub.py b/vps/owrt-remote-hub.py
index 8888443..3f8d2d9 100644
--- a/vps/owrt-remote-hub.py
+++ b/vps/owrt-remote-hub.py
@@ -16314,22 +16314,22 @@ class Handler(BaseHTTPRequestHandler):
return self.server.app
def log_message(self, fmt, *args):
- # Successful long-poll and terminal-read responses are expected to be
- # frequent. Nginx and journald already retain their failures; avoid
- # writing a line for every normal 200 response while keeping 4xx/5xx
- # diagnostics visible.
- path = urllib.parse.urlsplit(str(self.path or "")).path
+ # Only suppress ordinary successful access records. Application errors,
+ # audit events and 4xx/5xx responses must still reach stderr/journald.
+ path = urllib.parse.urlsplit(str(getattr(self, "path", "") or "")).path
status = 0
- if len(args) > 1:
+ if fmt == '"%s" %s %s' and len(args) == 3:
try:
status = int(str(args[1]))
except (TypeError, ValueError):
status = 0
quiet = (
path == "/api/notifications/wait"
+ or path in ("/api/heartbeat", "/api/router-logs")
or re.fullmatch(r"/api/ssh-session/[^/]+/read", path or "")
)
- if quiet and 200 <= status < 400:
+ debug = os.environ.get("OWRT_REMOTE_HTTP_DEBUG", "").lower() in ("1", "true", "yes")
+ if quiet and 200 <= status < 400 and not debug:
return
sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), fmt % args))