From 799ad8aad6478c6bc3bd7adb8e7466cb73da2b0e Mon Sep 17 00:00:00 2001 From: kzolotarev95 Date: Fri, 20 Nov 2026 00:31:35 +0400 Subject: [PATCH] Bound VPS log retention and quiet routine Hub traffic --- README.md | 6 + docs/VPS_LOG_MAINTENANCE.md | 195 +++++++++++ tests/test_log_maintenance.py | 189 +++++++++++ tests/test_notification_load.py | 34 ++ vps/deploy-log-maintenance.ps1 | 54 +++ vps/install-vps.sh | 51 +-- vps/log-maintenance.py | 563 ++++++++++++++++++++++++++++++++ vps/owrt-remote-hub.py | 14 +- 8 files changed, 1063 insertions(+), 43 deletions(-) create mode 100644 docs/VPS_LOG_MAINTENANCE.md create mode 100644 tests/test_log_maintenance.py create mode 100644 vps/deploy-log-maintenance.ps1 create mode 100644 vps/log-maintenance.py diff --git a/README.md b/README.md index e9f18ee..099db42 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,12 @@ Xray и недоступном SSH-туннеле, если агент прод Нужны обновлённые Hub и агент. Описание и проверка.

+

+Журналы VPS: ежедневная ротация в 03:20, ограничение journald, +фильтрация успешных частых HTTP-запросов и проверка заполнения диска. +Установка, проверка и откат. +

+ diff --git a/docs/VPS_LOG_MAINTENANCE.md b/docs/VPS_LOG_MAINTENANCE.md new file mode 100644 index 0000000..cc384ae --- /dev/null +++ b/docs/VPS_LOG_MAINTENANCE.md @@ -0,0 +1,195 @@ +# Обслуживание журналов VPS + +Исправление для v121. Оно меняет только HTTP-логирование Hub, +ротацию журналов и обслуживание диска. API, авторизация, Xray, heartbeat, +база данных и архивы журналов роутеров остаются в прежнем формате. + +## Причина + +`Handler.log_message()` фильтровал успешные long-poll и чтение SSH-терминала, +но записывал каждый успешный heartbeat и пакет `/api/router-logs`. +При большом количестве роутеров эти записи попадали в journald/syslog и Nginx. +Старая настройка Nginx могла добавлять второй `access_log`, сохраняя первый +без фильтра. Установщик также создавал правило `/etc/logrotate.d/owrt-remote` +для тех же файлов Nginx, которые уже обслуживает пакет Nginx. + +Частота SSH-перебора на рабочем VPS этим локальным тестом не установлена: +доступ к серверу и чтение его журналов не выполнялись. + +## Что установлено + +- Hub и стандартный `/var/log/nginx/access.log` пропускают успешные 2xx/3xx + для `/api/heartbeat`, `/api/router-logs`, `/api/notifications/wait` и + `/api/ssh-session/*/read`. HTTP 4xx/5xx, исключения и остальные действия + продолжают записываться. Содержимое доставленных журналов не фильтруется. +- Единственный планировщик — штатный `logrotate.timer`, ежедневно в **03:20 + по часовому поясу VPS**. `Persistent=true` запускает пропущенную задачу после + включения сервера. Отдельного cron или второго таймера нет. +- Штатные правила `/etc/logrotate.conf` и `/etc/logrotate.d/` сохраняют права, + владельцев и команды повторного открытия файлов Nginx/rsyslog. Временная + конфигурация добавляет `daily`, `maxsize 20M`, `rotate 1`, `maxage 1`, + `compress`, `nodelaycompress`. Активные журналы через `rm` не удаляются. +- Старые распознанные архивы настроенных журналов удаляются после 24 часов. + Очистка не представляет собой удаление всех `*.gz` на сервере: проверяются + имена архивов конкретных логов, тип файла, ссылки и открытые дескрипторы. +- journald: `SystemMaxUse=50M`, `SystemKeepFree=500M`, `MaxRetentionSec=7day`, + `Compress=yes`, файлы до 10 МиБ. Для непостоянного журнала задано 16 МиБ. + Ежедневно выполняются `journalctl --rotate` и очистка архивов старше суток + с ограничением 50 МиБ. Поэтому ежедневная политика строже предела 7 дней. +- Проверяются `/` и `/var/log`: предупреждение выше 75%, усиленная очистка + **закрытых архивов** выше 85%, критическое сообщение выше 90%. При усиленной + очистке допустимо удалить также свежие архивы, journald сокращается до 10 МиБ. +- Результат: `/var/lib/owrt-remote-log-maintenance/result.json`; собственный + журнал `/var/log/owrt-remote-maintenance.log` — до 64 КиБ плюс один архив + до 64 КиБ. Права состояния и резервных копий ограничены root. +- `flock` исключает параллельные обслуживание/установку/откат. Используется + также штатная блокировка состояния logrotate. Маркер попытки записывается + до работы; автоматический повтор в тот же день пропускается даже после + ошибки. Явный `Run` нужен для повторной попытки после исправления причины. + +`maxsize` проверяется при запуске logrotate, а не непрерывно. Ежедневная +задача не гарантирует, что сторонний сервис не заполнит диск между запусками. +Лимиты journald не ограничивают размер базы Hub или пользовательских файлов. + +## Установка подготовленной версии + +PowerShell Windows 11: + +```powershell +$Fix = "C:\Users\k.zolotarev95\Documents\openwrt overlay\vps\deploy-log-maintenance.ps1" +powershell.exe -NoProfile -ExecutionPolicy Bypass -File $Fix -Vps "root@132.243.19.232" -Action Install +``` + +Нужны Python 3, systemd, journald и установленный `logrotate`. Если последнего +нет, скрипт сообщит об этом до изменения конфигурации: + +```powershell +ssh root@132.243.19.232 'apt-get update && apt-get install -y logrotate' +``` + +Установка сохраняет исходные файлы, проверяет Python, Nginx, systemd и +`logrotate --debug`. При ошибке применения возвращает конфигурацию из копии. +Hub перезапускается; проверяется SHA256 именно запущенной сборки через `/health`. +Затем выполняется первое обслуживание. Если оно завершилось ошибкой, +изучить её и выполнить `Run` либо `Rollback`; уже удалённые архивы не вернуть. + +Повторный `Install` обновляет исправление, сохраняя первую резервную копию. +Основной `install-vps.sh` также подключает этот механизм после настройки HTTPS, +когда конфигурация Nginx уже существует. Для полного обновления существующего +VPS через GitHub с сохранением текущего логина выполнить на самом VPS: + +```sh +curl -fsSL "https://raw.githubusercontent.com/kzolotarev95/luci-app-owrt-remote/main/vps/install-vps.sh?v=$(date +%s)" | sudo env RESET_LOGIN=0 sh +``` + +Кнопка обновления панели заменяет только файл Hub. Для установки новых правил +обслуживания журналов нужен полный установщик выше или PowerShell-скрипт. + +Нестандартные правила с целями вне `/var/log`, `olddir`, неоднозначным синтаксисом +или пользовательским условием `access_log if=...` требуют ручного рассмотрения. +Установка остановится с причиной, не выполняя опасную очистку. Иные назначения +access_log, например отдельные пользовательские файлы, автоматически не меняются. + +## Проверка и ручной запуск + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File $Fix -Vps "root@132.243.19.232" -Action Status +powershell.exe -NoProfile -ExecutionPolicy Bypass -File $Fix -Vps "root@132.243.19.232" -Action Run +ssh root@132.243.19.232 'systemctl list-timers logrotate.timer --no-pager; journalctl -u logrotate.service -n 50 --no-pager; tail -n 20 /var/log/owrt-remote-maintenance.log; journalctl --disk-usage; nginx -t; systemctl is-active owrt-remote nginx rsyslog systemd-journald' +``` + +`Run` принудительно запускает ротацию повторно; для обычной проверки достаточно +`Status`. Не применять `Run` многократно без необходимости. + +Временно включить полный HTTP-лог Hub для диагностики можно через +`systemctl edit owrt-remote`, добавив: + +```ini +[Service] +Environment=OWRT_REMOTE_HTTP_DEBUG=1 +``` + +После изменения нужен `systemctl restart owrt-remote`. Для выключения удалить +эту строку из созданного drop-in и перезапустить Hub. На фильтр Nginx эта +переменная не влияет; ошибки Nginx всегда сохраняются. + +## SSH + +Проверить попытки входа без изменения доступа: + +```powershell +ssh root@132.243.19.232 'journalctl -u ssh -u sshd --since yesterday --no-pager | tail -n 100' +``` + +Возможные меры: ограничение SSH по доверенным адресам в firewall, Fail2ban, +`MaxAuthTries 3`, `LoginGraceTime 30`, `MaxStartups 10:30:30`. Подбирать после +просмотра реальной нагрузки и наличия резервного доступа. Перед применением +проверять `sshd -t`, сохранять текущий SSH-сеанс и проверять второй вход. +Исправление не меняет `PasswordAuthentication`, `PermitRootLogin`, SSH или firewall. + +## Откат + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File $Fix -Vps "root@132.243.19.232" -Action Rollback +``` + +Копия: `/root/owrt-remote-backup-before-log-maintenance-v121/manifest.json` +и файлы рядом. Откат восстанавливает исходный Hub, Nginx, настройки journald, +таймера и старое правило logrotate; возвращает прежнее состояние таймера. +База, сертификаты, SSH-ключи и данные пользователей не открываются обработчиком +очистки. Удалённые или уже ротированные журналы откат не восстанавливает. + +## Изменённые файлы и проверки + +- `vps/owrt-remote-hub.py` — фильтрация успешных HTTP-записей и DEBUG. +- `vps/log-maintenance.py` — настройка, обслуживание, проверки и откат. +- `vps/install-vps.sh` — подключение обслуживания вместо старых правил. +- `vps/deploy-log-maintenance.ps1` — установка локальной сборки и откат. +- `tests/test_log_maintenance.py`, `tests/test_notification_load.py` — регрессии. +- Этот документ — эксплуатация и ограничения. + +Тесты Python можно запустить отдельно: + +```powershell +python -m unittest discover -s tests -p test_log_maintenance.py -v +python -m unittest discover -s tests -p test_notification_load.py -v +``` + +Проверки POSIX-блокировок, ссылок и прав запускаются на Linux; на Windows +они явно помечены как пропущенные. Проверка на рабочем VPS выполняется после +установки пользователем; локальная Ubuntu не подтверждает состояние его диска +или фактическую интенсивность SSH-перебора. + +### Результаты локальной проверки + +На отдельной виртуальной машине Ubuntu 24.04 с настоящими systemd, logrotate, +Nginx, rsyslog и работающим Hub проверены: + +- 13 тестов обслуживания и 7 тестов HTTP-логирования — успешно на Linux. +- Установка, расписание 03:20, фактические настройки journald, ручное + обслуживание, повтор в тот же день и блокировка параллельного запуска. +- Успешные запросы к четырём частым API не попадают в access.log; + HTTP 404/503 и обычное действие остаются. Настоящий Hub возвращает и + записывает HTTP 401 при heartbeat без авторизации. +- Удаление старых архивов, сохранение постороннего `.gz`, запись новых событий + Nginx и rsyslog после ротации, ограничение собственного журнала. +- Переход со старого правила проекта и дублирующего access_log без сохранения + второго регистратора; исходные файлы возвращаются при откате. +- Повторная установка, ручной откат и автоматический откат при намеренно + ошибочной конфигурации Nginx. +- SQLite `integrity_check=ok`, тестовая запись в базе и SHA256 тестовых настроек, + токенов, сертификата, SSH-ключей и пользовательских файлов сохранены. +- Дополнительно 19 регрессий архива журналов роутеров и 31 проверка жизненного + цикла ресурсов прошли на Windows; 2 проверки POSIX в последнем наборе пропущены. +- Синтаксис Python, PowerShell, shell-установщика и `git diff --check` проверены. + +Пороги 75/85/90% проверяются с подменёнными показаниями диска в тестах: +заполнение настоящего раздела до 100% не выполнялось. Существующее предупреждение +Python `invalid escape sequence '\s'` в другом HTML-шаблоне Hub не относится к +этому исправлению и не мешает запуску; шаблон в рамках этой задачи не менялся. + +После установки пользователем на рабочий VPS получен успешный результат: +`ok=true`, удалены 263 архива, освобождены 2 365 938 байт, journald занимает +18,8 МиБ. Hub, Nginx, rsyslog и journald подтвердили состояние `active`. +Эта проверка выполнена по предоставленному выводу; независимое подключение +к рабочему серверу не выполнялось. diff --git a/tests/test_log_maintenance.py b/tests/test_log_maintenance.py new file mode 100644 index 0000000..e944cdb --- /dev/null +++ b/tests/test_log_maintenance.py @@ -0,0 +1,189 @@ +"""Retention policy, safe archive selection, disk levels and HTTP diagnostics.""" +import importlib.util +import io +import os +from pathlib import Path +import tempfile +import unittest +import subprocess +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("log_maintenance", ROOT / "vps/log-maintenance.py") +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) + + +class MaintenanceTests(unittest.TestCase): + def test_only_exact_old_project_rule_can_be_removed(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / 'owrt-remote' + rule = '/var/log/nginx/*.log {\n daily\n rotate 7\n size 50M\n missingok\n notifempty\n compress\n delaycompress\n copytruncate\n}\n' + path.write_text(rule) + self.assertTrue(m.legacy_policy(path)) + path.write_text(rule.replace('rotate 7', 'rotate 30')) + self.assertFalse(m.legacy_policy(path)) + path.write_text(rule.replace('copytruncate', 'postrotate\n echo custom hook\n endscript')) + self.assertFalse(m.legacy_policy(path)) + + def test_retention_preserves_native_reopen_hooks(self): + text = '''weekly +/var/log/syslog +/var/log/auth.log +{ + rotate 7 + weekly + delaycompress + postrotate + if true; then + echo "{ brace inside shell }" + fi + endscript +} +''' + result, patterns = m.retention_config(text) + self.assertEqual(patterns, ["/var/log/syslog", "/var/log/auth.log"]) + self.assertIn('echo "{ brace inside shell }"', result) + self.assertGreater(result.index("daily"), result.index("endscript")) + self.assertIn("maxsize 20M", result) + self.assertIn("nodelaycompress", result) + + def test_non_log_paths_and_ambiguous_rules_fail_before_rotation(self): + for source in ("/var/lib/owrt-remote/hub.db {\n}\n", "/var/log/../lib/data {\n}\n", + "/var/log/a { rotate 1 }\n", "/var/log/a {\nolddir /home/user\n}\n"): + with self.subTest(source=source), self.assertRaises(RuntimeError): + m.retention_config(source) + + def test_archive_names_do_not_match_active_or_user_files(self): + for suffix in (".1", ".20.gz", ".old", "-20261119.gz", ".20261119-032000.xz"): + self.assertTrue(m.archive_name("syslog", "syslog" + suffix), suffix) + for name in ("syslog", "syslog.db", "syslog.backup.tar.gz", "user.gz", "syslog.log", "syslog.old.key"): + self.assertFalse(m.archive_name("syslog", name), name) + + def test_prune_only_old_closed_archives_and_keep_active_inode(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + active = root / "access.log" + active.write_bytes(b"active") + old, fresh, opened, unrelated = [root / x for x in ("access.log.2.gz", "access.log.1", "access.log.old", "notes.gz")] + for p in (old, fresh, opened, unrelated): + p.write_bytes(b"archive") + for p in (old, opened, unrelated): + os.utime(p, (100, 100)) + st = opened.stat() + inode = active.stat().st_ino + removed, size = m.prune_archives([active], now=100000, opened={(st.st_dev, st.st_ino)}) + self.assertEqual((removed, size), (1, 7)) + self.assertFalse(old.exists()) + self.assertTrue(all(p.exists() for p in (active, fresh, opened, unrelated))) + self.assertEqual(active.stat().st_ino, inode) + self.assertEqual(m.prune_archives([active], emergency=True, opened={(st.st_dev, st.st_ino)})[0], 1) + self.assertTrue(opened.exists()) + + @unittest.skipIf(os.name == "nt", "Requires Unix link semantics") + def test_symlinks_and_hardlinks_are_not_deleted(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve(); active = root / "a.log"; active.write_text("active") + user = root / "user"; user.write_text("private") + (root / "a.log.1").symlink_to(user) + os.link(user, root / "a.log.2") + self.assertEqual(m.prune_archives([active], emergency=True, opened=set()), (0, 0)) + self.assertEqual(user.read_text(), "private") + + def test_disk_checks_root_and_separate_log_mount(self): + usage = type("Usage", (), {"total": 110, "used": 91, "free": 9})() + with mock.patch.object(m.shutil, "disk_usage", return_value=usage) as get: + levels = m.disk_levels() + self.assertEqual([x[0] for x in levels], ["/", "/var/log"]) + self.assertTrue(all(x[2] == "CRITICAL" for x in levels)) + self.assertTrue(all(x[1] == 91 for x in levels)) + self.assertEqual(get.call_count, 2) + for used, expected in ((75, 'OK'), (75.1, 'WARN'), (85, 'WARN'), (90, 'WARN'), (90.1, 'CRITICAL')): + usage = type('Usage', (), {'used': used, 'free': 100 - used})() + with self.subTest(used=used), mock.patch.object(m.shutil, 'disk_usage', return_value=usage): + self.assertEqual(m.disk_levels()[0][2], expected) + + @unittest.skipIf(os.name == 'nt', 'Requires fcntl lock') + def test_disk_pressure_selects_aggressive_archives_and_smaller_journal_budget(self): + for percent, emergency in ((84.9, False), (85, False), (85.1, True)): + with self.subTest(percent=percent), tempfile.TemporaryDirectory() as temp: + with mock.patch.object(m, 'STATE', Path(temp) / 'state'), \ + mock.patch.object(m, 'RUN', Path(temp) / 'run'), \ + mock.patch.object(m, 'flatten_config', return_value=''), \ + mock.patch.object(m, 'disk_levels', return_value=[('/', percent, 'WARN'), ('/var/log', 20, 'OK')]), \ + mock.patch.object(m, 'prune_archives', return_value=(0, 0)) as prune, \ + mock.patch.object(m, 'record'), mock.patch('sys.stdout', new=io.StringIO()), \ + mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, '')) as command: + m.run() + self.assertTrue(all(call.kwargs['emergency'] == emergency for call in prune.call_args_list)) + self.assertTrue(any('--vacuum-size=' + ('10M' if emergency else '50M') in call.args[0] for call in command.call_args_list)) + + def test_later_journald_override_cannot_silently_disable_limit(self): + text = '[Journal]\nSystemMaxUse=50M\nSystemKeepFree=500M\nMaxRetentionSec=7day\nCompress=yes\n' + with mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, text)): + m.validate_journal_config() + with mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, text + '\n[Journal]\nSystemMaxUse=1G\n')): + with self.assertRaisesRegex(RuntimeError, 'overrides SystemMaxUse'): + m.validate_journal_config() + + @unittest.skipIf(os.name == 'nt', 'Requires POSIX nginx paths') + def test_nginx_modifies_existing_logger_without_adding_duplicate(self): + text = 'http {\n access_log /var/log/nginx/access.log;\n access_log /var/log/nginx/custom.log;\n}\n' + path = Path('/etc/nginx/nginx.conf') + with mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, '# configuration file /etc/nginx/nginx.conf:\n')), \ + mock.patch.object(Path, 'resolve', return_value=path), \ + mock.patch.object(Path, 'is_file', return_value=True), \ + mock.patch.object(Path, 'read_text', return_value=text): + configs = m.nginx_policies() + changed = configs[str(path)] + self.assertEqual(changed.count('access_log '), 2) + self.assertIn('access.log combined if=$owrt_remote_access_log;', changed) + self.assertIn('access_log /var/log/nginx/custom.log;', changed) + + @unittest.skipIf(os.name == 'nt', 'Requires POSIX nginx paths') + def test_existing_custom_nginx_condition_is_not_silently_overwritten(self): + with mock.patch.object(m, 'command', return_value=subprocess.CompletedProcess([], 0, '# configuration file /etc/nginx/nginx.conf:\n')), \ + mock.patch.object(Path, 'resolve', return_value=Path('/etc/nginx/nginx.conf')), \ + mock.patch.object(Path, 'is_file', return_value=True), \ + mock.patch.object(Path, 'read_text', return_value='access_log /var/log/nginx/access.log combined if=$custom;'): + with self.assertRaisesRegex(RuntimeError, 'manual review'): + m.nginx_policies() + + @unittest.skipIf(os.name == 'nt', 'Requires POSIX file permissions') + def test_result_log_rollover_is_bounded(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / 'maintenance.log' + path.write_bytes(b'x' * 100000) + with mock.patch.object(m, 'RESULT', path), mock.patch('sys.stdout', new=io.StringIO()): + m.record('OK', 'done') + self.assertLess(path.stat().st_size, 200) + self.assertEqual(path.with_suffix('.log.1').stat().st_size, 65536) + + @unittest.skipIf(os.name == 'nt', 'Requires fcntl lock') + def test_daily_guard_parallel_lock_and_failed_run_are_explicit(self): + import datetime + import fcntl + with tempfile.TemporaryDirectory() as temp: + state, runtime = Path(temp) / 'state', Path(temp) / 'run' + state.mkdir(); runtime.mkdir() + with mock.patch.object(m, 'STATE', state), mock.patch.object(m, 'RUN', runtime), \ + mock.patch.object(m, 'record'), mock.patch('sys.stdout', new=io.StringIO()) as output, \ + mock.patch.object(m, 'flatten_config', side_effect=RuntimeError('invalid policy')) as parse: + with (runtime / 'lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + m.run() + self.assertFalse(parse.called) + with self.assertRaisesRegex(RuntimeError, 'invalid policy'): + m.run() + self.assertIn('"ok": false', (state / 'result.json').read_text()) + self.assertEqual((state / 'last-attempt').read_text().strip(), datetime.date.today().isoformat()) + parse.reset_mock() + m.run() + self.assertFalse(parse.called) + self.assertIn('already', output.getvalue()) + with self.assertRaisesRegex(RuntimeError, 'invalid policy'): + m.run(force=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_notification_load.py b/tests/test_notification_load.py index 911258d..e3a2806 100644 --- a/tests/test_notification_load.py +++ b/tests/test_notification_load.py @@ -94,6 +94,40 @@ class NotificationLoadTests(unittest.TestCase): self.assertIn("500", output.getvalue()) self.assertNotIn("200", output.getvalue()) + def test_heartbeat_and_log_upload_keep_errors_debug_and_application_messages(self): + class Handler: + path = "/api/heartbeat?unused=1" + def address_string(self): return "127.0.0.1" + def log_date_time_string(self): return "now" + handler = Handler() + for path in ("/api/heartbeat", "/api/router-logs", "/api/ssh-session/test/read"): + handler.path = path + "?test=1" + with self.subTest(path=path), mock.patch.dict(os.environ, {"OWRT_REMOTE_HTTP_DEBUG": "0"}): + output = io.StringIO() + with mock.patch.object(self.hub.sys, "stderr", output): + for status in (200, 201, 204, 301, 304): + self.hub.Handler.log_message(handler, '"%s" %s %s', "POST /test HTTP/1.1", str(status), "0") + for status in (400, 401, 403, 429, 500, 503, 599): + self.hub.Handler.log_message(handler, '"%s" %s %s', "POST /test HTTP/1.1", str(status), "0") + self.hub.Handler.log_message(handler, "database failed: %s", "disk full") + self.hub.Handler.log_message(handler, "custom numeric error: %s %s", "bad", 200) + lines = output.getvalue().splitlines() + self.assertEqual(len(lines), 9) + self.assertIn("database failed", output.getvalue()) + with mock.patch.dict(os.environ, {"OWRT_REMOTE_HTTP_DEBUG": "1"}), mock.patch.object(self.hub.sys, "stderr", output): + self.hub.Handler.log_message(handler, '"%s" %s %s', "POST /test HTTP/1.1", "200", "0") + self.assertEqual(len(output.getvalue().splitlines()), 10) + + def test_malformed_http_request_without_path_is_still_logged(self): + class Handler: + def address_string(self): return '127.0.0.1' + def log_date_time_string(self): return 'now' + output = io.StringIO() + with mock.patch.object(self.hub.sys, 'stderr', output): + self.hub.Handler.log_message(Handler(), 'code %d, message %s', 400, 'Bad request syntax') + self.assertIn('400', output.getvalue()) + self.assertIn('Bad request syntax', output.getvalue()) + if __name__ == "__main__": unittest.main() diff --git a/vps/deploy-log-maintenance.ps1 b/vps/deploy-log-maintenance.ps1 new file mode 100644 index 0000000..16ba3f9 --- /dev/null +++ b/vps/deploy-log-maintenance.ps1 @@ -0,0 +1,54 @@ +param( + [string]$Vps = 'root@132.243.19.232', + [ValidateSet('Install', 'Rollback', 'Status', 'Run')][string]$Action = 'Install' +) + +$ErrorActionPreference = 'Stop' +$RepoRoot = Split-Path -Parent $PSScriptRoot +$UploadName = 'owrt-log-maintenance-' + [guid]::NewGuid().ToString('N') +$StageRoot = Join-Path ([System.IO.Path]::GetTempPath()) $UploadName +$Utf8 = [System.Text.UTF8Encoding]::new($false) +$RemoteScript = @' +#!/bin/sh +set -eu +umask 077 +stage=/tmp/__UPLOAD_NAME__ +trap 'rm -rf "$stage"' EXIT +trap 'exit 1' INT TERM +case '__ACTION__' in + Install) + command -v logrotate >/dev/null || { echo 'Install prerequisite first: apt-get install logrotate' >&2; exit 1; } + python3 "$stage/log-maintenance.py" install --hub-candidate "$stage/hub.py" + if ! systemctl start logrotate.service; then + journalctl -u logrotate.service -n 40 --no-pager >&2 + exit 1 + fi + python3 /opt/owrt-remote/log-maintenance.py status + ;; + Rollback) python3 "$stage/log-maintenance.py" rollback ;; + Status) python3 /opt/owrt-remote/log-maintenance.py status ;; + Run) python3 /opt/owrt-remote/log-maintenance.py run --force ;; +esac +echo 'LOG_MAINTENANCE___ACTION___OK' +'@ +$RemoteScript = $RemoteScript.Replace('__UPLOAD_NAME__', $UploadName).Replace('__ACTION__', $Action) +try { + [System.IO.Directory]::CreateDirectory($StageRoot) | Out-Null + [System.IO.File]::WriteAllText((Join-Path $StageRoot 'run.sh'), $RemoteScript.Replace("`r`n", "`n") + "`n", $Utf8) + foreach ($Pair in @(@('vps\log-maintenance.py', 'log-maintenance.py'), @('vps\owrt-remote-hub.py', 'hub.py'))) { + if ($Pair[1] -eq 'hub.py' -and $Action -ne 'Install') { continue } + $Content = [System.IO.File]::ReadAllText((Join-Path $RepoRoot $Pair[0])).Replace("`r`n", "`n") + [System.IO.File]::WriteAllText((Join-Path $StageRoot $Pair[1]), $Content, $Utf8) + } + & scp -O -r -- $StageRoot "${Vps}:/tmp/" + if ($LASTEXITCODE -ne 0) { throw 'Upload failed. VPS was not changed.' } + & ssh -T -o ConnectTimeout=15 $Vps "sh /tmp/$UploadName/run.sh" + if ($LASTEXITCODE -ne 0) { throw "$Action failed; success was not confirmed. See the error above." } + Write-Host "Log maintenance $Action finished successfully." +} +finally { + $Resolved = [System.IO.Path]::GetFullPath($StageRoot) + $Expected = [System.IO.Path]::GetFullPath((Join-Path ([System.IO.Path]::GetTempPath()) $UploadName)) + if ($Resolved -ne $Expected -or (Split-Path -Leaf $Resolved) -ne $UploadName) { throw 'Unsafe staging cleanup path.' } + if (Test-Path -LiteralPath $Resolved) { Remove-Item -LiteralPath $Resolved -Recurse -Force } +} diff --git a/vps/install-vps.sh b/vps/install-vps.sh index c1f1408..03cfe41 100755 --- a/vps/install-vps.sh +++ b/vps/install-vps.sh @@ -153,41 +153,20 @@ PY } install_log_limits() { - $SUDO mkdir -p /etc/systemd/journald.conf.d /etc/nginx/conf.d - $SUDO tee /etc/systemd/journald.conf.d/owrt-remote-limits.conf >/dev/null <<'EOF' -[Journal] -SystemMaxUse=256M -SystemKeepFree=512M -MaxRetentionSec=7day -RateLimitIntervalSec=30s -RateLimitBurst=2000 -Compress=yes -EOF - $SUDO tee /etc/nginx/conf.d/owrt-remote-log-policy.conf >/dev/null <<'EOF' -# Successful long-poll and terminal reads are expected traffic. -# Keep 4xx/5xx responses in access.log for diagnostics. -map "$request_uri:$status" $owrt_remote_access_log { - default 1; - ~^/api/notifications/wait[^:]*:2[0-9][0-9]$ 0; - ~^/api/ssh-session/[^/]+/read[^:]*:2[0-9][0-9]$ 0; -} -access_log /var/log/nginx/access.log combined if=$owrt_remote_access_log; -EOF - $SUDO tee /etc/logrotate.d/owrt-remote >/dev/null <<'EOF' -/var/log/nginx/*.log { - daily - rotate 7 - size 50M - missingok - notifempty - compress - delaycompress - copytruncate -} -EOF - $SUDO systemctl daemon-reload >/dev/null 2>&1 || true - $SUDO systemctl reload-or-restart systemd-journald >/dev/null 2>&1 || true - $SUDO journalctl --vacuum-size=256M --vacuum-time=7d >/dev/null 2>&1 || true + info "Настраиваю ежедневную ротацию журналов и защиту диска..." + if ! command -v logrotate >/dev/null 2>&1; then + apt_get install -y logrotate || die "не удалось установить logrotate" + fi + maintenance_candidate="$(mktemp /tmp/owrt-log-maintenance.XXXXXX.py)" || die "не удалось создать временный файл" + if ! curl -fsSL --max-time 120 -o "$maintenance_candidate" "$RAW_BASE/vps/log-maintenance.py?v=$(date +%s)"; then + rm -f "$maintenance_candidate" + die "не удалось загрузить обслуживание журналов" + fi + if ! $SUDO python3 "$maintenance_candidate" install; then + rm -f "$maintenance_candidate" + die "не удалось настроить обслуживание журналов; смотри причину выше" + fi + rm -f "$maintenance_candidate" } install_xray_service() { @@ -329,12 +308,12 @@ main() { install_xray_binary install_files install_python_deps_v2 - install_log_limits install_xray_service open_firewall start_hub check_hub || die "Hub установлен, но сервис не поднялся. Лог выше." enable_https "$host" + install_log_limits print_result "$host" } diff --git a/vps/log-maintenance.py b/vps/log-maintenance.py new file mode 100644 index 0000000..f4d7003 --- /dev/null +++ b/vps/log-maintenance.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +"""Bounded VPS log maintenance. Never opens the Hub database or router archives. + +Uses the distribution's logrotate rules and reopen hooks, with retention +overrides in a temporary configuration. One owner: system logrotate.timer. +""" +import argparse +import ast +import datetime as dt +import glob +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import stat +import subprocess +import sys +import tempfile +import time + +MARKER = "# Managed by OpenWrt Remote log maintenance" +INSTALL_PATH = Path("/opt/owrt-remote/log-maintenance.py") +BACKUP = Path("/root/owrt-remote-backup-before-log-maintenance-v121") +STATE = Path("/var/lib/owrt-remote-log-maintenance") +RUN = Path("/run/owrt-remote-log-maintenance") +RESULT = Path("/var/log/owrt-remote-maintenance.log") +HUB = Path("/opt/owrt-remote/owrt-remote-hub.py") +LEGACY = Path("/etc/logrotate.d/owrt-remote") +POLICY = "\n daily\n maxsize 20M\n rotate 1\n maxage 1\n compress\n nodelaycompress\n" +CONFIGS = { + "/etc/systemd/journald.conf.d/owrt-remote-limits.conf": MARKER + """ +[Journal] +SystemMaxUse=50M +SystemKeepFree=500M +SystemMaxFileSize=10M +RuntimeMaxUse=16M +RuntimeKeepFree=64M +MaxRetentionSec=7day +Compress=yes +""", + "/etc/systemd/system/logrotate.timer.d/owrt-remote.conf": MARKER + """ +[Timer] +OnCalendar= +OnCalendar=*-*-* 03:20:00 +AccuracySec=1min +RandomizedDelaySec=0 +Persistent=true +""", + "/etc/systemd/system/logrotate.service.d/owrt-remote.conf": MARKER + """ +[Service] +ExecStart= +ExecStart=/usr/bin/python3 /opt/owrt-remote/log-maintenance.py run +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=30min +""", + "/etc/nginx/conf.d/owrt-remote-log-policy.conf": MARKER + """ +# Only routine successful traffic is quiet; errors and other actions remain. +map "$uri:$status" $owrt_remote_access_log { + default 1; + ~^/api/notifications/wait:[23][0-9][0-9]$ 0; + ~^/api/ssh-session/[^/]+/read:[23][0-9][0-9]$ 0; + ~^/api/heartbeat:[23][0-9][0-9]$ 0; + ~^/api/router-logs:[23][0-9][0-9]$ 0; +} +""", +} + + +def atomic(path, data, mode=0o600): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + raise RuntimeError("Refusing symlink: " + str(path)) + fd, name = tempfile.mkstemp(prefix="." + path.name + ".", dir=path.parent) + try: + with os.fdopen(fd, "wb") as stream: + os.fchmod(stream.fileno(), mode) + stream.write(data if isinstance(data, bytes) else data.encode()) + stream.flush() + os.fsync(stream.fileno()) + os.replace(name, path) + finally: + if os.path.exists(name): + os.unlink(name) + + +def command(args, timeout=120, check=True): + result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=timeout, text=True, errors="replace") + if check and result.returncode: + raise RuntimeError(f"{shlex.join(args)}: rc={result.returncode}; {result.stdout[-6000:]}") + return result + + +def record(level, message): + """One small result file, at most 64 KiB plus one 64 KiB previous file.""" + line = (dt.datetime.now().astimezone().isoformat(timespec="seconds") + + f" [{level}] " + str(message).replace("\n", " ")[:4000] + "\n").encode() + print(line.decode().rstrip(), flush=True) + if RESULT.is_symlink() or RESULT.with_suffix(".log.1").is_symlink(): + raise RuntimeError("Unsafe maintenance log path") + if RESULT.exists() and RESULT.stat().st_size + len(line) > 65536: + with RESULT.open("rb") as stream: + stream.seek(max(0, os.fstat(stream.fileno()).st_size - 65536)) + previous = stream.read(65536) + atomic(RESULT.with_suffix(".log.1"), previous) + # Keep the active inode available to readers. + with RESULT.open("r+b") as stream: + stream.truncate(0) + fd = os.open(RESULT, os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "ab") as stream: + stream.write(line) + + +def legacy_policy(path=LEGACY): + if not path.exists(): + return False + if path.is_symlink(): + return False + lines = [line.strip() for line in path.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#")] + expected = {"daily", "rotate 7", "size 50M", "missingok", "notifempty", + "compress", "delaycompress", "copytruncate"} + return (len(lines) == 10 and lines[0] == "/var/log/nginx/*.log {" + and lines[-1] == "}" and set(lines[1:-1]) == expected) + + +def nginx_policies(): + """Patch existing standard destinations; never add a second access logger. + + sites-enabled entries are commonly symlinks: backup/edit their resolved + targets in /etc/nginx, leaving the symlinks themselves intact. + """ + rendered = command(["nginx", "-T"]).stdout + configs = {} + for filename in re.findall(r"^# configuration file ([^\n]+):$", rendered, re.M): + path = Path(filename).resolve() + if str(path) == "/etc/nginx/conf.d/owrt-remote-log-policy.conf": + continue + if not str(path).startswith("/etc/nginx/") or not path.is_file(): + raise RuntimeError("Unexpected nginx configuration: " + str(path)) + text = path.read_text() + def add_condition(match): + directive = match.group(0) + if "$owrt_remote_access_log" in directive: + return directive + if re.search(r"\bif=", directive): + raise RuntimeError("Custom nginx log condition requires manual review: " + str(path)) + # nginx needs an explicit format before optional parameters. Ubuntu's + # default is simply `access_log /var/log/nginx/access.log;`. + prefix = directive[:-1] + if len(shlex.split(prefix)) == 2: + prefix += " combined" + return prefix + " if=$owrt_remote_access_log;" + updated = re.sub(r"(?m)^\s*access_log\s+/var/log/nginx/access\.log(?:\s+[^;\n]+)?;", add_condition, text) + if updated != text: + configs[str(path)] = updated + return configs + + +def flatten_config(path, seen=None): + """Read trusted distribution configs; never execute hooks during parsing.""" + path = Path(path) + seen = set() if seen is None else seen + if path in seen or len(seen) > 128: + raise RuntimeError("Recursive or oversized logrotate include: " + str(path)) + info = path.lstat() + if path.is_symlink() or not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022: + raise RuntimeError("Untrusted logrotate config: " + str(path)) + if info.st_size > 262144: + raise RuntimeError("Oversized logrotate config: " + str(path)) + seen.add(path) + lines, script = [], False + for line in path.read_text().splitlines(keepends=True): + word = line.strip().split(" ", 1)[0] + if word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"): + script = True + elif word == "endscript": + script = False + if not script and re.match(r"^\s*include\s+", line): + parts = shlex.split(line, comments=True) + if len(parts) != 2 or not (parts[1] == "/etc/logrotate.conf" or parts[1] == "/etc/logrotate.d" or parts[1].startswith("/etc/logrotate.d/")): + raise RuntimeError("Unsupported logrotate include: " + line.strip()) + include = Path(parts[1]) + children = sorted(include.iterdir()) if include.is_dir() else [include] + for child in children: + if child.is_dir() or re.search(r"(?:~|\.(?:bak|old|disabled|dpkg-.+|rpmnew|rpmsave))$", child.name): + continue + if child == LEGACY and legacy_policy(): + continue + lines.append(flatten_config(child, seen)) + else: + lines.append(line) + seen.remove(path) + result = "".join(lines) + if len(result) > 1048576: + raise RuntimeError("Logrotate config exceeds 1 MiB") + return result + "\n" + + +def retention_config(source): + """Keep native create/su/reopen hooks, override only retention per stanza. + + Refuse ambiguous syntax and targets outside /var/log before any rotation. + In particular, no arbitrary service file in /var/lib can enter cleanup. + """ + output, patterns, header, block = [], [], [], [] + script, inside = False, False + for line in source.splitlines(keepends=True): + stripped = line.strip() + if inside: + word = stripped.split(" ", 1)[0] + if not script and word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"): + script = True + elif script and word == "endscript": + script = False + if not script and stripped == "}": + # olddir may put archives outside /var/log: refuse to clean there. + check_script = False + for item in block: + word = item.strip().split(" ", 1)[0] + if word in ("firstaction", "prerotate", "postrotate", "lastaction", "preremove"): + check_script = True + elif word == "endscript": + check_script = False + if check_script: + continue + if re.match(r"\s*(?:olddir|include)\s+", item): + raise RuntimeError("olddir/include inside log stanza requires administrator review") + output.extend(block) + output.append(POLICY + "}\n") + inside, block = False, [] + elif not script and ("{" in stripped or stripped.startswith("}")): + raise RuntimeError("Unsupported logrotate block syntax: " + stripped) + else: + block.append(line) + continue + if header or stripped.startswith(("/", '"', "'")): + header.append(line) + if "{" in line: + combined = "".join(header) + if combined.count("{") != 1 or combined.split("{", 1)[1].strip(): + raise RuntimeError("Unsupported inline logrotate stanza") + targets = shlex.split(combined.split("{", 1)[0], comments=True) + for target in targets: + if not target.startswith("/var/log/") or ".." in Path(target).parts: + raise RuntimeError("Refusing log target outside /var/log: " + target) + patterns.extend(targets) + output.extend(header) + header, inside = [], True + else: + output.append(line) + if header or inside or script: + raise RuntimeError("Incomplete logrotate configuration") + return "".join(output), patterns + + +def archive_name(active, candidate): + if not candidate.startswith(active): + return False + suffix = candidate[len(active):] + return bool(re.fullmatch(r"(?:\.\d+|\.old|[-.]\d{8}(?:[-.]\d{4,6})?)(?:\.(?:gz|xz|bz2|zst))?", suffix)) + + +def open_inodes(): + opened = set() + for directory in Path("/proc").iterdir(): + if not directory.name.isdigit(): + continue + try: + for fd in (directory / "fd").iterdir(): + try: + info = fd.stat() + opened.add((info.st_dev, info.st_ino)) + except FileNotFoundError: + pass + except FileNotFoundError: + pass + return opened + + +def prune_archives(active_paths, emergency=False, now=None, opened=None): + now = time.time() if now is None else now + opened = open_inodes() if opened is None else opened + removed, freed = 0, 0 + for active in sorted(set(Path(p) for p in active_paths)): + if not active.is_file() or active.is_symlink() or active.resolve() != active: + continue + for candidate in active.parent.iterdir(): + if not archive_name(active.name, candidate.name): + continue + info = candidate.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + continue + if not emergency and now - info.st_mtime <= 86400: + continue + if (info.st_dev, info.st_ino) in opened: + continue + latest = candidate.lstat() + if (latest.st_dev, latest.st_ino, latest.st_mtime_ns) != (info.st_dev, info.st_ino, info.st_mtime_ns): + continue + candidate.unlink() # Only verified closed archives, never active logs. + removed += 1 + freed += info.st_size + return removed, freed + + +def disk_levels(): + result = [] + for path in ("/", "/var/log"): + usage = shutil.disk_usage(path) + # Match df: blocks reserved for root are not available to log writers. + available_total = usage.used + usage.free + percent = 100 * usage.used / available_total if available_total else 100 + level = "CRITICAL" if percent > 90 else "WARN" if percent > 75 else "OK" + result.append((path, percent, level)) + return result + + +def service_status(): + for name in ("owrt-remote", "nginx", "rsyslog", "systemd-journald"): + result = command(["systemctl", "is-active", name], check=False) + print(f"{name}: {result.stdout.strip()}") + + +def validate_journal_config(): + effective, section = {}, "" + for line in command(["systemd-analyze", "cat-config", "systemd/journald.conf"]).stdout.splitlines(): + text = line.strip() + if text.startswith("["): + section = text + elif section == "[Journal]" and not text.startswith(("#", ";")) and "=" in text: + key, value = text.split("=", 1) + effective[key.strip()] = value.strip() + for key, value in {"SystemMaxUse": "50M", "SystemKeepFree": "500M", + "MaxRetentionSec": "7day", "Compress": "yes"}.items(): + if effective.get(key) != value: + raise RuntimeError("Another journald configuration overrides " + key + "; administrator review required") + + +def run(force=False): + import fcntl + STATE.mkdir(mode=0o700, parents=True, exist_ok=True) + RUN.mkdir(mode=0o700, parents=True, exist_ok=True) + with (RUN / "lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("Maintenance already running; skipped") + return + today = dt.date.today().isoformat() + stamp = STATE / "last-attempt" + if not force and stamp.exists() and stamp.read_text().strip() == today: + print("Daily attempt already made; use run --force to retry an error") + return + # Write before work: even a process crash cannot cause a second daily run. + atomic(stamp, today + "\n") + try: + config, patterns = retention_config(flatten_config("/etc/logrotate.conf")) + active = [Path(p) for pattern in patterns for p in glob.glob(pattern)] + emergency = any(percent > 85 for _, percent, _ in disk_levels()) + before = disk_levels() + for path, percent, level in before: + record(level, f"before {path}: {percent:.1f}% used") + atomic(RUN / "logrotate.conf", config) + command(["logrotate", "--debug", str(RUN / "logrotate.conf")]) + removed, freed = prune_archives(active, emergency=emergency) + # Same state/lock as distro logrotate, not a private competing state. + command(["logrotate", "--wait-for-state-lock", "--force", str(RUN / "logrotate.conf")], timeout=1500) + command(["journalctl", "--rotate"], timeout=60) + vacuum = command(["journalctl", "--vacuum-time=1d", "--vacuum-size=" + ("10M" if emergency else "50M")]) + more, size = prune_archives(active, emergency=emergency) + removed += more + freed += size + after = disk_levels() + result = {"day": today, "ok": True, "archives_removed": removed, + "bytes_removed": freed, "emergency": emergency, + "disk_after": after, "finished_at": time.time()} + atomic(STATE / "result.json", json.dumps(result) + "\n") + record("OK", f"completed; archives_removed={removed}; bytes_removed={freed}; emergency={emergency}; {vacuum.stdout[-500:]}") + for path, percent, level in after: + record(level, f"after {path}: {percent:.1f}% used; database and user data excluded") + print(command(["journalctl", "--disk-usage"]).stdout.strip()) + except Exception as exc: + atomic(STATE / "result.json", json.dumps({"day": today, "ok": False, "error": str(exc)[-6000:]}) + "\n") + record("ERROR", exc) + raise + finally: + (RUN / "logrotate.conf").unlink(missing_ok=True) + + +def snapshot(paths, backup): + backup.mkdir(mode=0o700, parents=True, exist_ok=True) + manifest_path = backup / "manifest.json" + if manifest_path.exists(): + manifest = json.loads(manifest_path.read_text()) + else: + manifest = {"files": [], "timer_enabled": command(["systemctl", "is-enabled", "logrotate.timer"], check=False).stdout.strip(), + "timer_active": command(["systemctl", "is-active", "logrotate.timer"], check=False).stdout.strip()} + known = {item["path"] for item in manifest["files"]} + for path in paths: + path = Path(path) + if str(path) in known: + continue + if path.is_symlink(): + raise RuntimeError("Refusing to replace symlink: " + str(path)) + item = {"path": str(path), "exists": path.exists(), "copy": str(len(manifest["files"]))} + if path.exists(): + info = path.stat() + item.update(mode=stat.S_IMODE(info.st_mode), uid=info.st_uid, gid=info.st_gid) + atomic(backup / item["copy"], path.read_bytes()) + manifest["files"].append(item) + atomic(manifest_path, json.dumps(manifest)) + + +def restore(backup): + manifest = json.loads((backup / "manifest.json").read_text()) + command(["systemctl", "stop", "logrotate.timer"]) + for item in manifest["files"]: + path = Path(item["path"]) + if item["exists"]: + atomic(path, (backup / item["copy"]).read_bytes(), item["mode"]) + os.chown(path, item["uid"], item["gid"]) + else: + path.unlink(missing_ok=True) + command(["systemctl", "daemon-reload"]) + command(["systemctl", "restart", "systemd-journald"]) + if shutil.which("nginx"): + command(["nginx", "-t"]) + if command(["systemctl", "is-active", "nginx"], check=False).returncode == 0: + command(["systemctl", "reload", "nginx"]) + if manifest["timer_enabled"] == "enabled": + command(["systemctl", "enable", "logrotate.timer"]) + else: + command(["systemctl", "disable", "logrotate.timer"]) + if manifest["timer_active"] == "active": + command(["systemctl", "start", "logrotate.timer"]) + if any(item["path"] == str(HUB) for item in manifest["files"]): + command(["systemctl", "restart", "owrt-remote"]) + print("Configuration restored. Already rotated/deleted archives cannot be restored.") + + +def install(hub_candidate=None, backup=BACKUP): + for executable in ("logrotate", "journalctl", "systemctl", "systemd-analyze"): + if not shutil.which(executable): + raise RuntimeError("Missing prerequisite: " + executable) + if command(["systemctl", "is-active", "logrotate.service"], check=False).stdout.strip() in ("active", "activating"): + raise RuntimeError("Wait for the running system logrotate.service to finish") + if LEGACY.exists() and not legacy_policy(): + raise RuntimeError("Custom /etc/logrotate.d/owrt-remote found; manual review required") + retention_config(flatten_config("/etc/logrotate.conf")) + if hub_candidate: + text = Path(hub_candidate).read_text() + ast.parse(text) + if "OWRT_REMOTE_HTTP_DEBUG" not in text: + raise RuntimeError("Hub candidate does not contain the HTTP log fix") + configs = dict(CONFIGS) + if not shutil.which("nginx"): + configs.pop("/etc/nginx/conf.d/owrt-remote-log-policy.conf") + else: + configs.update(nginx_policies()) + paths = [*configs, str(INSTALL_PATH), str(LEGACY)] + ([str(HUB)] if hub_candidate else []) + snapshot(paths, backup) + try: + command(["systemctl", "stop", "logrotate.timer"]) + for path, content in configs.items(): + atomic(path, content, 0o644) + atomic(INSTALL_PATH, Path(__file__).read_bytes(), 0o755) + if legacy_policy(): + LEGACY.unlink() # Old duplicate configuration, not a log file. + if hub_candidate: + atomic(HUB, Path(hub_candidate).read_bytes(), 0o755) + config, _ = retention_config(flatten_config("/etc/logrotate.conf")) + RUN.mkdir(mode=0o700, parents=True, exist_ok=True) + atomic(RUN / "validate.conf", config) + try: + command(["logrotate", "--debug", str(RUN / "validate.conf")]) + finally: + (RUN / "validate.conf").unlink(missing_ok=True) + command(["systemctl", "daemon-reload"]) + command(["systemd-analyze", "verify", "logrotate.service", "logrotate.timer"]) + validate_journal_config() + if shutil.which("nginx"): + command(["nginx", "-t"]) + if command(["systemctl", "is-active", "nginx"], check=False).returncode == 0: + command(["systemctl", "reload", "nginx"]) + command(["systemctl", "restart", "systemd-journald"]) + if hub_candidate: + command(["systemctl", "restart", "owrt-remote"]) + check_health(hashlib.sha256(HUB.read_bytes()).hexdigest()) + # Start the persistent timer only after main releases our lock. A missed + # calendar event can otherwise immediately run and be skipped as busy. + command(["systemctl", "enable", "logrotate.timer"]) + print("Backup: " + str(backup)) + print("LOG_MAINTENANCE_INSTALL_OK") + except Exception: + restore(backup) + raise + + +def check_health(revision): + import urllib.request + # No proxy environment / shell wildcard expansion. Retry startup quietly. + client = urllib.request.build_opener(urllib.request.ProxyHandler({})) + for attempt in range(15): + try: + with client.open("http://127.0.0.1:8088/health?panel-update=1", timeout=3) as response: + data = json.load(response) + if data.get("ok") and data.get("revision") == revision: + return + except (OSError, ValueError): + pass + time.sleep(1) + raise RuntimeError("Hub health check failed after restart") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("install", "rollback", "run", "status")) + parser.add_argument("--hub-candidate") + parser.add_argument("--force", action="store_true", help="Explicit manual retry; daily timer never uses this") + args = parser.parse_args() + if os.geteuid() != 0: + parser.error("Run as root") + if args.action in ("install", "rollback"): + import fcntl + RUN.mkdir(mode=0o700, parents=True, exist_ok=True) + with (RUN / "lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise RuntimeError("Log maintenance/install already running; retry after completion") + if args.action == "install": + install(args.hub_candidate) + else: + restore(BACKUP) + if args.action == "install": + try: + command(["systemctl", "start", "logrotate.timer"]) + except Exception: + restore(BACKUP) + raise + elif args.action == "run": + run(args.force) + else: + print(command(["systemctl", "list-timers", "logrotate.timer", "--no-pager"]).stdout) + print(command(["journalctl", "--disk-usage"]).stdout) + print((STATE / "result.json").read_text() if (STATE / "result.json").exists() else "No maintenance result yet") + for path, percent, level in disk_levels(): + print(level, path, f"{percent:.1f}%") + service_status() + + +if __name__ == "__main__": + try: + main() + except Exception as error: + print("ERROR: " + str(error), file=sys.stderr) + sys.exit(1) diff --git a/vps/owrt-remote-hub.py b/vps/owrt-remote-hub.py index 8888443..3f8d2d9 100644 --- a/vps/owrt-remote-hub.py +++ b/vps/owrt-remote-hub.py @@ -16314,22 +16314,22 @@ class Handler(BaseHTTPRequestHandler): return self.server.app def log_message(self, fmt, *args): - # Successful long-poll and terminal-read responses are expected to be - # frequent. Nginx and journald already retain their failures; avoid - # writing a line for every normal 200 response while keeping 4xx/5xx - # diagnostics visible. - path = urllib.parse.urlsplit(str(self.path or "")).path + # Only suppress ordinary successful access records. Application errors, + # audit events and 4xx/5xx responses must still reach stderr/journald. + path = urllib.parse.urlsplit(str(getattr(self, "path", "") or "")).path status = 0 - if len(args) > 1: + if fmt == '"%s" %s %s' and len(args) == 3: try: status = int(str(args[1])) except (TypeError, ValueError): status = 0 quiet = ( path == "/api/notifications/wait" + or path in ("/api/heartbeat", "/api/router-logs") or re.fullmatch(r"/api/ssh-session/[^/]+/read", path or "") ) - if quiet and 200 <= status < 400: + debug = os.environ.get("OWRT_REMOTE_HTTP_DEBUG", "").lower() in ("1", "true", "yes") + if quiet and 200 <= status < 400 and not debug: return sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), fmt % args))