Install log maintenance through Hub self-update

This commit is contained in:
kzolotarev95 committed 2026-11-20 01:10:56 +04:00
1 parent 799ad8aad6
commit 780d036a9c
8 files changed
+536 -7

No files matched your search

+2 -1
View File
@@ -99,7 +99,8 @@ class LocalTestTests(unittest.TestCase):
self.start_server()
return SimpleNamespace(returncode=0)
with mock.patch.object(hub, 'hub_update_lock', side_effect=lambda *a, **k: nullcontext()), \
mock.patch.object(hub.subprocess, 'run', side_effect=restart), mock.patch.object(hub.time, 'sleep'):
mock.patch.object(hub.subprocess, 'run', side_effect=restart), mock.patch.object(hub.time, 'sleep'), \
mock.patch.object(hub, 'hub_install_log_maintenance'), mock.patch.object(hub, 'hub_prune_log_maintenance_backups'):
self.assertEqual(hub.run_hub_update_worker(job['id']), 0)
self.assertEqual(self.target.read_bytes(), (self.fixture / 'candidate.py').read_bytes())
new_manager = self.server.app.hub_update
+58
View File
@@ -46,6 +46,9 @@ class UpdateTests(unittest.TestCase):
self.manager = hub.HubUpdateManager(self.target, self.directory)
self.addCleanup(mock.patch.stopall)
mock.patch.object(hub, 'hub_update_lock', side_effect=lambda *a, **k: nullcontext()).start()
self.install_logs = mock.patch.object(hub, 'hub_install_log_maintenance').start()
self.rollback_logs = mock.patch.object(hub, 'hub_rollback_log_maintenance').start()
mock.patch.object(hub, 'hub_prune_log_maintenance_backups').start()
def response(self, data=NEW, url=None):
response = io.BytesIO(data)
@@ -164,6 +167,7 @@ class UpdateTests(unittest.TestCase):
self.assertEqual(manager_after_restart.snapshot()['job']['status'], 'succeeded')
self.assertFalse((self.directory / 'candidate.py').exists())
self.assertEqual(run.call_args.args[0], ['systemctl', 'restart', 'owrt-remote'])
self.install_logs.assert_called_once_with(self.directory, job['id'], source=NEW)
def test_failed_health_restores_previous_bytes_and_restarts(self):
job = self.prepare()
@@ -172,6 +176,7 @@ class UpdateTests(unittest.TestCase):
self.assertEqual(hub.run_hub_update_worker(job['id'], self.directory, self.target), 1)
self.assertEqual(self.target.read_bytes(), SOURCE)
self.assertEqual(run.call_count, 2)
self.rollback_logs.assert_called_once_with(self.directory, job['id'])
self.assertIn('Предыдущий файл восстановлен', hub.hub_update_state(self.directory)['message'])
def test_failure_to_write_rollback_progress_does_not_prevent_restoration(self):
@@ -222,6 +227,53 @@ class UpdateTests(unittest.TestCase):
self.assertEqual(self.target.read_bytes(), SOURCE)
self.assertFalse(list(self.root.glob('.hub.py-*')))
def test_failed_maintenance_does_not_replace_or_restart_hub(self):
job = self.prepare()
self.install_logs.side_effect = RuntimeError('nginx validation failed')
with mock.patch.object(hub.time, 'sleep'), mock.patch.object(hub.subprocess, 'run') as run:
self.assertEqual(hub.run_hub_update_worker(job['id'], self.directory, self.target), 1)
self.assertEqual(self.target.read_bytes(), SOURCE)
self.assertFalse(run.called)
self.assertIn('nginx validation failed', hub.hub_update_state(self.directory)['message'])
def test_embedded_payload_matches_source_and_tampering_is_rejected(self):
actual, digest = hub.hub_log_maintenance_payload(SOURCE)
self.assertEqual(actual, (ROOT / 'vps/log-maintenance.py').read_bytes().replace(b'\r\n', b'\n'))
self.assertEqual(hashlib.sha256(actual).hexdigest(), digest)
with self.assertRaisesRegex(ValueError, 'сумма'):
hub.hub_log_maintenance_payload(SOURCE.replace(digest.encode(), b'0' * 64, 1))
def test_first_update_from_old_worker_waits_for_maintenance(self):
job = {'id': 'a' * 32, 'status': 'restarting', 'revision': hub.HUB_LOADED_REVISION,
'started_at': hub.time.time()}
hub.hub_update_state(self.directory, job)
self.install_logs.side_effect = None
captured = {}
def thread(**kwargs):
captured['install'] = kwargs['target']
return mock.Mock()
with mock.patch.object(hub.threading, 'Thread', side_effect=thread):
self.manager.resume_log_maintenance()
self.assertEqual(self.manager.log_maintenance_status, 'installing')
self.assertEqual(self.manager.snapshot()['job']['status'], 'installing')
captured['install']()
self.assertEqual(self.manager.log_maintenance_status, 'ready')
self.install_logs.assert_called_once_with(self.directory, job['id'], budget=40)
self.install_logs.side_effect = RuntimeError('disk full')
captured['install']()
self.assertEqual(self.manager.snapshot()['job']['status'], 'failed')
self.assertIn('disk full', self.manager.snapshot()['job']['message'])
def test_new_worker_receipt_prevents_duplicate_startup_installation(self):
job = {'id': 'b' * 32, 'status': 'restarting', 'revision': hub.HUB_LOADED_REVISION,
'started_at': hub.time.time()}
hub.hub_update_state(self.directory, job)
with mock.patch.object(hub, 'hub_log_maintenance_receipt', return_value={
'status': 'ready', 'sha256': hub.HUB_LOG_MAINTENANCE_SHA256}), \
mock.patch.object(hub.threading, 'Thread') as thread:
self.manager.resume_log_maintenance()
self.assertFalse(thread.called)
class HttpTests(unittest.TestCase):
def test_owner_csrf_origin_validation_and_revision_health(self):
@@ -257,6 +309,12 @@ class HttpTests(unittest.TestCase):
start.assert_called_once_with('abc')
self.assertEqual(request('/health')[1], {'ok': True})
self.assertEqual(request('/health?panel-update=1')[1]['revision'], hub.HUB_LOADED_REVISION)
app.hub_update.log_maintenance_status = 'installing'
self.assertTrue(request('/health')[1]['ok'], 'normal heartbeat health is independent')
self.assertFalse(request('/health?panel-update=1')[1]['ok'])
app.hub_update.log_maintenance_status = 'failed'
app.hub_update.log_maintenance_error = 'test maintenance failure'
self.assertEqual(request('/health?panel-update=1')[1]['log_maintenance']['error'], 'test maintenance failure')
finally:
server.shutdown()
server.server_close()
+97
View File
@@ -0,0 +1,97 @@
"""The button's maintenance transaction, per-update rollback and bounded backups."""
import datetime
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location('component_update_hub', ROOT / 'vps/owrt-remote-hub.py')
hub = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = hub
with mock.patch.dict(os.environ, {'OWRT_REMOTE_STATE_DIR': tempfile.gettempdir()}):
spec.loader.exec_module(hub)
class ComponentTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.directory = Path(self.temp.name)
self.job_id = 'c' * 32
self.result = self.directory / 'result.json'
self.result.write_text(json.dumps({'day': datetime.date.today().isoformat(), 'ok': True}))
self.addCleanup(mock.patch.stopall)
mock.patch.object(hub, 'HUB_LOG_MAINTENANCE_RESULT', self.result).start()
mock.patch.object(hub.shutil, 'which', return_value='/usr/bin/command').start()
def test_success_verifies_cleanup_timer_and_exact_payload(self):
with mock.patch.object(hub, 'hub_log_maintenance_command') as command:
receipt = hub.hub_install_log_maintenance(self.directory, self.job_id)
stage = hub.hub_log_maintenance_directory(self.directory, self.job_id)
self.assertEqual((stage / 'log-maintenance.py').read_bytes(), hub.hub_log_maintenance_payload()[0])
calls = [call.args[0] for call in command.call_args_list]
self.assertEqual(calls[0][-3:], ['install', '--backup', str(stage / 'backup')])
self.assertEqual(calls[1][-1], 'run')
self.assertEqual(calls[-2:], [['systemctl', 'is-active', 'logrotate.timer'], ['systemctl', 'is-enabled', 'logrotate.timer']])
self.assertEqual(receipt['status'], 'ready')
self.assertEqual(hub.hub_log_maintenance_receipt(self.directory, self.job_id)['status'], 'ready')
self.assertFalse(any('owrt-remote' in call for call in calls), 'component installation never stops heartbeat/Hub')
def test_failure_restores_only_this_updates_snapshot_and_records_real_reason(self):
calls = []
def run(args, deadline):
calls.append(args)
if 'install' in args:
backup = Path(args[-1]); backup.mkdir()
(backup / 'manifest.json').write_text('{}')
if args[-1] == 'run':
raise RuntimeError('logrotate failed: disk full')
with mock.patch.object(hub, 'hub_log_maintenance_command', side_effect=run):
with self.assertRaisesRegex(RuntimeError, 'disk full'):
hub.hub_install_log_maintenance(self.directory, self.job_id)
self.assertIn('rollback', calls[-1])
self.assertIn(self.job_id, calls[-1][-1])
receipt = hub.hub_log_maintenance_receipt(self.directory, self.job_id)
self.assertEqual(receipt['status'], 'failed')
self.assertIn('disk full', receipt['error'])
def test_failed_or_old_cleanup_result_cannot_be_reported_as_success(self):
for result in ({'ok': False, 'error': 'UBIFS error'}, {'ok': True, 'day': '2000-01-01'}):
self.result.write_text(json.dumps(result))
with self.subTest(result=result), mock.patch.object(hub, 'hub_log_maintenance_command'):
with self.assertRaisesRegex(RuntimeError, 'не подтверждено'):
hub.hub_install_log_maintenance(self.directory, self.job_id)
def test_missing_logrotate_uses_bounded_package_installation(self):
with mock.patch.object(hub.shutil, 'which', side_effect=lambda name: None if name == 'logrotate' else '/usr/bin/apt-get'), \
mock.patch.object(hub, 'hub_log_maintenance_command') as command:
hub.hub_install_log_maintenance(self.directory, self.job_id)
args = command.call_args_list[0].args[0]
self.assertEqual(args, ['apt-get', '-o', 'DPkg::Lock::Timeout=5', 'install', '-y', 'logrotate'])
def test_command_errors_include_stderr_and_timeouts_are_bounded(self):
with mock.patch.object(hub.subprocess, 'run', return_value=subprocess.CompletedProcess([], 1, '', 'nginx configuration invalid')) as run:
with self.assertRaisesRegex(RuntimeError, 'nginx configuration invalid'):
hub.hub_log_maintenance_command(['test'], hub.time.monotonic() + 5)
self.assertLessEqual(run.call_args.kwargs['timeout'], 5)
def test_retention_keeps_latest_two_snapshots_and_ignores_other_data(self):
for number, char in enumerate('abc'):
stage = hub.hub_log_maintenance_directory(self.directory, char * 32)
stage.mkdir(); (stage / 'keep').write_text(char)
os.utime(stage, (100 + number, 100 + number))
other = self.directory / 'router-data'; other.mkdir()
hub.hub_prune_log_maintenance_backups(self.directory, 'c' * 32)
self.assertFalse(hub.hub_log_maintenance_directory(self.directory, 'a' * 32).exists())
self.assertTrue(hub.hub_log_maintenance_directory(self.directory, 'b' * 32).exists())
self.assertTrue(other.exists())
if __name__ == '__main__':
unittest.main()