Install log maintenance through Hub self-update
This commit is contained in:
1 parent
799ad8aad6
commit
780d036a9c
8 files changed
+536
-7
No files matched your search
@@ -99,7 +99,8 @@ class LocalTestTests(unittest.TestCase):
|
||||
self.start_server()
|
||||
return SimpleNamespace(returncode=0)
|
||||
with mock.patch.object(hub, 'hub_update_lock', side_effect=lambda *a, **k: nullcontext()), \
|
||||
mock.patch.object(hub.subprocess, 'run', side_effect=restart), mock.patch.object(hub.time, 'sleep'):
|
||||
mock.patch.object(hub.subprocess, 'run', side_effect=restart), mock.patch.object(hub.time, 'sleep'), \
|
||||
mock.patch.object(hub, 'hub_install_log_maintenance'), mock.patch.object(hub, 'hub_prune_log_maintenance_backups'):
|
||||
self.assertEqual(hub.run_hub_update_worker(job['id']), 0)
|
||||
self.assertEqual(self.target.read_bytes(), (self.fixture / 'candidate.py').read_bytes())
|
||||
new_manager = self.server.app.hub_update
|
||||
|
||||
@@ -46,6 +46,9 @@ class UpdateTests(unittest.TestCase):
|
||||
self.manager = hub.HubUpdateManager(self.target, self.directory)
|
||||
self.addCleanup(mock.patch.stopall)
|
||||
mock.patch.object(hub, 'hub_update_lock', side_effect=lambda *a, **k: nullcontext()).start()
|
||||
self.install_logs = mock.patch.object(hub, 'hub_install_log_maintenance').start()
|
||||
self.rollback_logs = mock.patch.object(hub, 'hub_rollback_log_maintenance').start()
|
||||
mock.patch.object(hub, 'hub_prune_log_maintenance_backups').start()
|
||||
|
||||
def response(self, data=NEW, url=None):
|
||||
response = io.BytesIO(data)
|
||||
@@ -164,6 +167,7 @@ class UpdateTests(unittest.TestCase):
|
||||
self.assertEqual(manager_after_restart.snapshot()['job']['status'], 'succeeded')
|
||||
self.assertFalse((self.directory / 'candidate.py').exists())
|
||||
self.assertEqual(run.call_args.args[0], ['systemctl', 'restart', 'owrt-remote'])
|
||||
self.install_logs.assert_called_once_with(self.directory, job['id'], source=NEW)
|
||||
|
||||
def test_failed_health_restores_previous_bytes_and_restarts(self):
|
||||
job = self.prepare()
|
||||
@@ -172,6 +176,7 @@ class UpdateTests(unittest.TestCase):
|
||||
self.assertEqual(hub.run_hub_update_worker(job['id'], self.directory, self.target), 1)
|
||||
self.assertEqual(self.target.read_bytes(), SOURCE)
|
||||
self.assertEqual(run.call_count, 2)
|
||||
self.rollback_logs.assert_called_once_with(self.directory, job['id'])
|
||||
self.assertIn('Предыдущий файл восстановлен', hub.hub_update_state(self.directory)['message'])
|
||||
|
||||
def test_failure_to_write_rollback_progress_does_not_prevent_restoration(self):
|
||||
@@ -222,6 +227,53 @@ class UpdateTests(unittest.TestCase):
|
||||
self.assertEqual(self.target.read_bytes(), SOURCE)
|
||||
self.assertFalse(list(self.root.glob('.hub.py-*')))
|
||||
|
||||
def test_failed_maintenance_does_not_replace_or_restart_hub(self):
|
||||
job = self.prepare()
|
||||
self.install_logs.side_effect = RuntimeError('nginx validation failed')
|
||||
with mock.patch.object(hub.time, 'sleep'), mock.patch.object(hub.subprocess, 'run') as run:
|
||||
self.assertEqual(hub.run_hub_update_worker(job['id'], self.directory, self.target), 1)
|
||||
self.assertEqual(self.target.read_bytes(), SOURCE)
|
||||
self.assertFalse(run.called)
|
||||
self.assertIn('nginx validation failed', hub.hub_update_state(self.directory)['message'])
|
||||
|
||||
def test_embedded_payload_matches_source_and_tampering_is_rejected(self):
|
||||
actual, digest = hub.hub_log_maintenance_payload(SOURCE)
|
||||
self.assertEqual(actual, (ROOT / 'vps/log-maintenance.py').read_bytes().replace(b'\r\n', b'\n'))
|
||||
self.assertEqual(hashlib.sha256(actual).hexdigest(), digest)
|
||||
with self.assertRaisesRegex(ValueError, 'сумма'):
|
||||
hub.hub_log_maintenance_payload(SOURCE.replace(digest.encode(), b'0' * 64, 1))
|
||||
|
||||
def test_first_update_from_old_worker_waits_for_maintenance(self):
|
||||
job = {'id': 'a' * 32, 'status': 'restarting', 'revision': hub.HUB_LOADED_REVISION,
|
||||
'started_at': hub.time.time()}
|
||||
hub.hub_update_state(self.directory, job)
|
||||
self.install_logs.side_effect = None
|
||||
captured = {}
|
||||
def thread(**kwargs):
|
||||
captured['install'] = kwargs['target']
|
||||
return mock.Mock()
|
||||
with mock.patch.object(hub.threading, 'Thread', side_effect=thread):
|
||||
self.manager.resume_log_maintenance()
|
||||
self.assertEqual(self.manager.log_maintenance_status, 'installing')
|
||||
self.assertEqual(self.manager.snapshot()['job']['status'], 'installing')
|
||||
captured['install']()
|
||||
self.assertEqual(self.manager.log_maintenance_status, 'ready')
|
||||
self.install_logs.assert_called_once_with(self.directory, job['id'], budget=40)
|
||||
self.install_logs.side_effect = RuntimeError('disk full')
|
||||
captured['install']()
|
||||
self.assertEqual(self.manager.snapshot()['job']['status'], 'failed')
|
||||
self.assertIn('disk full', self.manager.snapshot()['job']['message'])
|
||||
|
||||
def test_new_worker_receipt_prevents_duplicate_startup_installation(self):
|
||||
job = {'id': 'b' * 32, 'status': 'restarting', 'revision': hub.HUB_LOADED_REVISION,
|
||||
'started_at': hub.time.time()}
|
||||
hub.hub_update_state(self.directory, job)
|
||||
with mock.patch.object(hub, 'hub_log_maintenance_receipt', return_value={
|
||||
'status': 'ready', 'sha256': hub.HUB_LOG_MAINTENANCE_SHA256}), \
|
||||
mock.patch.object(hub.threading, 'Thread') as thread:
|
||||
self.manager.resume_log_maintenance()
|
||||
self.assertFalse(thread.called)
|
||||
|
||||
|
||||
class HttpTests(unittest.TestCase):
|
||||
def test_owner_csrf_origin_validation_and_revision_health(self):
|
||||
@@ -257,6 +309,12 @@ class HttpTests(unittest.TestCase):
|
||||
start.assert_called_once_with('abc')
|
||||
self.assertEqual(request('/health')[1], {'ok': True})
|
||||
self.assertEqual(request('/health?panel-update=1')[1]['revision'], hub.HUB_LOADED_REVISION)
|
||||
app.hub_update.log_maintenance_status = 'installing'
|
||||
self.assertTrue(request('/health')[1]['ok'], 'normal heartbeat health is independent')
|
||||
self.assertFalse(request('/health?panel-update=1')[1]['ok'])
|
||||
app.hub_update.log_maintenance_status = 'failed'
|
||||
app.hub_update.log_maintenance_error = 'test maintenance failure'
|
||||
self.assertEqual(request('/health?panel-update=1')[1]['log_maintenance']['error'], 'test maintenance failure')
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""The button's maintenance transaction, per-update rollback and bounded backups."""
|
||||
import datetime
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('component_update_hub', ROOT / 'vps/owrt-remote-hub.py')
|
||||
hub = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = hub
|
||||
with mock.patch.dict(os.environ, {'OWRT_REMOTE_STATE_DIR': tempfile.gettempdir()}):
|
||||
spec.loader.exec_module(hub)
|
||||
|
||||
|
||||
class ComponentTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.directory = Path(self.temp.name)
|
||||
self.job_id = 'c' * 32
|
||||
self.result = self.directory / 'result.json'
|
||||
self.result.write_text(json.dumps({'day': datetime.date.today().isoformat(), 'ok': True}))
|
||||
self.addCleanup(mock.patch.stopall)
|
||||
mock.patch.object(hub, 'HUB_LOG_MAINTENANCE_RESULT', self.result).start()
|
||||
mock.patch.object(hub.shutil, 'which', return_value='/usr/bin/command').start()
|
||||
|
||||
def test_success_verifies_cleanup_timer_and_exact_payload(self):
|
||||
with mock.patch.object(hub, 'hub_log_maintenance_command') as command:
|
||||
receipt = hub.hub_install_log_maintenance(self.directory, self.job_id)
|
||||
stage = hub.hub_log_maintenance_directory(self.directory, self.job_id)
|
||||
self.assertEqual((stage / 'log-maintenance.py').read_bytes(), hub.hub_log_maintenance_payload()[0])
|
||||
calls = [call.args[0] for call in command.call_args_list]
|
||||
self.assertEqual(calls[0][-3:], ['install', '--backup', str(stage / 'backup')])
|
||||
self.assertEqual(calls[1][-1], 'run')
|
||||
self.assertEqual(calls[-2:], [['systemctl', 'is-active', 'logrotate.timer'], ['systemctl', 'is-enabled', 'logrotate.timer']])
|
||||
self.assertEqual(receipt['status'], 'ready')
|
||||
self.assertEqual(hub.hub_log_maintenance_receipt(self.directory, self.job_id)['status'], 'ready')
|
||||
self.assertFalse(any('owrt-remote' in call for call in calls), 'component installation never stops heartbeat/Hub')
|
||||
|
||||
def test_failure_restores_only_this_updates_snapshot_and_records_real_reason(self):
|
||||
calls = []
|
||||
def run(args, deadline):
|
||||
calls.append(args)
|
||||
if 'install' in args:
|
||||
backup = Path(args[-1]); backup.mkdir()
|
||||
(backup / 'manifest.json').write_text('{}')
|
||||
if args[-1] == 'run':
|
||||
raise RuntimeError('logrotate failed: disk full')
|
||||
with mock.patch.object(hub, 'hub_log_maintenance_command', side_effect=run):
|
||||
with self.assertRaisesRegex(RuntimeError, 'disk full'):
|
||||
hub.hub_install_log_maintenance(self.directory, self.job_id)
|
||||
self.assertIn('rollback', calls[-1])
|
||||
self.assertIn(self.job_id, calls[-1][-1])
|
||||
receipt = hub.hub_log_maintenance_receipt(self.directory, self.job_id)
|
||||
self.assertEqual(receipt['status'], 'failed')
|
||||
self.assertIn('disk full', receipt['error'])
|
||||
|
||||
def test_failed_or_old_cleanup_result_cannot_be_reported_as_success(self):
|
||||
for result in ({'ok': False, 'error': 'UBIFS error'}, {'ok': True, 'day': '2000-01-01'}):
|
||||
self.result.write_text(json.dumps(result))
|
||||
with self.subTest(result=result), mock.patch.object(hub, 'hub_log_maintenance_command'):
|
||||
with self.assertRaisesRegex(RuntimeError, 'не подтверждено'):
|
||||
hub.hub_install_log_maintenance(self.directory, self.job_id)
|
||||
|
||||
def test_missing_logrotate_uses_bounded_package_installation(self):
|
||||
with mock.patch.object(hub.shutil, 'which', side_effect=lambda name: None if name == 'logrotate' else '/usr/bin/apt-get'), \
|
||||
mock.patch.object(hub, 'hub_log_maintenance_command') as command:
|
||||
hub.hub_install_log_maintenance(self.directory, self.job_id)
|
||||
args = command.call_args_list[0].args[0]
|
||||
self.assertEqual(args, ['apt-get', '-o', 'DPkg::Lock::Timeout=5', 'install', '-y', 'logrotate'])
|
||||
|
||||
def test_command_errors_include_stderr_and_timeouts_are_bounded(self):
|
||||
with mock.patch.object(hub.subprocess, 'run', return_value=subprocess.CompletedProcess([], 1, '', 'nginx configuration invalid')) as run:
|
||||
with self.assertRaisesRegex(RuntimeError, 'nginx configuration invalid'):
|
||||
hub.hub_log_maintenance_command(['test'], hub.time.monotonic() + 5)
|
||||
self.assertLessEqual(run.call_args.kwargs['timeout'], 5)
|
||||
|
||||
def test_retention_keeps_latest_two_snapshots_and_ignores_other_data(self):
|
||||
for number, char in enumerate('abc'):
|
||||
stage = hub.hub_log_maintenance_directory(self.directory, char * 32)
|
||||
stage.mkdir(); (stage / 'keep').write_text(char)
|
||||
os.utime(stage, (100 + number, 100 + number))
|
||||
other = self.directory / 'router-data'; other.mkdir()
|
||||
hub.hub_prune_log_maintenance_backups(self.directory, 'c' * 32)
|
||||
self.assertFalse(hub.hub_log_maintenance_directory(self.directory, 'a' * 32).exists())
|
||||
self.assertTrue(hub.hub_log_maintenance_directory(self.directory, 'b' * 32).exists())
|
||||
self.assertTrue(other.exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user