Files
luci-app-owrt-remote/tests/test_router_log_archive.py
T

324 lines
17 KiB
Python

"""Router log delivery, offline archive, authorization and retry regressions."""
import base64
import concurrent.futures
import importlib.util
import json
import os
from pathlib import Path
import sys
import tempfile
import threading
import unittest
import urllib.error
import urllib.request
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
STATE = tempfile.TemporaryDirectory(prefix='router-log-archive-')
spec = importlib.util.spec_from_file_location('log_archive_hub', ROOT / 'vps/owrt-remote-hub.py')
hub = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = hub
with mock.patch.dict(os.environ, {'OWRT_REMOTE_STATE_DIR': STATE.name}):
spec.loader.exec_module(hub)
def batch(raw, start=0, journal='journal-1', clear_ack=''):
return {'journal_id': journal, 'from': start, 'to': start + len(raw),
'data': base64.b64encode(raw).decode(), 'clear_ack': clear_ack}
class QuietHandler(hub.Handler):
def log_message(self, *args):
pass
class ArchiveTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(dir=STATE.name)
self.addCleanup(self.temp.cleanup)
self.app = hub.App(Path(self.temp.name) / 'hub.db', 'session-test', 'agent-test', '')
with self.app.conn() as conn:
hub.upsert_router(conn, {'id': 'main', 'name': 'Test router', 'vps_host': 'hub.example'})
hub.upsert_router(conn, {'id': 'other', 'name': 'Other router', 'vps_host': 'hub.example'})
hub.write_json_private(hub.AUTH_FILE, {'username': 'owner', 'password_hash': 'unused'})
self.server = hub.HubHTTPServer(('127.0.0.1', 0), QuietHandler)
self.server.app = self.app
self.server.is_tls = False
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.addCleanup(self.close)
def close(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
def request(self, path, payload=None, cookie=True, agent=False, method=None):
headers = {}
if cookie:
headers['Cookie'] = 'owrt_remote_session=session-test'
if agent:
headers['Authorization'] = 'Bearer agent-test'
raw = None
if payload is not None:
raw = json.dumps(payload).encode('utf-8')
headers['Content-Type'] = 'application/json'
request = urllib.request.Request(f'http://127.0.0.1:{self.server.server_port}{path}',
data=raw, headers=headers, method=method)
with urllib.request.urlopen(request, timeout=5) as response:
return response.read(), response.headers
def ingest(self, value, state=None):
payload = {'id': 'main', 'batch': value}
if state is not None:
payload['state'] = state
raw, _ = self.request('/api/router-logs', payload, cookie=False, agent=True)
return json.loads(raw)
def snapshot(self):
raw, _ = self.request('/api/router/main/logs')
return json.loads(raw)
def test_offline_router_page_download_and_archive_survive_hub_restart(self):
raw = '[ERROR] WAN: connection refused\n[WARN] Retry\n[OK] Before disconnect\n'.encode()
self.ingest(batch(raw))
snapshot = self.snapshot()
self.assertFalse(snapshot['online'])
self.assertEqual(snapshot['text'].encode(), raw)
page, _ = self.request('/router/main/logs')
self.assertIn('Глобал логи'.encode(), page)
self.assertIn('Роутер оффлайн'.encode(), page)
downloaded, headers = self.request('/api/router/main/logs/download')
self.assertEqual(downloaded, raw)
self.assertIn('attachment;', headers['Content-Disposition'])
self.server.app = hub.App(self.app.db_path, 'session-test', 'agent-test', '')
self.assertEqual(self.snapshot()['text'].encode(), raw)
with self.app.conn() as conn:
router = hub.router_with_access(hub.row_to_router(hub.get_router(conn, 'main')), {'is_owner': True})
self.assertTrue(router['permissions']['logs'])
self.assertEqual(router['logs_url'], '/router/main/logs')
self.assertFalse(router['online'])
def test_lost_ack_retries_and_overlapping_batches_do_not_duplicate(self):
first = b'[OK] first\n'
second = b'[ERROR] next\n'
self.ingest(batch(first))
self.ingest(batch(first))
self.ingest(batch(first + second))
self.assertEqual(self.snapshot()['text'].encode(), first + second)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
list(pool.map(lambda _: self.ingest(batch(first + second)), range(4)))
self.assertEqual(self.snapshot()['text'].encode(), first + second)
self.ingest(batch(b'[OK] after reboot\n', journal='journal-2'))
self.assertIn('новый журнал', self.snapshot()['text'])
self.assertIn('first', self.snapshot()['text'])
def test_empty_status_cannot_confirm_missing_records_or_invent_rotation_gap(self):
reply = self.ingest(batch(b'', start=298))
self.assertEqual(reply['offset'], 0)
self.assertEqual(self.snapshot()['text'], '')
raw = b'[OK] retained records arrive after status\n'
self.ingest(batch(raw))
reply = self.ingest(batch(b'', start=1000))
self.assertEqual(reply['offset'], len(raw))
self.assertEqual(self.snapshot()['text'].encode(), raw)
def test_actual_gap_with_data_is_reported_once_but_retries_keep_data(self):
value = batch(b'[OK] surviving records\n', start=50)
self.ingest(value)
self.ingest(value)
text = self.snapshot()['text']
self.assertEqual(text.count('не доставлена'), 1)
self.assertEqual(text.count('surviving records'), 1)
def test_empty_new_generation_cannot_erase_current_journal_cursor(self):
raw = b'[OK] previous generation\n'
self.ingest(batch(raw))
self.ingest(batch(b'', start=2000, journal='new-journal'))
self.ingest(batch(raw))
self.assertEqual(self.snapshot()['text'].encode(), raw)
def test_recovery_generation_replays_previously_falsely_confirmed_chunks(self):
# Reproduce the cursor left by the former empty-batch ingestion bug.
with self.app.conn() as conn:
conn.execute("insert into router_logs(router_id,journal_id,cursor) values ('main','broken',298)")
self.ingest(batch(b'[OK] retained restart event\n', journal='recovered-old'))
self.assertIn('retained restart event', self.snapshot()['text'])
def test_real_router_spool_recovers_bad_digest_and_delivers_after_restart(self):
from test_persistent_logs import PersistentLogsTests
client = PersistentLogsTests()
client.setUp()
self.addCleanup(client.doCleanups)
client.append('Панель: действие=stop; начало\nСервис: остановка туннеля\n')
client.flush()
original = client.export()
client.acknowledge(original); client.flush()
with self.app.conn() as conn:
conn.execute('insert into router_logs(router_id,journal_id,cursor) values (?,?,?)',
('main', original['journal_id'], original['to']))
part = next(client.journal.glob('part.*'))
part.rename(part.with_name(part.name.rsplit('.', 1)[0] + '.'))
(client.run_dir / 'recovered-v2').unlink()
repaired = client.export()
reply = self.ingest(repaired, {'enabled': False, 'xray': 'stopped'})
client.call('ack', '', reply['journal_id'], str(reply['offset']))
# A lost acknowledgment may replay, but the archive must stay unchanged.
self.ingest(repaired)
self.assertEqual(self.snapshot()['text'].count('Сервис: остановка туннеля'), 1)
client.append('Сервис: запуск\n'); client.flush()
second = client.export()
reply = self.ingest(second, {'enabled': True, 'xray': 'running'})
client.call('ack', '', reply['journal_id'], str(reply['offset']))
text = self.snapshot()['text']
self.assertIn('Сервис: запуск', text)
self.assertNotIn('не доставлена до ротации', text)
self.assertEqual(client.export()['data'], '')
def test_disk_mount_diagnostic_resolves_deepest_mount_and_escaped_paths(self):
path = self.app.db_path.resolve()
mount = str(path.parent).replace(' ', r'\040')
root = str(Path(path.anchor)).replace(' ', r'\040')
mountinfo = f'1 0 8:1 / {root} rw - ext4 /dev/vda1 rw\n'
mountinfo += f'2 1 0:1 / {mount} rw - tmpfs tmpfs rw\n'
with mock.patch.object(hub.sys, 'platform', 'linux'), mock.patch('builtins.open', mock.mock_open(read_data=mountinfo)):
result = hub.router_log_storage(path)
self.assertIs(result['persistent'], False)
self.assertEqual(result['filesystem'], 'tmpfs')
with mock.patch.object(hub.sys, 'platform', 'linux'), mock.patch('builtins.open', mock.mock_open(read_data=mountinfo.splitlines()[0])):
self.assertIs(hub.router_log_storage(path)['persistent'], True)
def test_offline_clear_does_not_restore_old_logs_on_reconnection(self):
old = b'[ERROR] old\n'
self.ingest(batch(old))
raw, _ = self.request('/api/router/main/logs/clear', {}, method='POST')
self.assertTrue(json.loads(raw)['pending_clear'])
self.assertEqual(self.snapshot()['text'], '')
reply = self.ingest(batch(old))
token = reply['clear_request']
self.assertTrue(token)
self.assertEqual(self.snapshot()['text'], '')
self.ingest(batch(b'', journal='after-clear', clear_ack=token))
self.assertFalse(self.snapshot()['pending_clear'])
fresh = b'[OK] fresh\n'
self.ingest(batch(fresh, journal='after-clear', clear_ack=token))
self.assertEqual(self.snapshot()['text'].encode(), fresh)
self.ingest(batch(fresh, journal='after-clear', clear_ack=token))
self.assertEqual(self.snapshot()['text'].encode(), fresh)
def test_credentials_are_redacted_and_retention_is_bounded(self):
self.ingest(batch(b'[ERROR] hub_token=SECRET\n'))
self.assertNotIn('SECRET', self.snapshot()['text'])
self.assertTrue(self.snapshot()['text'].startswith('[ERROR]'))
with mock.patch.object(hub, 'ROUTER_LOG_MAX_BYTES', 80):
self.ingest(batch(b'[OK] ' + b'X' * 150 + b'\n', journal='new'))
self.assertEqual(self.snapshot()['bytes'], 80)
def test_malformed_batches_rejected_without_touching_archive(self):
for value in (batch(b'abc', start=-1), {**batch(b'abc'), 'to': 9},
{**batch(b'abc'), 'data': 'invalid base64'}, batch(b'x' * 65537),
batch(b'', journal='../file')):
with self.subTest(value=list(value)):
with self.assertRaises(urllib.error.HTTPError) as error:
self.ingest(value)
self.assertEqual(error.exception.code, 400)
self.assertEqual(self.snapshot()['text'], '')
def test_agent_token_and_router_permission_are_required(self):
with self.assertRaises(urllib.error.HTTPError) as error:
self.request('/api/router-logs', {'id': 'main', 'batch': batch(b'')}, cookie=True)
self.assertEqual(error.exception.code, 401)
with self.assertRaises(urllib.error.HTTPError) as error:
self.request('/api/router/main/logs', cookie=False)
self.assertEqual(error.exception.code, 401)
delegated = {'username': 'viewer', 'is_owner': False, 'router_flags': {'other': ['G'], 'main': ['A']}}
with mock.patch.object(hub.Handler, 'current_account_profile', return_value=delegated):
for path, method in [('/router/main/logs', None), ('/api/router/main/logs', None),
('/api/router/main/logs/download', None), ('/api/router/main/logs/clear', 'POST')]:
with self.assertRaises(urllib.error.HTTPError) as error:
self.request(path, {} if method else None, method=method)
self.assertEqual(error.exception.code, 403)
def test_hub_records_offline_transition_once_and_delivery_does_not_mark_online(self):
with self.app.conn() as conn:
conn.execute('update routers set last_seen = ? where id = ?', (hub.now_ts(), 'main'))
self.app.prime_router_state_snapshot()
with self.app.conn() as conn:
conn.execute('update routers set last_seen = 1 where id = ?', ('main',))
with mock.patch.object(hub, 'notify_router_offline'):
self.app.check_router_state_changes()
self.app.check_router_state_changes()
self.assertEqual(self.snapshot()['text'].count('истекло время'), 1)
self.ingest(batch(b'[OK] live logs with no heartbeat\n'))
self.assertFalse(self.snapshot()['online'])
def test_stop_reason_arrives_while_offline_and_no_heartbeat_is_required(self):
initial = b'[OK] Heartbeat: OK; Hub=https://hub.example\n'
self.ingest(batch(initial))
stopped = '[OK] Панель: действие=stop; начало\n[OK] Сервис: остановка туннеля\n[WARN] Панель: Сервис остановлен; результат=warn\n'.encode()
self.ingest(batch(stopped, start=len(initial)), {'enabled': False, 'xray': 'stopped'})
snapshot = self.snapshot()
self.assertFalse(snapshot['online'])
self.assertIn('Сервис остановлен', snapshot['text'])
self.assertIn('Remote выключен', snapshot['diagnosis']['text'])
self.assertEqual(snapshot['diagnosis']['level'], 'warn')
# Empty periodic deliveries must still carry the state without new log lines.
self.ingest(batch(b'', start=len(initial) + len(stopped)), {'enabled': True, 'xray': 'stopped'})
self.assertEqual(self.snapshot()['diagnosis']['level'], 'error')
self.assertIn('Xray остановлен', self.snapshot()['diagnosis']['text'])
self.ingest(batch(b'', start=len(initial) + len(stopped)), {'enabled': True, 'xray': 'running'})
self.assertNotIn('Remote выключен', self.snapshot()['diagnosis']['text'])
def test_offline_monitor_states_known_stop_cause_instead_of_referring_to_router(self):
with self.app.conn() as conn:
conn.execute('update routers set last_seen = ? where id = ?', (hub.now_ts() - 1, 'main'))
self.app.prime_router_state_snapshot()
self.ingest(batch('[WARN] Панель: Сервис остановлен; результат=warn\n'.encode()),
{'enabled': False, 'xray': 'stopped'})
with self.app.conn() as conn:
conn.execute('update routers set last_seen = 1 where id = ?', ('main',))
with mock.patch.object(hub, 'notify_router_offline'):
self.app.check_router_state_changes()
text = self.snapshot()['text']
self.assertIn('Remote выключен', text)
self.assertNotIn('Причина на роутере', text)
def test_unknown_outage_is_not_reported_as_a_confirmed_cause(self):
self.ingest(batch(b'[OK] Heartbeat: OK; Hub=https://hub.example\n'))
self.assertIn('Причина обрыва не доставлена', self.snapshot()['diagnosis']['text'])
raw = b'[ERROR] WAN has no IPv4 default route on wan\n'
self.ingest(batch(raw, journal='network-outage'))
reason = self.snapshot()['diagnosis']['text']
self.assertIn('WAN has no IPv4 default route', reason)
self.assertIn('Последнее доставленное', reason)
def test_legacy_delivered_stop_reason_is_visible_without_new_agent_state(self):
self.ingest(batch('[WARN] Панель: Сервис остановлен; результат=warn\n'.encode()))
self.assertIn('остановлен через панель', self.snapshot()['diagnosis']['text'])
self.ingest(batch('[OK] Сервис: запуск\n'.encode(), journal='restart'))
self.assertNotIn('остановлен через панель', self.snapshot()['diagnosis']['text'])
def test_invalid_state_rolls_back_batch_and_does_not_change_archive(self):
self.ingest(batch(b'[OK] before\n'))
original = self.snapshot()['text']
with self.assertRaises(urllib.error.HTTPError):
self.ingest(batch(b'[ERROR] invalid\n', journal='invalid-state'), {'enabled': 'false', 'xray': 'running'})
self.assertEqual(self.snapshot()['text'], original)
def test_existing_archive_gains_state_columns_without_losing_logs(self):
raw = b'[OK] saved by the previous Hub\n'
self.ingest(batch(raw))
with self.app.conn() as conn:
conn.execute('alter table router_logs drop column state_json')
conn.execute('alter table router_logs drop column state_at')
self.server.app = hub.App(self.app.db_path, 'session-test', 'agent-test', '')
self.assertEqual(self.snapshot()['text'].encode(), raw)
self.ingest(batch(b'', start=len(raw)), {'enabled': False, 'xray': 'stopped'})
snapshot = self.snapshot()
self.assertIn('Remote выключен', snapshot['diagnosis']['text'])
self.assertEqual(snapshot['text'].encode(), raw)
if __name__ == '__main__':
unittest.main()