Release v117 with persistent logs and WAN tunnel recovery
This commit is contained in:
1 parent
5f65a3d811
commit
b62db40daf
61 files changed
+3080
-438
No files matched your search
@@ -0,0 +1,208 @@
|
||||
"""WAN outage/recovery using the real agent shell and a simulated procd."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
import test_host_policy as base
|
||||
from test_wan_failover import UPLINK_MOCKS, JSONFILTER
|
||||
|
||||
|
||||
@unittest.skipUnless(base.SH, "POSIX shell required")
|
||||
class WanRecoveryTests(unittest.TestCase):
|
||||
setUp = base.RouterShellTests.setUp
|
||||
config = base.RouterShellTests.config
|
||||
|
||||
def shell(self, commands):
|
||||
shell = os.environ.get("OWRT_RECOVERY_TEST_SHELL", base.SH)
|
||||
args = [shell, "ash"] if "busybox" in shell.lower() else [shell]
|
||||
return subprocess.run(args, input=self.agent + "\n" + base.MOCKS + "\n" + UPLINK_MOCKS + "\n" + commands,
|
||||
encoding="utf-8", capture_output=True, cwd=base.ROOT, env=self.env, timeout=40)
|
||||
|
||||
def prepare(self, enabled="1"):
|
||||
self.config(enabled=enabled, wan_direct="1", wan_interface="wan", vps_host="192.0.2.10",
|
||||
hub_url="https://hub.example", hub_token="secret-test-token")
|
||||
self.directory.joinpath("jsonfilter.py").write_text(JSONFILTER, encoding="utf-8")
|
||||
self.directory.joinpath("service.json").write_text(json.dumps({"owrt-remote": {"instances": {
|
||||
"heartbeat": {"running": True, "pid": 123}, "xray": {"running": False}
|
||||
}}}), encoding="utf-8")
|
||||
|
||||
PROCD = r'''
|
||||
xray_running() { return 1; }
|
||||
ubus() {
|
||||
case "$1 $2 $3" in
|
||||
'call service list') cat "$TEST_ROOT/service.json" ;;
|
||||
'call service add')
|
||||
printf '%s\n' "$4" >>"$TEST_ROOT/add-calls"
|
||||
[ ! -e "$TEST_ROOT/add-fail" ] ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
jsonfilter() {
|
||||
"$TEST_PYTHON" -c 'import json,sys
|
||||
state=json.load(sys.stdin)
|
||||
instance="heartbeat" if ".heartbeat." in sys.argv[1] else "xray"
|
||||
value=state.get("owrt-remote",{}).get("instances",{}).get(instance,{}).get("running","")
|
||||
print(str(value).lower())' "$2"
|
||||
}
|
||||
wan_select_interface() { [ ! -e "$TEST_ROOT/wan-down" ] || return 1; printf wan; }
|
||||
wan_device() { [ ! -e "$TEST_ROOT/wan-down" ] || return 1; printf eth0; }
|
||||
'''
|
||||
|
||||
def test_online_heartbeat_after_outage_recovers_dead_tunnel_on_same_uplink_once(self):
|
||||
self.prepare()
|
||||
result = self.shell(self.PROCD + r'''
|
||||
touch "$TEST_ROOT/wan-down"
|
||||
wan_reconcile_xray || true
|
||||
test ! -e "$TEST_ROOT/add-calls" || exit 1
|
||||
rm "$TEST_ROOT/wan-down"
|
||||
wan_reconcile_xray || exit
|
||||
wan_reconcile_xray
|
||||
''')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
requests = self.directory.joinpath("add-calls").read_text().splitlines()
|
||||
self.assertEqual(len(requests), 1)
|
||||
request = json.loads(requests[0])
|
||||
self.assertEqual(set(request["instances"]), {"xray"})
|
||||
self.assertEqual(request["instances"]["xray"]["command"], ["/usr/sbin/owrt-remote", "run"])
|
||||
self.assertEqual(request["instances"]["xray"]["respawn"], ["3600", "5", "5"])
|
||||
self.assertFalse(self.directory.joinpath("owrt-remote-xray-recovery").exists())
|
||||
self.assertNotIn("secret-test-token", result.stderr)
|
||||
|
||||
def test_initial_start_waiting_for_wan_is_left_to_procd(self):
|
||||
self.prepare()
|
||||
state = self.directory / "service.json"
|
||||
data = json.loads(state.read_text())
|
||||
data["owrt-remote"]["instances"]["xray"]["running"] = True
|
||||
state.write_text(json.dumps(data))
|
||||
result = self.shell(self.PROCD + "wan_reconcile_xray")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(self.directory.joinpath("add-calls").exists())
|
||||
|
||||
def test_successful_heartbeat_recovers_dead_xray_and_still_returns_hub_response(self):
|
||||
self.prepare()
|
||||
result = self.shell(self.PROCD + r'''
|
||||
wan_prepare() { return 0; }
|
||||
heartbeat_json() { printf '{}'; }
|
||||
heartbeat_candidate_endpoints() { printf 'https://hub.example/api/heartbeat'; }
|
||||
post_json() { printf '{"ok":true}'; }
|
||||
heartbeat_apply_hub_update() { return 0; }
|
||||
send_heartbeat
|
||||
''')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(json.loads(result.stdout), {"ok": True})
|
||||
request = json.loads(self.directory.joinpath("add-calls").read_text())
|
||||
self.assertEqual(set(request["instances"]), {"xray"})
|
||||
self.assertIn("Heartbeat: OK", result.stderr)
|
||||
|
||||
def test_healthy_xray_is_not_restarted_when_wan_returns(self):
|
||||
self.prepare()
|
||||
result = self.shell(self.PROCD + "xray_running() { return 0; }; wan_reconcile_xray")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(self.directory.joinpath("add-calls").exists())
|
||||
|
||||
def test_panel_stop_and_external_service_stop_are_respected(self):
|
||||
self.prepare(enabled="0")
|
||||
result = self.shell(self.PROCD + "wan_reconcile_xray")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(self.directory.joinpath("add-calls").exists())
|
||||
self.prepare()
|
||||
self.directory.joinpath("service.json").write_text('{}')
|
||||
result = self.shell(self.PROCD + "wan_reconcile_xray")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(self.directory.joinpath("add-calls").exists())
|
||||
|
||||
def test_parallel_requests_and_stale_lock_produce_one_recovery(self):
|
||||
self.prepare()
|
||||
lock = self.directory / "owrt-remote-xray-recovery"
|
||||
lock.mkdir()
|
||||
lock.joinpath("pid").write_text("2147483000")
|
||||
result = self.shell(self.PROCD + "wan_reconcile_xray & wan_reconcile_xray & wait")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(len(self.directory.joinpath("add-calls").read_text().splitlines()), 1)
|
||||
self.assertFalse(lock.exists())
|
||||
|
||||
def test_ubus_failure_reports_error_cleans_lock_and_honors_backoff(self):
|
||||
self.prepare()
|
||||
self.directory.joinpath("add-fail").touch()
|
||||
result = self.shell(self.PROCD + "wan_reconcile_xray; rc=$?; wan_reconcile_xray; exit $rc")
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
self.assertIn("procd", result.stderr)
|
||||
self.assertEqual(len(self.directory.joinpath("add-calls").read_text().splitlines()), 1)
|
||||
self.assertFalse(self.directory.joinpath("owrt-remote-xray-recovery").exists())
|
||||
|
||||
def test_empty_lock_and_dead_reaper_do_not_block_later_recovery(self):
|
||||
self.prepare()
|
||||
lock = self.directory / "owrt-remote-xray-recovery"
|
||||
lock.mkdir()
|
||||
(lock / "reap").mkdir()
|
||||
(lock / "reap/pid").write_text("2147483000")
|
||||
result = self.shell(self.PROCD + "wan_reconcile_xray")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(len(self.directory.joinpath("add-calls").read_text().splitlines()), 1)
|
||||
self.assertFalse(lock.exists())
|
||||
|
||||
def test_missing_wan_does_not_cause_repeated_service_restarts(self):
|
||||
self.prepare()
|
||||
self.directory.joinpath("wan-down").touch()
|
||||
result = self.shell(self.PROCD + r'''
|
||||
active_ssh_session() { return 1; }
|
||||
heartbeat_self_heal; heartbeat_self_heal; heartbeat_self_heal; heartbeat_self_heal
|
||||
''')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(self.directory.joinpath("owrt-remote-heartbeat.restart").exists())
|
||||
self.assertIn("IPv4 default route", result.stderr)
|
||||
|
||||
def test_supervised_start_waits_beyond_six_retries_then_launches_with_valid_config(self):
|
||||
self.prepare()
|
||||
self.directory.joinpath("wan-down").touch()
|
||||
executable = self.directory / "fake-xray"
|
||||
executable.write_text('#!/bin/sh\n[ "$1" != help ] || { echo run; exit; }; printf "%s\\n" "$*" >"$TEST_ROOT/xray-called"\n', newline="\n")
|
||||
executable.chmod(0o755)
|
||||
config = self.directory / "valid-config.json"
|
||||
config.write_text('{"outbounds":[]}')
|
||||
result = self.shell(r'''
|
||||
sleep() {
|
||||
local n
|
||||
n=$(cat "$TEST_ROOT/waits" 2>/dev/null || echo 0)
|
||||
n=$((n+1)); printf '%s' "$n" >"$TEST_ROOT/waits"
|
||||
[ "$n" -lt 8 ] || rm -f "$TEST_ROOT/wan-down"
|
||||
}
|
||||
ensure_xray_available_with_retry() { return 0; }
|
||||
xray_bin_path() { printf '%s' "$TEST_ROOT/fake-xray"; }
|
||||
render_client_config() { printf '%s' "$TEST_ROOT/valid-config.json"; }
|
||||
run_xray
|
||||
''')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(self.directory.joinpath("waits").read_text(), "8")
|
||||
self.assertEqual(self.directory.joinpath("xray-called").read_text().strip(), 'run -config ' + config.as_posix())
|
||||
self.assertEqual(result.stderr.count("конфигурация сохранена"), 1)
|
||||
self.assertIn("продолжается запуск", result.stderr)
|
||||
|
||||
def test_disabling_service_cancels_wait_without_rewriting_configuration(self):
|
||||
self.prepare()
|
||||
self.directory.joinpath("wan-down").touch()
|
||||
result = self.shell(r'''
|
||||
sleep() { uci set owrtremote.main.enabled=0; }
|
||||
render_client_config() { touch "$TEST_ROOT/rendered"; }
|
||||
run_xray
|
||||
''')
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
self.assertFalse(self.directory.joinpath("rendered").exists())
|
||||
|
||||
def test_configuration_error_with_available_wan_is_not_retried_as_wan_outage(self):
|
||||
self.prepare()
|
||||
result = self.shell(r'''
|
||||
sleep() { touch "$TEST_ROOT/slept"; }
|
||||
ensure_xray_available_with_retry() { return 0; }
|
||||
xray_bin_path() { printf /usr/bin/xray; }
|
||||
render_client_config() { printf 'invalid configuration\n' >&2; return 7; }
|
||||
run_xray
|
||||
''')
|
||||
self.assertEqual(result.returncode, 1, result.stderr)
|
||||
self.assertIn("invalid configuration", result.stderr)
|
||||
self.assertFalse(self.directory.joinpath("slept").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user