Dawnline 0.2.0
- Diagnostics page: Health and Log tabs in place of Health and Activity - Watchdog: repairs the AmneziaWG module, tunnels, nft table, firewall rules, uhttpd portal setting, portal, resolver and rpcd after two bad checks in a row, with backoff; after 3 failed repairs logs once in red, keeps repairing quietly and logs "restored" when it works again - Heartbeat cron job restarts a hung agent - Service log: tunnel and service starts with reasons (restarted, router start, update, crash), stops Dawnline made itself vs unexpected ones, certificate renewal and expiry, WAN address changes - Admin actions logged as "You ..." with their own icon and tint; every event has an icon tile; names kept for removed people and devices; server pill on events when there are several servers - Log filters: All, Connections, People, System, Problems; search and the command palette removed - Home network reaches people's devices: fw4 forwarding from the home zones, limited to people with home access - "Devices reach each other" per server, in the Access card - Portal: "This device" card first with a dotted border, key rotation and copy address in the menu, devices sorted Online/Offline/Not set up/Paused, over-limit slots in orange - People cards: slot bar, Online / Never connected, add chip always shown - Person and device pages: Add device tile, Connect at the bottom with File/Key/QR tabs, status text without dots; pulsing server tab dot - Server pill: Running / Not running / Turned off
This commit is contained in:
1 parent
d0955e44b8
commit
82ab86848e
50 files changed
+2166
-982
No files matched your search
@@ -5,17 +5,21 @@
|
||||
// * keeps the "like a home device" path and the Home-only resolver up
|
||||
// * logs connects, disconnects and roaming
|
||||
// * enforces end dates, quotas, schedules and pauses (peers + nft sets)
|
||||
// * watches the tunnel interfaces
|
||||
// * watches the tunnel interfaces and repairs what it can (dawnline.watchdog)
|
||||
'use strict';
|
||||
|
||||
import * as uloop from 'uloop';
|
||||
import { readfile, stat, unlink } from 'fs';
|
||||
import { read_json, write_json, ensure_dir, sh_out, q, DATA_DIR, RUN_DIR } from 'dawnline.util';
|
||||
import { readfile, writefile, stat, unlink } from 'fs';
|
||||
import {
|
||||
read_json, write_json, ensure_dir, sh_out, q, slog, slog_trim, slog_restore, DATA_DIR, RUN_DIR, VERSION
|
||||
} from 'dawnline.util';
|
||||
import * as core from 'dawnline.core';
|
||||
import * as live from 'dawnline.live';
|
||||
import * as apply from 'dawnline.apply';
|
||||
import * as ops from 'dawnline.ops';
|
||||
import * as geo from 'dawnline.geo';
|
||||
import * as watchdog from 'dawnline.watchdog';
|
||||
import * as net from 'dawnline.net';
|
||||
|
||||
const TICK = 60 * 1000;
|
||||
const FLUSH_EVERY = 3600;
|
||||
@@ -30,7 +34,6 @@ let presence = read_json(`${RUN_DIR}/presence.json`, {});
|
||||
let last_online = presence.online ?? {};
|
||||
let last_endpoint = presence.endpoint ?? {};
|
||||
let offline_ticks = {};
|
||||
let iface_down = {};
|
||||
let last_flush = time();
|
||||
let last_log_flush = time();
|
||||
let last_net = null;
|
||||
@@ -103,7 +106,10 @@ function check_people(m, states, events) {
|
||||
let ts = time();
|
||||
for (let id, p in m.people) {
|
||||
let st = states[id], prev = last_states?.[id];
|
||||
if (prev != null && prev != st)
|
||||
// The admin's own pauses are logged by rpcd when they happen; here only
|
||||
// what changes on its own (end date, data limit, allowed hours).
|
||||
let by_admin = (x) => x == 'paused' || x == 'paused_all';
|
||||
if (prev != null && prev != st && !by_admin(prev) && !by_admin(st))
|
||||
push(events, { type: st == 'active' ? 'resumed' : 'paused', person: id, detail: REASON[st] ?? st });
|
||||
// "Remove their devices" when the end date passes.
|
||||
if (core.person_expired(p, ts) && p.expire_action == 'remove' && length(p.devices)) {
|
||||
@@ -182,24 +188,22 @@ function check_devices(m, lv, events) {
|
||||
write_json(`${RUN_DIR}/presence.json`, { online: last_online, endpoint: last_endpoint }, 0o600);
|
||||
}
|
||||
|
||||
function check_servers(m, dmp, events) {
|
||||
for (let id, srv in m.servers) {
|
||||
if (!srv.enabled || !srv.private_key)
|
||||
continue;
|
||||
let up = dmp.ifaces[srv.ifname] != null;
|
||||
if (!up) {
|
||||
iface_down[id] = (iface_down[id] ?? 0) + 1;
|
||||
if (iface_down[id] == 2) {
|
||||
push(events, { type: 'server_down', server: id, detail: srv.ifname });
|
||||
}
|
||||
}
|
||||
else {
|
||||
if ((iface_down[id] ?? 0) >= 2) {
|
||||
push(events, { type: 'server_up', server: id, detail: srv.ifname });
|
||||
}
|
||||
iface_down[id] = 0;
|
||||
}
|
||||
}
|
||||
// What the service log was already told. On flash: the public address often
|
||||
// changes with a reboot (PPPoE), and that is when it matters. Written rarely.
|
||||
const NOTES = `${DATA_DIR}/notes.json`;
|
||||
let notes = read_json(NOTES, {});
|
||||
let last_cert_check = 0;
|
||||
|
||||
function note(k, v) {
|
||||
notes[k] = v;
|
||||
ensure_dir(DATA_DIR, 0o700);
|
||||
write_json(NOTES, notes, 0o600);
|
||||
}
|
||||
|
||||
/** "Jan 1 2027" from the certificate, or null. */
|
||||
function cert_end(path) {
|
||||
let m = match(sh_out(`openssl x509 -enddate -noout -in ${q(path)} 2>/dev/null`) ?? '', /notAfter=([A-Za-z]+) +([0-9]+) [0-9:]+ ([0-9]+)/);
|
||||
return m ? `${m[1]} ${m[2]} ${m[3]}` : null;
|
||||
}
|
||||
|
||||
function check_cert(m) {
|
||||
@@ -207,9 +211,42 @@ function check_cert(m) {
|
||||
return;
|
||||
let c = ops.portal_cert(m.settings);
|
||||
let mt = stat(c.cert)?.mtime;
|
||||
if (cert_mtime != null && mt != null && mt != cert_mtime)
|
||||
if (cert_mtime != null && mt != null && mt != cert_mtime) {
|
||||
slog('portal', 'up', `certificate renewed · valid until ${cert_end(c.cert) ?? '?'}`, null, 'Personal portal');
|
||||
system('/etc/init.d/dawnline reload >/dev/null 2>&1 &');
|
||||
}
|
||||
cert_mtime = mt;
|
||||
// Running out soon (ACME did not renew it): once per certificate, checked hourly.
|
||||
if (mt == null || c.self_signed || time() - last_cert_check < 3600)
|
||||
return;
|
||||
last_cert_check = time();
|
||||
if (notes.cert_warned != mt && system(`openssl x509 -checkend 604800 -noout -in ${q(c.cert)} >/dev/null 2>&1`) != 0) {
|
||||
slog('portal', 'error', `certificate expires ${cert_end(c.cert) ?? 'within 7 days'}`,
|
||||
'Browsers will refuse the portal after that. Renew it with ACME, or choose another certificate in Settings.', 'Personal portal');
|
||||
note('cert_warned', mt);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The router's public IPv4 changed. Devices whose configuration holds the
|
||||
* address itself (a server's public address is an IP) cannot connect until
|
||||
* they get a new one; with a hostname only the DNS record must follow.
|
||||
*/
|
||||
function check_wan(m) {
|
||||
let srv = filter(values(m.servers), (x) => x.enabled && x.private_key)[0];
|
||||
let ip = srv ? net.wan_address(srv) : null;
|
||||
if (!ip || ip == notes.wan)
|
||||
return;
|
||||
let was = notes.wan;
|
||||
note('wan', ip);
|
||||
if (!was)
|
||||
return;
|
||||
let stuck = map(filter(values(m.servers), (x) => core.endpoint_host(m, x) == was), (x) => x.name);
|
||||
if (length(stuck))
|
||||
slog('dawnline', 'error', `public address changed · ${was} → ${ip}`,
|
||||
`${join(', ', stuck)} still give${length(stuck) == 1 ? 's' : ''} devices the old address: set the public address in the server settings (a DDNS name keeps working), then devices need new configurations.`, 'Router');
|
||||
else
|
||||
slog('dawnline', 'info', `public address changed · ${was} → ${ip}`, null, 'Router');
|
||||
}
|
||||
|
||||
let last_backup_day = null;
|
||||
@@ -247,6 +284,8 @@ function check_usage_reset(m) {
|
||||
}
|
||||
|
||||
function tick() {
|
||||
ensure_dir(RUN_DIR);
|
||||
writefile(watchdog.BEAT, `${time()}`);
|
||||
// A restore or import drops the live counters; start over from flash.
|
||||
if (!stat(`${RUN_DIR}/usage.json`) && usage.updated)
|
||||
usage = live.load_usage();
|
||||
@@ -264,7 +303,7 @@ function tick() {
|
||||
|
||||
check_devices(m, lv, events);
|
||||
check_people(m, states, events);
|
||||
check_servers(m, dmp, events);
|
||||
// Tunnels going down and up are in the service log (hotplug, watchdog), with names and reasons.
|
||||
|
||||
let changed = last_states == null || sprintf('%J', states) != sprintf('%J', last_states);
|
||||
if (changed || time() - last_sync > RESYNC_EVERY) {
|
||||
@@ -277,6 +316,7 @@ function tick() {
|
||||
|
||||
let persist = reset || time() - last_flush >= FLUSH_EVERY;
|
||||
live.save_usage(usage, persist);
|
||||
slog_trim(persist);
|
||||
if (persist)
|
||||
last_flush = time();
|
||||
if (pol.persist == 'off' || time() - last_log_flush >= LOG_FLUSH[pol.persist]) {
|
||||
@@ -288,16 +328,26 @@ function tick() {
|
||||
// The router's network changed under us: re-derive everything.
|
||||
let sig = apply.net_signature(m);
|
||||
if (last_net != null && sig != last_net) {
|
||||
system(`logger -t dawnline-agent 'network changed, re-applying'`);
|
||||
slog('dawnline', 'info', 'network changed — Dawnline set itself up again', null, 'Router');
|
||||
apply.apply();
|
||||
sig = apply.net_signature(core.load());
|
||||
}
|
||||
last_net = sig;
|
||||
}
|
||||
catch (e) {
|
||||
system(`logger -t dawnline-agent ${q('like-home: ' + e.message)}`);
|
||||
slog('dawnline', 'error', `failed: ${e.message}`, null, 'Internet like a home device');
|
||||
}
|
||||
try {
|
||||
watchdog.run(core.load());
|
||||
}
|
||||
catch (e) {
|
||||
slog('dawnline', 'error', `failed: ${e.message}`, null, 'Watchdog');
|
||||
}
|
||||
check_cert(m);
|
||||
try {
|
||||
check_wan(m);
|
||||
}
|
||||
catch (e) {}
|
||||
nightly_backup();
|
||||
try {
|
||||
check_geo(m);
|
||||
@@ -310,8 +360,36 @@ function tick() {
|
||||
function shutdown() {
|
||||
live.save_usage(usage, true);
|
||||
live.persist_events();
|
||||
slog_trim(true);
|
||||
// A clean stop: the next start is not a crash.
|
||||
unlink(watchdog.BEAT);
|
||||
}
|
||||
|
||||
// Why this start: the heartbeat file outlives an agent that died (crash, out
|
||||
// of memory, kill -9); the init script notes a stop; the version on flash
|
||||
// changes with an update; a fresh uptime means the router just booted.
|
||||
let crashed = stat(watchdog.BEAT) != null;
|
||||
let stopped = +(readfile(`${RUN_DIR}/agent.stopped`) ?? 0);
|
||||
unlink(`${RUN_DIR}/agent.stopped`);
|
||||
let was = trim(readfile(`${DATA_DIR}/version`) ?? '');
|
||||
if (was != VERSION) {
|
||||
ensure_dir(DATA_DIR, 0o700);
|
||||
writefile(`${DATA_DIR}/version`, VERSION);
|
||||
}
|
||||
slog_restore();
|
||||
if (was && was != VERSION)
|
||||
slog('dawnline', 'up', `updated · ${was} → ${VERSION}`, null, 'Dawnline');
|
||||
else if (crashed)
|
||||
slog('dawnline', 'repair', 'stopped unexpectedly — started again',
|
||||
// Why, when syslog knows: procd, the kernel (out of memory, segfault), ucode errors.
|
||||
sh_out('logread -l 400 2>/dev/null | grep -iE "kernel:.*(out of memory|killed process|oom-kill|segfault)|procd:.*dawnline|dawnline/agent" | tail -n 6'), 'Dawnline');
|
||||
else if (stopped && time() - stopped < 300)
|
||||
slog('dawnline', 'info', `restarted · version ${VERSION}`, null, 'Dawnline');
|
||||
else if (live.uptime() < 600)
|
||||
slog('dawnline', 'info', `started with the router · version ${VERSION}`, null, 'Dawnline');
|
||||
else
|
||||
slog('dawnline', 'info', `started · version ${VERSION}`, null, 'Dawnline');
|
||||
|
||||
uloop.init();
|
||||
|
||||
let timer;
|
||||
@@ -320,7 +398,7 @@ timer = uloop.timer(1000, function() {
|
||||
tick();
|
||||
}
|
||||
catch (e) {
|
||||
system(`logger -t dawnline-agent ${q('tick failed: ' + e.message)}`);
|
||||
slog('dawnline', 'error', `background check failed: ${e.message}`, e.stacktrace?.[0]?.context, 'Dawnline');
|
||||
}
|
||||
timer.set(TICK);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
# Cron, every 5 minutes: the watchdog lives in the agent, so this looks after
|
||||
# the agent itself and kills it when it runs but has stopped ticking (procd
|
||||
# starts it again). Off with the watchdog (Health → Watchdog).
|
||||
[ "$(uci -q get dawnline.settings.watchdog)" = 0 ] && exit 0
|
||||
pid="$(ubus call service list '{"name":"dawnline"}' 2>/dev/null | jsonfilter -q -e '@.dawnline.instances.agent.pid')"
|
||||
[ -n "$pid" ] && [ -r "/proc/$pid/stat" ] || exit 0
|
||||
|
||||
now="$(date +%s)"
|
||||
last="$(cat /var/run/dawnline/agent.beat 2>/dev/null)"
|
||||
# Seconds the agent has been running: uptime minus its start time (field 22, in 1/100 s).
|
||||
age="$(awk -v up="$(cut -d. -f1 /proc/uptime)" '{ print up - int($22 / 100) }' "/proc/$pid/stat")"
|
||||
[ "${age:-0}" -gt 300 ] || exit 0
|
||||
[ $((now - ${last:-0})) -gt 300 ] || exit 0
|
||||
|
||||
kill -9 "$pid"
|
||||
# Logged here; the restarted agent should not report a crash as well.
|
||||
rm -f /var/run/dawnline/agent.beat
|
||||
if [ -n "$last" ]; then detail="No check for $((now - last)) s."
|
||||
else detail="It never completed a check."; fi
|
||||
msg="stopped responding — restarted"
|
||||
mkdir -p /var/run/dawnline
|
||||
printf '{"ts":%s,"src":"dawnline","level":"repair","who":"Dawnline","msg":"%s","detail":"%s"}\n' "$now" "$msg" "$detail" >> /var/run/dawnline/service.log
|
||||
logger -t dawnline "Dawnline $msg"
|
||||
@@ -11,7 +11,7 @@
|
||||
import * as fs from 'fs';
|
||||
import { cursor } from 'uci';
|
||||
import { connect } from 'ubus';
|
||||
import { VERSION, REPO } from 'dawnline.util';
|
||||
import { VERSION, REPO, slog, iface_intent } from 'dawnline.util';
|
||||
|
||||
const STATE_DIR = '/var/run/dawnline';
|
||||
const JOB_FILE = `${STATE_DIR}/update-job.json`;
|
||||
@@ -358,8 +358,10 @@ function install_awg(arg) {
|
||||
push(ifaces, s['.name']);
|
||||
});
|
||||
phase(`Reloading the kernel module (${length(ifaces)} tunnel(s) restart)`);
|
||||
for (let i in ifaces)
|
||||
for (let i in ifaces) {
|
||||
iface_intent(i, 'update', c.get('network', i, '_dawnline_name') ?? i);
|
||||
run(`ifdown ${q(i)}`);
|
||||
}
|
||||
run('rmmod amneziawg');
|
||||
run_logged('modprobe amneziawg');
|
||||
for (let i in ifaces)
|
||||
@@ -472,6 +474,19 @@ function install_pkgs(arg) {
|
||||
|
||||
const CHECKS = { app: check_app, amneziawg: check_awg, packages: check_pkgs };
|
||||
const INSTALLS = { app: install_app, amneziawg: install_awg, pkgs: install_pkgs };
|
||||
const UPDATE_WHO = { app: 'Dawnline', amneziawg: 'AmneziaWG', pkgs: 'System packages' };
|
||||
|
||||
/** A line in the service log for each finished update (nothing when there was nothing to do). */
|
||||
function log_update(name, r) {
|
||||
// Dawnline's own update is logged by the new agent when it starts (also after install.sh).
|
||||
if (name == 'amneziawg') {
|
||||
let b = r?.before?.['kmod-amneziawg'], a = r?.after?.['kmod-amneziawg'];
|
||||
slog('awg', 'up', b && a && b != a ? `updated · ${b} → ${a}` : `installed the ${AWG_SOURCES[r?.build]?.label ?? r?.build} build${a ? ' · ' + a : ''}`,
|
||||
null, 'AmneziaWG');
|
||||
}
|
||||
else if (name == 'pkgs' && length([ ...(r?.updated ?? []), ...(r?.installed ?? []) ]))
|
||||
slog('dawnline', 'up', `updated · ${join(', ', [ ...(r.updated ?? []), ...(r.installed ?? []) ])}`, null, 'System packages');
|
||||
}
|
||||
|
||||
|
||||
if (!fs.access(STATE_DIR))
|
||||
@@ -503,6 +518,7 @@ try {
|
||||
die(`Unknown component ${name}`);
|
||||
phase(`Updating ${name}`);
|
||||
job.result[name] = INSTALLS[name](arg);
|
||||
log_update(name, job.result[name]);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -514,6 +530,10 @@ catch (e) {
|
||||
job.state = 'failed';
|
||||
job.error = e.message;
|
||||
log(`Error: ${e.message}`);
|
||||
if (job.action == 'install') {
|
||||
let name = split(job.component, ':')[0];
|
||||
slog(name == 'amneziawg' ? 'awg' : 'dawnline', 'error', 'update failed', e.message, UPDATE_WHO[name] ?? 'Dawnline');
|
||||
}
|
||||
}
|
||||
|
||||
run(`rm -rf ${WORK_DIR}`);
|
||||
|
||||
Reference in new issue
Block a user