Files
dawnline/web/i18n/extract.ts
T
Aemeath 2f2aaf6c34
Build packages / CI (push) Blocked by required conditions
Build packages / Version (push) Waiting to run
CI / Web (format, types, tests, bundles) (push) Blocked by required conditions
CI / ShellCheck (push) Blocked by required conditions
CI / ucode static checks (push) Blocked by required conditions
CI / ucode compiles and tests pass (OpenWrt 24.10) (push) Blocked by required conditions
CI / ucode compiles and tests pass (OpenWrt 25.12) (push) Blocked by required conditions
Dawnline 0.11.0
- Russian translation. The admin pages, the LuCI menu and the router's
  messages (errors, Health checks, update steps, the service log) come in
  Russian with the new luci-i18n-dawnline-ru package, released beside the
  app. LuCI shows it when its language is Russian, or "auto" with a Russian
  browser
- The translation installs with install.sh when LuCI is in Russian (or with
  --lang ru), follows every Dawnline update and rollback, and can be added
  or removed in Components → Dawnline → Translation
- The portal speaks the visitor's language: it opens in the system's
  language and switches between English and Russian in place, from the
  language pill beside its QR code. It carries its own translation and needs
  no package
- Set-up steps quote the apps' own menus word for word, in Russian too:
  AmneziaVPN is "+ → Insert → Continue" (it has no "Add server"), and the
  Mac and Windows menus of WireGuard and AmneziaWG have their real names
- Questions in the portal (Reconnect, Remove) read from the left edge and
  close with Cancel, without ×
- Numbers, sizes and dates follow the page's language. A server that can't
  start says why in one sentence; event details and service log lines read
  as whole sentences
- README in Russian (README_ru.md). rules/i18n.md: how strings are written
  and translated; CI checks the catalogs (npm run i18n:check)
2026-10-20 19:33:52 +03:00

419 lines
19 KiB
TypeScript

// Dawnline's string catalogs, collected from the code (run in web/):
//
// node i18n/extract.ts rewrite the .pot templates and merge the .po translations
// node i18n/extract.ts --check CI: fail when a catalog is out of date or a string is untranslated
// node i18n/extract.ts --import a.json … also take translations from drafts: [{ id, ctx?, plural?, str }],
// str a string or, for a plural, one per form
// node i18n/extract.ts --lint [--import a.json …] [--only src/admin/x.ts …]
// write nothing: report string problems, bad or stray drafts
// and, with --only, the strings of those files left untranslated
//
// Three catalogs (rules/i18n.md):
// dawnline — the admin UI, its shared code, the backend's messages and the
// LuCI menu: package/luci-app-dawnline/po (luci.mk builds
// luci-i18n-dawnline-<lang> from it)
// portal — the personal portal, the shared code and the messages of the
// backend modules the portal runs: src/portal/i18n, bundled into portal.js
// demo — what only the portal demo says: src/portal/demo/i18n
//
// A string is collected from _('…'), N_(n, '…', '…') and plural(n, '…', '…')
// with literal arguments, and from the backend's L_('…'): a service log line's
// words, written in English and put into the reader's language when shown (they
// also go into src/admin/events/log-templates.ts, the list the admin page
// matches log lines against). Anything else under those names is an error, as is
// a string LuCI could not find again (leading, trailing or doubled whitespace,
// newlines) and, in the portal and the shared code, a call at module level
// (the portal changes language in place, so text is made when it is drawn).
// The catalogs keep the files a string comes from, not lines: they change
// only when strings do.
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import * as prettier from 'prettier';
import ts from 'typescript';
import { keyOf, parsePo, pluralForms, translated, writePo, type Entry, type Po } from './po.ts';
const WEB = path.resolve(import.meta.dirname, '..');
const REPO = path.resolve(WEB, '..');
const PKG = path.join(REPO, 'package/luci-app-dawnline');
/** The translations and their gettext plural rules (po2lmo hands the formula to LuCI's N_()). */
const LANGS: Record<string, { team: string; plural: string }> = {
ru: {
team: 'Russian',
plural: 'nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);',
},
};
const walk = (dir: string, keep: (f: string) => boolean): string[] =>
!fs.existsSync(dir)
? []
: fs
.readdirSync(dir, { withFileTypes: true })
.flatMap((d) =>
d.isDirectory() ? walk(path.join(dir, d.name), keep) : keep(path.join(dir, d.name)) ? [path.join(dir, d.name)] : [],
)
.sort();
const tsFiles = (dir: string, skip?: string) =>
walk(path.join(WEB, dir), (f) => f.endsWith('.ts') && !f.endsWith('.d.ts') && !(skip && f.startsWith(path.join(WEB, skip) + path.sep)));
/** The backend: ucode modules and the ucode programs (by their shebang) in the package. */
const ucodeFiles = () =>
walk(path.join(PKG, 'root'), (f) => f.endsWith('.uc') || (!path.extname(f) && fs.readFileSync(f, 'utf8').startsWith('#!/usr/bin/ucode')));
/**
* The backend modules whose messages reach the portal's visitors (its own
* refusals, what refuses a device change, the keys): the page translates them when
* the router sends them in English (no translation package installed).
*/
const PORTAL_UCODE = ['portal.uc', 'devices.uc', 'core.uc', 'awg.uc'].map((f) => path.join(PKG, 'root/usr/share/ucode/dawnline', f));
const menuFiles = () => walk(path.join(PKG, 'root/usr/share/luci/menu.d'), (f) => f.endsWith('.json'));
interface CatalogSpec {
name: string;
pot: string;
po: (lang: string) => string;
sources: () => string[];
menus?: () => string[];
/** Strings here are made when the page is drawn (the portal switches language in place). */
lazy: (file: string) => boolean;
/** The service log's words (L_) are in it: only the admin page shows the log. */
logs?: boolean;
}
const CATALOGS: CatalogSpec[] = [
{
name: 'dawnline',
pot: path.join(PKG, 'po/templates/dawnline.pot'),
po: (l) => path.join(PKG, `po/${l}/dawnline.po`),
sources: () => [...tsFiles('src/admin'), ...tsFiles('src/shared'), ...ucodeFiles()],
menus: menuFiles,
logs: true,
// rpcd keeps the backend's modules loaded while the language changes per call.
lazy: (f) => f.startsWith(path.join(WEB, 'src/shared')) || f.startsWith(path.join(PKG, 'root')),
},
{
name: 'portal',
pot: path.join(WEB, 'src/portal/i18n/portal.pot'),
po: (l) => path.join(WEB, `src/portal/i18n/${l}.po`),
sources: () => [...tsFiles('src/portal', 'src/portal/demo'), ...tsFiles('src/shared'), ...PORTAL_UCODE],
lazy: () => true,
},
{
name: 'demo',
pot: path.join(WEB, 'src/portal/demo/i18n/demo.pot'),
po: (l) => path.join(WEB, `src/portal/demo/i18n/${l}.po`),
sources: () => tsFiles('src/portal/demo'),
lazy: () => true,
},
];
/** Where the wrappers themselves pass strings on: not literals, by design. */
const WRAPPERS = new Set([path.join(WEB, 'src/shared/i18n.ts'), path.join(WEB, 'src/portal/i18n.ts')]);
/** The string arguments of each translation call: [msgid, msgid_plural?, msgctxt?]; `log`: a log line's words (L_). */
const CALLS: Record<string, { id: number; plural?: number; ctx?: number; log?: boolean }> = {
_: { id: 0, ctx: 1 },
N_: { id: 1, plural: 2, ctx: 3 },
plural: { id: 1, plural: 2 },
L_: { id: 0, log: true },
};
/** The generated list of log line templates (L_), for the admin page. */
const LOG_TEMPLATES_FILE = path.join(WEB, 'src/admin/events/log-templates.ts');
const logTemplates = new Set<string>();
const rel = (f: string) => path.relative(REPO, f).split(path.sep).join('/');
/** Why LuCI would not find this string again, or null. Its lookup squeezes whitespace (cbi.js trimws). */
export function badText(s: string): string | null {
if (!s) return 'is empty';
if (/[\n\r\t]/.test(s)) return 'has a newline or a tab (po2lmo cannot carry them)';
if (/^\s|\s$/.test(s)) return 'starts or ends with a space';
if (/\s\s/.test(s)) return 'has two spaces in a row';
return null;
}
interface Found extends Entry {
files: Set<string>;
}
class Errors {
list: string[] = [];
add(where: string, msg: string) {
this.list.push(`${where}: ${msg}`);
}
}
function isFunctionLike(n: ts.Node): boolean {
return ts.isFunctionLike(n) || ts.isPropertyDeclaration(n) || ts.isClassStaticBlockDeclaration(n);
}
/** Every translation call in one file. */
function scan(file: string, lazy: boolean, logs: boolean, errors: Errors, out: Found[]) {
const text = fs.readFileSync(file, 'utf8');
const kind = file.endsWith('.ts') ? ts.ScriptKind.TS : ts.ScriptKind.JS;
const sf = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, true, kind);
const wrapper = WRAPPERS.has(file);
const visit = (node: ts.Node) => {
if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && CALLS[node.expression.text] && !wrapper) {
const name = node.expression.text;
const spec = CALLS[name];
const where = `${rel(file)}:${sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1}`;
const lit = (i: number | undefined): string | undefined | null => {
if (i === undefined) return undefined;
const a = node.arguments[i];
if (!a) return undefined;
return ts.isStringLiteral(a) || ts.isNoSubstitutionTemplateLiteral(a) ? a.text : null;
};
const id = lit(spec.id);
const pl = lit(spec.plural);
const ctx = lit(spec.ctx);
if (id == null || pl === null || (spec.plural !== undefined && pl === undefined) || ctx === null) {
errors.add(
where,
`${name}() needs string literals: a variable or a template with \${…} cannot be collected (fill placeholders with fmt())`,
);
} else {
for (const [what, s] of [
['msgid', id],
['plural', pl],
['context', ctx],
] as const)
if (s !== undefined) {
const why = badText(s);
if (why) errors.add(where, `the ${what} ${JSON.stringify(s)} ${why}`);
}
// A log line's words are not translated where they are written: anywhere goes.
if (spec.log) logTemplates.add(id);
else if (lazy && !wrapper) {
let p: ts.Node | undefined = node.parent;
while (p && !isFunctionLike(p)) p = p.parent;
if (!p) errors.add(where, `${name}() at module level: here text is made when it is drawn (the portal changes language in place)`);
}
const e: Found = { id, str: [], refs: [], files: new Set([rel(file)]) };
if (spec.log && !logs) return;
if (pl !== undefined) e.plural = pl;
if (ctx !== undefined) e.ctx = ctx;
out.push(e);
}
}
ts.forEachChild(node, visit);
};
visit(sf);
}
/** LuCI's menu titles (translated by the theme's menu). */
function scanMenu(file: string, out: Found[]) {
const menu = JSON.parse(fs.readFileSync(file, 'utf8')) as Record<string, { title?: string }>;
for (const node of Object.values(menu)) if (node.title) out.push({ id: node.title, str: [], refs: [], files: new Set([rel(file)]) });
}
/** One catalog's strings, deduplicated and sorted (by code unit: the same order on every machine). */
function collect(spec: CatalogSpec, errors: Errors): Entry[] {
const found: Found[] = [];
for (const f of spec.sources()) scan(f, spec.lazy(f), !!spec.logs, errors, found);
for (const f of spec.menus?.() ?? []) scanMenu(f, found);
const byKey = new Map<string, Found>();
for (const e of found) {
const k = keyOf(e);
const had = byKey.get(k);
if (!had) byKey.set(k, e);
else if (had.plural !== e.plural)
errors.add(
[...e.files][0],
`${JSON.stringify(e.id)} is used ${had.plural === undefined || e.plural === undefined ? 'both as a plural and as a plain string' : `with two plurals (${JSON.stringify(had.plural)}, ${JSON.stringify(e.plural)})`}: give one of them a context`,
);
else e.files.forEach((f) => had.files.add(f));
}
return [...byKey.values()]
.map(({ files, ...e }) => ({ ...e, refs: [...files].sort() }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : (a.ctx ?? '') < (b.ctx ?? '') ? -1 : (a.ctx ?? '') > (b.ctx ?? '') ? 1 : 0));
}
const HEADER: [string, string][] = [
['Project-Id-Version', 'Dawnline'],
['MIME-Version', '1.0'],
['Content-Type', 'text/plain; charset=UTF-8'],
['Content-Transfer-Encoding', '8bit'],
];
const read = (f: string): Po | null => (fs.existsSync(f) ? parsePo(fs.readFileSync(f, 'utf8')) : null);
interface Draft {
id: string;
ctx?: string;
plural?: string;
str: string | string[];
}
/** The placeholders of a string, in order: what a translation must keep. */
export const holes = (s: string) => (s.match(/%[sd]/g) ?? []).join(' ');
/** Problems in one translation, for --check and the summary. */
export function badTranslation(e: Entry): string | null {
const want = holes(e.plural ?? e.id);
if (e.plural !== undefined && holes(e.id) !== want) return `the singular and the plural have different placeholders`;
for (const s of e.str) {
if (!s) continue;
if (s.includes('\\')) return `${JSON.stringify(s)} has a backslash (po2lmo would show it as written)`;
if (s.includes('—')) return `${JSON.stringify(s)} has an em dash: use a plain dash (rules/i18n.md)`;
const why = badText(s);
if (why) return `${JSON.stringify(s)} ${why}`;
if (holes(s) !== want) return `${JSON.stringify(s)} has the placeholders "${holes(s)}", the English "${want}"`;
}
return null;
}
/** The command line: --check, --lint, and the files after --import and --only. */
function options(argv: string[]) {
const o = { check: false, lint: false, imports: [] as string[], only: [] as string[] };
let list: string[] | null = null;
for (const a of argv) {
if (a === '--check' || a === '--lint') {
o[a === '--check' ? 'check' : 'lint'] = true;
list = null;
} else if (a === '--import') list = o.imports;
else if (a === '--only') list = o.only;
else if (list) list.push(a);
else throw new Error(`unknown argument ${a} (see the top of i18n/extract.ts)`);
}
return o;
}
const draftKey = (d: { ctx?: string; id: string; plural?: string }) => keyOf(d) + (d.plural !== undefined ? '\u0002' : '');
/** The list of log line templates as a module (formatted as the repo formats code). */
async function logTemplatesModule(): Promise<string> {
const list = [...logTemplates].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
const code = [
"// Generated by web/i18n/extract.ts from the backend's L_() strings (npm run i18n): do not edit.",
'// The words of service log lines, as written; events/model.ts matches log lines against them.',
'',
`export const LOG_TEMPLATES: string[] = ${JSON.stringify(list)};`,
'',
].join('\n');
return prettier.format(code, { ...(await prettier.resolveConfig(LOG_TEMPLATES_FILE)), filepath: LOG_TEMPLATES_FILE });
}
async function main() {
const opt = options(process.argv.slice(2));
const errors = new Errors();
const problems: string[] = [];
const drafts = new Map<string, Draft & { from: string }>();
for (const f of opt.imports)
for (const d of JSON.parse(fs.readFileSync(path.resolve(f), 'utf8')) as Draft[]) {
const had = drafts.get(draftKey(d));
if (had && JSON.stringify(had.str) !== JSON.stringify(d.str))
problems.push(
`drafts disagree on ${JSON.stringify(d.id)}: ${JSON.stringify(had.str)} (${had.from}), ${JSON.stringify(d.str)} (${f})`,
);
drafts.set(draftKey(d), { ...d, from: f });
}
const usedDrafts = new Set<string>();
const collected = CATALOGS.map((spec) => ({ spec, entries: collect(spec, errors) }));
const only = opt.only.map((f) => rel(path.resolve(f)));
const under = (r: string) => only.some((o) => r === o || r.startsWith(`${o}/`));
// Linting some files: the problems of those (other files may be half-way through a change).
if (opt.lint) problems.push(...errors.list.filter((e) => !only.length || under(e.replace(/:\d+: [\s\S]*$/, ''))));
else if (errors.list.length) {
console.error(errors.list.join('\n'));
console.error(`\n${errors.list.length} string problem(s): fix them in the code (rules/i18n.md).`);
process.exit(1);
}
// What each language already has, across catalogs: a string translated in one is reused in the others.
const known = new Map<string, Map<string, string[]>>();
for (const lang of Object.keys(LANGS)) {
const m = new Map<string, string[]>();
for (const { spec } of collected)
for (const e of read(spec.po(lang))?.entries ?? [])
if (translated(e)) m.set(keyOf(e) + (e.plural !== undefined ? `\u0002${e.plural}` : ''), e.str);
known.set(lang, m);
}
const inOnly = (e: Entry) => e.refs.some(under);
const out = new Map<string, string>();
const report: string[] = [];
const missingOnly = new Set<string>();
for (const { spec, entries } of collected) {
out.set(
spec.pot,
writePo({
header: [...HEADER, ['Plural-Forms', 'nplurals=2; plural=(n != 1);']],
entries: entries.map((e) => ({ ...e, str: e.plural !== undefined ? ['', ''] : [''] })),
}),
);
for (const [lang, info] of Object.entries(LANGS)) {
const file = spec.po(lang);
const old = new Map((read(file)?.entries ?? []).map((e) => [draftKey(e), e]));
const header: [string, string][] = [...HEADER, ['Language', lang], ['Language-Team', info.team], ['Plural-Forms', info.plural]];
const forms = pluralForms({ header, entries: [] });
const merged = entries.map((e): Entry => {
const k = draftKey(e);
const blank = e.plural !== undefined ? Array<string>(forms).fill('') : [''];
const draft = drafts.get(k);
if (draft) {
usedDrafts.add(k);
const str = Array.isArray(draft.str) ? draft.str : [draft.str];
if (str.length === blank.length) return { ...e, str };
problems.push(`${spec.name}/${lang}: the draft for ${JSON.stringify(e.id)} has ${str.length} form(s), ${blank.length} needed`);
}
const was = old.get(k);
// A changed plural keeps its forms only when the English plural stayed the same.
if (was && translated(was) && was.plural === e.plural) return { ...e, str: was.str };
const other = known.get(lang)!.get(keyOf(e) + (e.plural !== undefined ? `\u0002${e.plural}` : ''));
return { ...e, str: other && other.length === blank.length ? other : blank };
});
out.set(file, writePo({ header, entries: merged }));
const missing = merged.filter((e) => !translated(e));
for (const e of merged) {
const why = badTranslation(e);
if (why && (!only.length || inOnly(e))) problems.push(`${rel(file)}: ${JSON.stringify(e.id)}: ${why}`);
}
report.push(`${spec.name}/${lang}: ${merged.length - missing.length} of ${merged.length} translated`);
if (opt.check)
for (const e of missing) problems.push(`${rel(file)}: untranslated ${JSON.stringify(e.id)}${e.ctx ? ` (${e.ctx})` : ''}`);
for (const e of missing)
if (inOnly(e))
missingOnly.add(
`no translation for ${JSON.stringify(e.id)}${e.ctx ? ` (context ${e.ctx})` : ''}${e.plural !== undefined ? ` (plural ${JSON.stringify(e.plural)})` : ''} [${e.refs.filter(under).join(' ')}]`,
);
}
}
for (const [k, d] of drafts)
if (!usedDrafts.has(k))
problems.push(`draft ${JSON.stringify(d.id)}${d.plural !== undefined ? ' (plural)' : ''} is not in the code (${d.from})`);
problems.push(...missingOnly);
out.set(LOG_TEMPLATES_FILE, await logTemplatesModule());
if (opt.check || opt.lint) {
if (opt.check)
for (const [f, text] of out)
if (!fs.existsSync(f) || fs.readFileSync(f, 'utf8') !== text)
problems.unshift(`${rel(f)} is out of date: run "npm run i18n" in web/ and commit the result`);
if (problems.length) {
console.error(problems.join('\n'));
process.exit(1);
}
console.log(opt.lint ? 'No problems.' : report.join('\n'));
return;
}
for (const [f, text] of out) {
if (fs.existsSync(f) && fs.readFileSync(f, 'utf8') === text) continue;
fs.mkdirSync(path.dirname(f), { recursive: true });
fs.writeFileSync(f, text);
}
console.log(report.join('\n'));
if (problems.length) console.warn(`\n${problems.join('\n')}`);
}
// Run as a program; imported (by the tests), only the checks above.
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();