Dawnline 0.11.0
Build packages / CI (push) Blocked by required conditions
Build packages / Version (push) Waiting to run
CI / Web (format, types, tests, bundles) (push) Blocked by required conditions
CI / ShellCheck (push) Blocked by required conditions
CI / ucode static checks (push) Blocked by required conditions
CI / ucode compiles and tests pass (OpenWrt 24.10) (push) Blocked by required conditions
CI / ucode compiles and tests pass (OpenWrt 25.12) (push) Blocked by required conditions
Build packages / CI (push) Blocked by required conditions
Build packages / Version (push) Waiting to run
CI / Web (format, types, tests, bundles) (push) Blocked by required conditions
CI / ShellCheck (push) Blocked by required conditions
CI / ucode static checks (push) Blocked by required conditions
CI / ucode compiles and tests pass (OpenWrt 24.10) (push) Blocked by required conditions
CI / ucode compiles and tests pass (OpenWrt 25.12) (push) Blocked by required conditions
- Russian translation. The admin pages, the LuCI menu and the router's messages (errors, Health checks, update steps, the service log) come in Russian with the new luci-i18n-dawnline-ru package, released beside the app. LuCI shows it when its language is Russian, or "auto" with a Russian browser - The translation installs with install.sh when LuCI is in Russian (or with --lang ru), follows every Dawnline update and rollback, and can be added or removed in Components → Dawnline → Translation - The portal speaks the visitor's language: it opens in the system's language and switches between English and Russian in place, from the language pill beside its QR code. It carries its own translation and needs no package - Set-up steps quote the apps' own menus word for word, in Russian too: AmneziaVPN is "+ → Insert → Continue" (it has no "Add server"), and the Mac and Windows menus of WireGuard and AmneziaWG have their real names - Questions in the portal (Reconnect, Remove) read from the left edge and close with Cancel, without × - Numbers, sizes and dates follow the page's language. A server that can't start says why in one sentence; event details and service log lines read as whole sentences - README in Russian (README_ru.md). rules/i18n.md: how strings are written and translated; CI checks the catalogs (npm run i18n:check)
This commit is contained in:
1 parent
caed976e63
commit
2f2aaf6c34
173 files changed
+25645
-2823
No files matched your search
@@ -0,0 +1,418 @@
|
||||
// Dawnline's string catalogs, collected from the code (run in web/):
|
||||
//
|
||||
// node i18n/extract.ts rewrite the .pot templates and merge the .po translations
|
||||
// node i18n/extract.ts --check CI: fail when a catalog is out of date or a string is untranslated
|
||||
// node i18n/extract.ts --import a.json … also take translations from drafts: [{ id, ctx?, plural?, str }],
|
||||
// str a string or, for a plural, one per form
|
||||
// node i18n/extract.ts --lint [--import a.json …] [--only src/admin/x.ts …]
|
||||
// write nothing: report string problems, bad or stray drafts
|
||||
// and, with --only, the strings of those files left untranslated
|
||||
//
|
||||
// Three catalogs (rules/i18n.md):
|
||||
// dawnline — the admin UI, its shared code, the backend's messages and the
|
||||
// LuCI menu: package/luci-app-dawnline/po (luci.mk builds
|
||||
// luci-i18n-dawnline-<lang> from it)
|
||||
// portal — the personal portal, the shared code and the messages of the
|
||||
// backend modules the portal runs: src/portal/i18n, bundled into portal.js
|
||||
// demo — what only the portal demo says: src/portal/demo/i18n
|
||||
//
|
||||
// A string is collected from _('…'), N_(n, '…', '…') and plural(n, '…', '…')
|
||||
// with literal arguments, and from the backend's L_('…'): a service log line's
|
||||
// words, written in English and put into the reader's language when shown (they
|
||||
// also go into src/admin/events/log-templates.ts, the list the admin page
|
||||
// matches log lines against). Anything else under those names is an error, as is
|
||||
// a string LuCI could not find again (leading, trailing or doubled whitespace,
|
||||
// newlines) and, in the portal and the shared code, a call at module level
|
||||
// (the portal changes language in place, so text is made when it is drawn).
|
||||
// The catalogs keep the files a string comes from, not lines: they change
|
||||
// only when strings do.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import * as prettier from 'prettier';
|
||||
import ts from 'typescript';
|
||||
import { keyOf, parsePo, pluralForms, translated, writePo, type Entry, type Po } from './po.ts';
|
||||
|
||||
const WEB = path.resolve(import.meta.dirname, '..');
|
||||
const REPO = path.resolve(WEB, '..');
|
||||
const PKG = path.join(REPO, 'package/luci-app-dawnline');
|
||||
|
||||
/** The translations and their gettext plural rules (po2lmo hands the formula to LuCI's N_()). */
|
||||
const LANGS: Record<string, { team: string; plural: string }> = {
|
||||
ru: {
|
||||
team: 'Russian',
|
||||
plural: 'nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);',
|
||||
},
|
||||
};
|
||||
|
||||
const walk = (dir: string, keep: (f: string) => boolean): string[] =>
|
||||
!fs.existsSync(dir)
|
||||
? []
|
||||
: fs
|
||||
.readdirSync(dir, { withFileTypes: true })
|
||||
.flatMap((d) =>
|
||||
d.isDirectory() ? walk(path.join(dir, d.name), keep) : keep(path.join(dir, d.name)) ? [path.join(dir, d.name)] : [],
|
||||
)
|
||||
.sort();
|
||||
|
||||
const tsFiles = (dir: string, skip?: string) =>
|
||||
walk(path.join(WEB, dir), (f) => f.endsWith('.ts') && !f.endsWith('.d.ts') && !(skip && f.startsWith(path.join(WEB, skip) + path.sep)));
|
||||
|
||||
/** The backend: ucode modules and the ucode programs (by their shebang) in the package. */
|
||||
const ucodeFiles = () =>
|
||||
walk(path.join(PKG, 'root'), (f) => f.endsWith('.uc') || (!path.extname(f) && fs.readFileSync(f, 'utf8').startsWith('#!/usr/bin/ucode')));
|
||||
|
||||
/**
|
||||
* The backend modules whose messages reach the portal's visitors (its own
|
||||
* refusals, what refuses a device change, the keys): the page translates them when
|
||||
* the router sends them in English (no translation package installed).
|
||||
*/
|
||||
const PORTAL_UCODE = ['portal.uc', 'devices.uc', 'core.uc', 'awg.uc'].map((f) => path.join(PKG, 'root/usr/share/ucode/dawnline', f));
|
||||
|
||||
const menuFiles = () => walk(path.join(PKG, 'root/usr/share/luci/menu.d'), (f) => f.endsWith('.json'));
|
||||
|
||||
interface CatalogSpec {
|
||||
name: string;
|
||||
pot: string;
|
||||
po: (lang: string) => string;
|
||||
sources: () => string[];
|
||||
menus?: () => string[];
|
||||
/** Strings here are made when the page is drawn (the portal switches language in place). */
|
||||
lazy: (file: string) => boolean;
|
||||
/** The service log's words (L_) are in it: only the admin page shows the log. */
|
||||
logs?: boolean;
|
||||
}
|
||||
|
||||
const CATALOGS: CatalogSpec[] = [
|
||||
{
|
||||
name: 'dawnline',
|
||||
pot: path.join(PKG, 'po/templates/dawnline.pot'),
|
||||
po: (l) => path.join(PKG, `po/${l}/dawnline.po`),
|
||||
sources: () => [...tsFiles('src/admin'), ...tsFiles('src/shared'), ...ucodeFiles()],
|
||||
menus: menuFiles,
|
||||
logs: true,
|
||||
// rpcd keeps the backend's modules loaded while the language changes per call.
|
||||
lazy: (f) => f.startsWith(path.join(WEB, 'src/shared')) || f.startsWith(path.join(PKG, 'root')),
|
||||
},
|
||||
{
|
||||
name: 'portal',
|
||||
pot: path.join(WEB, 'src/portal/i18n/portal.pot'),
|
||||
po: (l) => path.join(WEB, `src/portal/i18n/${l}.po`),
|
||||
sources: () => [...tsFiles('src/portal', 'src/portal/demo'), ...tsFiles('src/shared'), ...PORTAL_UCODE],
|
||||
lazy: () => true,
|
||||
},
|
||||
{
|
||||
name: 'demo',
|
||||
pot: path.join(WEB, 'src/portal/demo/i18n/demo.pot'),
|
||||
po: (l) => path.join(WEB, `src/portal/demo/i18n/${l}.po`),
|
||||
sources: () => tsFiles('src/portal/demo'),
|
||||
lazy: () => true,
|
||||
},
|
||||
];
|
||||
|
||||
/** Where the wrappers themselves pass strings on: not literals, by design. */
|
||||
const WRAPPERS = new Set([path.join(WEB, 'src/shared/i18n.ts'), path.join(WEB, 'src/portal/i18n.ts')]);
|
||||
|
||||
/** The string arguments of each translation call: [msgid, msgid_plural?, msgctxt?]; `log`: a log line's words (L_). */
|
||||
const CALLS: Record<string, { id: number; plural?: number; ctx?: number; log?: boolean }> = {
|
||||
_: { id: 0, ctx: 1 },
|
||||
N_: { id: 1, plural: 2, ctx: 3 },
|
||||
plural: { id: 1, plural: 2 },
|
||||
L_: { id: 0, log: true },
|
||||
};
|
||||
|
||||
/** The generated list of log line templates (L_), for the admin page. */
|
||||
const LOG_TEMPLATES_FILE = path.join(WEB, 'src/admin/events/log-templates.ts');
|
||||
const logTemplates = new Set<string>();
|
||||
|
||||
const rel = (f: string) => path.relative(REPO, f).split(path.sep).join('/');
|
||||
|
||||
/** Why LuCI would not find this string again, or null. Its lookup squeezes whitespace (cbi.js trimws). */
|
||||
export function badText(s: string): string | null {
|
||||
if (!s) return 'is empty';
|
||||
if (/[\n\r\t]/.test(s)) return 'has a newline or a tab (po2lmo cannot carry them)';
|
||||
if (/^\s|\s$/.test(s)) return 'starts or ends with a space';
|
||||
if (/\s\s/.test(s)) return 'has two spaces in a row';
|
||||
return null;
|
||||
}
|
||||
|
||||
interface Found extends Entry {
|
||||
files: Set<string>;
|
||||
}
|
||||
|
||||
class Errors {
|
||||
list: string[] = [];
|
||||
add(where: string, msg: string) {
|
||||
this.list.push(`${where}: ${msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
function isFunctionLike(n: ts.Node): boolean {
|
||||
return ts.isFunctionLike(n) || ts.isPropertyDeclaration(n) || ts.isClassStaticBlockDeclaration(n);
|
||||
}
|
||||
|
||||
/** Every translation call in one file. */
|
||||
function scan(file: string, lazy: boolean, logs: boolean, errors: Errors, out: Found[]) {
|
||||
const text = fs.readFileSync(file, 'utf8');
|
||||
const kind = file.endsWith('.ts') ? ts.ScriptKind.TS : ts.ScriptKind.JS;
|
||||
const sf = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, true, kind);
|
||||
const wrapper = WRAPPERS.has(file);
|
||||
const visit = (node: ts.Node) => {
|
||||
if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && CALLS[node.expression.text] && !wrapper) {
|
||||
const name = node.expression.text;
|
||||
const spec = CALLS[name];
|
||||
const where = `${rel(file)}:${sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1}`;
|
||||
const lit = (i: number | undefined): string | undefined | null => {
|
||||
if (i === undefined) return undefined;
|
||||
const a = node.arguments[i];
|
||||
if (!a) return undefined;
|
||||
return ts.isStringLiteral(a) || ts.isNoSubstitutionTemplateLiteral(a) ? a.text : null;
|
||||
};
|
||||
const id = lit(spec.id);
|
||||
const pl = lit(spec.plural);
|
||||
const ctx = lit(spec.ctx);
|
||||
if (id == null || pl === null || (spec.plural !== undefined && pl === undefined) || ctx === null) {
|
||||
errors.add(
|
||||
where,
|
||||
`${name}() needs string literals: a variable or a template with \${…} cannot be collected (fill placeholders with fmt())`,
|
||||
);
|
||||
} else {
|
||||
for (const [what, s] of [
|
||||
['msgid', id],
|
||||
['plural', pl],
|
||||
['context', ctx],
|
||||
] as const)
|
||||
if (s !== undefined) {
|
||||
const why = badText(s);
|
||||
if (why) errors.add(where, `the ${what} ${JSON.stringify(s)} ${why}`);
|
||||
}
|
||||
// A log line's words are not translated where they are written: anywhere goes.
|
||||
if (spec.log) logTemplates.add(id);
|
||||
else if (lazy && !wrapper) {
|
||||
let p: ts.Node | undefined = node.parent;
|
||||
while (p && !isFunctionLike(p)) p = p.parent;
|
||||
if (!p) errors.add(where, `${name}() at module level: here text is made when it is drawn (the portal changes language in place)`);
|
||||
}
|
||||
const e: Found = { id, str: [], refs: [], files: new Set([rel(file)]) };
|
||||
if (spec.log && !logs) return;
|
||||
if (pl !== undefined) e.plural = pl;
|
||||
if (ctx !== undefined) e.ctx = ctx;
|
||||
out.push(e);
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
visit(sf);
|
||||
}
|
||||
|
||||
/** LuCI's menu titles (translated by the theme's menu). */
|
||||
function scanMenu(file: string, out: Found[]) {
|
||||
const menu = JSON.parse(fs.readFileSync(file, 'utf8')) as Record<string, { title?: string }>;
|
||||
for (const node of Object.values(menu)) if (node.title) out.push({ id: node.title, str: [], refs: [], files: new Set([rel(file)]) });
|
||||
}
|
||||
|
||||
/** One catalog's strings, deduplicated and sorted (by code unit: the same order on every machine). */
|
||||
function collect(spec: CatalogSpec, errors: Errors): Entry[] {
|
||||
const found: Found[] = [];
|
||||
for (const f of spec.sources()) scan(f, spec.lazy(f), !!spec.logs, errors, found);
|
||||
for (const f of spec.menus?.() ?? []) scanMenu(f, found);
|
||||
const byKey = new Map<string, Found>();
|
||||
for (const e of found) {
|
||||
const k = keyOf(e);
|
||||
const had = byKey.get(k);
|
||||
if (!had) byKey.set(k, e);
|
||||
else if (had.plural !== e.plural)
|
||||
errors.add(
|
||||
[...e.files][0],
|
||||
`${JSON.stringify(e.id)} is used ${had.plural === undefined || e.plural === undefined ? 'both as a plural and as a plain string' : `with two plurals (${JSON.stringify(had.plural)}, ${JSON.stringify(e.plural)})`}: give one of them a context`,
|
||||
);
|
||||
else e.files.forEach((f) => had.files.add(f));
|
||||
}
|
||||
return [...byKey.values()]
|
||||
.map(({ files, ...e }) => ({ ...e, refs: [...files].sort() }))
|
||||
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : (a.ctx ?? '') < (b.ctx ?? '') ? -1 : (a.ctx ?? '') > (b.ctx ?? '') ? 1 : 0));
|
||||
}
|
||||
|
||||
const HEADER: [string, string][] = [
|
||||
['Project-Id-Version', 'Dawnline'],
|
||||
['MIME-Version', '1.0'],
|
||||
['Content-Type', 'text/plain; charset=UTF-8'],
|
||||
['Content-Transfer-Encoding', '8bit'],
|
||||
];
|
||||
|
||||
const read = (f: string): Po | null => (fs.existsSync(f) ? parsePo(fs.readFileSync(f, 'utf8')) : null);
|
||||
|
||||
interface Draft {
|
||||
id: string;
|
||||
ctx?: string;
|
||||
plural?: string;
|
||||
str: string | string[];
|
||||
}
|
||||
|
||||
/** The placeholders of a string, in order: what a translation must keep. */
|
||||
export const holes = (s: string) => (s.match(/%[sd]/g) ?? []).join(' ');
|
||||
|
||||
/** Problems in one translation, for --check and the summary. */
|
||||
export function badTranslation(e: Entry): string | null {
|
||||
const want = holes(e.plural ?? e.id);
|
||||
if (e.plural !== undefined && holes(e.id) !== want) return `the singular and the plural have different placeholders`;
|
||||
for (const s of e.str) {
|
||||
if (!s) continue;
|
||||
if (s.includes('\\')) return `${JSON.stringify(s)} has a backslash (po2lmo would show it as written)`;
|
||||
if (s.includes('—')) return `${JSON.stringify(s)} has an em dash: use a plain dash (rules/i18n.md)`;
|
||||
const why = badText(s);
|
||||
if (why) return `${JSON.stringify(s)} ${why}`;
|
||||
if (holes(s) !== want) return `${JSON.stringify(s)} has the placeholders "${holes(s)}", the English "${want}"`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The command line: --check, --lint, and the files after --import and --only. */
|
||||
function options(argv: string[]) {
|
||||
const o = { check: false, lint: false, imports: [] as string[], only: [] as string[] };
|
||||
let list: string[] | null = null;
|
||||
for (const a of argv) {
|
||||
if (a === '--check' || a === '--lint') {
|
||||
o[a === '--check' ? 'check' : 'lint'] = true;
|
||||
list = null;
|
||||
} else if (a === '--import') list = o.imports;
|
||||
else if (a === '--only') list = o.only;
|
||||
else if (list) list.push(a);
|
||||
else throw new Error(`unknown argument ${a} (see the top of i18n/extract.ts)`);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
const draftKey = (d: { ctx?: string; id: string; plural?: string }) => keyOf(d) + (d.plural !== undefined ? '\u0002' : '');
|
||||
|
||||
/** The list of log line templates as a module (formatted as the repo formats code). */
|
||||
async function logTemplatesModule(): Promise<string> {
|
||||
const list = [...logTemplates].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
|
||||
const code = [
|
||||
"// Generated by web/i18n/extract.ts from the backend's L_() strings (npm run i18n): do not edit.",
|
||||
'// The words of service log lines, as written; events/model.ts matches log lines against them.',
|
||||
'',
|
||||
`export const LOG_TEMPLATES: string[] = ${JSON.stringify(list)};`,
|
||||
'',
|
||||
].join('\n');
|
||||
return prettier.format(code, { ...(await prettier.resolveConfig(LOG_TEMPLATES_FILE)), filepath: LOG_TEMPLATES_FILE });
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const opt = options(process.argv.slice(2));
|
||||
const errors = new Errors();
|
||||
const problems: string[] = [];
|
||||
|
||||
const drafts = new Map<string, Draft & { from: string }>();
|
||||
for (const f of opt.imports)
|
||||
for (const d of JSON.parse(fs.readFileSync(path.resolve(f), 'utf8')) as Draft[]) {
|
||||
const had = drafts.get(draftKey(d));
|
||||
if (had && JSON.stringify(had.str) !== JSON.stringify(d.str))
|
||||
problems.push(
|
||||
`drafts disagree on ${JSON.stringify(d.id)}: ${JSON.stringify(had.str)} (${had.from}), ${JSON.stringify(d.str)} (${f})`,
|
||||
);
|
||||
drafts.set(draftKey(d), { ...d, from: f });
|
||||
}
|
||||
const usedDrafts = new Set<string>();
|
||||
|
||||
const collected = CATALOGS.map((spec) => ({ spec, entries: collect(spec, errors) }));
|
||||
const only = opt.only.map((f) => rel(path.resolve(f)));
|
||||
const under = (r: string) => only.some((o) => r === o || r.startsWith(`${o}/`));
|
||||
// Linting some files: the problems of those (other files may be half-way through a change).
|
||||
if (opt.lint) problems.push(...errors.list.filter((e) => !only.length || under(e.replace(/:\d+: [\s\S]*$/, ''))));
|
||||
else if (errors.list.length) {
|
||||
console.error(errors.list.join('\n'));
|
||||
console.error(`\n${errors.list.length} string problem(s): fix them in the code (rules/i18n.md).`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// What each language already has, across catalogs: a string translated in one is reused in the others.
|
||||
const known = new Map<string, Map<string, string[]>>();
|
||||
for (const lang of Object.keys(LANGS)) {
|
||||
const m = new Map<string, string[]>();
|
||||
for (const { spec } of collected)
|
||||
for (const e of read(spec.po(lang))?.entries ?? [])
|
||||
if (translated(e)) m.set(keyOf(e) + (e.plural !== undefined ? `\u0002${e.plural}` : ''), e.str);
|
||||
known.set(lang, m);
|
||||
}
|
||||
|
||||
const inOnly = (e: Entry) => e.refs.some(under);
|
||||
const out = new Map<string, string>();
|
||||
const report: string[] = [];
|
||||
const missingOnly = new Set<string>();
|
||||
for (const { spec, entries } of collected) {
|
||||
out.set(
|
||||
spec.pot,
|
||||
writePo({
|
||||
header: [...HEADER, ['Plural-Forms', 'nplurals=2; plural=(n != 1);']],
|
||||
entries: entries.map((e) => ({ ...e, str: e.plural !== undefined ? ['', ''] : [''] })),
|
||||
}),
|
||||
);
|
||||
for (const [lang, info] of Object.entries(LANGS)) {
|
||||
const file = spec.po(lang);
|
||||
const old = new Map((read(file)?.entries ?? []).map((e) => [draftKey(e), e]));
|
||||
const header: [string, string][] = [...HEADER, ['Language', lang], ['Language-Team', info.team], ['Plural-Forms', info.plural]];
|
||||
const forms = pluralForms({ header, entries: [] });
|
||||
const merged = entries.map((e): Entry => {
|
||||
const k = draftKey(e);
|
||||
const blank = e.plural !== undefined ? Array<string>(forms).fill('') : [''];
|
||||
const draft = drafts.get(k);
|
||||
if (draft) {
|
||||
usedDrafts.add(k);
|
||||
const str = Array.isArray(draft.str) ? draft.str : [draft.str];
|
||||
if (str.length === blank.length) return { ...e, str };
|
||||
problems.push(`${spec.name}/${lang}: the draft for ${JSON.stringify(e.id)} has ${str.length} form(s), ${blank.length} needed`);
|
||||
}
|
||||
const was = old.get(k);
|
||||
// A changed plural keeps its forms only when the English plural stayed the same.
|
||||
if (was && translated(was) && was.plural === e.plural) return { ...e, str: was.str };
|
||||
const other = known.get(lang)!.get(keyOf(e) + (e.plural !== undefined ? `\u0002${e.plural}` : ''));
|
||||
return { ...e, str: other && other.length === blank.length ? other : blank };
|
||||
});
|
||||
out.set(file, writePo({ header, entries: merged }));
|
||||
const missing = merged.filter((e) => !translated(e));
|
||||
for (const e of merged) {
|
||||
const why = badTranslation(e);
|
||||
if (why && (!only.length || inOnly(e))) problems.push(`${rel(file)}: ${JSON.stringify(e.id)}: ${why}`);
|
||||
}
|
||||
report.push(`${spec.name}/${lang}: ${merged.length - missing.length} of ${merged.length} translated`);
|
||||
if (opt.check)
|
||||
for (const e of missing) problems.push(`${rel(file)}: untranslated ${JSON.stringify(e.id)}${e.ctx ? ` (${e.ctx})` : ''}`);
|
||||
for (const e of missing)
|
||||
if (inOnly(e))
|
||||
missingOnly.add(
|
||||
`no translation for ${JSON.stringify(e.id)}${e.ctx ? ` (context ${e.ctx})` : ''}${e.plural !== undefined ? ` (plural ${JSON.stringify(e.plural)})` : ''} [${e.refs.filter(under).join(' ')}]`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const [k, d] of drafts)
|
||||
if (!usedDrafts.has(k))
|
||||
problems.push(`draft ${JSON.stringify(d.id)}${d.plural !== undefined ? ' (plural)' : ''} is not in the code (${d.from})`);
|
||||
problems.push(...missingOnly);
|
||||
|
||||
out.set(LOG_TEMPLATES_FILE, await logTemplatesModule());
|
||||
|
||||
if (opt.check || opt.lint) {
|
||||
if (opt.check)
|
||||
for (const [f, text] of out)
|
||||
if (!fs.existsSync(f) || fs.readFileSync(f, 'utf8') !== text)
|
||||
problems.unshift(`${rel(f)} is out of date: run "npm run i18n" in web/ and commit the result`);
|
||||
if (problems.length) {
|
||||
console.error(problems.join('\n'));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(opt.lint ? 'No problems.' : report.join('\n'));
|
||||
return;
|
||||
}
|
||||
for (const [f, text] of out) {
|
||||
if (fs.existsSync(f) && fs.readFileSync(f, 'utf8') === text) continue;
|
||||
fs.mkdirSync(path.dirname(f), { recursive: true });
|
||||
fs.writeFileSync(f, text);
|
||||
}
|
||||
console.log(report.join('\n'));
|
||||
if (problems.length) console.warn(`\n${problems.join('\n')}`);
|
||||
}
|
||||
|
||||
// Run as a program; imported (by the tests), only the checks above.
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
// The LMO catalog LuCI loads (/usr/lib/lua/luci/i18n/<name>.<lang>.lmo), made
|
||||
// from a .po byte for byte as LuCI's po2lmo makes it. The release packages are
|
||||
// built by the SDK's own po2lmo (luci.mk); this one is for the dev deploy
|
||||
// (scripts/build-release.sh), which has no SDK.
|
||||
//
|
||||
// Layout: the translations, each padded with zeros to 4 bytes, in the .po's
|
||||
// order (the Plural-Forms formula first, under key 0); then an index entry per
|
||||
// string, sorted by key: key hash, number of forms, offset, length (big-endian
|
||||
// uint32s); then the index's offset. A key is the msgid, with "ctx\x01" before
|
||||
// it and "\x02<n>" after it for plural form n; its hash is Paul Hsieh's
|
||||
// SuperFastHash seeded with the length (lmo.c sfh_hash, cbi.js sfh).
|
||||
//
|
||||
// node i18n/lmo.ts in.po out.lmo
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { parsePo, type Po } from './po.ts';
|
||||
|
||||
/** SuperFastHash of the bytes, seeded with their length, as a uint32. */
|
||||
export function sfh(data: Uint8Array): number {
|
||||
const get16 = (i: number) => data[i] | (data[i + 1] << 8);
|
||||
const s8 = (i: number) => (data[i] << 24) >> 24;
|
||||
let hash = data.length >>> 0;
|
||||
if (!data.length) return 0;
|
||||
let i = 0;
|
||||
for (let n = data.length >>> 2; n > 0; n--, i += 4) {
|
||||
hash = (hash + get16(i)) >>> 0;
|
||||
const tmp = ((get16(i + 2) << 11) ^ hash) >>> 0;
|
||||
hash = ((hash << 16) ^ tmp) >>> 0;
|
||||
hash = (hash + (hash >>> 11)) >>> 0;
|
||||
}
|
||||
switch (data.length & 3) {
|
||||
case 3:
|
||||
hash = (hash + get16(i)) >>> 0;
|
||||
hash = (hash ^ (hash << 16)) >>> 0;
|
||||
hash = (hash ^ (s8(i + 2) << 18)) >>> 0;
|
||||
hash = (hash + (hash >>> 11)) >>> 0;
|
||||
break;
|
||||
case 2:
|
||||
hash = (hash + get16(i)) >>> 0;
|
||||
hash = (hash ^ (hash << 11)) >>> 0;
|
||||
hash = (hash + (hash >>> 17)) >>> 0;
|
||||
break;
|
||||
case 1:
|
||||
hash = (hash + s8(i)) >>> 0;
|
||||
hash = (hash ^ (hash << 10)) >>> 0;
|
||||
hash = (hash + (hash >>> 1)) >>> 0;
|
||||
}
|
||||
hash = (hash ^ (hash << 3)) >>> 0;
|
||||
hash = (hash + (hash >>> 5)) >>> 0;
|
||||
hash = (hash ^ (hash << 4)) >>> 0;
|
||||
hash = (hash + (hash >>> 17)) >>> 0;
|
||||
hash = (hash ^ (hash << 25)) >>> 0;
|
||||
hash = (hash + (hash >>> 6)) >>> 0;
|
||||
return hash;
|
||||
}
|
||||
|
||||
const utf8 = (s: string) => new TextEncoder().encode(s);
|
||||
|
||||
export function lmo(po: Po): Uint8Array {
|
||||
const chunks: Uint8Array[] = [];
|
||||
const index: { key: number; forms: number; offset: number; length: number }[] = [];
|
||||
let offset = 0;
|
||||
const add = (key: number, forms: number, value: Uint8Array) => {
|
||||
const padded = new Uint8Array(value.length + ((4 - (value.length % 4)) % 4));
|
||||
padded.set(value);
|
||||
index.push({ key, forms, offset, length: value.length });
|
||||
chunks.push(padded);
|
||||
offset += padded.length;
|
||||
};
|
||||
const pf = po.header.find(([k]) => k === 'Plural-Forms');
|
||||
if (pf) add(0, 0, utf8(pf[1]));
|
||||
for (const e of po.entries) {
|
||||
if (!e.str[0]) continue;
|
||||
e.str.forEach((s, i) => {
|
||||
if (!s) return;
|
||||
const key = utf8(`${e.ctx !== undefined ? `${e.ctx}\u0001` : ''}${e.id}${e.plural !== undefined ? `\u0002${i}` : ''}`);
|
||||
const value = utf8(s);
|
||||
const k = sfh(key);
|
||||
// An unchanged string is left out: LuCI shows the English anyway.
|
||||
if (k !== sfh(value)) add(k, e.str.length, value);
|
||||
});
|
||||
}
|
||||
index.sort((a, b) => a.key - b.key);
|
||||
const out = new Uint8Array(offset + index.length * 16 + (offset > 0 ? 4 : 0));
|
||||
const view = new DataView(out.buffer);
|
||||
let at = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, at);
|
||||
at += c.length;
|
||||
}
|
||||
for (const x of index) {
|
||||
for (const v of [x.key, x.forms, x.offset, x.length]) {
|
||||
view.setUint32(at, v);
|
||||
at += 4;
|
||||
}
|
||||
}
|
||||
if (offset > 0) view.setUint32(at, offset);
|
||||
return out;
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
const [src, dst] = process.argv.slice(2);
|
||||
if (!src || !dst) {
|
||||
console.error('usage: node i18n/lmo.ts in.po out.lmo');
|
||||
process.exit(2);
|
||||
}
|
||||
fs.writeFileSync(dst, lmo(parsePo(fs.readFileSync(src, 'utf8'))));
|
||||
}
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
// Gettext catalogs (.pot / .po) as Dawnline writes them: the subset LuCI's
|
||||
// po2lmo reads. Entries have an optional msgctxt, a msgid, an optional
|
||||
// msgid_plural and the msgstr (or msgstr[n], one per plural form), with their
|
||||
// "#:" source references. Strings are one line each and know two escapes, \"
|
||||
// and \\: po2lmo copies any other escape as written ("\n" stays a backslash and
|
||||
// an n), so extract.ts refuses newlines and tabs in strings, and backslashes in
|
||||
// translations. No dates in the header: a catalog changes only when its strings do.
|
||||
//
|
||||
// Pure: no files. extract.ts reads and writes the catalogs; the portal's
|
||||
// bundle turns a .po into a lookup table with catalogOf().
|
||||
|
||||
export interface Entry {
|
||||
ctx?: string;
|
||||
id: string;
|
||||
plural?: string;
|
||||
/** The translation: one string, or one per plural form. '' where there is none yet. */
|
||||
str: string[];
|
||||
/** Where the string is used: "src/admin/pages/people.ts:120". */
|
||||
refs: string[];
|
||||
}
|
||||
|
||||
export interface Po {
|
||||
/** Header fields in their order ("Language", "Plural-Forms", …). */
|
||||
header: [string, string][];
|
||||
entries: Entry[];
|
||||
}
|
||||
|
||||
/** An entry's identity: its context and msgid (the plural form shares its msgid's). */
|
||||
export const keyOf = (e: { ctx?: string; id: string }) => (e.ctx ? `${e.ctx}\u0004${e.id}` : e.id);
|
||||
|
||||
export const translated = (e: Entry) => e.str.length > 0 && e.str.every((s) => s !== '');
|
||||
|
||||
/** A quoted po string's content: \" and \\ read, any other escape kept as written (the header's \n). */
|
||||
function unquote(s: string, line: number): string {
|
||||
const m = /^"((?:[^"\\]|\\.)*)"$/.exec(s);
|
||||
if (!m) throw new Error(`po line ${line}: expected a quoted string, got ${s}`);
|
||||
return m[1].replace(/\\(.)/g, (all, c: string) => (c === '"' || c === '\\' ? c : all));
|
||||
}
|
||||
|
||||
const quote = (s: string) => `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
||||
|
||||
type Draft = { ctx?: string; id?: string; plural?: string; str: string[]; refs: string[] };
|
||||
|
||||
export function parsePo(text: string): Po {
|
||||
const po: Po = { header: [], entries: [] };
|
||||
let cur: Draft = { str: [], refs: [] };
|
||||
// Where a continuation line ("…") goes: the field the last keyword opened.
|
||||
let append: ((s: string) => void) | null = null;
|
||||
const flush = () => {
|
||||
if (cur.id === '' && cur.ctx === undefined) {
|
||||
for (const f of (cur.str[0] ?? '').split('\\n')) {
|
||||
const i = f.indexOf(':');
|
||||
if (i > 0) po.header.push([f.slice(0, i).trim(), f.slice(i + 1).trim()]);
|
||||
}
|
||||
} else if (cur.id !== undefined) {
|
||||
const e: Entry = { id: cur.id, str: cur.str.map((s) => s ?? ''), refs: cur.refs };
|
||||
if (cur.ctx !== undefined) e.ctx = cur.ctx;
|
||||
if (cur.plural !== undefined) e.plural = cur.plural;
|
||||
po.entries.push(e);
|
||||
}
|
||||
cur = { str: [], refs: [] };
|
||||
};
|
||||
text.split(/\r?\n/).forEach((raw, n) => {
|
||||
const line = raw.trim();
|
||||
if (!line) return void (append = null);
|
||||
if (line.startsWith('#')) {
|
||||
if (cur.id !== undefined) flush();
|
||||
if (line.startsWith('#:')) cur.refs.push(...line.slice(2).trim().split(/\s+/).filter(Boolean));
|
||||
append = null;
|
||||
return;
|
||||
}
|
||||
if (line.startsWith('"')) {
|
||||
if (!append) throw new Error(`po line ${n + 1}: a string outside an entry`);
|
||||
return append(unquote(line, n + 1));
|
||||
}
|
||||
const m = /^(msgctxt|msgid_plural|msgid|msgstr(?:\[(\d)\])?)\s+(".*")$/.exec(line);
|
||||
if (!m) throw new Error(`po line ${n + 1}: cannot read ${line}`);
|
||||
const [, kw, idx, val] = m;
|
||||
const v = unquote(val, n + 1);
|
||||
if ((kw === 'msgctxt' || kw === 'msgid') && cur.id !== undefined) flush();
|
||||
if (kw === 'msgctxt') {
|
||||
cur.ctx = v;
|
||||
append = (s) => (cur.ctx += s);
|
||||
} else if (kw === 'msgid') {
|
||||
cur.id = v;
|
||||
append = (s) => (cur.id += s);
|
||||
} else if (kw === 'msgid_plural') {
|
||||
cur.plural = v;
|
||||
append = (s) => (cur.plural += s);
|
||||
} else {
|
||||
const i = idx === undefined ? 0 : Number(idx);
|
||||
cur.str[i] = v;
|
||||
append = (s) => (cur.str[i] += s);
|
||||
}
|
||||
});
|
||||
flush();
|
||||
return po;
|
||||
}
|
||||
|
||||
export function writePo(po: Po): string {
|
||||
const out = ['msgid ""', 'msgstr ""', ...po.header.map(([k, v]) => `${quote(`${k}: ${v}`).slice(0, -1)}\\n"`), ''];
|
||||
for (const e of po.entries) {
|
||||
for (const r of e.refs) out.push(`#: ${r}`);
|
||||
if (e.ctx !== undefined) out.push(`msgctxt ${quote(e.ctx)}`);
|
||||
out.push(`msgid ${quote(e.id)}`);
|
||||
if (e.plural !== undefined) {
|
||||
out.push(`msgid_plural ${quote(e.plural)}`);
|
||||
e.str.forEach((s, i) => out.push(`msgstr[${i}] ${quote(s)}`));
|
||||
} else out.push(`msgstr ${quote(e.str[0] ?? '')}`);
|
||||
out.push('');
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
/** The number of plural forms a catalog's header declares (2 when it says nothing). */
|
||||
export function pluralForms(po: Po): number {
|
||||
const pf = po.header.find(([k]) => k === 'Plural-Forms')?.[1] ?? '';
|
||||
const m = /nplurals\s*=\s*(\d+)/.exec(pf);
|
||||
return m ? Number(m[1]) : 2;
|
||||
}
|
||||
|
||||
/** What the portal looks strings up in: translated strings by key, plural forms by the singular's key. */
|
||||
export interface Catalog {
|
||||
s: Record<string, string>;
|
||||
p: Record<string, string[]>;
|
||||
}
|
||||
|
||||
/** A .po as a lookup table: translated entries only (an untranslated string stays English). */
|
||||
export function catalogOf(po: Po): Catalog {
|
||||
const c: Catalog = { s: {}, p: {} };
|
||||
for (const e of po.entries) {
|
||||
if (!translated(e)) continue;
|
||||
if (e.plural !== undefined) c.p[keyOf(e)] = e.str;
|
||||
else c.s[keyOf(e)] = e.str[0];
|
||||
}
|
||||
return c;
|
||||
}
|
||||
Reference in new issue
Block a user