fix(i18n): the untranslated ratchet sees plural copies and copied phrases

Two holes let English through the ratchet unseen.

A plural form en has no key for (Russian .few and .many, Arabic .two)
was compared with en's value for that same key, which never exists, so a
locale could copy en's text into exactly the forms a new plural group
needs and the count stayed put. Such a form is now compared with every
form en spells out for the group. A form en does declare keeps the exact
comparison: Italian "{count} file" is the Italian plural, not a copy of
en's "{count} file".

The // same-as-en marker excused any value in a Latin-script locale, so
one comment waved a whole copied sentence through. It is honoured only
on a value holding a single plain word, the case shared/CLAUDE.md
already describes. No locale carries the marker today, so nothing that
passed before fails now; the counts are unchanged.
This commit is contained in:
Maurice
2026-10-08 12:41:31 +02:00
parent d4673ca4dd
commit 5ca6afa789
9 changed files with 90 additions and 9 deletions
@@ -0,0 +1,6 @@
// Fixture for the untranslated ratchet's plural rule: en spells out only `one`.
const a = {
'a.pack': '{count} items to pack',
'a.pack.one': '{count} item to pack',
};
export default a;
@@ -0,0 +1,8 @@
// Fixture: the general form, `few` and `many` are English copies; en declares no `few` or `many`.
const a = {
'a.pack': '{count} items to pack',
'a.pack.one': 'Собрать {count} вещь',
'a.pack.few': '{count} items to pack',
'a.pack.many': '{count} item to pack',
};
export default a;
@@ -1,9 +1,11 @@
// Fixture: "Status" is the German word too and says so; the name and the unit need no marker.
// The marker on a copied sentence is not honoured: a.delete still counts.
const a = {
'a.settings': 'Einstellungen',
'a.appearance': 'Erscheinungsbild',
'a.status': 'Status', // same-as-en
'a.addon': 'Atlas',
'a.distance': '{count} km',
'a.delete': 'Delete this trip and all its places permanently', // same-as-en
};
export default a;
@@ -5,5 +5,6 @@ const a = {
'a.status': 'Status',
'a.addon': 'Atlas',
'a.distance': '{count} km',
'a.delete': 'Delete this trip and all its places permanently',
};
export default a;
@@ -5,5 +5,6 @@ const a = {
'a.status': 'Status',
'a.addon': 'Atlas',
'a.distance': '{count} km',
'a.delete': 'Supprimer définitivement ce voyage et tous ses lieux',
};
export default a;
@@ -5,5 +5,6 @@ const a = {
'a.status': 'Status', // same-as-en
'a.addon': 'Atlas',
'a.distance': '{count} km',
'a.delete': 'この旅行とすべての場所を完全に削除',
};
export default a;
+58 -8
View File
@@ -10,8 +10,10 @@
* translator leaves on a string they could not translate yet
* identical an unmarked value equal to en's, unless the rule in
* `isInvariant` says the text reads the same in every language,
* or a translator confirmed it with `// same-as-en` in a
* Latin-script locale (`isExcused`)
* or a translator confirmed a single word with `// same-as-en`
* in a Latin-script locale (`isExcused`). A plural form en has
* no key for (a Russian `.few` or `.many`) is compared with
* every form en spells out for its group (`enReference`).
*
* Both counts are held at scripts/i18n-untranslated-baseline.json. The check
* fails when either grows past its entry (a file without one may hold none),
@@ -30,6 +32,7 @@
* files, or a value the catalogue reader cannot parse is an error, never a
* pass.
*/
import { pluralFormOf, pluralGroups } from '../src/i18n/plural.ts';
import { asPath, I18N_ROOT, listDomainFiles, listLocales, readCatalog } from './i18n-catalog.mjs';
import { existsSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
@@ -134,8 +137,13 @@ export const isPlainWord = (word) => word.length >= 2 && !/\p{Lu}/u.test(word.sl
* and that is for a translator to confirm, not for the rule to assume.
*/
export function isInvariant(value) {
return plainWords(value).length === 0;
}
/** The plain words `value` holds once placeholders, markup, addresses, names and units are dropped. */
export function plainWords(value) {
const text = value.replace(PLACEHOLDER_RE, ' ').replace(TAG_RE, ' ').replace(ADDRESS_RE, ' ').replace(NAME_RE, ' ');
return !(text.match(WORD_RE) ?? []).some(isPlainWord);
return (text.match(WORD_RE) ?? []).filter(isPlainWord);
}
/** Whether `locale` is written in Latin script. Throws for a locale on neither list. */
@@ -149,9 +157,46 @@ export function isLatinLocale(locale) {
/**
* Whether an unmarked value equal to en's is excused: it is invariant, or a
* translator marked it `// same-as-en` in a Latin-script locale.
* translator marked it `// same-as-en` in a Latin-script locale and it holds
* one plain word. The marker is for a word that really is the locale's own
* (German "Status"); a phrase that reads the same as en's is a copy, and the
* marker on it is ignored rather than trusted.
*/
export const isExcused = (value, same, locale) => isInvariant(value) || (same && isLatinLocale(locale));
export const isExcused = (value, same, locale) => {
const words = plainWords(value);
return words.length === 0 || (same && isLatinLocale(locale) && words.length === 1);
};
/**
* Every en value a locale's `key` may not repeat. A key en declares has its
* en value. A form of a plural group that en does not declare (Russian
* `.few`, Arabic `.two`) has every form en spells out for the group, since
* it has no en counterpart of its own and a copy of any of them is English.
* A form en declares keeps the exact comparison: Italian "{count} file" for
* en's general "{count} files" is the Italian plural, not en's `.one`.
*/
export function enReference(en, groups) {
const forms = new Map();
for (const [key, value] of en) {
const base = groupOf(key, groups);
if (base !== null) {
if (!forms.has(base)) forms.set(base, new Set());
forms.get(base).add(value);
}
}
return (key) => {
if (en.has(key)) return new Set([en.get(key)]);
const base = groupOf(key, groups);
return base === null ? new Set() : (forms.get(base) ?? new Set());
};
}
/** The plural group `key` belongs to (its base, `key.other` or a category form), else null. */
function groupOf(key, groups) {
if (groups.has(key)) return key;
if (key.endsWith('.other') && groups.has(key.slice(0, -'.other'.length))) return key.slice(0, -'.other'.length);
return pluralFormOf(key, groups)?.base ?? null;
}
/**
* Today's counts: `{ [locale]: { [file]: { marked, identical } } }`, only
@@ -162,7 +207,12 @@ export function countUntranslated(root = I18N_ROOT) {
const locales = listLocales(root);
if (!locales.includes('en')) throw new Error(`${asPath(root)}/en is required as the reference locale`);
const enFiles = listDomainFiles('en', root);
const enValues = new Map(enFiles.map((f) => [f, new Map(readCatalog('en', f, root).map((e) => [e.key, e.value]))]));
const enValues = new Map(
enFiles.map((f) => {
const en = new Map(readCatalog('en', f, root).map((e) => [e.key, e.value]));
return [f, enReference(en, pluralGroups(en.keys()))];
}),
);
const counts = {};
for (const locale of locales) {
// Classify every folder first: an unlisted locale is an error, not a pass.
@@ -171,12 +221,12 @@ export function countUntranslated(root = I18N_ROOT) {
const files = new Set(listDomainFiles(locale, root));
for (const file of enFiles) {
if (!files.has(file)) throw new Error(`${locale}/${file} is missing; run i18n-parity for the file report`);
const en = enValues.get(file);
const enValuesOf = enValues.get(file);
let marked = 0;
let identical = 0;
for (const { key, value, marked: isMarked, same } of readCatalog(locale, file, root)) {
if (isMarked) marked++;
else if (en.get(key) === value && !isExcused(value, same, locale)) identical++;
else if (enValuesOf(key).has(value) && !isExcused(value, same, locale)) identical++;
}
if (marked || identical) (counts[locale] ??= {})[file] = { marked, identical };
}