From 822f5bcda2a9a993a179021c0ab483b99a0059c3 Mon Sep 17 00:00:00 2001 From: Bert Verhelst Date: Fri, 12 Jun 2026 21:36:11 +0200 Subject: [PATCH 1/2] feat(translations): parallelize translation extraction + show progress --- ui/package-lock.json | 47 +++ ui/package.json | 4 +- .../extract-and-replace-translations.ts | 289 ++++++++++++------ 3 files changed, 247 insertions(+), 93 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 4334cace..1e5004cd 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -20,6 +20,7 @@ "@testing-library/jest-dom": "6.9.1", "@testing-library/react": "16.3.2", "@testing-library/user-event": "14.6.1", + "@types/cli-progress": "^3.11.6", "@types/dompurify": "^3.0.5", "@types/enzyme": "3.10.13", "@types/file-saver": "2.0.5", @@ -37,6 +38,7 @@ "ajv": "8.11.0", "autoprefixer": "10.4.2", "bump-package-versions": "^1.0.7", + "cli-progress": "^3.12.0", "console-log-colors": "^0.5.0", "cross-env": "7.0.3", "dependency-cruiser": "13.1.5", @@ -4533,6 +4535,16 @@ "@types/node": "*" } }, + "node_modules/@types/cli-progress": { + "version": "3.11.6", + "resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.6.tgz", + "integrity": "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -6087,6 +6099,41 @@ "dev": true, "license": "MIT" }, + "node_modules/cli-progress": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cli-progress/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-progress/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/cliui": { "version": "8.0.1", "dev": true, diff --git a/ui/package.json b/ui/package.json index a7607da7..2d08ae22 100644 --- a/ui/package.json +++ b/ui/package.json @@ -72,10 +72,10 @@ }, "peerDependencies": { "@emotion/react": "11.11.1", + "@floating-ui/react": "0.27.18", "@hookform/resolvers": "2.9.11", "@meemoo/react-components": "5.1.38", "@studiohyperdrive/pagination": "1.0.0", - "@floating-ui/react": "0.27.18", "@tanstack/react-query": "5.90.8", "@viaa/avo2-components": "6.2.15", "@viaa/avo2-types": "5.0.16", @@ -125,6 +125,7 @@ "@testing-library/jest-dom": "6.9.1", "@testing-library/react": "16.3.2", "@testing-library/user-event": "14.6.1", + "@types/cli-progress": "^3.11.6", "@types/dompurify": "^3.0.5", "@types/enzyme": "3.10.13", "@types/file-saver": "2.0.5", @@ -142,6 +143,7 @@ "ajv": "8.11.0", "autoprefixer": "10.4.2", "bump-package-versions": "^1.0.7", + "cli-progress": "^3.12.0", "console-log-colors": "^0.5.0", "cross-env": "7.0.3", "dependency-cruiser": "13.1.5", diff --git a/ui/scripts/extract-and-replace-translations.ts b/ui/scripts/extract-and-replace-translations.ts index 3f89b8c0..39fc6f09 100644 --- a/ui/scripts/extract-and-replace-translations.ts +++ b/ui/scripts/extract-and-replace-translations.ts @@ -1,10 +1,14 @@ // noinspection ES6PreferShortImport -import { execSync } from 'node:child_process'; +import { exec } from 'node:child_process'; import * as fs from 'node:fs/promises'; import * as path from 'node:path'; +import { promisify } from 'node:util'; +import cliProgress from 'cli-progress'; import { green, grey, red, yellow } from 'console-log-colors'; import { compact, intersection, kebabCase, lowerCase, trim, upperFirst, without } from 'lodash-es'; + +const execAsync = promisify(exec); /** This script runs over all the code and looks for either: tHtml('Aanvraagformulier') @@ -45,6 +49,8 @@ import { const ALL_APPS = `[${App.AVO}, ${App.HET_ARCHIEF}]`; +type ProgressCallback = (pct: number, status: string) => void; + export function getFullKey( translationEntry: TranslationEntry | MultiLanguageTranslationEntry ): `${Component}${typeof TRANSLATION_SEPARATOR}${Location}${typeof TRANSLATION_SEPARATOR}${Key}` { @@ -162,31 +168,28 @@ async function extractTranslationsFromCodeFiles( component: Component, oldTranslations: Record, oldTranslationsJsonPath: string, - tsConfigFilePath?: string + tsConfigFilePath?: string, + onProgress?: ProgressCallback ): Promise { const tsProject = new Project({ tsConfigFilePath, }); const sourceCodeTranslations: TranslationEntry[] = []; - const sourceFiles = tsProject.getSourceFiles(); - for (const sourceFile of sourceFiles) { - if (!(sourceFile.getBaseName().endsWith('.ts') || sourceFile.getBaseName().endsWith('.tsx'))) { - continue; // Skip non-typescript files - } - if ( - sourceFile.getBaseNameWithoutExtension().includes('.test') || - sourceFile.getBaseNameWithoutExtension().includes('.spec') || - sourceFile.isDeclarationFile() - ) { - continue; // Skip test and declaration files - } - - if (!sourceFile.getFilePath().startsWith(rootFolderPath)) { - continue; // Skip files outside the root folder - } + const sourceFiles = tsProject.getSourceFiles().filter((sourceFile) => { + return ( + (sourceFile.getBaseName().endsWith('.ts') || sourceFile.getBaseName().endsWith('.tsx')) && + !sourceFile.getBaseNameWithoutExtension().includes('.test') && + !sourceFile.getBaseNameWithoutExtension().includes('.spec') && + !sourceFile.isDeclarationFile() && + sourceFile.getFilePath().startsWith(rootFolderPath) + ); + }); - // Find and extract translations, replace strings with translation keys + const total = sourceFiles.length; + for (let i = 0; i < total; i++) { + const sourceFile = sourceFiles[i]; + onProgress?.(Math.round((i / total) * 80), sourceFile.getBaseName()); // Find all tHtml() and tText() function calls const translationFunctionCalls = sourceFile @@ -248,6 +251,7 @@ async function extractTranslationsFromCodeFiles( } }); } + onProgress?.(85, 'saving...'); await tsProject.save(); return sourceCodeTranslations; @@ -418,13 +422,14 @@ async function updateTranslations( app: App, component: Component, outputJsonFile: string, - tsConfigPath?: string + allOnlineTranslations: TranslationEntry[], + tsConfigPath?: string, + onProgress?: ProgressCallback ): Promise { try { - const onlineTranslations = (await getOnlineTranslations(app)).filter( - (t) => t.component === component - ); + const onlineTranslations = allOnlineTranslations.filter((t) => t.component === component); + onProgress?.(5, 'reading existing translations...'); const nlJsonTranslations: Record = JSON.parse( (await fs.readFile(path.resolve(rootFolderPath, outputJsonFile))).toString() ); @@ -449,16 +454,20 @@ async function updateTranslations( component, nlJsonTranslations, resolvePath(rootFolderPath, outputJsonFile), - tsConfigPath + tsConfigPath, + onProgress ); - return await combineTranslations( + onProgress?.(90, 'combining translations...'); + const result = await combineTranslations( nlJsonTranslationEntries, sourceCodeTranslations, onlineTranslations, path.join(rootFolderPath, outputJsonFile), app ); + onProgress?.(95, 'done'); + return result; } catch (err) { throw new Error( JSON.stringify({ @@ -479,94 +488,122 @@ function resolvePath(...filePaths: string[]): string { return path.resolve(getDirName(), ...filePaths).replace(/\\/g, '/'); } -function formatCode(path: string) { - process.stdout.write(grey('Formatting code...')); - execSync(`cd ${path} && npm run format`); - console.info(green('done\n')); +async function formatCode(codePath: string) { + await execAsync(`cd ${codePath} && npm run format`); } -async function extractAvoAdminCoreTranslations() { - // AVO admin-core - console.info('Extracting AVO admin-core translations...'); - const avoAdminCoreTranslations = await updateTranslations( +async function extractAvoAdminCoreTranslations( + allOnlineTranslations: TranslationEntry[], + onProgress?: ProgressCallback +) { + const translations = await updateTranslations( resolvePath('../src/react-admin'), App.AVO, Component.ADMIN_CORE, '../shared/translations/avo/nl.json', - resolvePath('../tsconfig.json') + allOnlineTranslations, + resolvePath('../tsconfig.json'), + onProgress ); - formatCode(resolvePath('../')); - return avoAdminCoreTranslations; + onProgress?.(97, 'formatting...'); + await formatCode(resolvePath('../')); + onProgress?.(100, 'done'); + return translations; } -async function extractAvoClientTranslations() { - // AVO client - console.info('Extracting AVO client translations...'); - const avoClientTranslations = await updateTranslations( +async function extractAvoClientTranslations( + allOnlineTranslations: TranslationEntry[], + onProgress?: ProgressCallback +) { + const translations = await updateTranslations( resolvePath('../../../avo2-client/src'), App.AVO, Component.FRONTEND, 'shared/translations/nl.json', - resolvePath('../../../avo2-client/tsconfig.json') + allOnlineTranslations, + resolvePath('../../../avo2-client/tsconfig.json'), + onProgress ); - formatCode(resolvePath('../../../avo2-client')); - return avoClientTranslations; + onProgress?.(97, 'formatting...'); + await formatCode(resolvePath('../../../avo2-client')); + onProgress?.(100, 'done'); + return translations; } -async function extractAvoProxyTranslations() { - // AVO proxy - console.info('Extracting AVO admin-core translations...'); - const avoProxyTranslations = await updateTranslations( +async function extractAvoProxyTranslations( + allOnlineTranslations: TranslationEntry[], + onProgress?: ProgressCallback +) { + const translations = await updateTranslations( resolvePath('../../../avo2-proxy/server/src'), App.AVO, Component.BACKEND, 'shared/translations/nl.json', - resolvePath('../../../avo2-proxy/server/tsconfig.json') + allOnlineTranslations, + resolvePath('../../../avo2-proxy/server/tsconfig.json'), + onProgress ); - formatCode(resolvePath('../../../avo2-proxy/server')); - return avoProxyTranslations; + onProgress?.(97, 'formatting...'); + await formatCode(resolvePath('../../../avo2-proxy/server')); + onProgress?.(100, 'done'); + return translations; } -async function extractHetArchiefAdminCoreTranslations() { - // HetArchief admin-core - console.info('Extracting HET_ARCHIEF admin-core translations...'); - const hetArchiefAdminCoreTranslations = await updateTranslations( +async function extractHetArchiefAdminCoreTranslations( + allOnlineTranslations: TranslationEntry[], + onProgress?: ProgressCallback +) { + const translations = await updateTranslations( resolvePath('../src/react-admin'), App.HET_ARCHIEF, Component.ADMIN_CORE, '../shared/translations/hetArchief/nl.json', - resolvePath('../tsconfig.json') + allOnlineTranslations, + resolvePath('../tsconfig.json'), + onProgress ); - formatCode(resolvePath('../')); - return hetArchiefAdminCoreTranslations; + onProgress?.(97, 'formatting...'); + await formatCode(resolvePath('../')); + onProgress?.(100, 'done'); + return translations; } -async function extractHetArchiefClientTranslations() { - // HetArchief client - console.info('Extracting HET_ARCHIEF client translations...'); - const hetArchiefClientTranslations = await updateTranslations( +async function extractHetArchiefClientTranslations( + allOnlineTranslations: TranslationEntry[], + onProgress?: ProgressCallback +) { + const translations = await updateTranslations( resolvePath('../../../hetarchief-client/src'), App.HET_ARCHIEF, Component.FRONTEND, '../public/locales/nl/common.json', - resolvePath('../../../hetarchief-client/tsconfig.json') + allOnlineTranslations, + resolvePath('../../../hetarchief-client/tsconfig.json'), + onProgress ); - formatCode(resolvePath('../../../hetarchief-client')); - return hetArchiefClientTranslations; + onProgress?.(97, 'formatting...'); + await formatCode(resolvePath('../../../hetarchief-client')); + onProgress?.(100, 'done'); + return translations; } -async function extractHetArchiefProxyTranslations() { - // HetArchief proxy - console.info('Extracting HET_ARCHIEF proxy translations...'); - const hetArchiefProxyTranslations = await updateTranslations( +async function extractHetArchiefProxyTranslations( + allOnlineTranslations: TranslationEntry[], + onProgress?: ProgressCallback +) { + const translations = await updateTranslations( resolvePath('../../../hetarchief-proxy/src'), App.HET_ARCHIEF, Component.BACKEND, 'shared/i18n/locales/nl.json', - resolvePath('../../../hetarchief-proxy/tsconfig.json') + allOnlineTranslations, + resolvePath('../../../hetarchief-proxy/tsconfig.json'), + onProgress ); - formatCode(resolvePath('../../../hetarchief-proxy')); - return hetArchiefProxyTranslations; + onProgress?.(97, 'formatting...'); + await formatCode(resolvePath('../../../hetarchief-proxy')); + onProgress?.(100, 'done'); + return translations; } async function extractTranslations() { @@ -577,28 +614,96 @@ async function extractTranslations() { ); } + const labels = + app === App.AVO + ? ['admin-core', 'avo-client', 'avo-proxy'] + : ['admin-core', 'hetarchief-client', 'hetarchief-proxy']; + + const labelWidth = Math.max('total'.length, ...labels.map((l) => l.length)); + const pad = (s: string) => s.padEnd(labelWidth); + const pct = (n: number) => `${String(n).padStart(3)}%`; + + const multiBar = new cliProgress.MultiBar( + { + clearOnComplete: false, + hideCursor: true, + format: ' {bar} {pct} | {label} | {status}', + }, + cliProgress.Presets.shades_classic + ); + + const DIM = '\x1b[2m'; + const RESET = '\x1b[0m'; + const dimFormat = ` ${DIM}{bar} {pct} | {label} | {status}${RESET}`; + + const totalBar = multiBar.create(100, 0, { pct: pct(0), label: pad('total'), status: 'fetching online translations...' }); + const bars = labels.map((label) => + multiBar.create(100, 0, { pct: pct(0), label: pad(label), status: 'waiting...' }, { format: dimFormat }) + ); + + // Track each bar's percentage so we can compute a total + const pcts = [0, 0, 0]; + const makeOnProgress = + (index: number): ProgressCallback => + (value, status) => { + pcts[index] = value; + bars[index].update(value, { pct: pct(value), label: pad(labels[index]), status }); + const total = Math.round(pcts.reduce((a, b) => a + b, 0) / pcts.length); + totalBar.update(total, { + pct: pct(total), + label: pad('total'), + status: `${pcts.filter((p) => p === 100).length}/${pcts.length} done`, + }); + }; + + // Buffer console output so it doesn't trample the progress bars mid-render + const logBuffer: string[] = []; + const origLog = console.log.bind(console); + const origInfo = console.info.bind(console); + const origWarn = console.warn.bind(console); + const origError = console.error.bind(console); + // biome-ignore lint/suspicious/noExplicitAny: intentional console override + const capture = (...args: any[]) => logBuffer.push(args.map(String).join(' ')); + console.log = capture; + console.info = capture; + console.warn = capture; + console.error = capture; + let allTranslations: TranslationEntry[]; - if (app === App.AVO) { - const avoAdminCoreTranslations = await extractAvoAdminCoreTranslations(); - const avoClientTranslations = await extractAvoClientTranslations(); - const avoProxyTranslations = await extractAvoProxyTranslations(); - - allTranslations = [ - ...avoAdminCoreTranslations, - ...avoClientTranslations, - ...avoProxyTranslations, - ]; - } else { - // HET_ARCHIEF - const hetArchiefAdminCoreTranslations = await extractHetArchiefAdminCoreTranslations(); - const hetArchiefClientTranslations = await extractHetArchiefClientTranslations(); - const hetArchiefProxyTranslations = await extractHetArchiefProxyTranslations(); - - allTranslations = [ - ...hetArchiefAdminCoreTranslations, - ...hetArchiefClientTranslations, - ...hetArchiefProxyTranslations, - ]; + try { + const allOnlineTranslations = await getOnlineTranslations(app); + totalBar.update(0, { pct: pct(0), label: pad('total'), status: '0/3 done' }); + + if (app === App.AVO) { + [...Array(3).keys()].forEach((i) => makeOnProgress(i)(0, 'starting...')); + const [adminCore, client, proxy] = await Promise.all([ + extractAvoAdminCoreTranslations(allOnlineTranslations, makeOnProgress(0)), + extractAvoClientTranslations(allOnlineTranslations, makeOnProgress(1)), + extractAvoProxyTranslations(allOnlineTranslations, makeOnProgress(2)), + ]); + allTranslations = [...adminCore, ...client, ...proxy]; + } else { + // HET_ARCHIEF + [...Array(3).keys()].forEach((i) => makeOnProgress(i)(0, 'starting...')); + const [adminCore, client, proxy] = await Promise.all([ + extractHetArchiefAdminCoreTranslations(allOnlineTranslations, makeOnProgress(0)), + extractHetArchiefClientTranslations(allOnlineTranslations, makeOnProgress(1)), + extractHetArchiefProxyTranslations(allOnlineTranslations, makeOnProgress(2)), + ]); + allTranslations = [...adminCore, ...client, ...proxy]; + } + } finally { + console.log = origLog; + console.info = origInfo; + console.warn = origWarn; + console.error = origError; + } + + multiBar.stop(); + + // Flush buffered output now that the progress bars are done + for (const msg of logBuffer) { + origInfo(msg); } // Output all translations as sql file From 1e31f472f479d39e4475aa1b8c16dba5e4281769 Mon Sep 17 00:00:00 2001 From: Bert Verhelst Date: Fri, 12 Jun 2026 22:43:25 +0200 Subject: [PATCH 2/2] fix(translations): fix pr remarks for extract translations script --- ui/package.json | 4 ++-- ui/scripts/extract-and-replace-translations.ts | 17 +++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/ui/package.json b/ui/package.json index 2d08ae22..bf03956e 100644 --- a/ui/package.json +++ b/ui/package.json @@ -60,8 +60,8 @@ "backup-translations-avo-qas": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/backup-translations.ts -- AVO", "translations:extract:hetarchief": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/extract-and-replace-translations.ts -- HET_ARCHIEF", "translations:extract:avo": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/extract-and-replace-translations.ts -- AVO", - "list-of-changed-translations-hetarchief": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/get-list-of-changed-translations.ts -- HET_ARCHIEF v5.2.77 v5.2.79", - "list-commit-messages-between-tags-hetarchief": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/get-list-of-commit-messages-between-tags.ts -- v5.2.77 v5.2.79", + "list-of-changed-translations-hetarchief": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/get-list-of-changed-translations.ts -- HET_ARCHIEF v5.2.79 v5.2.83", + "list-commit-messages-between-tags-hetarchief": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/get-list-of-commit-messages-between-tags.ts -- v5.2.79 v5.2.83", "list-of-changed-translations-avo": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/get-list-of-changed-translations.ts -- AVO v5.2.26 v5.2.32", "list-commit-messages-between-tags-avo": "dotenv -e .env/.env.local tsx --project ./scripts/tsconfig.json scripts/get-list-of-commit-messages-between-tags.ts -- v5.2.26 v5.2.32", "check-bundled-files": "npx depcruise src/react-admin --config bundle-import-rules.json --include-only \"^src\" --exclude \"(.spec.tsx?|.test.tsx?|node_modules)\"", diff --git a/ui/scripts/extract-and-replace-translations.ts b/ui/scripts/extract-and-replace-translations.ts index 39fc6f09..a851545b 100644 --- a/ui/scripts/extract-and-replace-translations.ts +++ b/ui/scripts/extract-and-replace-translations.ts @@ -189,7 +189,7 @@ async function extractTranslationsFromCodeFiles( const total = sourceFiles.length; for (let i = 0; i < total; i++) { const sourceFile = sourceFiles[i]; - onProgress?.(Math.round((i / total) * 80), sourceFile.getBaseName()); + onProgress?.(10 + Math.round(((i + 1) / total) * 70), sourceFile.getBaseName()); // Find all tHtml() and tText() function calls const translationFunctionCalls = sourceFile @@ -489,7 +489,7 @@ function resolvePath(...filePaths: string[]): string { } async function formatCode(codePath: string) { - await execAsync(`cd ${codePath} && npm run format`); + await execAsync('npm run format', { cwd: codePath }); } async function extractAvoAdminCoreTranslations( @@ -669,7 +669,7 @@ async function extractTranslations() { console.warn = capture; console.error = capture; - let allTranslations: TranslationEntry[]; + let allTranslations: TranslationEntry[] = []; try { const allOnlineTranslations = await getOnlineTranslations(app); totalBar.update(0, { pct: pct(0), label: pad('total'), status: '0/3 done' }); @@ -697,13 +697,10 @@ async function extractTranslations() { console.info = origInfo; console.warn = origWarn; console.error = origError; - } - - multiBar.stop(); - - // Flush buffered output now that the progress bars are done - for (const msg of logBuffer) { - origInfo(msg); + multiBar.stop(); + for (const msg of logBuffer) { + origInfo(msg); + } } // Output all translations as sql file