From d8525fa5fd3fe2c36b6cf8a0b7a30a4689889bd7 Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Wed, 12 Nov 2025 18:08:12 +0545 Subject: [PATCH 1/3] fix(translatte): update commands --- .../translatte/commands/clearServerStrings.ts | 12 +- .../commands/exportServerStringsToExcel.ts | 4 +- .../translatte/commands/pushMigration.ts | 10 +- .../translatte/commands/pushStringsDref.ts | 153 ++++++++++++++++++ .../commands/pushStringsFromExcel.ts | 94 ++++++++--- app/scripts/translatte/main.ts | 32 ++++ 6 files changed, 273 insertions(+), 32 deletions(-) create mode 100644 app/scripts/translatte/commands/pushStringsDref.ts diff --git a/app/scripts/translatte/commands/clearServerStrings.ts b/app/scripts/translatte/commands/clearServerStrings.ts index 6df5d923b5..d368922403 100644 --- a/app/scripts/translatte/commands/clearServerStrings.ts +++ b/app/scripts/translatte/commands/clearServerStrings.ts @@ -1,11 +1,11 @@ -import { listToGroupList } from "@togglecorp/fujs"; +import { isTruthyString, listToGroupList } from "@togglecorp/fujs"; import { fetchServerState, postLanguageStrings, writeFilePromisify } from "../utils"; async function clearServerStrings(apiUrl: string, authToken: string) { const serverStrings = await fetchServerState(apiUrl, authToken); const bulkActions = listToGroupList( - serverStrings, + serverStrings.filter(({ page_name }) => isTruthyString(page_name)), ({ language }) => language, ({ key, page_name }) => ({ action: "delete" as const, @@ -19,7 +19,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { response: object, }[] = []; - console.log('Pusing delete actions for en...') + console.log('Pushing delete actions for en...') const enResponse = await postLanguageStrings( 'en', bulkActions.en, @@ -31,7 +31,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { logs.push({ responseFor: 'en', response: enResponseJson }); - console.log('Pusing delete actions for fr...') + console.log('Pushing delete actions for fr...') const frResponse = await postLanguageStrings( 'fr', bulkActions.fr, @@ -42,7 +42,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { const frResponseJson = await frResponse.json(); logs.push({ responseFor: 'fr', response: frResponseJson }); - console.log('Pusing delete actions for es...') + console.log('Pushing delete actions for es...') const esResponse = await postLanguageStrings( 'es', bulkActions.es, @@ -52,7 +52,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { const esResponseJson = await esResponse.json(); logs.push({ responseFor: 'es', response: esResponseJson }); - console.log('Pusing delete actions for ar...') + console.log('Pushing delete actions for ar...') const arResponse = await postLanguageStrings( 'ar', bulkActions.ar, diff --git a/app/scripts/translatte/commands/exportServerStringsToExcel.ts b/app/scripts/translatte/commands/exportServerStringsToExcel.ts index bcfa7f8e93..73b8f966f1 100644 --- a/app/scripts/translatte/commands/exportServerStringsToExcel.ts +++ b/app/scripts/translatte/commands/exportServerStringsToExcel.ts @@ -1,7 +1,7 @@ import xlsx from 'exceljs'; import { fetchServerState } from "../utils"; -import { isFalsyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { isFalsyString, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; async function exportServerStringsToExcel( apiUrl: string, @@ -35,7 +35,7 @@ async function exportServerStringsToExcel( const keyGroupedStrings = mapToList( listToGroupList( - serverStrings, + serverStrings.filter(({ page_name, key }) => isTruthyString(page_name) && isTruthyString(key)), ({ page_name, key }) => `${page_name}:${key}`, ), (list) => { diff --git a/app/scripts/translatte/commands/pushMigration.ts b/app/scripts/translatte/commands/pushMigration.ts index bce31ef3a7..74969d1e9e 100644 --- a/app/scripts/translatte/commands/pushMigration.ts +++ b/app/scripts/translatte/commands/pushMigration.ts @@ -102,6 +102,10 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke const serverActionsForCurrentLanguage = actions.flatMap((actionItem) => { if (language === 'en') { if (actionItem.action === 'add') { + if (isFalsyString(actionItem.value)) { + return undefined; + } + return { action: 'set' as const, key: actionItem.key, @@ -168,7 +172,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke ); await writeFilePromisify( - `server-actions.json`, + `/tmp/server-actions.json`, JSON.stringify(serverActions, null, 2), 'utf8', ); @@ -193,7 +197,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke const setActions = actions.filter(({ action }) => action === 'set'); const deleteActions = actions.filter(({ action }) => action === 'delete'); - console.log(`Pusing deleted actions for ${lang}...`) + console.log(`Pushing deleted actions for ${lang}...`) const deleteResponse = await postLanguageStrings( lang, deleteActions, @@ -221,7 +225,7 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke await applyAction(serverActions.ar.language, serverActions.ar.actions); await writeFilePromisify( - `push-migration-logs.json`, + `/tmp/push-migration-logs.json`, JSON.stringify(logs, null, 2), 'utf8', ); diff --git a/app/scripts/translatte/commands/pushStringsDref.ts b/app/scripts/translatte/commands/pushStringsDref.ts new file mode 100644 index 0000000000..82d3e5c7d3 --- /dev/null +++ b/app/scripts/translatte/commands/pushStringsDref.ts @@ -0,0 +1,153 @@ +import xlsx, { CellValue } from 'exceljs'; +import { fetchServerState, postLanguageStrings } from "../utils"; +import { encodeDate, isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { Language, ServerActionItem, SourceStringItem } from '../types'; + + +function getValueFromCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return getValueFromCellValue(cellValue.result); +} + +async function pushStringsDref(importFilePath: string, apiUrl: string, accessToken: string) { + const strings = await fetchServerState(apiUrl); + const enStrings = strings.filter((string) => string.language === 'en'); + + const workbook = new xlsx.Workbook(); + + await workbook.xlsx.readFile(importFilePath); + + const firstSheet = workbook.worksheets[0]; + const columns = firstSheet.columns.map( + (column) => { + const key = column.values?.[1]?.toString(); + if (isNotDefined(key)) { + return undefined; + } + return { key, column: column.number } + } + ).filter(isDefined); + + const columnMap = listToMap( + columns, + ({ key }) => key, + ({ column }) => column, + ); + + const updatedStrings: SourceStringItem[] = []; + + firstSheet.eachRow((row) => { + const enColumnKey = columnMap['EN']; + const frColumnKey = columnMap['FR']; + const esColumnKey = columnMap['ES']; + const arColumnKey = columnMap['AR']; + + const enValue = isDefined(enColumnKey) ? getValueFromCellValue(row.getCell(enColumnKey).value) : undefined; + + const string = enStrings.find(({ value }) => value === enValue); + + if (string) { + const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; + const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; + const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + + updatedStrings.push({ + ...string, + language: 'fr', + value: String(frValue), + }); + + updatedStrings.push({ + ...string, + language: 'es', + value: String(esValue), + }); + + updatedStrings.push({ + ...string, + language: 'ar', + value: String(arValue), + }); + } + }); + + const languageGroupedActions = mapToList( + listToGroupList( + updatedStrings, + ({ language }) => language, + (languageString) => { + const serverAction: ServerActionItem = { + action: 'set', + key: languageString.key, + page_name: languageString.page_name, + value: languageString.value, + hash: languageString.hash, + } + + return serverAction; + }, + ), + (actions, language) => ({ + language: language as Language, + actions, + }) + ); + + for (let i = 0; i < languageGroupedActions.length; i++) { + const action = languageGroupedActions[i]; + + console.log(`posting ${action.language} actions...`); + const result = await postLanguageStrings( + action.language, + action.actions, + apiUrl, + accessToken, + ) + + const resultJson = await result.json(); + console.info(resultJson); + } +} + +export default pushStringsDref; diff --git a/app/scripts/translatte/commands/pushStringsFromExcel.ts b/app/scripts/translatte/commands/pushStringsFromExcel.ts index b9da7b38b5..0cf234a60e 100644 --- a/app/scripts/translatte/commands/pushStringsFromExcel.ts +++ b/app/scripts/translatte/commands/pushStringsFromExcel.ts @@ -1,10 +1,56 @@ -import xlsx from 'exceljs'; +import xlsx, { CellValue } from 'exceljs'; import { Md5 } from 'ts-md5'; -import { isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { encodeDate, isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; import { Language, ServerActionItem } from '../types'; import { postLanguageStrings } from '../utils'; +function getValueFromCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return getValueFromCellValue(cellValue.result); +} + async function pushStringsFromExcel(importFilePath: string, apiUrl: string, accessToken: string) { const workbook = new xlsx.Workbook(); @@ -37,27 +83,27 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce firstSheet.eachRow( (row) => { - const keyColumn = columnMap['key']; - const key = isDefined(keyColumn) ? row.getCell(keyColumn).value?.toString() : undefined; + const keyColumn = columnMap['Key']; + const key = isDefined(keyColumn) ? String(getValueFromCellValue(row.getCell(keyColumn).value)) : undefined; - const namespaceColumn = columnMap['namespace']; - const namespace = isDefined(namespaceColumn) ? row.getCell(namespaceColumn).value?.toString() : undefined; + const namespaceColumn = columnMap['Namespace']; + const namespace = isDefined(namespaceColumn) ? String(getValueFromCellValue(row.getCell(namespaceColumn).value)) : undefined; if (isNotDefined(key) || isNotDefined(namespace)) { return; } - const enColumn = columnMap['en']; - const en = isDefined(enColumn) ? row.getCell(enColumn).value?.toString() : undefined; + const enColumn = columnMap['EN']; + const en = isDefined(enColumn) ? String(getValueFromCellValue(row.getCell(enColumn).value)) : undefined; - const arColumn = columnMap['ar']; - const ar = isDefined(arColumn) ? row.getCell(arColumn).value?.toString() : undefined; + const arColumn = columnMap['AR']; + const ar = isDefined(arColumn) ? String(getValueFromCellValue(row.getCell(arColumn).value)) : undefined; - const frColumn = columnMap['fr']; - const fr = isDefined(frColumn) ? row.getCell(frColumn).value?.toString() : undefined; + const frColumn = columnMap['FR']; + const fr = isDefined(frColumn) ? String(getValueFromCellValue(row.getCell(frColumn).value)) : undefined; - const esColumn = columnMap['es']; - const es = isDefined(esColumn) ? row.getCell(esColumn).value?.toString() : undefined; + const esColumn = columnMap['ES']; + const es = isDefined(esColumn) ? String(getValueFromCellValue(row.getCell(esColumn).value)) : undefined; if (isNotDefined(en)) { return; @@ -65,6 +111,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce const hash = Md5.hashStr(en); + /* strings.push({ key, namespace, @@ -72,6 +119,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce value: en, hash, }); + */ if (isDefined(ar)) { strings.push({ @@ -127,16 +175,20 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }) ); - const postPromises = languageGroupedActions.map( - (languageStrings) => postLanguageStrings( - languageStrings.language, - languageStrings.actions, + for (let i = 0; i < languageGroupedActions.length; i++) { + const action = languageGroupedActions[i]; + + console.log(`posting ${action.language} actions...`); + const result = await postLanguageStrings( + action.language, + action.actions, apiUrl, accessToken, - ) - ) + ); - await Promise.all(postPromises); + const resultJson = await result.text(); + console.info(resultJson); + } } export default pushStringsFromExcel; diff --git a/app/scripts/translatte/main.ts b/app/scripts/translatte/main.ts index f8dd65a23f..905c6a04b9 100644 --- a/app/scripts/translatte/main.ts +++ b/app/scripts/translatte/main.ts @@ -14,6 +14,7 @@ import pushMigration from './commands/pushMigration'; import pushStringsFromExcel from './commands/pushStringsFromExcel'; import exportServerStringsToExcel from './commands/exportServerStringsToExcel'; import clearServerStrings from './commands/clearServerStrings'; +import pushStringsDref from './commands/pushStringsDref'; const currentDir = cwd(); @@ -254,6 +255,37 @@ yargs(hideBin(process.argv)) ); }, ) + .command( + 'push-strings-dref ', + 'IMPORTANT!!! Temporary command, do not use!', + (yargs) => { + yargs.positional('IMPORT_FILE_PATH', { + type: 'string', + describe: 'Find the import file on IMPORT_FILE_PATH', + }); + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + } + }); + }, + async (argv) => { + const importFilePath = (argv.IMPORT_FILE_PATH as string); + + await pushStringsDref( + importFilePath, + argv.apiUrl as string, + argv.authToken as string, + ); + }, + ) .command( 'export-server-strings ', 'Export server strings to excel file', From 0f4155fc25fb1ea2e5e1a08abe05f903a6a52a49 Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Thu, 20 Nov 2025 16:30:34 +0545 Subject: [PATCH 2/3] feat(translatte): add command to sync en strings --- .../translatte/commands/pushStringsDref.ts | 63 +++++++++++++++++-- .../commands/pushStringsFromExcel.ts | 30 ++++++--- .../translatte/commands/syncEnStrings.ts | 39 ++++++++++++ app/scripts/translatte/main.ts | 31 +++++++++ 4 files changed, 149 insertions(+), 14 deletions(-) create mode 100644 app/scripts/translatte/commands/syncEnStrings.ts diff --git a/app/scripts/translatte/commands/pushStringsDref.ts b/app/scripts/translatte/commands/pushStringsDref.ts index 82d3e5c7d3..062ca3820e 100644 --- a/app/scripts/translatte/commands/pushStringsDref.ts +++ b/app/scripts/translatte/commands/pushStringsDref.ts @@ -2,6 +2,7 @@ import xlsx, { CellValue } from 'exceljs'; import { fetchServerState, postLanguageStrings } from "../utils"; import { encodeDate, isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; import { Language, ServerActionItem, SourceStringItem } from '../types'; +import { Md5 } from 'ts-md5'; function getValueFromCellValue(cellValue: CellValue) { @@ -78,6 +79,16 @@ async function pushStringsDref(importFilePath: string, apiUrl: string, accessTok const updatedStrings: SourceStringItem[] = []; firstSheet.eachRow((row) => { + const keyColumn = columnMap['Key']; + const key = isDefined(keyColumn) ? String(getValueFromCellValue(row.getCell(keyColumn).value)) : undefined; + + const namespaceColumn = columnMap['Namespace']; + const namespace = isDefined(namespaceColumn) ? String(getValueFromCellValue(row.getCell(namespaceColumn).value)) : undefined; + + if (isNotDefined(key) || isNotDefined(namespace)) { + return; + } + const enColumnKey = columnMap['EN']; const frColumnKey = columnMap['FR']; const esColumnKey = columnMap['ES']; @@ -85,27 +96,69 @@ async function pushStringsDref(importFilePath: string, apiUrl: string, accessTok const enValue = isDefined(enColumnKey) ? getValueFromCellValue(row.getCell(enColumnKey).value) : undefined; - const string = enStrings.find(({ value }) => value === enValue); + const strings = enStrings.filter(({ value }) => value === enValue); + + if (strings.length > 0) { + strings.forEach((string) => { + const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; + const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; + const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + + updatedStrings.push({ + ...string, + language: 'fr', + value: String(frValue), + }); + + updatedStrings.push({ + ...string, + language: 'es', + value: String(esValue), + }); + + updatedStrings.push({ + ...string, + language: 'ar', + value: String(arValue), + }); + }); + } - if (string) { + if (strings.length === 0) { const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + const hash = Md5.hashStr(String(enValue)); + + updatedStrings.push({ + key, + page_name: namespace, + hash, + language: 'en', + value: String(enValue), + }); + updatedStrings.push({ - ...string, + key, + page_name: namespace, + hash, language: 'fr', value: String(frValue), }); updatedStrings.push({ - ...string, + key, + page_name: namespace, + hash, language: 'es', value: String(esValue), }); updatedStrings.push({ - ...string, + key, + page_name: namespace, + hash, language: 'ar', value: String(arValue), }); diff --git a/app/scripts/translatte/commands/pushStringsFromExcel.ts b/app/scripts/translatte/commands/pushStringsFromExcel.ts index 0cf234a60e..b0ed5ccab5 100644 --- a/app/scripts/translatte/commands/pushStringsFromExcel.ts +++ b/app/scripts/translatte/commands/pushStringsFromExcel.ts @@ -1,9 +1,9 @@ import xlsx, { CellValue } from 'exceljs'; import { Md5 } from 'ts-md5'; -import { encodeDate, isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { encodeDate, isDefined, isFalsyString, isNotDefined, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; import { Language, ServerActionItem } from '../types'; -import { postLanguageStrings } from '../utils'; +import { postLanguageStrings, writeFilePromisify } from '../utils'; function getValueFromCellValue(cellValue: CellValue) { if (isNotDefined(cellValue)) { @@ -82,20 +82,28 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }[] = []; firstSheet.eachRow( - (row) => { + (row, i) => { + if (i === 0) { + return; + } + const keyColumn = columnMap['Key']; const key = isDefined(keyColumn) ? String(getValueFromCellValue(row.getCell(keyColumn).value)) : undefined; const namespaceColumn = columnMap['Namespace']; const namespace = isDefined(namespaceColumn) ? String(getValueFromCellValue(row.getCell(namespaceColumn).value)) : undefined; - if (isNotDefined(key) || isNotDefined(namespace)) { + if (isFalsyString(key) || isFalsyString(namespace)) { return; } const enColumn = columnMap['EN']; const en = isDefined(enColumn) ? String(getValueFromCellValue(row.getCell(enColumn).value)) : undefined; + if (isFalsyString(en)) { + return; + } + const arColumn = columnMap['AR']; const ar = isDefined(arColumn) ? String(getValueFromCellValue(row.getCell(arColumn).value)) : undefined; @@ -111,7 +119,6 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce const hash = Md5.hashStr(en); - /* strings.push({ key, namespace, @@ -119,9 +126,8 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce value: en, hash, }); - */ - if (isDefined(ar)) { + if (isTruthyString(ar)) { strings.push({ key, namespace, @@ -131,7 +137,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }); } - if (isDefined(fr)) { + if (isTruthyString(fr)) { strings.push({ key, namespace, @@ -141,7 +147,7 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }); } - if (isDefined(es)) { + if (isTruthyString(es)) { strings.push({ key, namespace, @@ -175,6 +181,12 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce }) ); + await writeFilePromisify( + '/tmp/language-grouped-actions.json', + JSON.stringify(languageGroupedActions, null, 2), + 'utf8', + ); + for (let i = 0; i < languageGroupedActions.length; i++) { const action = languageGroupedActions[i]; diff --git a/app/scripts/translatte/commands/syncEnStrings.ts b/app/scripts/translatte/commands/syncEnStrings.ts new file mode 100644 index 0000000000..3b3b7982ef --- /dev/null +++ b/app/scripts/translatte/commands/syncEnStrings.ts @@ -0,0 +1,39 @@ +import { isDefined, isFalsyString } from "@togglecorp/fujs"; +import { fetchServerState, postLanguageStrings, writeFilePromisify } from "../utils"; + +async function syncEnStrings(sourceApiUrl: string, desinationApiUrl: string, authToken: string) { + const serverStrings = await fetchServerState(sourceApiUrl, authToken); + const enStrings = serverStrings.filter((string) => string.language === 'en'); + + const actions = enStrings.map((string) => { + if (isFalsyString(string.key) || isFalsyString(string.page_name) || isFalsyString(string.value)) { + return undefined; + } + + return { + action: 'set' as const, + key: string.key, + page_name: string.page_name, + value: string.value, + hash: string.hash, + }; + }).filter(isDefined); + + console.log("posting en actions..."); + const result = await postLanguageStrings( + 'en', + actions, + desinationApiUrl, + authToken, + ) + + const resultJson = await result.json(); + console.info(resultJson); + await writeFilePromisify( + '/tmp/sync-en-strings-logs.json', + JSON.stringify(resultJson, null, 2), + 'utf8', + ); +} + +export default syncEnStrings; diff --git a/app/scripts/translatte/main.ts b/app/scripts/translatte/main.ts index 905c6a04b9..9a2f34c02f 100644 --- a/app/scripts/translatte/main.ts +++ b/app/scripts/translatte/main.ts @@ -15,6 +15,7 @@ import pushStringsFromExcel from './commands/pushStringsFromExcel'; import exportServerStringsToExcel from './commands/exportServerStringsToExcel'; import clearServerStrings from './commands/clearServerStrings'; import pushStringsDref from './commands/pushStringsDref'; +import syncEnStrings from './commands/syncEnStrings'; const currentDir = cwd(); @@ -286,6 +287,36 @@ yargs(hideBin(process.argv)) ); }, ) + .command( + 'sync-en-strings', + 'IMPORTANT!!! Temporary command, do not use!', + (yargs) => { + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'source-api-url': { + type: 'string', + describe: 'URL for the source API server', + require: true, + }, + 'destination-api-url': { + type: 'string', + describe: 'URL for the destination', + require: true, + } + }); + }, + async (argv) => { + await syncEnStrings( + argv.sourceApiUrl as string, + argv.destinationApiUrl as string, + argv.authToken as string, + ); + }, + ) .command( 'export-server-strings ', 'Export server strings to excel file', From ecd4b8e4cadadf9aacc4011f7ba203904b4225ec Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Thu, 20 Nov 2025 16:30:34 +0545 Subject: [PATCH 3/3] feat(translatte): add command to sync en strings --- .../translatte/commands/pushStringsDref.ts | 239 ++++++++++++++---- .../commands/pushStringsFromExcel.ts | 62 +++-- app/scripts/translatte/commands/testExcel.ts | 235 +++++++++++++++++ app/scripts/translatte/main.ts | 46 +++- 4 files changed, 507 insertions(+), 75 deletions(-) create mode 100644 app/scripts/translatte/commands/testExcel.ts diff --git a/app/scripts/translatte/commands/pushStringsDref.ts b/app/scripts/translatte/commands/pushStringsDref.ts index 062ca3820e..3cc25d18e2 100644 --- a/app/scripts/translatte/commands/pushStringsDref.ts +++ b/app/scripts/translatte/commands/pushStringsDref.ts @@ -1,6 +1,6 @@ import xlsx, { CellValue } from 'exceljs'; -import { fetchServerState, postLanguageStrings } from "../utils"; -import { encodeDate, isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { fetchServerState, getTranslationFileNames, postLanguageStrings, readTranslations, writeFilePromisify } from "../utils"; +import { encodeDate, isDefined, isFalsyString, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; import { Language, ServerActionItem, SourceStringItem } from '../types'; import { Md5 } from 'ts-md5'; @@ -51,12 +51,85 @@ function getValueFromCellValue(cellValue: CellValue) { return getValueFromCellValue(cellValue.result); } -async function pushStringsDref(importFilePath: string, apiUrl: string, accessToken: string) { - const strings = await fetchServerState(apiUrl); - const enStrings = strings.filter((string) => string.language === 'en'); +function getCombinedKey(namespace: string, key: string) { + return `${namespace}:${key}`; +} +async function createExcel(groupedStrings: Record) { const workbook = new xlsx.Workbook(); + const now = new Date(); + workbook.created = now; + + const yyyy = now.getFullYear(); + const mm = (now.getMonth() + 1).toString().padStart(2, '0'); + const dd = now.getDate().toString().padStart(2, '0'); + const HH = now.getHours().toString().padStart(2, '0'); + const MM = now.getMinutes().toString().padStart(2, '0'); + + const worksheet = workbook.addWorksheet( + `${yyyy}-${mm}-${dd} ${HH}-${MM}` + ); + + worksheet.columns = [ + { header: 'Namespace', key: 'namespace' }, + { header: 'Key', key: 'key' }, + { header: 'EN', key: 'en' }, + { header: 'FR', key: 'fr' }, + { header: 'ES', key: 'es' }, + { header: 'AR', key: 'ar' }, + ]; + + Object.values(groupedStrings).map((translations) => { + const translationByLang = listToMap( + translations, + ({ language }) => language, + ); + + if (isFalsyString(translationByLang.en)) { + console.info(JSON.stringify(translationByLang, null, 2)); + } else { + worksheet.addRow({ + namespace: translationByLang.en.page_name, + key: translationByLang.en.key, + en: translationByLang.en.value, + fr: translationByLang.fr?.value, + es: translationByLang.es?.value, + ar: translationByLang.ar?.value, + }); + } + }); + + const fileName = `go-dref-updated-strings-${yyyy}-${mm}-${dd}`; + + await workbook.xlsx.writeFile(`${fileName}.xlsx`); +} +async function pushStringsDref( + projectPath: string, + importFilePath: string, + translationFileNames: string[], + apiUrl: string, + accessToken: string, +) { + const serverState = await fetchServerState(apiUrl); + + const groupedServerStateMapping = listToGroupList( + serverState, + ({ page_name, key }) => getCombinedKey(page_name, key), + ); + + const serverEnStringItems = serverState.filter((string) => string.language === 'en'); + + const translationFiles = await getTranslationFileNames( + projectPath, + Array.isArray(translationFileNames) ? translationFileNames : [translationFileNames], + ); + const { translations } = await readTranslations(translationFiles); + const fileState = translations.map((item) => ({ + ...item, + })); + + const workbook = new xlsx.Workbook(); await workbook.xlsx.readFile(importFilePath); const firstSheet = workbook.worksheets[0]; @@ -78,14 +151,8 @@ async function pushStringsDref(importFilePath: string, apiUrl: string, accessTok const updatedStrings: SourceStringItem[] = []; - firstSheet.eachRow((row) => { - const keyColumn = columnMap['Key']; - const key = isDefined(keyColumn) ? String(getValueFromCellValue(row.getCell(keyColumn).value)) : undefined; - - const namespaceColumn = columnMap['Namespace']; - const namespace = isDefined(namespaceColumn) ? String(getValueFromCellValue(row.getCell(namespaceColumn).value)) : undefined; - - if (isNotDefined(key) || isNotDefined(namespace)) { + firstSheet.eachRow((row, i) => { + if (i === 0) { return; } @@ -95,74 +162,140 @@ async function pushStringsDref(importFilePath: string, apiUrl: string, accessTok const arColumnKey = columnMap['AR']; const enValue = isDefined(enColumnKey) ? getValueFromCellValue(row.getCell(enColumnKey).value) : undefined; + const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; + const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; + const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + + const serverMatchedStrings = serverEnStringItems.filter(({ value }) => value === enValue); + + serverMatchedStrings.forEach((matchedItem) => { + const combinedKey = getCombinedKey(matchedItem.page_name, matchedItem.key); + + groupedServerStateMapping[combinedKey] = groupedServerStateMapping[combinedKey].map((translationItem) => { + if (translationItem.language === 'fr') { + return { + ...matchedItem, + language: 'fr', + value: String(frValue), + } + } + + if (translationItem.language === 'es') { + return { + ...matchedItem, + language: 'es', + value: String(esValue), + } + } - const strings = enStrings.filter(({ value }) => value === enValue); + if (translationItem.language === 'ar') { + return { + ...matchedItem, + language: 'ar', + value: String(esValue), + } + } + + return translationItem; + }); + + updatedStrings.push({ + ...matchedItem, + language: 'fr', + value: String(frValue), + }); - if (strings.length > 0) { - strings.forEach((string) => { - const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; - const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; - const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + updatedStrings.push({ + ...matchedItem, + language: 'es', + value: String(esValue), + }); - updatedStrings.push({ - ...string, + updatedStrings.push({ + ...matchedItem, + language: 'ar', + value: String(arValue), + }); + }); + + const serverMatchedStringsMapping = listToMap( + serverMatchedStrings, + ({ key, page_name }) => getCombinedKey(page_name, key), + () => true, + ); + + const fileMatchedEnStrings = fileState.filter( + ({ value, key, namespace }) => value === enValue && !serverMatchedStringsMapping[getCombinedKey(namespace, key)] + ); + + fileMatchedEnStrings.forEach((matchedItem) => { + const hash = Md5.hashStr(matchedItem.value); + const combinedKey = getCombinedKey(matchedItem.namespace, matchedItem.key); + + groupedServerStateMapping[combinedKey] = [ + { + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, + language: 'en', + value: matchedItem.value, + }, + { + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, language: 'fr', value: String(frValue), - }); - - updatedStrings.push({ - ...string, + }, + { + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, language: 'es', value: String(esValue), - }); - - updatedStrings.push({ - ...string, + }, + { + key: matchedItem.key, + page_name: matchedItem.namespace, + hash, language: 'ar', value: String(arValue), - }); - }); - } - - if (strings.length === 0) { - const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; - const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; - const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + }, + ]; - const hash = Md5.hashStr(String(enValue)); updatedStrings.push({ - key, - page_name: namespace, + key: matchedItem.key, + page_name: matchedItem.namespace, hash, language: 'en', - value: String(enValue), + value: matchedItem.value, }); updatedStrings.push({ - key, - page_name: namespace, + key: matchedItem.key, + page_name: matchedItem.namespace, hash, language: 'fr', value: String(frValue), }); updatedStrings.push({ - key, - page_name: namespace, + key: matchedItem.key, + page_name: matchedItem.namespace, hash, language: 'es', value: String(esValue), }); updatedStrings.push({ - key, - page_name: namespace, + key: matchedItem.key, + page_name: matchedItem.namespace, hash, language: 'ar', value: String(arValue), }); - } + }); }); const languageGroupedActions = mapToList( @@ -187,6 +320,9 @@ async function pushStringsDref(importFilePath: string, apiUrl: string, accessTok }) ); + await createExcel(groupedServerStateMapping); + + /* for (let i = 0; i < languageGroupedActions.length; i++) { const action = languageGroupedActions[i]; @@ -199,8 +335,13 @@ async function pushStringsDref(importFilePath: string, apiUrl: string, accessTok ) const resultJson = await result.json(); - console.info(resultJson); + await writeFilePromisify( + `/tmp/push-${action.language}-strings-dref-logs.json`, + JSON.stringify(resultJson, null, 2), + 'utf8', + ); } + */ } export default pushStringsDref; diff --git a/app/scripts/translatte/commands/pushStringsFromExcel.ts b/app/scripts/translatte/commands/pushStringsFromExcel.ts index b0ed5ccab5..addb405ee4 100644 --- a/app/scripts/translatte/commands/pushStringsFromExcel.ts +++ b/app/scripts/translatte/commands/pushStringsFromExcel.ts @@ -1,6 +1,6 @@ import xlsx, { CellValue } from 'exceljs'; import { Md5 } from 'ts-md5'; -import { encodeDate, isDefined, isFalsyString, isNotDefined, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { encodeDate, isDefined, isFalsyString, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; import { Language, ServerActionItem } from '../types'; import { postLanguageStrings, writeFilePromisify } from '../utils'; @@ -57,15 +57,23 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce await workbook.xlsx.readFile(importFilePath); const firstSheet = workbook.worksheets[0]; - const columns = firstSheet.columns.map( - (column) => { - const key = column.values?.[1]?.toString(); - if (isNotDefined(key)) { - return undefined; - } - return { key, column: column.number } + console.info(firstSheet.columnCount); + + const columns: { + key: string; + column: number | undefined; + }[] = []; + + for (let i = 0; i < firstSheet.columnCount; i++) { + const column = firstSheet.columns[i]; + const key = column.values?.[1]?.toString(); + + if (isNotDefined(key)) { + return; } - ).filter(isDefined); + + columns.push({ key, column: column.number }) + } const columnMap = listToMap( columns, @@ -98,67 +106,71 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce } const enColumn = columnMap['EN']; - const en = isDefined(enColumn) ? String(getValueFromCellValue(row.getCell(enColumn).value)) : undefined; + const en = isDefined(enColumn) ? getValueFromCellValue(row.getCell(enColumn).value) : undefined; if (isFalsyString(en)) { return; } const arColumn = columnMap['AR']; - const ar = isDefined(arColumn) ? String(getValueFromCellValue(row.getCell(arColumn).value)) : undefined; + const ar = isDefined(arColumn) ? getValueFromCellValue(row.getCell(arColumn).value) : undefined; const frColumn = columnMap['FR']; - const fr = isDefined(frColumn) ? String(getValueFromCellValue(row.getCell(frColumn).value)) : undefined; + const fr = isDefined(frColumn) ? getValueFromCellValue(row.getCell(frColumn).value) : undefined; const esColumn = columnMap['ES']; - const es = isDefined(esColumn) ? String(getValueFromCellValue(row.getCell(esColumn).value)) : undefined; + const es = isDefined(esColumn) ? getValueFromCellValue(row.getCell(esColumn).value) : undefined; if (isNotDefined(en)) { return; } - const hash = Md5.hashStr(en); + const hash = Md5.hashStr(String(en)); strings.push({ key, namespace, language: 'en', - value: en, + value: String(en), hash, }); - if (isTruthyString(ar)) { + if (isDefined(fr)) { strings.push({ key, namespace, - language: 'ar', - value: ar, - hash, + language: 'fr', + value: String(fr), + hash, }); } - if (isTruthyString(fr)) { + + if (isDefined(es)) { strings.push({ key, namespace, - language: 'fr', - value: fr, + language: 'es', + value: String(es), hash, }); } - if (isTruthyString(es)) { + if (isDefined(ar)) { strings.push({ key, namespace, - language: 'es', - value: es, + language: 'ar', + value: String(ar), hash, }); } + } ); + console.info(`Total ${strings.length} actions`); + const languageGroupedActions = mapToList( listToGroupList( strings, diff --git a/app/scripts/translatte/commands/testExcel.ts b/app/scripts/translatte/commands/testExcel.ts new file mode 100644 index 0000000000..4f75059b21 --- /dev/null +++ b/app/scripts/translatte/commands/testExcel.ts @@ -0,0 +1,235 @@ +import xlsx, { CellValue } from 'exceljs'; +import { encodeDate, isDefined, isFalsyString, isNotDefined, isTruthyString, listToGroupList, listToMap } from "@togglecorp/fujs"; +import { fetchServerState, getTranslationFileNames, readTranslations, writeFilePromisify } from "../utils"; +import { Md5 } from "ts-md5"; + +function getValueFromCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return getValueFromCellValue(cellValue.result); +} + +export function isTranslatedTemplateValid( + source: string, + translation: string, +): boolean { + const extract = (s: string): Set => { + const set = new Set(); + const re = /\{([^{}]+)\}/g; + let match: RegExpExecArray | null; + + while ((match = re.exec(s)) !== null) { + const key = match[1].trim(); + if (key) set.add(key); + } + return set; + }; + + const sourceVars = extract(source); + const translationVars = extract(translation); + + if (sourceVars.size !== translationVars.size) return false; + + for (const v of sourceVars) { + if (!translationVars.has(v)) return false; + } + + return true; +} + +async function createExcel( + items: { + key: string, + namespace: string, + en: string, + fr: string | undefined, + es: string | undefined, + ar: string | undefined, + }[] +) { + const workbook = new xlsx.Workbook(); + const now = new Date(); + workbook.created = now; + + const yyyy = now.getFullYear(); + const mm = (now.getMonth() + 1).toString().padStart(2, '0'); + const dd = now.getDate().toString().padStart(2, '0'); + const HH = now.getHours().toString().padStart(2, '0'); + const MM = now.getMinutes().toString().padStart(2, '0'); + + const worksheet = workbook.addWorksheet( + `${yyyy}-${mm}-${dd} ${HH}-${MM}` + ); + + worksheet.columns = [ + { header: 'Namespace', key: 'namespace' }, + { header: 'Key', key: 'key' }, + { header: 'EN', key: 'en' }, + { header: 'FR', key: 'fr' }, + { header: 'ES', key: 'es' }, + { header: 'AR', key: 'ar' }, + ]; + + items.forEach((item) => { + const itemSafe = {}; + + for (const [key, value] of Object.entries(item)) { + if (isTruthyString(value)) { + itemSafe[key] = value; + } + } + + worksheet.addRow(itemSafe); + }); + + const fileName = `go-strings-${yyyy}-${mm}-${dd}`; + + await workbook.xlsx.writeFile(`${fileName}.xlsx`); +} + +async function testExcel( + importFilePath: string, + projectPath: string, + translationFileNames: string[], + apiUrl: string, + accessToken?: string, +) { + const workbook = new xlsx.Workbook(); + await workbook.xlsx.readFile(importFilePath); + + const firstSheet = workbook.worksheets[0]; + const columns = firstSheet.columns.map( + (column) => { + const key = column.values?.[1]?.toString(); + if (isNotDefined(key)) { + return undefined; + } + return { key, column: column.number } + } + ).filter(isDefined); + + const columnMap = listToMap( + columns, + ({ key }) => key, + ({ column }) => column, + ); + + const xlsxRowsHashMap: Record = { + }; + + firstSheet.eachRow((row, i) => { + if (i === 0) { + return; + } + + const enColumnKey = columnMap['EN']; + const frColumnKey = columnMap['FR']; + const esColumnKey = columnMap['ES']; + const arColumnKey = columnMap['AR']; + + const enValue = isDefined(enColumnKey) ? getValueFromCellValue(row.getCell(enColumnKey).value) : undefined; + const frValue = isDefined(frColumnKey) ? getValueFromCellValue(row.getCell(frColumnKey).value) : undefined; + const esValue = isDefined(esColumnKey) ? getValueFromCellValue(row.getCell(esColumnKey).value) : undefined; + const arValue = isDefined(arColumnKey) ? getValueFromCellValue(row.getCell(arColumnKey).value) : undefined; + + const hash = isDefined(enValue) ? Md5.hashStr(String(enValue)) : undefined; + + if (isDefined(hash) && isDefined(enValue)) { + xlsxRowsHashMap[hash] = { + en: String(enValue), + fr: isDefined(frValue) ? String(frValue) : undefined, + es: isDefined(esValue) ? String(esValue) : undefined, + ar: isDefined(arValue) ? String(arValue) : undefined, + } + } + }); + + const translationFiles = await getTranslationFileNames( + projectPath, + Array.isArray(translationFileNames) ? translationFileNames : [], + ); + + const { translations } = await readTranslations(translationFiles); + + const fileState = translations.map((item) => ({ + ...item, + hash: Md5.hashStr(item.value), + })); + + const hashGroupedFileStateMapping = listToGroupList( + fileState, + ({ hash }) => hash, + ); + + const hashKeys = Object.keys(hashGroupedFileStateMapping); + const newStrings = hashKeys.flatMap((hash) => { + const strings = hashGroupedFileStateMapping[hash]; + const xlsxString = xlsxRowsHashMap[hash]; + + return strings.map((stringItem) => ({ + key: stringItem.key, + namespace: stringItem.namespace, + en: stringItem.value, + fr: xlsxString?.fr, + es: xlsxString?.es, + ar: xlsxString?.ar, + })); + }); + + await createExcel(newStrings); + + /* + await writeFilePromisify( + `/tmp/local-strings-logs.json`, + JSON.stringify(newStrings, null, 2), + 'utf8', + ); + */ +} + +export default testExcel; diff --git a/app/scripts/translatte/main.ts b/app/scripts/translatte/main.ts index 9a2f34c02f..af0c92a819 100644 --- a/app/scripts/translatte/main.ts +++ b/app/scripts/translatte/main.ts @@ -16,6 +16,7 @@ import exportServerStringsToExcel from './commands/exportServerStringsToExcel'; import clearServerStrings from './commands/clearServerStrings'; import pushStringsDref from './commands/pushStringsDref'; import syncEnStrings from './commands/syncEnStrings'; +import testExcel from './commands/testExcel'; const currentDir = cwd(); @@ -257,13 +258,17 @@ yargs(hideBin(process.argv)) }, ) .command( - 'push-strings-dref ', + 'push-strings-dref ', 'IMPORTANT!!! Temporary command, do not use!', (yargs) => { yargs.positional('IMPORT_FILE_PATH', { type: 'string', describe: 'Find the import file on IMPORT_FILE_PATH', }); + yargs.positional('TRANSLATION_FILES', { + type: 'string', + describe: 'Read the files from TRANSLATION_FILES', + }); yargs.options({ 'auth-token': { type: 'string', @@ -281,7 +286,46 @@ yargs(hideBin(process.argv)) const importFilePath = (argv.IMPORT_FILE_PATH as string); await pushStringsDref( + currentDir, + importFilePath, + argv.TRANSLATION_FILES as string[], + argv.apiUrl as string, + argv.authToken as string, + ); + }, + ) + .command( + 'test-excel ', + 'IMPORTANT!!! Temporary command, do not use!', + (yargs) => { + yargs.positional('IMPORT_FILE_PATH', { + type: 'string', + describe: 'Find the import file on IMPORT_FILE_PATH', + }); + yargs.positional('TRANSLATION_FILES', { + type: 'string', + describe: 'Read the files from TRANSLATION_FILES', + }); + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + // require: true, + }, + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + } + }); + }, + async (argv) => { + const importFilePath = (argv.IMPORT_FILE_PATH as string); + + await testExcel( importFilePath, + currentDir, + argv.TRANSLATION_FILES as string[], argv.apiUrl as string, argv.authToken as string, );