From 8d83ec68cc510ec1ebd1152d4b420b8bc07331f8 Mon Sep 17 00:00:00 2001 From: yaroslav8765 Date: Wed, 25 Feb 2026 16:33:27 +0200 Subject: [PATCH 1/3] feat: add p-limit dependency and implement token length management for translation requests --- index.ts | 155 ++++++++++++++++++++++++++++++++-------------- package-lock.json | 42 ++++++++++--- package.json | 3 +- types.ts | 7 +++ 4 files changed, 153 insertions(+), 54 deletions(-) diff --git a/index.ts b/index.ts index 70eb1b8..d54cca4 100644 --- a/index.ts +++ b/index.ts @@ -9,6 +9,7 @@ import chokidar from 'chokidar'; import { AsyncQueue } from '@sapphire/async-queue'; import getFlagEmoji from 'country-flag-svg'; import { parse } from 'bcp-47'; +import pLimit from 'p-limit'; const processFrontendMessagesQueue = new AsyncQueue(); @@ -494,51 +495,17 @@ export default class I18nPlugin extends AdminForthPlugin { }); } - async translateToLang ( - langIsoCode: SupportedLanguage, - strings: { en_string: string, category: string }[], - plurals=false, - translations: any[], - updateStrings: Record = {} - ): Promise { - const maxKeysInOneReq = 10; - if (strings.length === 0) { - return []; - } + async generateAndSaveBunch ( + prompt: String, + strings: { en_string: string, category: string }[], + translations: any[], + updateStrings: Record = {}, + lang: String, + ): Promise{ - const replacedLanguageCodeForTranslations = this.options.translateLangAsBCP47Code && langIsoCode.length === 2 ? this.options.translateLangAsBCP47Code[langIsoCode as any] : null; - if (strings.length > maxKeysInOneReq) { - let totalTranslated = []; - for (let i = 0; i < strings.length; i += maxKeysInOneReq) { - const slicedStrings = strings.slice(i, i + maxKeysInOneReq); - process.env.HEAVY_DEBUG && console.log('🪲🔪slicedStrings len', slicedStrings.length); - const madeKeys = await this.translateToLang(langIsoCode, slicedStrings, plurals, translations, updateStrings); - totalTranslated = totalTranslated.concat(madeKeys); - } - return totalTranslated; - } - const langCode = replacedLanguageCodeForTranslations ? replacedLanguageCodeForTranslations : langIsoCode; - const lang = langIsoCode; - const primaryLang = getPrimaryLanguageCode(lang); - const langName = iso6391.getName(primaryLang); - const requestSlavicPlurals = Object.keys(SLAVIC_PLURAL_EXAMPLES).includes(primaryLang) && plurals; - const region = String(lang).split('-')[1]?.toUpperCase() || ''; - const prompt = ` - I need to translate strings in JSON to ${langName} language ${replacedLanguageCodeForTranslations || lang.length > 2 ? `BCP-47 code ${langCode}` : `ISO 639-1 code ${langIsoCode}`} from English for my web app. - ${region ? `Use the regional conventions for ${langCode} (region ${region}), including spelling, punctuation, and formatting.` : ''} - ${requestSlavicPlurals ? `You should provide 4 slavic forms (in format "zero count | singular count | 2-4 | 5+") e.g. "apple | apples" should become "${SLAVIC_PLURAL_EXAMPLES[lang]}"` : ''} - Keep keys, as is, write translation into values! If keys have variables (in curly brackets), then translated strings should have them as well (variables itself should not be translated). Here are the strings: - - \`\`\`json - ${ - JSON.stringify(strings.reduce((acc: object, s: { en_string: string }): object => { - acc[s.en_string] = ''; - return acc; - }, {}), null, 2) - } - \`\`\` - `; + console.log('Call generateAndSaveBunch with prompt --->', prompt) + // return []; const jsonSchemaProperties = {}; strings.forEach(s => { jsonSchemaProperties[s.en_string] = { @@ -548,7 +515,6 @@ export default class I18nPlugin extends AdminForthPlugin { }); const jsonSchemaRequired = strings.map(s => s.en_string); - // call OpenAI const resp = await this.options.completeAdapter.complete( prompt, @@ -582,17 +548,18 @@ export default class I18nPlugin extends AdminForthPlugin { res = resp.content//.split("```json")[1].split("```")[0]; } catch (e) { console.error(`Error in parsing LLM resp: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, ); - return []; + return null; } try { res = JSON.parse(res); } catch (e) { console.error(`Error in parsing LLM resp json: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, ); - return []; + return null; } + console.log('Translation response:', res) for (const [enStr, translatedStr] of Object.entries(res) as [string, string][]) { const translationsTargeted = translations.filter(t => t[this.enFieldName] === enStr); // might be several with same en_string @@ -619,7 +586,103 @@ export default class I18nPlugin extends AdminForthPlugin { ].updates[this.trFieldNames[lang]] = translatedStr; } } + } + + async translateToLang ( + langIsoCode: SupportedLanguage, + strings: { en_string: string, category: string }[], + plurals=false, + translations: any[], + updateStrings: Record = {} + ): Promise { + + const maxInputTokens = this.options.inputTokensPerBatch ?? 30000; + const limit = pLimit(10); + const enStringsTokenLengthCache = {}; + + const tokenLengthPerString = async (str: string): Promise => { + if (!enStringsTokenLengthCache[str]) { + enStringsTokenLengthCache[str] = await this.options.completeAdapter.measureTokensCount(`"${str}":"",`); + } + return enStringsTokenLengthCache[str]; + } + const promises = strings.map(s => limit(() => tokenLengthPerString(s.en_string))); + + await Promise.all(promises); + + if (strings.length === 0) { + return []; + } + + const replacedLanguageCodeForTranslations = this.options.translateLangAsBCP47Code && langIsoCode.length === 2 ? this.options.translateLangAsBCP47Code[langIsoCode as any] : null; + + const langCode = replacedLanguageCodeForTranslations ? replacedLanguageCodeForTranslations : langIsoCode; + const lang = langIsoCode; + const primaryLang = getPrimaryLanguageCode(lang); + const langName = iso6391.getName(primaryLang); + const requestSlavicPlurals = Object.keys(SLAVIC_PLURAL_EXAMPLES).includes(primaryLang) && plurals; + const region = String(lang).split('-')[1]?.toUpperCase() || ''; + const basePrompt = ` + I need to translate strings in JSON to ${langName} language ${replacedLanguageCodeForTranslations || lang.length > 2 ? `BCP-47 code ${langCode}` : `ISO 639-1 code ${langIsoCode}`} from English for my web app. + ${region ? `Use the regional conventions for ${langCode} (region ${region}), including spelling, punctuation, and formatting.` : ''} + ${requestSlavicPlurals ? `You should provide 4 slavic forms (in format "zero count | singular count | 2-4 | 5+") e.g. "apple | apples" should become "${SLAVIC_PLURAL_EXAMPLES[lang]}"` : ''} + Keep keys, as is, write translation into values! If keys have variables (in curly brackets), then translated strings should have them as well (variables itself should not be translated). Here are the strings: + \`\`\`json + + \`\`\` + `; + + const basePromptTokenLength = await this.options.completeAdapter.measureTokensCount(basePrompt); + const allowedTokensAmountForFields = maxInputTokens - basePromptTokenLength; + const stringsToTranslate = strings.map( s => s.en_string); + const generationTasks = [] + while (stringsToTranslate.length !== 0) { + let stringBanch = []; + let banchTokens = 0; + for (const string of stringsToTranslate) { + if( banchTokens + enStringsTokenLengthCache[string] <= allowedTokensAmountForFields ) { + stringBanch.push(string); + banchTokens += enStringsTokenLengthCache[string]; + } else { + continue; + } + } + if ( stringBanch.length === 0 ) { + break; + } + for ( const string of stringBanch) { + const index = stringsToTranslate.indexOf(string); + if (index !== -1) { + stringsToTranslate.splice(index, 1); + } + } + const promptToGenerate = basePrompt.split(`\`\`\`json`)[0] + + `\`\`\`json${ + stringBanch.map(s => `"${s}": ""`) + } + \`\`\``; + // await new Promise(resolve => setTimeout(resolve, 1000)); + const stringBanchCopy = [...stringBanch]; + generationTasks.push( + this.generateAndSaveBunch( + promptToGenerate, + strings.filter(s => stringBanchCopy.includes(s.en_string)), + translations.filter(t => stringBanchCopy.includes(t.en_string)), + updateStrings, + lang + ) + ) + } + + await Promise.all(generationTasks); + + + + + if (allowedTokensAmountForFields < 0) { + throw new AiTranslateError("Not enought input generation tokens") + } return Object.keys(updateStrings); } diff --git a/package-lock.json b/package-lock.json index 32c0ace..4b57a82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,8 @@ "country-flag-svg": "^1.0.19", "fs-extra": "^11.3.2", "iso-3166": "^4.3.0", - "iso-639-1": "^3.1.3" + "iso-639-1": "^3.1.3", + "p-limit": "^7.3.0" }, "devDependencies": { "@types/node": "^22.10.7", @@ -8822,16 +8823,18 @@ } }, "node_modules/p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", "license": "MIT", "dependencies": { - "p-try": "^1.0.0" + "yocto-queue": "^1.2.1" }, "engines": { - "node": ">=4" + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-locate": { @@ -8847,6 +8850,19 @@ "node": ">=4" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/p-map": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.3.tgz", @@ -11101,6 +11117,18 @@ "node": ">=12" } }, + "node_modules/yocto-queue": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/yoctocolors": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.1.2.tgz", diff --git a/package.json b/package.json index 9b9cbad..facbdaa 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,8 @@ "country-flag-svg": "^1.0.19", "fs-extra": "^11.3.2", "iso-3166": "^4.3.0", - "iso-639-1": "^3.1.3" + "iso-639-1": "^3.1.3", + "p-limit": "^7.3.0" }, "peerDependencies": { "adminforth": "next" diff --git a/types.ts b/types.ts index 87c0a9d..6b183fa 100644 --- a/types.ts +++ b/types.ts @@ -74,4 +74,11 @@ export interface PluginOptions { * key - one of the values form supportedLanguages, value -BCP47 tag */ translateLangAsBCP47Code?: Partial>; + + /** + * Batch size of one translation generation request. + * This is an optional parameter that can be used to control the size of strings sent in a single request to the completion adapter. + * Default value is 30000 tokens + */ + inputTokensPerBatch?: number; } \ No newline at end of file From 6a6fed22ae061a2cb5dbd7a8c1a078c9b7ebc035 Mon Sep 17 00:00:00 2001 From: yaroslav8765 Date: Wed, 25 Feb 2026 16:34:31 +0200 Subject: [PATCH 2/3] chore: remove debug logs --- index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/index.ts b/index.ts index d54cca4..73153d9 100644 --- a/index.ts +++ b/index.ts @@ -503,7 +503,6 @@ export default class I18nPlugin extends AdminForthPlugin { lang: String, ): Promise{ - console.log('Call generateAndSaveBunch with prompt --->', prompt) // return []; const jsonSchemaProperties = {}; @@ -559,7 +558,6 @@ export default class I18nPlugin extends AdminForthPlugin { } - console.log('Translation response:', res) for (const [enStr, translatedStr] of Object.entries(res) as [string, string][]) { const translationsTargeted = translations.filter(t => t[this.enFieldName] === enStr); // might be several with same en_string From b29fccc27eaf1e34803cc5fbed7db00e3c5cb1c4 Mon Sep 17 00:00:00 2001 From: yaroslav8765 Date: Thu, 26 Feb 2026 09:25:41 +0200 Subject: [PATCH 3/3] feat: enhance translation error handling and reporting in bulk translation process --- custom/BulkActionButton.vue | 2 +- index.ts | 92 ++++++++++++++++++++++++++++--------- 2 files changed, 71 insertions(+), 23 deletions(-) diff --git a/custom/BulkActionButton.vue b/custom/BulkActionButton.vue index 335b3a1..b4e4c0b 100644 --- a/custom/BulkActionButton.vue +++ b/custom/BulkActionButton.vue @@ -119,7 +119,7 @@ adminforth.list.refresh(); props.clearCheckboxes(); if (res.ok) { - adminforth.alert({ message: res.successMessage, variant: 'success' }); + adminforth.alert({ message: `${res.successMessage}. ${res.failedToTranslate.length > 0 ? `${t('Failed to translate')}: ${res.failedToTranslate.length}` : ''}`, variant: 'success' }); } else { adminforth.alert({ message: res.errorMessage || t('Failed to translate selected items. Please, try again.'), variant: 'danger' }); } diff --git a/index.ts b/index.ts index 73153d9..a4bd3b2 100644 --- a/index.ts +++ b/index.ts @@ -10,6 +10,7 @@ import { AsyncQueue } from '@sapphire/async-queue'; import getFlagEmoji from 'country-flag-svg'; import { parse } from 'bcp-47'; import pLimit from 'p-limit'; +import { randomUUID } from 'crypto'; const processFrontendMessagesQueue = new AsyncQueue(); @@ -69,6 +70,12 @@ function isValidSupportedLanguageTag(langCode: SupportedLanguage): boolean { } } +interface IFailedTranslation { + lang: SupportedLanguage; + en_string: string; + failedReason: string; +} + interface ICachingAdapter { get(key: string): Promise; @@ -496,11 +503,12 @@ export default class I18nPlugin extends AdminForthPlugin { } async generateAndSaveBunch ( - prompt: String, + prompt: string, strings: { en_string: string, category: string }[], translations: any[], updateStrings: Record = {}, - lang: String, + lang: string, + failedToTranslate: IFailedTranslation[], ): Promise{ @@ -547,6 +555,13 @@ export default class I18nPlugin extends AdminForthPlugin { res = resp.content//.split("```json")[1].split("```")[0]; } catch (e) { console.error(`Error in parsing LLM resp: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, ); + strings.forEach(s => { + failedToTranslate.push({ + lang: lang as SupportedLanguage, + en_string: s.en_string, + failedReason: "Error in parsing LLM response" + }); + }); return null; } @@ -554,6 +569,13 @@ export default class I18nPlugin extends AdminForthPlugin { res = JSON.parse(res); } catch (e) { console.error(`Error in parsing LLM resp json: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, ); + strings.forEach(s => { + failedToTranslate.push({ + lang: lang as SupportedLanguage, + en_string: s.en_string, + failedReason: "Error in parsing LLM response JSON" + }); + }); return null; } @@ -592,25 +614,27 @@ export default class I18nPlugin extends AdminForthPlugin { plurals=false, translations: any[], updateStrings: Record = {} - ): Promise { + ): Promise<{ updatedKeys: string[], failedToTranslate: IFailedTranslation[] }> { const maxInputTokens = this.options.inputTokensPerBatch ?? 30000; const limit = pLimit(10); - const enStringsTokenLengthCache = {}; + const enStringsTokenLengthCache: Record[] = []; + - const tokenLengthPerString = async (str: string): Promise => { - if (!enStringsTokenLengthCache[str]) { - enStringsTokenLengthCache[str] = await this.options.completeAdapter.measureTokensCount(`"${str}":"",`); - } - return enStringsTokenLengthCache[str]; + const tokenLengthPerString = async (str: string): Promise => { + const objectToPush = { + en_string: str, + numOfTokens: await this.options.completeAdapter.measureTokensCount(`"${str}":"",`) + }; + enStringsTokenLengthCache.push(objectToPush); } const promises = strings.map(s => limit(() => tokenLengthPerString(s.en_string))); await Promise.all(promises); if (strings.length === 0) { - return []; + return { updatedKeys: [], failedToTranslate: [] }; } const replacedLanguageCodeForTranslations = this.options.translateLangAsBCP47Code && langIsoCode.length === 2 ? this.options.translateLangAsBCP47Code[langIsoCode as any] : null; @@ -631,22 +655,31 @@ export default class I18nPlugin extends AdminForthPlugin { \`\`\` `; + const failedToTranslate: IFailedTranslation[] = []; const basePromptTokenLength = await this.options.completeAdapter.measureTokensCount(basePrompt); const allowedTokensAmountForFields = maxInputTokens - basePromptTokenLength; const stringsToTranslate = strings.map( s => s.en_string); const generationTasks = [] while (stringsToTranslate.length !== 0) { - let stringBanch = []; + const stringBanch = []; let banchTokens = 0; for (const string of stringsToTranslate) { - if( banchTokens + enStringsTokenLengthCache[string] <= allowedTokensAmountForFields ) { + const numberOfTokensForString = enStringsTokenLengthCache.find(cache => cache.en_string === string)?.numOfTokens || 0; + if( banchTokens + numberOfTokensForString <= allowedTokensAmountForFields ) { stringBanch.push(string); - banchTokens += enStringsTokenLengthCache[string]; + banchTokens += numberOfTokensForString; } else { continue; } } if ( stringBanch.length === 0 ) { + stringsToTranslate.forEach(s => { + failedToTranslate.push({ + lang, + en_string: s, + failedReason: "Not enough input generation tokens" + }); + }); break; } for ( const string of stringBanch) { @@ -668,7 +701,8 @@ export default class I18nPlugin extends AdminForthPlugin { strings.filter(s => stringBanchCopy.includes(s.en_string)), translations.filter(t => stringBanchCopy.includes(t.en_string)), updateStrings, - lang + lang, + failedToTranslate ) ) } @@ -681,12 +715,21 @@ export default class I18nPlugin extends AdminForthPlugin { if (allowedTokensAmountForFields < 0) { throw new AiTranslateError("Not enought input generation tokens") } - return Object.keys(updateStrings); + return { updatedKeys: Object.keys(updateStrings), failedToTranslate }; } - // returns translated count - async bulkTranslate({ selectedIds, selectedLanguages }: { selectedIds: string[], selectedLanguages?: SupportedLanguage[] }): Promise { + // returns translated count + async bulkTranslate({ selectedIds, selectedLanguages }: + { + selectedIds: string[], + selectedLanguages?: SupportedLanguage[] + }): + Promise<{ + totalTranslated: number, + failedToTranslate: IFailedTranslation[] + }> + { const needToTranslateByLang : Partial< Record< SupportedLanguage, @@ -729,20 +772,22 @@ export default class I18nPlugin extends AdminForthPlugin { const langsInvolved = new Set(Object.keys(needToTranslateByLang)); let totalTranslated = []; + let failedToTranslate: IFailedTranslation[] = []; await Promise.all( Object.entries(needToTranslateByLang).map( async ([lang, strings]: [SupportedLanguage, { en_string: string, category: string }[]]) => { // first translate without plurals const stringsWithoutPlurals = strings.filter(s => !s.en_string.includes('|')); - const noPluralKeys = await this.translateToLang(lang, stringsWithoutPlurals, false, translations, updateStrings); + const { updatedKeys: noPluralKeys, failedToTranslate: failedNoPlural } = await this.translateToLang(lang, stringsWithoutPlurals, false, translations, updateStrings); const stringsWithPlurals = strings.filter(s => s.en_string.includes('|')); - const pluralKeys = await this.translateToLang(lang, stringsWithPlurals, true, translations, updateStrings); + const { updatedKeys: pluralKeys, failedToTranslate: failedPlural } = await this.translateToLang(lang, stringsWithPlurals, true, translations, updateStrings); totalTranslated = totalTranslated.concat(noPluralKeys, pluralKeys); + failedToTranslate = failedToTranslate.concat(failedNoPlural, failedPlural); } ) ); @@ -774,8 +819,7 @@ export default class I18nPlugin extends AdminForthPlugin { await this.cache.clear(`${this.resourceConfig.resourceId}:${category}:${lang}`); } } - - return new Set(totalTranslated).size; + return { totalTranslated: new Set(totalTranslated).size, failedToTranslate: failedToTranslate }; } async processExtractedMessages(adminforth: IAdminForth, filePath: string) { @@ -1137,8 +1181,11 @@ export default class I18nPlugin extends AdminForthPlugin { const selectedIds = body.selectedIds; let translatedCount = 0; + let failedToTranslate: IFailedTranslation[] = []; try { - translatedCount = await this.bulkTranslate({ selectedIds, selectedLanguages }); + const result = await this.bulkTranslate({ selectedIds, selectedLanguages }); + translatedCount = result.totalTranslated; + failedToTranslate = result.failedToTranslate; } catch (e) { process.env.HEAVY_DEBUG && console.error('🪲⛔ bulkTranslate error', e); if (e instanceof AiTranslateError) { @@ -1153,6 +1200,7 @@ export default class I18nPlugin extends AdminForthPlugin { successMessage: await tr(`Translated {count} items`, 'backend', { count: translatedCount, }), + failedToTranslate, }; } });