From 1d393c2c44722add07430eaa2ef4886854d18a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 5 Sep 2024 17:28:25 +0200 Subject: [PATCH 01/32] wip: i18next import support --- .../in/GenericStructuredProcessor.kt | 69 ++++++++++++++++++- ...GenericStructuredRawDataToTextConvertor.kt | 14 ++-- .../formats/i18next/I18nextParameterParser.kt | 24 +++++++ .../formats/i18next/ParsedI18nextKey.kt | 7 ++ .../formats/i18next/ParsedI18nextParam.kt | 8 +++ .../i18next/PluralsI18nextKeyParser.kt | 23 +++++++ .../formats/importCommon/ImportFormat.kt | 7 ++ .../json/in/JsonImportFormatDetector.kt | 7 ++ .../in/I18nextToIcuPlaceholderConvertor.kt | 68 ++++++++++++++++++ .../io/tolgee/formats/pluralFormsUtil.kt | 9 +++ .../json/in/JsonFormatProcessorTest.kt | 28 ++++++++ .../json/in/JsonImportFormatDetectorTest.kt | 5 ++ .../test/resources/import/json/i18next.json | 21 ++++++ .../test/resources/import/json/i18next2.json | 65 +++++++++++++++++ 14 files changed, 347 insertions(+), 8 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/i18next/I18nextParameterParser.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextKey.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextParam.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/i18next/PluralsI18nextKeyParser.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt create mode 100644 backend/data/src/test/resources/import/json/i18next.json create mode 100644 backend/data/src/test/resources/import/json/i18next2.json diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index 9c7ef1c719..518ae7d6be 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -2,6 +2,9 @@ package io.tolgee.formats.genericStructuredFile.`in` import io.tolgee.formats.ImportFileProcessor import io.tolgee.formats.MessageConvertorResult +import io.tolgee.formats.allPluralKeywords +import io.tolgee.formats.i18next.ParsedI18nextKey +import io.tolgee.formats.i18next.PluralsI18nextKeyParser import io.tolgee.formats.importCommon.ImportFormat import io.tolgee.service.dataImport.processors.FileProcessorContext @@ -12,8 +15,72 @@ class GenericStructuredProcessor( private val languageTag: String? = null, private val format: ImportFormat, ) : ImportFileProcessor() { + private val keyParser = PluralsI18nextKeyParser() + override fun process() { - data.import("") + data.preprocess().import("") + } + + private fun Any?.preprocess(): Any? { + if (this == null) { + return null + } + + (this as? List<*>)?.let { + return it.preprocessList() + } + + (this as? Map<*, *>)?.let { + return it.preprocessMap() + } + + return this + } + + private fun List<*>.preprocessList(): List<*> { + return this.map { it.preprocess() } + } + + private fun Map<*, *>.groupByPlurals(keyRegex: Regex): Map>> { + return this.entries.mapIndexedNotNull { idx, (key, value) -> + if (key !is String) { + context.fileEntity.addKeyIsNotStringIssue(key.toString(), idx) + return@mapIndexedNotNull null + } + val default = ParsedI18nextKey(null, null, key) + + val match = keyRegex.find(key) ?: return@mapIndexedNotNull default to value + val parsedKey = keyParser.parse(match) + + if (parsedKey?.key == null || parsedKey.plural == null || parsedKey.plural !in allPluralKeywords) { + return@mapIndexedNotNull default to value + } + + return@mapIndexedNotNull parsedKey to value + }.groupBy { (parsedKey, _) -> + parsedKey.key + }.toMap() + } + + private fun Map<*, *>.preprocessMap(): Map<*, *> { + if (format.pluralsViaSuffixesRegex == null) { + return this.mapValues { (_, value) -> value.preprocess() } + } + + val plurals = this.groupByPlurals(format.pluralsViaSuffixesRegex) + + return plurals.flatMap { (key, values) -> + if (key == null || values.size < 2) { + // Fallback for non-plural keys + values.map { (parsedKey, value) -> + parsedKey.fullMatch to value + } + } else { + listOf(key to values.map { (parsedKey, value) -> + parsedKey.plural to value + }.toMap()) + } + }.toMap() } private fun Any?.import(key: String) { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt index 7145da1e18..276fdc0eca 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt @@ -2,6 +2,7 @@ package io.tolgee.formats.genericStructuredFile.`in` import com.ibm.icu.text.PluralRules import io.tolgee.formats.MessageConvertorResult +import io.tolgee.formats.allPluralKeywords import io.tolgee.formats.importCommon.ImportFormat import io.tolgee.formats.importCommon.unwrapString import java.util.* @@ -10,11 +11,6 @@ class GenericStructuredRawDataToTextConvertor( private val format: ImportFormat, private val languageTag: String, ) : StructuredRawDataConvertor { - private val availablePluralKeywords by lazy { - val locale = Locale.forLanguageTag(languageTag) - PluralRules.forLocale(locale).keywords.toSet() - } - override fun convert( rawData: Any?, projectIcuPlaceholdersEnabled: Boolean, @@ -77,11 +73,15 @@ class GenericStructuredRawDataToTextConvertor( ): List? { val map = rawData as? Map<*, *> ?: return null - if (!format.pluralsViaNesting) { + if (!format.pluralsViaNesting && format.pluralsViaSuffixesRegex == null) { + return null + } + + if (!map.keys.all { it in allPluralKeywords }) { return null } - if (!map.keys.all { it in availablePluralKeywords }) { + if (map.size < 2) { return null } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/I18nextParameterParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/I18nextParameterParser.kt new file mode 100644 index 0000000000..d86b479fea --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/I18nextParameterParser.kt @@ -0,0 +1,24 @@ +package io.tolgee.formats.i18next + +class I18nextParameterParser { + fun parse(match: MatchResult): ParsedI18nextParam? { + return ParsedI18nextParam( + key = match.groups.getGroupOrNull("key")?.value, + nestedKey = match.groups.getGroupOrNull("nestedKey")?.value, + format = match.groups.getGroupOrNull("format")?.value, + fullMatch = match.value, + ) + } + +// FIXME: move somewhere shared + private fun MatchGroupCollection.getGroupOrNull(name: String): MatchGroup? { + try { + return this[name] + } catch (e: IllegalArgumentException) { + if (e.message?.contains("No group with name") != true) { + throw e + } + return null + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextKey.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextKey.kt new file mode 100644 index 0000000000..d8d523eb0c --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextKey.kt @@ -0,0 +1,7 @@ +package io.tolgee.formats.i18next + +data class ParsedI18nextKey( + val key: String? = null, + val plural: String? = null, + val fullMatch: String, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextParam.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextParam.kt new file mode 100644 index 0000000000..38f3864b60 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextParam.kt @@ -0,0 +1,8 @@ +package io.tolgee.formats.i18next + +data class ParsedI18nextParam( + val key: String? = null, + val nestedKey: String? = null, + val format: String? = null, + val fullMatch: String, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/PluralsI18nextKeyParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/PluralsI18nextKeyParser.kt new file mode 100644 index 0000000000..90cbc7cc17 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/PluralsI18nextKeyParser.kt @@ -0,0 +1,23 @@ +package io.tolgee.formats.i18next + +class PluralsI18nextKeyParser { + fun parse(match: MatchResult): ParsedI18nextKey? { + return ParsedI18nextKey( + key = match.groups.getGroupOrNull("key")?.value, + plural = match.groups.getGroupOrNull("plural")?.value, + fullMatch = match.value, + ) + } + + // FIXME: move somewhere shared + private fun MatchGroupCollection.getGroupOrNull(name: String): MatchGroup? { + try { + return this[name] + } catch (e: IllegalArgumentException) { + if (e.message?.contains("No group with name") != true) { + throw e + } + return null + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt index c85422eede..98b61e2fd9 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt @@ -2,6 +2,7 @@ package io.tolgee.formats.importCommon import io.tolgee.formats.paramConvertors.`in`.AppleToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.CToIcuPlaceholderConvertor +import io.tolgee.formats.paramConvertors.`in`.I18nextToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor @@ -10,9 +11,15 @@ import io.tolgee.formats.po.`in`.PoToIcuMessageConvertor enum class ImportFormat( val fileFormat: ImportFileFormat, val pluralsViaNesting: Boolean = false, + val pluralsViaSuffixesRegex: Regex? = null, val messageConvertorOrNull: ImportMessageConvertor? = null, val rootKeyIsLanguageTag: Boolean = false, ) { + JSON_I18NEXT( + ImportFileFormat.JSON, + messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { I18nextToIcuPlaceholderConvertor() }, + pluralsViaSuffixesRegex = I18nextToIcuPlaceholderConvertor.I18NEXT_PLURAL_SUFFIX_REGEX, + ), JSON_ICU( ImportFileFormat.JSON, messageConvertorOrNull = diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt b/backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt index a414e394e7..8194c04979 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt @@ -6,6 +6,7 @@ import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.detectFr import io.tolgee.formats.importCommon.ImportFormat import io.tolgee.formats.paramConvertors.`in`.CToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor +import io.tolgee.formats.paramConvertors.`in`.I18nextToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor @@ -45,6 +46,12 @@ class JsonImportFormatDetector { 0.6, ), ), + ImportFormat.JSON_I18NEXT to + arrayOf( + FormatDetectionUtil.regexFactor( + I18nextToIcuPlaceholderConvertor.I18NEXT_DETECTION_REGEX, + ), + ), ) } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt new file mode 100644 index 0000000000..4493400811 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -0,0 +1,68 @@ +package io.tolgee.formats.paramConvertors.`in` + +import io.tolgee.formats.ToIcuPlaceholderConvertor +import io.tolgee.formats.escapeIcu +import io.tolgee.formats.i18next.I18nextParameterParser + +class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { + private val parser = I18nextParameterParser() + + override val regex: Regex + get() = I18NEXT_PLACEHOLDER_REGEX + + override val pluralArgName: String? = null + + override fun convert( + matchResult: MatchResult, + isInPlural: Boolean, + ): String { + val parsed = parser.parse(matchResult) ?: return matchResult.value.escapeIcu(isInPlural) + + if (parsed.nestedKey != null) { + // TODO: nested keys are not yet supported + return matchResult.value.escapeIcu(isInPlural) + } + + return when (parsed.format) { + null -> "{${parsed.key}}" + "number" -> "{${parsed.key}, number}" + else -> matchResult.value.escapeIcu(isInPlural) + } + } + + companion object { + val I18NEXT_PLACEHOLDER_REGEX = + """ + (?x) + ( + \{\{ + (?:-\ *)? + (?\w+)(?:,\ *(?[^}]+))? + }} + | + \\${'$'}t\( + (?[^)]+) + \) + ) + """.trimIndent().toRegex() + + val I18NEXT_DETECTION_REGEX = + """ + (?x) + (^|\W+) + ( + \{\{ + (?:-\ *)? + (?\w+)(?:,\ *(?[^}]+))? + }} + | + \\${'$'}t\( + (?[^)]+) + \) + ) + """.trimIndent().toRegex() + + val I18NEXT_PLURAL_SUFFIX_REGEX = """^(?\w+)_(?\w+)$""".toRegex() + + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/pluralFormsUtil.kt b/backend/data/src/main/kotlin/io/tolgee/formats/pluralFormsUtil.kt index d2daaf9a8d..0e36e64413 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/pluralFormsUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/pluralFormsUtil.kt @@ -6,6 +6,15 @@ import io.tolgee.formats.escaping.IcuUnescper import io.tolgee.formats.escaping.PluralFormIcuEscaper import io.tolgee.util.nullIfEmpty +val allPluralKeywords = listOf( + PluralRules.KEYWORD_ZERO, + PluralRules.KEYWORD_ONE, + PluralRules.KEYWORD_TWO, + PluralRules.KEYWORD_FEW, + PluralRules.KEYWORD_MANY, + PluralRules.KEYWORD_OTHER, +) + fun getPluralFormsForLocale(languageTag: String): MutableSet { val uLocale = getULocaleFromTag(languageTag) val pluralRules = PluralRules.forLocale(uLocale) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt index 983573f926..94c95f2ce7 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt @@ -177,6 +177,34 @@ class JsonFormatProcessorTest { ) } + @Test + fun `returns correct parsed result for i18next`() { + mockUtil.mockIt("i18next.json", "src/test/resources/import/json/i18next.json") + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(1) + mockUtil.fileProcessorContext.assertTranslations("i18next", "keyDeep.inner") + mockUtil.fileProcessorContext.assertTranslations("i18next", "keyWithArrayValue[0]") + mockUtil.fileProcessorContext.assertTranslations("i18next", "keyWithArrayValue[1]") + .assertSingle { + hasText("things") + } + mockUtil.fileProcessorContext.assertTranslations("i18next", "keyNesting") + .assertSingle { + hasText("reuse ${'$'}t(keyDeep.inner) (is not supported)") + } + mockUtil.fileProcessorContext.assertTranslations("i18next", "keyContext_male") + .assertSingle { + hasText("the male variant (is parsed as normal key and context is ignored)") + } + mockUtil.fileProcessorContext.assertTranslations("i18next", "keyPluralSimple") + .assertSingle { + hasText("{value, plural,\n" + + "one {the singular (is parsed as plural under one key - keyPluralSimple)}\n" + + "other {the plural (is parsed as plural under one key - keyPluralSimple)}\n" + + "}") + } + } + @Test fun `respects provided format`() { mockUtil.mockIt("en.json", "src/test/resources/import/json/icu.json") diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt index 50c03db831..1a5dd7bfb9 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt @@ -19,6 +19,11 @@ class JsonImportFormatDetectorTest { mockUtil = FileProcessorContextMockUtil() } + @Test + fun `detected i18next`() { + "src/test/resources/import/json/i18next.json".assertDetected(ImportFormat.JSON_I18NEXT) + } + @Test fun `detected icu`() { "src/test/resources/import/json/icu.json".assertDetected(ImportFormat.JSON_ICU) diff --git a/backend/data/src/test/resources/import/json/i18next.json b/backend/data/src/test/resources/import/json/i18next.json new file mode 100644 index 0000000000..c890a06331 --- /dev/null +++ b/backend/data/src/test/resources/import/json/i18next.json @@ -0,0 +1,21 @@ +{ + "key": "value", + "keyDeep": { + "inner": "value" + }, + "keyNesting": "reuse $t(keyDeep.inner) (is not supported)", + "keyInterpolate": "replace this {{value}}", + "keyInterpolateUnescaped": "replace this {{- value}} (we ignore the -)", + "keyInterpolateWithFormatting": "replace this {{value, number}} (only number is supported)", + "keyContext_male": "the male variant (is parsed as normal key and context is ignored)", + "keyContext_female": "the female variant (is parsed as normal key and context is ignored)", + "keyPluralSimple_one": "the singular (is parsed as plural under one key - keyPluralSimple)", + "keyPluralSimple_other": "the plural (is parsed as plural under one key - keyPluralSimple)", + "keyPluralMultipleEgArabic_one": "the plural form 1", + "keyPluralMultipleEgArabic_two": "the plural form 2", + "keyPluralMultipleEgArabic_few": "the plural form 3", + "keyPluralMultipleEgArabic_many": "the plural form 4", + "keyPluralMultipleEgArabic_other": "the plural form 5", + "keyWithArrayValue": ["multipe", "things"], + "keyWithObjectValue": { "valueA": "return this with valueB", "valueB": "more text" } +} \ No newline at end of file diff --git a/backend/data/src/test/resources/import/json/i18next2.json b/backend/data/src/test/resources/import/json/i18next2.json new file mode 100644 index 0000000000..7955a04bac --- /dev/null +++ b/backend/data/src/test/resources/import/json/i18next2.json @@ -0,0 +1,65 @@ +{ + "note": "Most features in this example are not supported by the tolgee yet", + "translation": { + "key": "Hello World", + + "interpolation_example": "Hello {{name}}", + + "plural_example": { + "one": "You have one message", + "other": "You have {{count}} messages" + }, + + "context_example": { + "male": "He is a teacher", + "female": "She is a teacher" + }, + + "nested_example": "This is a {{type}} message", + "type": "nested", + + "formatted_value": "The price is {{value, currency}}", + + "array_example": [ + "Apples", + "Oranges", + "Bananas" + ], + + "select_example": { + "morning": "Good morning", + "afternoon": "Good afternoon", + "evening": "Good evening" + }, + + "multiline_example": "This is line one.\nThis is line two.", + + "gender_with_plural": { + "male": { + "one": "He has one cat", + "other": "He has {{count}} cats" + }, + "female": { + "one": "She has one cat", + "other": "She has {{count}} cats" + } + }, + + "rich_text_example": "Welcome to our application!", + + "json_value_example": { + "key": "This is a value inside a JSON object" + }, + + "conditional_translations": "{{isLoggedIn, select, true {Welcome back, {{name}}!} false {Please log in}}}", + + "language_switch": { + "en": "English", + "es": "Spanish", + "fr": "French" + }, + + "missing_key_fallback": "This is the default value if the key is missing." + } +} + From a102293bfc8903480beea158f8cee75ace3af232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 6 Sep 2024 11:56:02 +0200 Subject: [PATCH 02/32] fix: structural and readability fixes + removing duplicates --- .../in/GenericStructuredProcessor.kt | 63 ++++++++++--------- .../io/tolgee/formats/getGroupOrNull.kt | 12 ++++ .../formats/i18next/ParsedI18nextKey.kt | 7 --- .../i18next/PluralsI18nextKeyParser.kt | 23 ------- .../{ => in}/I18nextParameterParser.kt | 16 +---- .../i18next/{ => in}/ParsedI18nextParam.kt | 2 +- .../i18next/in/PluralsI18nextKeyParser.kt | 16 +++++ .../formats/importCommon/ImportFormat.kt | 4 +- .../formats/importCommon/ParsedPluralsKey.kt | 7 +++ .../formats/importCommon/PluralsKeyParser.kt | 5 ++ .../in/I18nextToIcuPlaceholderConvertor.kt | 5 +- .../formats/po/in/CLikeParameterParser.kt | 13 +--- 12 files changed, 85 insertions(+), 88 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/getGroupOrNull.kt delete mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextKey.kt delete mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/i18next/PluralsI18nextKeyParser.kt rename backend/data/src/main/kotlin/io/tolgee/formats/i18next/{ => in}/I18nextParameterParser.kt (50%) rename backend/data/src/main/kotlin/io/tolgee/formats/i18next/{ => in}/ParsedI18nextParam.kt (79%) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/PluralsI18nextKeyParser.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ParsedPluralsKey.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/importCommon/PluralsKeyParser.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index 518ae7d6be..7d6164c6ed 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -3,9 +3,9 @@ package io.tolgee.formats.genericStructuredFile.`in` import io.tolgee.formats.ImportFileProcessor import io.tolgee.formats.MessageConvertorResult import io.tolgee.formats.allPluralKeywords -import io.tolgee.formats.i18next.ParsedI18nextKey -import io.tolgee.formats.i18next.PluralsI18nextKeyParser import io.tolgee.formats.importCommon.ImportFormat +import io.tolgee.formats.importCommon.ParsedPluralsKey +import io.tolgee.formats.importCommon.PluralsKeyParser import io.tolgee.service.dataImport.processors.FileProcessorContext class GenericStructuredProcessor( @@ -15,8 +15,6 @@ class GenericStructuredProcessor( private val languageTag: String? = null, private val format: ImportFormat, ) : ImportFileProcessor() { - private val keyParser = PluralsI18nextKeyParser() - override fun process() { data.preprocess().import("") } @@ -41,44 +39,49 @@ class GenericStructuredProcessor( return this.map { it.preprocess() } } - private fun Map<*, *>.groupByPlurals(keyRegex: Regex): Map>> { + private fun Any?.parsePluralsKey(keyParser: PluralsKeyParser): ParsedPluralsKey? { + val key = this as? String ?: return null + return keyParser.parse(key).takeIf { + it.key != null && it.plural in allPluralKeywords + } ?: ParsedPluralsKey(null, null, key) + } + + private fun Map<*, *>.groupByPlurals( + keyParser: PluralsKeyParser + ): Map>> { return this.entries.mapIndexedNotNull { idx, (key, value) -> - if (key !is String) { - context.fileEntity.addKeyIsNotStringIssue(key.toString(), idx) - return@mapIndexedNotNull null + key.parsePluralsKey(keyParser)?.let { it to value }.also { + if (it == null) { + context.fileEntity.addKeyIsNotStringIssue(key.toString(), idx) + } } - val default = ParsedI18nextKey(null, null, key) - - val match = keyRegex.find(key) ?: return@mapIndexedNotNull default to value - val parsedKey = keyParser.parse(match) + }.groupBy { (parsedKey, _) -> parsedKey.key }.toMap() + } - if (parsedKey?.key == null || parsedKey.plural == null || parsedKey.plural !in allPluralKeywords) { - return@mapIndexedNotNull default to value - } + private fun List>.useOriginalKey(): List> { + return map { (parsedKey, value) -> + parsedKey.originalKey to value + } + } - return@mapIndexedNotNull parsedKey to value - }.groupBy { (parsedKey, _) -> - parsedKey.key - }.toMap() + private fun List>.usePluralsKey(commonKey: String): List> { + return listOf(commonKey to this.associate { (parsedKey, value) -> + parsedKey.plural to value + }) } private fun Map<*, *>.preprocessMap(): Map<*, *> { - if (format.pluralsViaSuffixesRegex == null) { + if (format.pluralsViaSuffixesParser == null) { return this.mapValues { (_, value) -> value.preprocess() } } - val plurals = this.groupByPlurals(format.pluralsViaSuffixesRegex) + val plurals = this.groupByPlurals(format.pluralsViaSuffixesParser) - return plurals.flatMap { (key, values) -> - if (key == null || values.size < 2) { - // Fallback for non-plural keys - values.map { (parsedKey, value) -> - parsedKey.fullMatch to value - } + return plurals.flatMap { (commonKey, values) -> + if (commonKey == null || values.size < 2) { + values.useOriginalKey() } else { - listOf(key to values.map { (parsedKey, value) -> - parsedKey.plural to value - }.toMap()) + values.usePluralsKey(commonKey) } }.toMap() } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/getGroupOrNull.kt b/backend/data/src/main/kotlin/io/tolgee/formats/getGroupOrNull.kt new file mode 100644 index 0000000000..8fddc3ee5a --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/getGroupOrNull.kt @@ -0,0 +1,12 @@ +package io.tolgee.formats + +fun MatchGroupCollection.getGroupOrNull(name: String): MatchGroup? { + try { + return this[name] + } catch (e: IllegalArgumentException) { + if (e.message?.contains("No group with name") != true) { + throw e + } + return null + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextKey.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextKey.kt deleted file mode 100644 index d8d523eb0c..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextKey.kt +++ /dev/null @@ -1,7 +0,0 @@ -package io.tolgee.formats.i18next - -data class ParsedI18nextKey( - val key: String? = null, - val plural: String? = null, - val fullMatch: String, -) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/PluralsI18nextKeyParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/PluralsI18nextKeyParser.kt deleted file mode 100644 index 90cbc7cc17..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/PluralsI18nextKeyParser.kt +++ /dev/null @@ -1,23 +0,0 @@ -package io.tolgee.formats.i18next - -class PluralsI18nextKeyParser { - fun parse(match: MatchResult): ParsedI18nextKey? { - return ParsedI18nextKey( - key = match.groups.getGroupOrNull("key")?.value, - plural = match.groups.getGroupOrNull("plural")?.value, - fullMatch = match.value, - ) - } - - // FIXME: move somewhere shared - private fun MatchGroupCollection.getGroupOrNull(name: String): MatchGroup? { - try { - return this[name] - } catch (e: IllegalArgumentException) { - if (e.message?.contains("No group with name") != true) { - throw e - } - return null - } - } -} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/I18nextParameterParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/I18nextParameterParser.kt similarity index 50% rename from backend/data/src/main/kotlin/io/tolgee/formats/i18next/I18nextParameterParser.kt rename to backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/I18nextParameterParser.kt index d86b479fea..085fb49fe5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/I18nextParameterParser.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/I18nextParameterParser.kt @@ -1,4 +1,6 @@ -package io.tolgee.formats.i18next +package io.tolgee.formats.i18next.`in` + +import io.tolgee.formats.getGroupOrNull class I18nextParameterParser { fun parse(match: MatchResult): ParsedI18nextParam? { @@ -9,16 +11,4 @@ class I18nextParameterParser { fullMatch = match.value, ) } - -// FIXME: move somewhere shared - private fun MatchGroupCollection.getGroupOrNull(name: String): MatchGroup? { - try { - return this[name] - } catch (e: IllegalArgumentException) { - if (e.message?.contains("No group with name") != true) { - throw e - } - return null - } - } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextParam.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/ParsedI18nextParam.kt similarity index 79% rename from backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextParam.kt rename to backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/ParsedI18nextParam.kt index 38f3864b60..dfe969ad76 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/ParsedI18nextParam.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/ParsedI18nextParam.kt @@ -1,4 +1,4 @@ -package io.tolgee.formats.i18next +package io.tolgee.formats.i18next.`in` data class ParsedI18nextParam( val key: String? = null, diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/PluralsI18nextKeyParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/PluralsI18nextKeyParser.kt new file mode 100644 index 0000000000..87163e558f --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/PluralsI18nextKeyParser.kt @@ -0,0 +1,16 @@ +package io.tolgee.formats.i18next.`in` + +import io.tolgee.formats.getGroupOrNull +import io.tolgee.formats.importCommon.ParsedPluralsKey +import io.tolgee.formats.importCommon.PluralsKeyParser + +class PluralsI18nextKeyParser(private val keyRegex: Regex) : PluralsKeyParser { + override fun parse(key: String): ParsedPluralsKey { + val match = keyRegex.find(key) + return ParsedPluralsKey( + key = match?.groups?.getGroupOrNull("key")?.value, + plural = match?.groups?.getGroupOrNull("plural")?.value, + originalKey = key, + ) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt index 98b61e2fd9..c59c828578 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt @@ -11,14 +11,14 @@ import io.tolgee.formats.po.`in`.PoToIcuMessageConvertor enum class ImportFormat( val fileFormat: ImportFileFormat, val pluralsViaNesting: Boolean = false, - val pluralsViaSuffixesRegex: Regex? = null, + val pluralsViaSuffixesParser: PluralsKeyParser? = null, val messageConvertorOrNull: ImportMessageConvertor? = null, val rootKeyIsLanguageTag: Boolean = false, ) { JSON_I18NEXT( ImportFileFormat.JSON, messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { I18nextToIcuPlaceholderConvertor() }, - pluralsViaSuffixesRegex = I18nextToIcuPlaceholderConvertor.I18NEXT_PLURAL_SUFFIX_REGEX, + pluralsViaSuffixesParser = I18nextToIcuPlaceholderConvertor.I18NEXT_PLURAL_SUFFIX_KEY_PARSER, ), JSON_ICU( ImportFileFormat.JSON, diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ParsedPluralsKey.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ParsedPluralsKey.kt new file mode 100644 index 0000000000..13af6cdf9a --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ParsedPluralsKey.kt @@ -0,0 +1,7 @@ +package io.tolgee.formats.importCommon + +data class ParsedPluralsKey( + val key: String? = null, + val plural: String? = null, + val originalKey: String, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/PluralsKeyParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/PluralsKeyParser.kt new file mode 100644 index 0000000000..4f08f13a53 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/PluralsKeyParser.kt @@ -0,0 +1,5 @@ +package io.tolgee.formats.importCommon + +interface PluralsKeyParser { + fun parse(key: String): ParsedPluralsKey +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index 4493400811..5853275e5d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -2,7 +2,8 @@ package io.tolgee.formats.paramConvertors.`in` import io.tolgee.formats.ToIcuPlaceholderConvertor import io.tolgee.formats.escapeIcu -import io.tolgee.formats.i18next.I18nextParameterParser +import io.tolgee.formats.i18next.`in`.I18nextParameterParser +import io.tolgee.formats.i18next.`in`.PluralsI18nextKeyParser class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { private val parser = I18nextParameterParser() @@ -64,5 +65,7 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { val I18NEXT_PLURAL_SUFFIX_REGEX = """^(?\w+)_(?\w+)$""".toRegex() + val I18NEXT_PLURAL_SUFFIX_KEY_PARSER = PluralsI18nextKeyParser(I18NEXT_PLURAL_SUFFIX_REGEX) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/po/in/CLikeParameterParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/po/in/CLikeParameterParser.kt index d7449d0852..b6673cce14 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/po/in/CLikeParameterParser.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/po/in/CLikeParameterParser.kt @@ -1,5 +1,7 @@ package io.tolgee.formats.po.`in` +import io.tolgee.formats.getGroupOrNull + class CLikeParameterParser { fun parse(match: MatchResult): ParsedCLikeParam? { val specifierGroup = match.groups["specifier"] @@ -18,15 +20,4 @@ class CLikeParameterParser { fullMatch = match.value, ) } - - private fun MatchGroupCollection.getGroupOrNull(name: String): MatchGroup? { - try { - return this[name] - } catch (e: IllegalArgumentException) { - if (e.message?.contains("No group with name") != true) { - throw e - } - return null - } - } } From 7bb61275dba9345b95a15a8bbef08606f8c46ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 6 Sep 2024 19:20:54 +0200 Subject: [PATCH 03/32] fix: missed variable usage during refractoring --- .../in/GenericStructuredRawDataToTextConvertor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt index 276fdc0eca..b69d11f863 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt @@ -73,7 +73,7 @@ class GenericStructuredRawDataToTextConvertor( ): List? { val map = rawData as? Map<*, *> ?: return null - if (!format.pluralsViaNesting && format.pluralsViaSuffixesRegex == null) { + if (!format.pluralsViaNesting && format.pluralsViaSuffixesParser == null) { return null } From 756701962ffdd892aeae40a8c7167e00a7bc2843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Mon, 16 Sep 2024 11:38:40 +0200 Subject: [PATCH 04/32] fix: lint --- .../in/GenericStructuredProcessor.kt | 13 +++++++------ .../GenericStructuredRawDataToTextConvertor.kt | 2 -- .../formats/json/in/JsonImportFormatDetector.kt | 2 +- .../in/I18nextToIcuPlaceholderConvertor.kt | 1 - .../kotlin/io/tolgee/formats/pluralFormsUtil.kt | 17 +++++++++-------- .../formats/json/in/JsonFormatProcessorTest.kt | 10 ++++++---- 6 files changed, 23 insertions(+), 22 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index 7d6164c6ed..f8a1ed556c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -46,9 +46,7 @@ class GenericStructuredProcessor( } ?: ParsedPluralsKey(null, null, key) } - private fun Map<*, *>.groupByPlurals( - keyParser: PluralsKeyParser - ): Map>> { + private fun Map<*, *>.groupByPlurals(keyParser: PluralsKeyParser): Map>> { return this.entries.mapIndexedNotNull { idx, (key, value) -> key.parsePluralsKey(keyParser)?.let { it to value }.also { if (it == null) { @@ -65,9 +63,12 @@ class GenericStructuredProcessor( } private fun List>.usePluralsKey(commonKey: String): List> { - return listOf(commonKey to this.associate { (parsedKey, value) -> - parsedKey.plural to value - }) + return listOf( + commonKey to + this.associate { (parsedKey, value) -> + parsedKey.plural to value + }, + ) } private fun Map<*, *>.preprocessMap(): Map<*, *> { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt index b69d11f863..6325ee721f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt @@ -1,11 +1,9 @@ package io.tolgee.formats.genericStructuredFile.`in` -import com.ibm.icu.text.PluralRules import io.tolgee.formats.MessageConvertorResult import io.tolgee.formats.allPluralKeywords import io.tolgee.formats.importCommon.ImportFormat import io.tolgee.formats.importCommon.unwrapString -import java.util.* class GenericStructuredRawDataToTextConvertor( private val format: ImportFormat, diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt b/backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt index 8194c04979..bdfa427e47 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt @@ -5,8 +5,8 @@ import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.ICU_DETE import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.detectFromPossibleFormats import io.tolgee.formats.importCommon.ImportFormat import io.tolgee.formats.paramConvertors.`in`.CToIcuPlaceholderConvertor -import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.I18nextToIcuPlaceholderConvertor +import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index 5853275e5d..3134a88256 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -66,6 +66,5 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { val I18NEXT_PLURAL_SUFFIX_REGEX = """^(?\w+)_(?\w+)$""".toRegex() val I18NEXT_PLURAL_SUFFIX_KEY_PARSER = PluralsI18nextKeyParser(I18NEXT_PLURAL_SUFFIX_REGEX) - } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/pluralFormsUtil.kt b/backend/data/src/main/kotlin/io/tolgee/formats/pluralFormsUtil.kt index 0e36e64413..1c946fe93e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/pluralFormsUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/pluralFormsUtil.kt @@ -6,14 +6,15 @@ import io.tolgee.formats.escaping.IcuUnescper import io.tolgee.formats.escaping.PluralFormIcuEscaper import io.tolgee.util.nullIfEmpty -val allPluralKeywords = listOf( - PluralRules.KEYWORD_ZERO, - PluralRules.KEYWORD_ONE, - PluralRules.KEYWORD_TWO, - PluralRules.KEYWORD_FEW, - PluralRules.KEYWORD_MANY, - PluralRules.KEYWORD_OTHER, -) +val allPluralKeywords = + listOf( + PluralRules.KEYWORD_ZERO, + PluralRules.KEYWORD_ONE, + PluralRules.KEYWORD_TWO, + PluralRules.KEYWORD_FEW, + PluralRules.KEYWORD_MANY, + PluralRules.KEYWORD_OTHER, + ) fun getPluralFormsForLocale(languageTag: String): MutableSet { val uLocale = getULocaleFromTag(languageTag) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt index 94c95f2ce7..42796f3716 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt @@ -198,10 +198,12 @@ class JsonFormatProcessorTest { } mockUtil.fileProcessorContext.assertTranslations("i18next", "keyPluralSimple") .assertSingle { - hasText("{value, plural,\n" + - "one {the singular (is parsed as plural under one key - keyPluralSimple)}\n" + - "other {the plural (is parsed as plural under one key - keyPluralSimple)}\n" + - "}") + hasText( + "{value, plural,\n" + + "one {the singular (is parsed as plural under one key - keyPluralSimple)}\n" + + "other {the plural (is parsed as plural under one key - keyPluralSimple)}\n" + + "}", + ) } } From bf21c5ffa71ce39cecf2e5433afff006e3033c37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 17 Sep 2024 15:59:27 +0200 Subject: [PATCH 05/32] feat: add i18next export support --- .../kotlin/io/tolgee/formats/ExportFormat.kt | 1 + .../io/tolgee/formats/ExportMessageFormat.kt | 2 + .../out/GenericStructuredFileExporter.kt | 26 +- .../formats/json/out/JsonFileExporter.kt | 1 + .../out/IcuToI18nextPlaceholderConvertor.kt | 53 ++ .../service/export/FileExporterFactory.kt | 2 +- .../formats/json/out/JsonFileExporterTest.kt | 25 +- e2e/cypress/common/export.ts | 15 + webapp/src/service/apiSchema.generated.ts | 706 ++++-------------- .../export/components/formatGroups.tsx | 34 + .../components/messageFormatTranslation.tsx | 1 + 11 files changed, 282 insertions(+), 584 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt b/backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt index 56ed5621d3..ba14b1d3b4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt @@ -9,6 +9,7 @@ enum class ExportFormat( ) { JSON("json", "application/json"), JSON_TOLGEE("json", "application/json"), + JSON_I18NEXT("json", "application/json"), XLIFF("xliff", "application/x-xliff+xml"), PO("po", "text/x-gettext-translation"), APPLE_STRINGS_STRINGSDICT( diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/ExportMessageFormat.kt b/backend/data/src/main/kotlin/io/tolgee/formats/ExportMessageFormat.kt index e1e745dc6b..fc9618d919 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/ExportMessageFormat.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/ExportMessageFormat.kt @@ -2,6 +2,7 @@ package io.tolgee.formats import io.tolgee.formats.paramConvertors.out.IcuToApplePlaceholderConvertor import io.tolgee.formats.paramConvertors.out.IcuToCPlaceholderConvertor +import io.tolgee.formats.paramConvertors.out.IcuToI18nextPlaceholderConvertor import io.tolgee.formats.paramConvertors.out.IcuToJavaPlaceholderConvertor import io.tolgee.formats.paramConvertors.out.IcuToPhpPlaceholderConvertor import io.tolgee.formats.paramConvertors.out.IcuToRubyPlaceholderConvertor @@ -13,6 +14,7 @@ enum class ExportMessageFormat(val paramConvertorFactory: () -> FromIcuPlacehold JAVA_STRING_FORMAT(paramConvertorFactory = { IcuToJavaPlaceholderConvertor() }), APPLE_SPRINTF(paramConvertorFactory = { IcuToApplePlaceholderConvertor() }), RUBY_SPRINTF(paramConvertorFactory = { IcuToRubyPlaceholderConvertor() }), + I18NEXT(paramConvertorFactory = { IcuToI18nextPlaceholderConvertor() }), ICU(paramConvertorFactory = { IcuToIcuPlaceholderConvertor() }), // PYTHON_SPRINTF, } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt index 95ad0a8912..34999a1b37 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt @@ -55,8 +55,11 @@ class GenericStructuredFileExporter( ) } + private val pluralsViaSuffixes + get() = messageFormat == ExportMessageFormat.I18NEXT + private val pluralsViaNesting - get() = messageFormat != ExportMessageFormat.ICU + get() = !pluralsViaSuffixes && messageFormat != ExportMessageFormat.ICU private val placeholderConvertorFactory get() = messageFormat.paramConvertorFactory @@ -65,6 +68,9 @@ class GenericStructuredFileExporter( if (pluralsViaNesting) { return addNestedPlural(translation) } + if (pluralsViaSuffixes) { + return addSuffixedPlural(translation) + } return addSingularTranslation(translation) } @@ -84,6 +90,24 @@ class GenericStructuredFileExporter( ) } + private fun addSuffixedPlural(translation: ExportTranslationView) { + val pluralForms = + convertMessageForNestedPlural(translation.text) ?: let { + // this should never happen, but if it does, it's better to add a null key then crash or ignore it + addNullValue(translation) + return + } + + val builder = getFileContentResultBuilder(translation) + pluralForms.forEach { (keyword, form) -> + builder.addValue( + translation.languageTag, + "${translation.key.name}_$keyword", + form, + ) + } + } + private fun addNullValue(translation: ExportTranslationView) { val builder = getFileContentResultBuilder(translation) builder.addValue( diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/json/out/JsonFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/json/out/JsonFileExporter.kt index 59862c93cc..f2a2867c04 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/json/out/JsonFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/json/out/JsonFileExporter.kt @@ -23,6 +23,7 @@ class JsonFileExporter( private val messageFormat = when (exportParams.format) { ExportFormat.JSON_TOLGEE -> ExportMessageFormat.ICU + ExportFormat.JSON_I18NEXT -> ExportMessageFormat.I18NEXT else -> exportParams.messageFormat ?: ExportMessageFormat.ICU } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt new file mode 100644 index 0000000000..1b4c359e28 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt @@ -0,0 +1,53 @@ +package io.tolgee.formats.paramConvertors.out + +import com.ibm.icu.text.MessagePattern +import io.tolgee.formats.FromIcuPlaceholderConvertor +import io.tolgee.formats.MessagePatternUtil + +class IcuToI18nextPlaceholderConvertor : FromIcuPlaceholderConvertor { + override fun convert(node: MessagePatternUtil.ArgNode): String { + val type = node.argType + + if (type == MessagePattern.ArgType.SIMPLE) { + when (node.typeName) { + "number" -> return convertNumber(node) + } + } + + return "{{${node.name}}}" + } + + override fun convertText(string: String): String { + // TODO: escape - there doesn't seem to be a documented way how to escape either {{ or $t in i18next + return string + } + + override fun convertReplaceNumber( + node: MessagePatternUtil.MessageContentsNode, + argName: String?, + ): String { + return "{{$argName, number}}" + } + + private fun convertNumber(node: MessagePatternUtil.ArgNode): String { + if (node.simpleStyle.trim() == "scientific") { + return "{{${node.name}, number}}" + } + val precision = node.getPrecision() + if (precision != null) { + return "{{${node.name}, number(minimumFractionDigits: $precision; maximumFractionDigits: $precision)}}" + } + + return "{{${node.name}, number}}" + } + + private fun MessagePatternUtil.ArgNode.getPrecision(): Int? { + val precisionMatch = ICU_PRECISION_REGEX.matchEntire(this.simpleStyle) + precisionMatch ?: return null + return precisionMatch.groups["precision"]?.value?.length + } + + companion object { + val ICU_PRECISION_REGEX = """.*\.(?0+)""".toRegex() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/export/FileExporterFactory.kt b/backend/data/src/main/kotlin/io/tolgee/service/export/FileExporterFactory.kt index fe8b9d2ce2..e52863ef78 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/export/FileExporterFactory.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/export/FileExporterFactory.kt @@ -34,7 +34,7 @@ class FileExporterFactory( projectIcuPlaceholdersSupport: Boolean, ): FileExporter { return when (exportParams.format) { - ExportFormat.JSON, ExportFormat.JSON_TOLGEE -> + ExportFormat.JSON, ExportFormat.JSON_TOLGEE, ExportFormat.JSON_I18NEXT -> JsonFileExporter( data, exportParams, diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/out/JsonFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/out/JsonFileExporterTest.kt index 56ecb39744..b3a0dac38b 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/out/JsonFileExporterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/out/JsonFileExporterTest.kt @@ -164,6 +164,23 @@ class JsonFileExporterTest { ) } + @Test + fun `exports i18next correctly`() { + val exporter = getIcuPlaceholdersEnabledExporter(ExportMessageFormat.I18NEXT) + val data = getExported(exporter) + data.assertFile( + "cs.json", + """ + |{ + | "key3_one": "{{count, number}} den {{icuParam, number}}", + | "key3_few": "{{count, number}} dny", + | "key3_other": "{{count, number}} dní", + | "item": "I will be first '{'icuParam'}' {{hello, number}}" + |} + """.trimMargin(), + ) + } + @Test fun `correct exports translation with colon`() { val exporter = getExporter(getTranslationWithColon()) @@ -190,7 +207,7 @@ class JsonFileExporterTest { return built.translations } - private fun getIcuPlaceholdersEnabledExporter(): JsonFileExporter { + private fun getIcuPlaceholdersEnabledExporter(messageFormat: ExportMessageFormat? = null): JsonFileExporter { val built = buildExportTranslationList { add( @@ -206,7 +223,11 @@ class JsonFileExporterTest { text = "I will be first '{'icuParam'}' {hello, number}", ) } - return getExporter(built.translations, true) + return getExporter( + built.translations, + true, + exportParams = ExportParams(messageFormat = messageFormat), + ) } @Test diff --git a/e2e/cypress/common/export.ts b/e2e/cypress/common/export.ts index 67d3ae8ffd..d80bc76f4d 100644 --- a/e2e/cypress/common/export.ts +++ b/e2e/cypress/common/export.ts @@ -207,6 +207,21 @@ export const testExportFormats = ( format: 'YAML_RUBY', }, }); + + testFormat(interceptFn, submitFn, clearCheckboxesAfter, afterFn, { + format: 'Flat i18next .json', + expectedParams: { + format: 'JSON_I18NEXT', + }, + }); + + testFormat(interceptFn, submitFn, clearCheckboxesAfter, afterFn, { + format: 'Structured i18next .json', + expectedParams: { + format: 'JSON_I18NEXT', + structureDelimiter: '.', + }, + }); }; const testFormat = ( diff --git a/webapp/src/service/apiSchema.generated.ts b/webapp/src/service/apiSchema.generated.ts index e2ea62dc71..1f0c0b7aa8 100644 --- a/webapp/src/service/apiSchema.generated.ts +++ b/webapp/src/service/apiSchema.generated.ts @@ -318,12 +318,6 @@ export interface paths { /** Pairs user account with slack account. */ post: operations["userLogin"]; }; - "/v2/public/translator/translate": { - post: operations["translate"]; - }; - "/v2/public/telemetry/report": { - post: operations["report"]; - }; "/v2/public/slack": { post: operations["slackCommand"]; }; @@ -339,26 +333,8 @@ export interface paths { */ post: operations["fetchBotEvent"]; }; - "/v2/public/licensing/subscription": { - post: operations["getMySubscription"]; - }; - "/v2/public/licensing/set-key": { - post: operations["onLicenceSetKey"]; - }; - "/v2/public/licensing/report-usage": { - post: operations["reportUsage"]; - }; - "/v2/public/licensing/report-error": { - post: operations["reportError"]; - }; - "/v2/public/licensing/release-key": { - post: operations["releaseKey"]; - }; - "/v2/public/licensing/prepare-set-key": { - post: operations["prepareSetLicenseKey"]; - }; "/v2/public/business-events/report": { - post: operations["report_1"]; + post: operations["report"]; }; "/v2/public/business-events/identify": { post: operations["identify"]; @@ -427,7 +403,7 @@ export interface paths { }; "/v2/projects/{projectId}/start-batch-job/pre-translate-by-tm": { /** Pre-translate provided keys to provided languages by TM. */ - post: operations["translate_1"]; + post: operations["translate"]; }; "/v2/projects/{projectId}/start-batch-job/machine-translate": { /** Translate provided keys to provided languages through primary MT provider. */ @@ -513,7 +489,7 @@ export interface paths { }; "/v2/ee-license/prepare-set-license-key": { /** Get info about the upcoming EE subscription. This will show, how much the subscription will cost when key is applied. */ - post: operations["prepareSetLicenseKey_1"]; + post: operations["prepareSetLicenseKey"]; }; "/v2/api-keys": { get: operations["allByUser"]; @@ -1062,7 +1038,12 @@ export interface components { | "tolgee_account_already_connected" | "slack_not_configured" | "slack_workspace_already_connected" - | "slack_connection_error"; + | "slack_connection_error" + | "email_verification_code_not_valid" + | "cannot_subscribe_to_free_plan" + | "plan_auto_assignment_only_for_free_plans" + | "plan_auto_assignment_only_for_private_plans" + | "plan_auto_assignment_organization_ids_not_in_for_organization_ids"; params?: { [key: string]: unknown }[]; }; ErrorResponseBody: { @@ -1135,6 +1116,16 @@ export interface components { | "SERVER_ADMIN"; /** @description The user's permission type. This field is null if uses granular permissions */ type?: "NONE" | "VIEW" | "TRANSLATE" | "REVIEW" | "EDIT" | "MANAGE"; + /** + * @description List of languages user can translate to. If null, all languages editing is permitted. + * @example 200001,200004 + */ + translateLanguageIds?: number[]; + /** + * @description List of languages user can change state to. If null, changing state of all language values is permitted. + * @example 200001,200004 + */ + stateChangeLanguageIds?: number[]; /** * @deprecated * @description Deprecated (use translateLanguageIds). @@ -1143,21 +1134,11 @@ export interface components { * @example 200001,200004 */ permittedLanguageIds?: number[]; - /** - * @description List of languages user can translate to. If null, all languages editing is permitted. - * @example 200001,200004 - */ - translateLanguageIds?: number[]; /** * @description List of languages user can view. If null, all languages view is permitted. * @example 200001,200004 */ viewLanguageIds?: number[]; - /** - * @description List of languages user can change state to. If null, changing state of all language values is permitted. - * @example 200001,200004 - */ - stateChangeLanguageIds?: number[]; /** * @description Granted scopes to the user. When user has type permissions, this field contains permission scopes of the type. * @example KEYS_EDIT,TRANSLATIONS_VIEW @@ -1718,8 +1699,8 @@ export interface components { secretKey?: string; endpoint: string; signingRegion: string; - enabled?: boolean; contentStorageType?: "S3" | "AZURE"; + enabled?: boolean; }; AzureContentStorageConfigModel: { containerName?: string; @@ -1771,6 +1752,7 @@ export interface components { format: | "JSON" | "JSON_TOLGEE" + | "JSON_I18NEXT" | "XLIFF" | "PO" | "APPLE_STRINGS_STRINGSDICT" @@ -1829,6 +1811,7 @@ export interface components { | "JAVA_STRING_FORMAT" | "APPLE_SPRINTF" | "RUBY_SPRINTF" + | "I18NEXT" | "ICU"; /** * @description This is a template that defines the structure of the resulting .zip file content. @@ -1868,6 +1851,7 @@ export interface components { format: | "JSON" | "JSON_TOLGEE" + | "JSON_I18NEXT" | "XLIFF" | "PO" | "APPLE_STRINGS_STRINGSDICT" @@ -1920,6 +1904,7 @@ export interface components { | "JAVA_STRING_FORMAT" | "APPLE_SPRINTF" | "RUBY_SPRINTF" + | "I18NEXT" | "ICU"; /** * @description If true, for structured formats (like JSON) arrays are supported. @@ -1983,7 +1968,7 @@ export interface components { overrideKeyDescriptions: boolean; /** @description If true, placeholders from other formats will be converted to ICU when possible */ convertPlaceholdersToIcu: boolean; - /** @description If true, only updates keys, skipping the creation of new keys */ + /** @description If false, only updates keys, skipping the creation of new keys */ createNewKeys: boolean; }; ImportSettingsModel: { @@ -1991,7 +1976,7 @@ export interface components { convertPlaceholdersToIcu: boolean; /** @description If true, key descriptions will be overridden by the import */ overrideKeyDescriptions: boolean; - /** @description If true, only updates keys, skipping the creation of new keys */ + /** @description If false, only updates keys, skipping the creation of new keys */ createNewKeys: boolean; }; /** @description User who created the comment */ @@ -2306,17 +2291,17 @@ export interface components { key: string; /** Format: int64 */ id: number; - projectName: string; userFullName?: string; + projectName: string; description: string; username?: string; + scopes: string[]; /** Format: int64 */ projectId: number; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ lastUsedAt?: number; - scopes: string[]; }; SuperTokenRequest: { /** @description Has to be provided when TOTP enabled */ @@ -2328,49 +2313,6 @@ export interface components { name: string; oldSlug?: string; }; - ExampleItem: { - source: string; - target: string; - key: string; - keyNamespace?: string; - }; - Metadata: { - examples: components["schemas"]["ExampleItem"][]; - closeItems: components["schemas"]["ExampleItem"][]; - keyDescription?: string; - projectDescription?: string; - languageDescription?: string; - }; - TolgeeTranslateParams: { - text: string; - keyName?: string; - sourceTag: string; - targetTag: string; - metadata?: components["schemas"]["Metadata"]; - formality?: "FORMAL" | "INFORMAL" | "DEFAULT"; - isBatch: boolean; - pluralForms?: { [key: string]: string }; - pluralFormExamples?: { [key: string]: string }; - }; - MtResult: { - translated?: string; - /** Format: int32 */ - price: number; - contextDescription?: string; - }; - TelemetryReportRequest: { - instanceId: string; - /** Format: int64 */ - projectsCount: number; - /** Format: int64 */ - translationsCount: number; - /** Format: int64 */ - languagesCount: number; - /** Format: int64 */ - distinctLanguagesCount: number; - /** Format: int64 */ - usersCount: number; - }; SlackCommandDto: { token?: string; team_id: string; @@ -2383,126 +2325,6 @@ export interface components { trigger_id?: string; team_domain: string; }; - GetMySubscriptionDto: { - licenseKey: string; - instanceId: string; - }; - PlanIncludedUsageModel: { - /** Format: int64 */ - seats: number; - /** Format: int64 */ - translationSlots: number; - /** Format: int64 */ - translations: number; - /** Format: int64 */ - mtCredits: number; - }; - PlanPricesModel: { - perSeat: number; - perThousandTranslations?: number; - perThousandMtCredits?: number; - subscriptionMonthly: number; - subscriptionYearly: number; - }; - SelfHostedEePlanModel: { - /** Format: int64 */ - id: number; - name: string; - public: boolean; - enabledFeatures: ( - | "GRANULAR_PERMISSIONS" - | "PRIORITIZED_FEATURE_REQUESTS" - | "PREMIUM_SUPPORT" - | "DEDICATED_SLACK_CHANNEL" - | "ASSISTED_UPDATES" - | "DEPLOYMENT_ASSISTANCE" - | "BACKUP_CONFIGURATION" - | "TEAM_TRAINING" - | "ACCOUNT_MANAGER" - | "STANDARD_SUPPORT" - | "PROJECT_LEVEL_CONTENT_STORAGES" - | "WEBHOOKS" - | "MULTIPLE_CONTENT_DELIVERY_CONFIGS" - | "AI_PROMPT_CUSTOMIZATION" - | "SLACK_INTEGRATION" - )[]; - prices: components["schemas"]["PlanPricesModel"]; - includedUsage: components["schemas"]["PlanIncludedUsageModel"]; - hasYearlyPrice: boolean; - free: boolean; - }; - SelfHostedEeSubscriptionModel: { - /** Format: int64 */ - id: number; - /** Format: int64 */ - currentPeriodStart?: number; - /** Format: int64 */ - currentPeriodEnd?: number; - currentBillingPeriod: "MONTHLY" | "YEARLY"; - /** Format: int64 */ - createdAt: number; - plan: components["schemas"]["SelfHostedEePlanModel"]; - status: - | "ACTIVE" - | "CANCELED" - | "PAST_DUE" - | "UNPAID" - | "ERROR" - | "KEY_USED_BY_ANOTHER_INSTANCE"; - licenseKey?: string; - estimatedCosts?: number; - }; - SetLicenseKeyLicensingDto: { - licenseKey: string; - /** Format: int64 */ - seats: number; - instanceId: string; - }; - ReportUsageDto: { - licenseKey: string; - /** Format: int64 */ - seats: number; - }; - ReportErrorDto: { - stackTrace: string; - licenseKey: string; - }; - ReleaseKeyDto: { - licenseKey: string; - }; - PrepareSetLicenseKeyDto: { - licenseKey: string; - /** Format: int64 */ - seats: number; - }; - AverageProportionalUsageItemModel: { - total: number; - unusedQuantity: number; - usedQuantity: number; - usedQuantityOverPlan: number; - }; - PrepareSetEeLicenceKeyModel: { - plan: components["schemas"]["SelfHostedEePlanModel"]; - usage: components["schemas"]["UsageModel"]; - }; - SumUsageItemModel: { - total: number; - /** Format: int64 */ - unusedQuantity: number; - /** Format: int64 */ - usedQuantity: number; - /** Format: int64 */ - usedQuantityOverPlan: number; - }; - UsageModel: { - subscriptionPrice?: number; - /** @description Relevant for invoices only. When there are applied stripe credits, we need to reduce the total price by this amount. */ - appliedStripeCredits?: number; - seats: components["schemas"]["AverageProportionalUsageItemModel"]; - translations: components["schemas"]["AverageProportionalUsageItemModel"]; - credits?: components["schemas"]["SumUsageItemModel"]; - total: number; - }; BusinessEventReportRequest: { eventName: string; anonymousUserId?: string; @@ -2867,7 +2689,12 @@ export interface components { | "tolgee_account_already_connected" | "slack_not_configured" | "slack_workspace_already_connected" - | "slack_connection_error"; + | "slack_connection_error" + | "email_verification_code_not_valid" + | "cannot_subscribe_to_free_plan" + | "plan_auto_assignment_only_for_free_plans" + | "plan_auto_assignment_only_for_private_plans" + | "plan_auto_assignment_organization_ids_not_in_for_organization_ids"; params?: { [key: string]: unknown }[]; }; UntagKeysRequest: { @@ -2976,6 +2803,7 @@ export interface components { * It is recommended to provide these values to prevent any issues with format detection. */ format?: + | "JSON_I18NEXT" | "JSON_ICU" | "JSON_JAVA" | "JSON_PHP" @@ -3063,7 +2891,7 @@ export interface components { overrideKeyDescriptions: boolean; /** @description If true, placeholders from other formats will be converted to ICU when possible */ convertPlaceholdersToIcu: boolean; - /** @description If true, only updates keys, skipping the creation of new keys */ + /** @description If false, only updates keys, skipping the creation of new keys */ createNewKeys: boolean; /** @description Definition of mapping for each file to import. */ fileMappings: components["schemas"]["ImportFileMapping"][]; @@ -3126,6 +2954,7 @@ export interface components { format: | "JSON" | "JSON_TOLGEE" + | "JSON_I18NEXT" | "XLIFF" | "PO" | "APPLE_STRINGS_STRINGSDICT" @@ -3179,6 +3008,7 @@ export interface components { | "JAVA_STRING_FORMAT" | "APPLE_SPRINTF" | "RUBY_SPRINTF" + | "I18NEXT" | "ICU"; /** * @description This is a template that defines the structure of the resulting .zip file content. @@ -3232,6 +3062,7 @@ export interface components { isPlural?: boolean; /** @description List of services to use. If null, then all enabled services are used. */ services?: ("GOOGLE" | "AWS" | "DEEPL" | "AZURE" | "BAIDU" | "TOLGEE")[]; + plural?: boolean; }; PagedModelTranslationMemoryItemModel: { _embedded?: { @@ -3304,6 +3135,78 @@ export interface components { createdAt: string; location?: string; }; + AverageProportionalUsageItemModel: { + total: number; + unusedQuantity: number; + usedQuantity: number; + usedQuantityOverPlan: number; + }; + PlanIncludedUsageModel: { + /** Format: int64 */ + seats: number; + /** Format: int64 */ + translationSlots: number; + /** Format: int64 */ + translations: number; + /** Format: int64 */ + mtCredits: number; + }; + PlanPricesModel: { + perSeat: number; + perThousandTranslations?: number; + perThousandMtCredits?: number; + subscriptionMonthly: number; + subscriptionYearly: number; + }; + PrepareSetEeLicenceKeyModel: { + plan: components["schemas"]["SelfHostedEePlanModel"]; + usage: components["schemas"]["UsageModel"]; + }; + SelfHostedEePlanModel: { + /** Format: int64 */ + id: number; + name: string; + public: boolean; + enabledFeatures: ( + | "GRANULAR_PERMISSIONS" + | "PRIORITIZED_FEATURE_REQUESTS" + | "PREMIUM_SUPPORT" + | "DEDICATED_SLACK_CHANNEL" + | "ASSISTED_UPDATES" + | "DEPLOYMENT_ASSISTANCE" + | "BACKUP_CONFIGURATION" + | "TEAM_TRAINING" + | "ACCOUNT_MANAGER" + | "STANDARD_SUPPORT" + | "PROJECT_LEVEL_CONTENT_STORAGES" + | "WEBHOOKS" + | "MULTIPLE_CONTENT_DELIVERY_CONFIGS" + | "AI_PROMPT_CUSTOMIZATION" + | "SLACK_INTEGRATION" + )[]; + prices: components["schemas"]["PlanPricesModel"]; + includedUsage: components["schemas"]["PlanIncludedUsageModel"]; + hasYearlyPrice: boolean; + free: boolean; + }; + SumUsageItemModel: { + total: number; + /** Format: int64 */ + unusedQuantity: number; + /** Format: int64 */ + usedQuantity: number; + /** Format: int64 */ + usedQuantityOverPlan: number; + }; + UsageModel: { + subscriptionPrice?: number; + /** @description Relevant for invoices only. When there are applied stripe credits, we need to reduce the total price by this amount. */ + appliedStripeCredits?: number; + seats: components["schemas"]["AverageProportionalUsageItemModel"]; + translations: components["schemas"]["AverageProportionalUsageItemModel"]; + credits?: components["schemas"]["SumUsageItemModel"]; + total: number; + }; CreateApiKeyDto: { /** Format: int64 */ projectId: number; @@ -3472,18 +3375,18 @@ export interface components { name: string; /** Format: int64 */ id: number; - basePermissions: components["schemas"]["PermissionModel"]; - /** @example btforg */ - slug: string; - /** @example This is a beautiful organization full of beautiful and clever people */ - description?: string; /** * @description The role of currently authorized user. * * Can be null when user has direct access to one of the projects owned by the organization. */ currentUserRole?: "MEMBER" | "OWNER"; + basePermissions: components["schemas"]["PermissionModel"]; + /** @example This is a beautiful organization full of beautiful and clever people */ + description?: string; avatar?: components["schemas"]["Avatar"]; + /** @example btforg */ + slug: string; }; PublicBillingConfigurationDTO: { enabled: boolean; @@ -3530,6 +3433,7 @@ export interface components { format: | "JSON" | "JSON_TOLGEE" + | "JSON_I18NEXT" | "XLIFF" | "PO" | "APPLE_STRINGS_STRINGSDICT" @@ -3620,9 +3524,9 @@ export interface components { name: string; /** Format: int64 */ id: number; + baseTranslation?: string; namespace?: string; description?: string; - baseTranslation?: string; translation?: string; }; KeySearchSearchResultModel: { @@ -3630,9 +3534,9 @@ export interface components { name: string; /** Format: int64 */ id: number; + baseTranslation?: string; namespace?: string; description?: string; - baseTranslation?: string; translation?: string; }; PagedModelKeySearchSearchResultModel: { @@ -4316,17 +4220,17 @@ export interface components { permittedLanguageIds?: number[]; /** Format: int64 */ id: number; - projectName: string; userFullName?: string; + projectName: string; description: string; username?: string; + scopes: string[]; /** Format: int64 */ projectId: number; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ lastUsedAt?: number; - scopes: string[]; }; PagedModelUserAccountModel: { _embedded?: { @@ -9626,96 +9530,6 @@ export interface operations { }; }; }; - translate: { - responses: { - /** OK */ - 200: { - content: { - "application/json": components["schemas"]["MtResult"]; - }; - }; - /** Bad Request */ - 400: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Unauthorized */ - 401: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Forbidden */ - 403: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Not Found */ - 404: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - }; - requestBody: { - content: { - "application/json": components["schemas"]["TolgeeTranslateParams"]; - }; - }; - }; - report: { - responses: { - /** OK */ - 200: unknown; - /** Bad Request */ - 400: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Unauthorized */ - 401: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Forbidden */ - 403: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Not Found */ - 404: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - }; - requestBody: { - content: { - "application/json": components["schemas"]["TelemetryReportRequest"]; - }; - }; - }; slackCommand: { parameters: { header: { @@ -9880,277 +9694,7 @@ export interface operations { }; }; }; - getMySubscription: { - responses: { - /** OK */ - 200: { - content: { - "application/json": components["schemas"]["SelfHostedEeSubscriptionModel"]; - }; - }; - /** Bad Request */ - 400: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Unauthorized */ - 401: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Forbidden */ - 403: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Not Found */ - 404: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - }; - requestBody: { - content: { - "application/json": components["schemas"]["GetMySubscriptionDto"]; - }; - }; - }; - onLicenceSetKey: { - responses: { - /** OK */ - 200: { - content: { - "application/json": components["schemas"]["SelfHostedEeSubscriptionModel"]; - }; - }; - /** Bad Request */ - 400: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Unauthorized */ - 401: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Forbidden */ - 403: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Not Found */ - 404: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - }; - requestBody: { - content: { - "application/json": components["schemas"]["SetLicenseKeyLicensingDto"]; - }; - }; - }; - reportUsage: { - responses: { - /** OK */ - 200: unknown; - /** Bad Request */ - 400: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Unauthorized */ - 401: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Forbidden */ - 403: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Not Found */ - 404: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - }; - requestBody: { - content: { - "application/json": components["schemas"]["ReportUsageDto"]; - }; - }; - }; - reportError: { - responses: { - /** OK */ - 200: unknown; - /** Bad Request */ - 400: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Unauthorized */ - 401: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Forbidden */ - 403: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Not Found */ - 404: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - }; - requestBody: { - content: { - "application/json": components["schemas"]["ReportErrorDto"]; - }; - }; - }; - releaseKey: { - responses: { - /** OK */ - 200: unknown; - /** Bad Request */ - 400: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Unauthorized */ - 401: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Forbidden */ - 403: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Not Found */ - 404: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - }; - requestBody: { - content: { - "application/json": components["schemas"]["ReleaseKeyDto"]; - }; - }; - }; - prepareSetLicenseKey: { - responses: { - /** OK */ - 200: { - content: { - "application/json": components["schemas"]["PrepareSetEeLicenceKeyModel"]; - }; - }; - /** Bad Request */ - 400: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Unauthorized */ - 401: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Forbidden */ - 403: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - /** Not Found */ - 404: { - content: { - "application/json": - | components["schemas"]["ErrorResponseTyped"] - | components["schemas"]["ErrorResponseBody"]; - }; - }; - }; - requestBody: { - content: { - "application/json": components["schemas"]["PrepareSetLicenseKeyDto"]; - }; - }; - }; - report_1: { + report: { responses: { /** OK */ 200: unknown; @@ -11386,7 +10930,7 @@ export interface operations { }; }; /** Pre-translate provided keys to provided languages by TM. */ - translate_1: { + translate: { parameters: { path: { projectId: number; @@ -11817,6 +11361,7 @@ export interface operations { format?: | "JSON" | "JSON_TOLGEE" + | "JSON_I18NEXT" | "XLIFF" | "PO" | "APPLE_STRINGS_STRINGSDICT" @@ -11880,6 +11425,7 @@ export interface operations { | "JAVA_STRING_FORMAT" | "APPLE_SPRINTF" | "RUBY_SPRINTF" + | "I18NEXT" | "ICU"; /** * This is a template that defines the structure of the resulting .zip file content. @@ -12897,7 +12443,7 @@ export interface operations { }; }; /** Get info about the upcoming EE subscription. This will show, how much the subscription will cost when key is applied. */ - prepareSetLicenseKey_1: { + prepareSetLicenseKey: { responses: { /** OK */ 200: { diff --git a/webapp/src/views/projects/export/components/formatGroups.tsx b/webapp/src/views/projects/export/components/formatGroups.tsx index 0a7084cb3c..7e35d7f7b9 100644 --- a/webapp/src/views/projects/export/components/formatGroups.tsx +++ b/webapp/src/views/projects/export/components/formatGroups.tsx @@ -196,6 +196,40 @@ export const formatGroups: FormatGroup[] = [ }, ], }, + { + name: 'i18next', + formats: [ + { + id: 'i18next_flat_json', + extension: 'json', + messageFormat: 'I18NEXT', + defaultStructureDelimiter: '', + structured: false, + showSupportArrays: true, + defaultSupportArrays: true, + name: , + format: 'JSON_I18NEXT', + matchByExportParams: (params) => + params.format === 'JSON_I18NEXT' && + (params.structureDelimiter === '' || + params.structureDelimiter == null) && + !params.supportArrays, + }, + { + id: 'i18next_structured_json', + extension: 'json', + messageFormat: 'I18NEXT', + defaultStructureDelimiter: '.', + structured: true, + showSupportArrays: true, + defaultSupportArrays: true, + name: , + format: 'JSON_I18NEXT', + matchByExportParams: (params) => + params.format === 'JSON_I18NEXT' && params.structureDelimiter === '.', + } + ] + } ]; type ExportParamsWithoutZip = Omit< diff --git a/webapp/src/views/projects/export/components/messageFormatTranslation.tsx b/webapp/src/views/projects/export/components/messageFormatTranslation.tsx index 829f713c21..ef48badb84 100644 --- a/webapp/src/views/projects/export/components/messageFormatTranslation.tsx +++ b/webapp/src/views/projects/export/components/messageFormatTranslation.tsx @@ -9,6 +9,7 @@ export const messageFormatTranslation: Record = { ), RUBY_SPRINTF: , + I18NEXT: , ICU: , APPLE_SPRINTF: , }; From 6459984f62ef91333bfd4db8574a067d82da74a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 19 Sep 2024 10:47:58 +0200 Subject: [PATCH 06/32] fix: lint --- .../src/views/projects/export/components/formatGroups.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/src/views/projects/export/components/formatGroups.tsx b/webapp/src/views/projects/export/components/formatGroups.tsx index 7e35d7f7b9..bb4bd5f897 100644 --- a/webapp/src/views/projects/export/components/formatGroups.tsx +++ b/webapp/src/views/projects/export/components/formatGroups.tsx @@ -227,9 +227,9 @@ export const formatGroups: FormatGroup[] = [ format: 'JSON_I18NEXT', matchByExportParams: (params) => params.format === 'JSON_I18NEXT' && params.structureDelimiter === '.', - } - ] - } + }, + ], + }, ]; type ExportParamsWithoutZip = Omit< From a733cdc2e8098f0bb31ddd5e13f5305136f3c07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 19 Sep 2024 10:48:21 +0200 Subject: [PATCH 07/32] fix: backend test - index changed after adding additional export formats --- .../io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt index 7aaaef3b0f..09afb1bdec 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt @@ -26,7 +26,7 @@ class ExportInfoControllerTest : AbstractControllerTest() { node("defaultFileStructureTemplate") .isString.isEqualTo("{namespace}/{languageTag}.{extension}") } - node("[4]") { + node("[5]") { node("extension").isEqualTo("") node("mediaType").isEqualTo("") node("defaultFileStructureTemplate") From 38077aea88d72ad40bd8edb0477a84decd54873e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 19 Sep 2024 16:08:11 +0200 Subject: [PATCH 08/32] fix: move preprocessing logic to separate file --- .../in/GenericStructuredProcessor.kt | 77 +++--------------- .../in/GenericSuffixedPluralsPreprocessor.kt | 79 +++++++++++++++++++ 2 files changed, 88 insertions(+), 68 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index f8a1ed556c..f1cd731107 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -16,75 +16,16 @@ class GenericStructuredProcessor( private val format: ImportFormat, ) : ImportFileProcessor() { override fun process() { - data.preprocess().import("") - } - - private fun Any?.preprocess(): Any? { - if (this == null) { - return null - } - - (this as? List<*>)?.let { - return it.preprocessList() - } - - (this as? Map<*, *>)?.let { - return it.preprocessMap() - } - - return this - } - - private fun List<*>.preprocessList(): List<*> { - return this.map { it.preprocess() } - } - - private fun Any?.parsePluralsKey(keyParser: PluralsKeyParser): ParsedPluralsKey? { - val key = this as? String ?: return null - return keyParser.parse(key).takeIf { - it.key != null && it.plural in allPluralKeywords - } ?: ParsedPluralsKey(null, null, key) - } - - private fun Map<*, *>.groupByPlurals(keyParser: PluralsKeyParser): Map>> { - return this.entries.mapIndexedNotNull { idx, (key, value) -> - key.parsePluralsKey(keyParser)?.let { it to value }.also { - if (it == null) { - context.fileEntity.addKeyIsNotStringIssue(key.toString(), idx) - } - } - }.groupBy { (parsedKey, _) -> parsedKey.key }.toMap() - } - - private fun List>.useOriginalKey(): List> { - return map { (parsedKey, value) -> - parsedKey.originalKey to value - } - } - - private fun List>.usePluralsKey(commonKey: String): List> { - return listOf( - commonKey to - this.associate { (parsedKey, value) -> - parsedKey.plural to value - }, - ) - } - - private fun Map<*, *>.preprocessMap(): Map<*, *> { - if (format.pluralsViaSuffixesParser == null) { - return this.mapValues { (_, value) -> value.preprocess() } + var data = data + if (format.pluralsViaSuffixesParser != null) { + data = GenericSuffixedPluralsPreprocessor( + context = context, + data = data, + pluralsViaSuffixesParser = format.pluralsViaSuffixesParser, + ).preprocess() + return } - - val plurals = this.groupByPlurals(format.pluralsViaSuffixesParser) - - return plurals.flatMap { (commonKey, values) -> - if (commonKey == null || values.size < 2) { - values.useOriginalKey() - } else { - values.usePluralsKey(commonKey) - } - }.toMap() + data.import("") } private fun Any?.import(key: String) { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt new file mode 100644 index 0000000000..fb622daa93 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt @@ -0,0 +1,79 @@ +package io.tolgee.formats.genericStructuredFile.`in` + +import io.tolgee.formats.allPluralKeywords +import io.tolgee.formats.importCommon.ParsedPluralsKey +import io.tolgee.formats.importCommon.PluralsKeyParser +import io.tolgee.service.dataImport.processors.FileProcessorContext + +class GenericSuffixedPluralsPreprocessor( + val context: FileProcessorContext, + private val data: Any?, + private val pluralsViaSuffixesParser: PluralsKeyParser, +) { + + fun preprocess(): Any? { + return data.preprocess() + } + + private fun Any?.preprocess(): Any? { + if (this == null) { + return null + } + + (this as? List<*>)?.let { + return it.preprocessList() + } + + (this as? Map<*, *>)?.let { + return it.preprocessMap() + } + + return this + } + + private fun List<*>.preprocessList(): List<*> { + return this.map { it.preprocess() } + } + + private fun Any?.parsePluralsKey(keyParser: PluralsKeyParser): ParsedPluralsKey? { + val key = this as? String ?: return null + return keyParser.parse(key).takeIf { + it.key != null && it.plural in allPluralKeywords + } ?: ParsedPluralsKey(null, null, key) + } + + private fun Map<*, *>.groupByPlurals(keyParser: PluralsKeyParser): Map>> { + return this.entries.mapIndexedNotNull { idx, (key, value) -> + key.parsePluralsKey(keyParser)?.let { it to value }.also { + if (it == null) { + context.fileEntity.addKeyIsNotStringIssue(key.toString(), idx) + } + } + }.groupBy { (parsedKey, _) -> parsedKey.key }.toMap() + } + + private fun List>.useOriginalKey(): List> { + return map { (parsedKey, value) -> + parsedKey.originalKey to value + } + } + + private fun List>.usePluralsKey(commonKey: String): List> { + return listOf( + commonKey to + this.associate { (parsedKey, value) -> + parsedKey.plural to value + }, + ) + } + + private fun Map<*, *>.preprocessMap(): Map<*, *> { + return this.groupByPlurals(pluralsViaSuffixesParser).flatMap { (commonKey, values) -> + if (commonKey == null || values.size < 2) { + values.useOriginalKey() + } else { + values.usePluralsKey(commonKey) + } + }.toMap() + } +} From c8fb9d31361e3fca6136753f29b2a45788c773e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 19 Sep 2024 16:19:31 +0200 Subject: [PATCH 09/32] fix: lint --- .../in/GenericStructuredProcessor.kt | 14 ++++++-------- .../in/GenericSuffixedPluralsPreprocessor.kt | 1 - 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index f1cd731107..1738f3dc80 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -2,10 +2,7 @@ package io.tolgee.formats.genericStructuredFile.`in` import io.tolgee.formats.ImportFileProcessor import io.tolgee.formats.MessageConvertorResult -import io.tolgee.formats.allPluralKeywords import io.tolgee.formats.importCommon.ImportFormat -import io.tolgee.formats.importCommon.ParsedPluralsKey -import io.tolgee.formats.importCommon.PluralsKeyParser import io.tolgee.service.dataImport.processors.FileProcessorContext class GenericStructuredProcessor( @@ -18,11 +15,12 @@ class GenericStructuredProcessor( override fun process() { var data = data if (format.pluralsViaSuffixesParser != null) { - data = GenericSuffixedPluralsPreprocessor( - context = context, - data = data, - pluralsViaSuffixesParser = format.pluralsViaSuffixesParser, - ).preprocess() + data = + GenericSuffixedPluralsPreprocessor( + context = context, + data = data, + pluralsViaSuffixesParser = format.pluralsViaSuffixesParser, + ).preprocess() return } data.import("") diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt index fb622daa93..58cc546910 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt @@ -10,7 +10,6 @@ class GenericSuffixedPluralsPreprocessor( private val data: Any?, private val pluralsViaSuffixesParser: PluralsKeyParser, ) { - fun preprocess(): Any? { return data.preprocess() } From df1517b257eefd2cf241613b1646868b5e1821c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 19 Sep 2024 16:32:53 +0200 Subject: [PATCH 10/32] fix: premature return --- .../genericStructuredFile/in/GenericStructuredProcessor.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index 1738f3dc80..83509b8804 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -13,17 +13,16 @@ class GenericStructuredProcessor( private val format: ImportFormat, ) : ImportFileProcessor() { override fun process() { - var data = data + var processedData = data if (format.pluralsViaSuffixesParser != null) { - data = + processedData = GenericSuffixedPluralsPreprocessor( context = context, data = data, pluralsViaSuffixesParser = format.pluralsViaSuffixesParser, ).preprocess() - return } - data.import("") + processedData.import("") } private fun Any?.import(key: String) { From 6ed82f2f013d368f449b5d9fac90d4e09fe339da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 19 Sep 2024 17:06:41 +0200 Subject: [PATCH 11/32] feat: initial handling of keep unescaped flag for potential later use --- .../io/tolgee/formats/i18next/in/I18nextParameterParser.kt | 1 + .../io/tolgee/formats/i18next/in/ParsedI18nextParam.kt | 1 + .../paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/I18nextParameterParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/I18nextParameterParser.kt index 085fb49fe5..b413d8e885 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/I18nextParameterParser.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/I18nextParameterParser.kt @@ -8,6 +8,7 @@ class I18nextParameterParser { key = match.groups.getGroupOrNull("key")?.value, nestedKey = match.groups.getGroupOrNull("nestedKey")?.value, format = match.groups.getGroupOrNull("format")?.value, + keepUnescaped = (match.groups.getGroupOrNull("unescapedflag")?.value?.length ?: 0) > 0, fullMatch = match.value, ) } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/ParsedI18nextParam.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/ParsedI18nextParam.kt index dfe969ad76..5f66d80a90 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/ParsedI18nextParam.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/ParsedI18nextParam.kt @@ -4,5 +4,6 @@ data class ParsedI18nextParam( val key: String? = null, val nestedKey: String? = null, val format: String? = null, + val keepUnescaped: Boolean = false, val fullMatch: String, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index 3134a88256..359e771a6e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -24,6 +24,8 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { return matchResult.value.escapeIcu(isInPlural) } + // TODO: handle unescaped flag by adding info about it to the metadata and using this info when exporting + return when (parsed.format) { null -> "{${parsed.key}}" "number" -> "{${parsed.key}, number}" @@ -37,7 +39,7 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { (?x) ( \{\{ - (?:-\ *)? + (?-\ *)? (?\w+)(?:,\ *(?[^}]+))? }} | @@ -53,7 +55,7 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { (^|\W+) ( \{\{ - (?:-\ *)? + (?-\ *)? (?\w+)(?:,\ *(?[^}]+))? }} | From 210d3109ce1c8fcd56dfc772f1c0c10ec87132bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 20 Sep 2024 11:54:07 +0200 Subject: [PATCH 12/32] fix: suggested changes from code review --- .../in/GenericSuffixedPluralsPreprocessor.kt | 5 ++--- .../paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt index 58cc546910..484e6d619d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt @@ -69,10 +69,9 @@ class GenericSuffixedPluralsPreprocessor( private fun Map<*, *>.preprocessMap(): Map<*, *> { return this.groupByPlurals(pluralsViaSuffixesParser).flatMap { (commonKey, values) -> if (commonKey == null || values.size < 2) { - values.useOriginalKey() - } else { - values.usePluralsKey(commonKey) + return@flatMap values.useOriginalKey() } + return@flatMap values.usePluralsKey(commonKey) }.toMap() } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index 359e771a6e..f2d8466ff9 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -20,7 +20,7 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { val parsed = parser.parse(matchResult) ?: return matchResult.value.escapeIcu(isInPlural) if (parsed.nestedKey != null) { - // TODO: nested keys are not yet supported + // Nested keys are not supported return matchResult.value.escapeIcu(isInPlural) } From f78fe06077a5302bc99d20e0e9e1a5f1c2aa1d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 20 Sep 2024 12:02:28 +0200 Subject: [PATCH 13/32] fix: suggested changes from code revirw --- .../paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt index 1b4c359e28..bd10057736 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt @@ -18,7 +18,8 @@ class IcuToI18nextPlaceholderConvertor : FromIcuPlaceholderConvertor { } override fun convertText(string: String): String { - // TODO: escape - there doesn't seem to be a documented way how to escape either {{ or $t in i18next + // We should escape {{ and $t, but there doesn't seem to be a documented + // way how to escape either {{ or $t in i18next return string } From e424b30652ee72e1313458c96a9ae8622270b0ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Wed, 25 Sep 2024 16:12:19 +0200 Subject: [PATCH 14/32] feat: handle unescaped flag by saving it to the custom values --- .../tolgee/formats/BaseIcuMessageConvertor.kt | 7 ++- .../formats/FromIcuPlaceholderConvertor.kt | 23 ++++++++ .../tolgee/formats/MessageConvertorFactory.kt | 2 + .../tolgee/formats/MessageConvertorResult.kt | 6 +- .../formats/ToIcuPlaceholderConvertor.kt | 3 + .../IcuToGenericFormatMessageConvertor.kt | 2 + .../in/GenericStructuredProcessor.kt | 58 ++++++++++++------- .../out/GenericStructuredFileExporter.kt | 18 ++++-- .../io/tolgee/formats/i18next/constants.kt | 3 + .../BaseImportRawDataConverter.kt | 2 +- .../GenericMapPluralImportRawDataConvertor.kt | 6 +- .../io/tolgee/formats/paramConversionUtil.kt | 11 +++- .../in/I18nextToIcuPlaceholderConvertor.kt | 17 +++++- .../out/IcuToI18nextPlaceholderConvertor.kt | 25 +++++++- .../properties/out/PropertiesFileExporter.kt | 4 +- .../formats/xliff/out/XliffFileExporter.kt | 5 +- .../kotlin/io/tolgee/model/key/KeyMeta.kt | 19 ++++++ .../processors/FileProcessorContext.kt | 19 ++++++ 18 files changed, 191 insertions(+), 39 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/i18next/constants.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/BaseIcuMessageConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/BaseIcuMessageConvertor.kt index f7106fdad1..abcf5aa0fd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/BaseIcuMessageConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/BaseIcuMessageConvertor.kt @@ -6,6 +6,7 @@ import io.tolgee.constants.Message class BaseIcuMessageConvertor( private val message: String, private val argumentConvertorFactory: () -> FromIcuPlaceholderConvertor, + private val customValues: Map? = null, private val keepEscaping: Boolean = false, private val forceIsPlural: Boolean? = null, ) { @@ -137,7 +138,7 @@ class BaseIcuMessageConvertor( is MessagePatternUtil.MessageContentsNode -> { if (node.type == MessagePatternUtil.MessageContentsNode.Type.REPLACE_NUMBER) { - addToResult(getFormPlaceholderConvertor(form).convertReplaceNumber(node, pluralArgName), form) + addToResult(getFormPlaceholderConvertor(form).convertReplaceNumber(node, customValues, pluralArgName), form) } } @@ -151,7 +152,7 @@ class BaseIcuMessageConvertor( form: String?, ) { val formPlaceholderConvertor = getFormPlaceholderConvertor(form) - val convertedText = formPlaceholderConvertor.convertText(node, keepEscaping) + val convertedText = formPlaceholderConvertor.convertText(node, keepEscaping, customValues) addToResult(convertedText, form) } @@ -164,7 +165,7 @@ class BaseIcuMessageConvertor( } when (node.argType) { MessagePattern.ArgType.SIMPLE, MessagePattern.ArgType.NONE -> { - addToResult(getFormPlaceholderConvertor(form).convert(node), form) + addToResult(getFormPlaceholderConvertor(form).convert(node, customValues), form) } MessagePattern.ArgType.PLURAL -> { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/FromIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/FromIcuPlaceholderConvertor.kt index 88a81f1bc5..d15ba553ea 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/FromIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/FromIcuPlaceholderConvertor.kt @@ -1,11 +1,26 @@ package io.tolgee.formats interface FromIcuPlaceholderConvertor { + fun convert( + node: MessagePatternUtil.ArgNode, + customValues: Map?, + ): String { + return convert(node) + } + fun convert(node: MessagePatternUtil.ArgNode): String /** * This method is called on the text parts (not argument parts) of the message */ + fun convertText( + node: MessagePatternUtil.TextNode, + keepEscaping: Boolean, + customValues: Map?, + ): String { + return convertText(node, keepEscaping) + } + fun convertText( node: MessagePatternUtil.TextNode, keepEscaping: Boolean, @@ -14,6 +29,14 @@ interface FromIcuPlaceholderConvertor { /** * How to # in ICU plural form */ + fun convertReplaceNumber( + node: MessagePatternUtil.MessageContentsNode, + customValues: Map?, + argName: String? = null, + ): String { + return convertReplaceNumber(node, argName) + } + fun convertReplaceNumber( node: MessagePatternUtil.MessageContentsNode, argName: String? = null, diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorFactory.kt b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorFactory.kt index c8a365071f..b0d6a3abae 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorFactory.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorFactory.kt @@ -4,6 +4,7 @@ class MessageConvertorFactory( private val message: String, private val forceIsPlural: Boolean, private val isProjectIcuPlaceholdersEnabled: Boolean = false, + private val customValues: Map? = null, private val paramConvertorFactory: () -> FromIcuPlaceholderConvertor, ) { fun create(): BaseIcuMessageConvertor { @@ -12,6 +13,7 @@ class MessageConvertorFactory( argumentConvertorFactory = getParamConvertorFactory(), forceIsPlural = forceIsPlural, keepEscaping = keepEscaping, + customValues = customValues, ) } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt index 2f27e85f83..2da1889342 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt @@ -1,3 +1,7 @@ package io.tolgee.formats -data class MessageConvertorResult(val message: String?, val pluralArgName: String?) +data class MessageConvertorResult( + val message: String?, + val pluralArgName: String?, + val customValues: Map? = null, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt index 4729e3dd71..569b2f809f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt @@ -9,4 +9,7 @@ interface ToIcuPlaceholderConvertor { val regex: Regex val pluralArgName: String? + + val customValues: Map? + get() = null } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/generic/IcuToGenericFormatMessageConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/generic/IcuToGenericFormatMessageConvertor.kt index 8462867fec..fbe0d1341a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/generic/IcuToGenericFormatMessageConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/generic/IcuToGenericFormatMessageConvertor.kt @@ -16,6 +16,7 @@ class IcuToGenericFormatMessageConvertor( private val message: String?, private val forceIsPlural: Boolean, private val isProjectIcuPlaceholdersEnabled: Boolean, + private val customValues: Map?, private val paramConvertorFactory: () -> FromIcuPlaceholderConvertor, ) { fun convert(): String? { @@ -45,6 +46,7 @@ class IcuToGenericFormatMessageConvertor( message = message, forceIsPlural = forceIsPlural, isProjectIcuPlaceholdersEnabled = isProjectIcuPlaceholdersEnabled, + customValues = customValues, paramConvertorFactory = paramConvertorFactory, ).create().convert() } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index 83509b8804..0953243436 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -28,17 +28,8 @@ class GenericStructuredProcessor( private fun Any?.import(key: String) { // Convertor handles strings and possible nested plurals, if convertor returns null, // it means that it's not a string or nested plurals, so we need to parse it further - convert(this)?.let { result -> - result.forEach { - context.addTranslation( - key, - languageTagOrGuess, - it.message, - rawData = this@import, - convertedBy = format, - pluralArgName = it.pluralArgName, - ) - } + convert(this)?.let { + it.applyAll(key, this@import) return } @@ -52,16 +43,8 @@ class GenericStructuredProcessor( return } - convert(this)?.firstOrNull()?.let { - context.addTranslation( - keyName = key, - languageName = languageTagOrGuess, - value = it.message, - pluralArgName = it.pluralArgName, - rawData = this@import, - convertedBy = format, - ) - } + // FIXME: I believe when this line is reached the convert function will always return null, am I missing something? + convert(this)?.firstOrNull()?.apply(key, this@import) } private fun convert(data: Any?): List? { @@ -95,6 +78,39 @@ class GenericStructuredProcessor( } } + private fun FileProcessorContext.mergeCustomAll( + key: String, + customValues: Map?, + ) { + customValues?.forEach { (cKey, cValue) -> + mergeCustom(key, cKey, cValue) + } + } + + private fun MessageConvertorResult.apply( + key: String, + rawData: Any?, + ) { + context.addTranslation( + keyName = key, + languageName = languageTagOrGuess, + value = message, + rawData = rawData, + convertedBy = format, + pluralArgName = pluralArgName, + ) + context.mergeCustomAll(key, customValues) + } + + private fun List.applyAll( + key: String, + rawData: Any?, + ) { + forEach { + it.apply(key, rawData) + } + } + private val languageTagOrGuess: String by lazy { languageTag ?: firstLanguageTagGuessOrUnknown } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt index 34999a1b37..6eb3b7e3b4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt @@ -51,7 +51,7 @@ class GenericStructuredFileExporter( builder.addValue( translation.languageTag, translation.key.name, - convertMessage(translation.text, translation.key.isPlural), + convertMessage(translation.text, translation.key.isPlural, translation.key.custom), ) } @@ -76,7 +76,7 @@ class GenericStructuredFileExporter( private fun addNestedPlural(translation: ExportTranslationView) { val pluralForms = - convertMessageForNestedPlural(translation.text) ?: let { + convertMessageForNestedPlural(translation.text, translation.key.custom) ?: let { // this should never happen, but if it does, it's better to add a null key then crash or ignore it addNullValue(translation) return @@ -92,7 +92,7 @@ class GenericStructuredFileExporter( private fun addSuffixedPlural(translation: ExportTranslationView) { val pluralForms = - convertMessageForNestedPlural(translation.text) ?: let { + convertMessageForNestedPlural(translation.text, translation.key.custom) ?: let { // this should never happen, but if it does, it's better to add a null key then crash or ignore it addNullValue(translation) return @@ -120,22 +120,28 @@ class GenericStructuredFileExporter( private fun convertMessage( text: String?, isPlural: Boolean, + customValues: Map?, ): String? { - return getMessageConvertor(text, isPlural).convert() + return getMessageConvertor(text, isPlural, customValues).convert() } private fun getMessageConvertor( text: String?, isPlural: Boolean, + customValues: Map?, ) = IcuToGenericFormatMessageConvertor( text, isPlural, isProjectIcuPlaceholdersEnabled = projectIcuPlaceholdersSupport, + customValues = customValues, paramConvertorFactory = placeholderConvertorFactory, ) - private fun convertMessageForNestedPlural(text: String?): Map? { - return getMessageConvertor(text, true).getForcedPluralForms() + private fun convertMessageForNestedPlural( + text: String?, + customValues: Map?, + ): Map? { + return getMessageConvertor(text, true, customValues).getForcedPluralForms() } private fun getFileContentResultBuilder(translation: ExportTranslationView): StructureModelBuilder { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/i18next/constants.kt b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/constants.kt new file mode 100644 index 0000000000..a53c469030 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/i18next/constants.kt @@ -0,0 +1,3 @@ +package io.tolgee.formats.i18next + +const val I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY = "_i18nextUnescapedPlaceholders" diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt index a910df031e..08f3779bcd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt @@ -35,7 +35,7 @@ class BaseImportRawDataConverter( } val converted = convertMessage(stringValue, false) - return MessageConvertorResult(converted.message, null) + return MessageConvertorResult(converted.message, null, converted.customValues) } private val doesNotNeedConversion = diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt index 2572b002bc..b38549e434 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt @@ -51,6 +51,7 @@ class GenericMapPluralImportRawDataConvertor( baseImportRawDataConverter: BaseImportRawDataConverter, ): MessageConvertorResult? { var pluralArgName = DEFAULT_PLURAL_ARGUMENT_NAME + val customValues = mutableMapOf() val converted = rawData.mapNotNull { (key, value) -> if (key !is String || value !is String?) { @@ -64,9 +65,12 @@ class GenericMapPluralImportRawDataConvertor( convertedMessage.pluralArgName?.let { pluralArgName = it } + convertedMessage.customValues?.let { + customValues.putAll(it) + } key to message }.toMap().toIcuPluralString(optimize = optimizePlurals, argName = pluralArgName) - return MessageConvertorResult(converted, pluralArgName) + return MessageConvertorResult(converted, pluralArgName, customValues) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt index 407454d1d0..cf6c4f3301 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt @@ -59,7 +59,14 @@ fun convertMessage( ) val pluralArgName = if (isInPlural) convertor.pluralArgName ?: DEFAULT_PLURAL_ARGUMENT_NAME else null - return converted.toConvertorResult(pluralArgName) + return converted.toConvertorResult(pluralArgName, convertor.customValues) } -private fun String?.toConvertorResult(pluralArgName: String? = null) = MessageConvertorResult(this, pluralArgName) +private fun String?.toConvertorResult( + pluralArgName: String? = null, + customValues: Map? = null, +) = MessageConvertorResult( + this, + pluralArgName, + customValues, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index f2d8466ff9..693c4b0ac5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -2,6 +2,7 @@ package io.tolgee.formats.paramConvertors.`in` import io.tolgee.formats.ToIcuPlaceholderConvertor import io.tolgee.formats.escapeIcu +import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY import io.tolgee.formats.i18next.`in`.I18nextParameterParser import io.tolgee.formats.i18next.`in`.PluralsI18nextKeyParser @@ -13,6 +14,18 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { override val pluralArgName: String? = null + private val unescapedKeys = mutableListOf() + + private val _customValues = mapOf(I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY to unescapedKeys) + + override val customValues: Map? + get() { + if (unescapedKeys.isEmpty()) { + return null + } + return _customValues + } + override fun convert( matchResult: MatchResult, isInPlural: Boolean, @@ -24,7 +37,9 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { return matchResult.value.escapeIcu(isInPlural) } - // TODO: handle unescaped flag by adding info about it to the metadata and using this info when exporting + if (parsed.key != null && parsed.keepUnescaped) { + unescapedKeys.add(parsed.key) + } return when (parsed.format) { null -> "{${parsed.key}}" diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt index bd10057736..9c4974732d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt @@ -3,9 +3,13 @@ package io.tolgee.formats.paramConvertors.out import com.ibm.icu.text.MessagePattern import io.tolgee.formats.FromIcuPlaceholderConvertor import io.tolgee.formats.MessagePatternUtil +import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY class IcuToI18nextPlaceholderConvertor : FromIcuPlaceholderConvertor { - override fun convert(node: MessagePatternUtil.ArgNode): String { + override fun convert( + node: MessagePatternUtil.ArgNode, + customValues: Map?, + ): String { val type = node.argType if (type == MessagePattern.ArgType.SIMPLE) { @@ -14,9 +18,28 @@ class IcuToI18nextPlaceholderConvertor : FromIcuPlaceholderConvertor { } } + if (customValues.hasUnescapedFlag(node.name)) { + return "{{- ${node.name}]}" + } + return "{{${node.name}}}" } + override fun convert(node: MessagePatternUtil.ArgNode): String { + return convert(node, null) + } + + private fun Map?.hasUnescapedFlag(name: String?): Boolean { + if (this == null || name == null) { + return false + } + return this[I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY] + ?.let { it as? List<*> } + ?.mapNotNull { it as? String } + ?.contains(name) + ?: false + } + override fun convertText(string: String): String { // We should escape {{ and $t, but there doesn't seem to be a documented // way how to escape either {{ or $t in i18next diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/properties/out/PropertiesFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/properties/out/PropertiesFileExporter.kt index 29b7f6fdda..2d4afe6b4b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/properties/out/PropertiesFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/properties/out/PropertiesFileExporter.kt @@ -32,7 +32,7 @@ class PropertiesFileExporter( translations.forEach { translation -> val fileName = computeFileName(translation) val keyName = translation.key.name - val value = convertMessage(translation.text, translation.key.isPlural) + val value = convertMessage(translation.text, translation.key.isPlural, translation.key.custom) val properties = result.getOrPut(fileName) { PropertiesConfiguration() } properties.setProperty(keyName, value) properties.layout.setComment(keyName, translation.key.description) @@ -42,11 +42,13 @@ class PropertiesFileExporter( private fun convertMessage( text: String?, plural: Boolean, + customValues: Map?, ): String? { return IcuToGenericFormatMessageConvertor( text, plural, projectIcuPlaceholdersSupport, + customValues = customValues, paramConvertorFactory = messageFormat.paramConvertorFactory, ).convert() } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/xliff/out/XliffFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/xliff/out/XliffFileExporter.kt index e9b271f15c..949e5c1b49 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/xliff/out/XliffFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/xliff/out/XliffFileExporter.kt @@ -55,8 +55,9 @@ class XliffFileExporter( convertMessage( baseTranslations[translation.key.namespace to translation.key.name]?.text, translation.key.isPlural, + translation.key.custom, ) - this.target = convertMessage(translation.text, translation.key.isPlural) + this.target = convertMessage(translation.text, translation.key.isPlural, translation.key.custom) this.note = translation.key.description }, ) @@ -65,11 +66,13 @@ class XliffFileExporter( private fun convertMessage( text: String?, plural: Boolean, + customValues: Map?, ): String? { return IcuToGenericFormatMessageConvertor( text, plural, projectIcuPlaceholdersSupport, + customValues = customValues, paramConvertorFactory = messageFormat.paramConvertorFactory, ).convert() } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/key/KeyMeta.kt b/backend/data/src/main/kotlin/io/tolgee/model/key/KeyMeta.kt index 4305e19e03..e4c49d12c7 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/key/KeyMeta.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/key/KeyMeta.kt @@ -85,6 +85,25 @@ class KeyMeta( custom[key] = value } + fun mergeCustom( + key: String, + value: Any, + ) { + val custom = + custom ?: mutableMapOf() + .also { + custom = it + } + + val currentValue = custom[key] + when { + currentValue == null -> custom[key] = value + currentValue is List<*> && value is List<*> -> custom[key] = currentValue + value + currentValue is Map<*, *> && value is Map<*, *> -> custom[key] = currentValue + value + else -> custom[key] = value + } + } + companion object { class KeyMetaListener { @PrePersist diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt index 030471f26f..c5a572fa8d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt @@ -152,6 +152,25 @@ data class FileProcessorContext( } } + fun mergeCustom( + translationKey: String, + customMapKey: String, + value: Any, + ) { + val keyMeta = getOrCreateKeyMeta(translationKey) + val previousValue = keyMeta.custom?.get(customMapKey) + keyMeta.mergeCustom(customMapKey, value) + keyMeta.custom?.let { + if (!customValuesValidator.isValid(it)) { + keyMeta.custom?.remove(customMapKey) + if (previousValue != null) { + keyMeta.custom?.put(customMapKey, previousValue) + } + fileEntity.addIssue(FileIssueType.INVALID_CUSTOM_VALUES, mapOf(FileIssueParamType.KEY_NAME to translationKey)) + } + } + } + fun setCustom( translationKey: String, customMapKey: String, From 742a6eceaf09f95c69b8de03f888b158c3e96a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 26 Sep 2024 12:06:07 +0200 Subject: [PATCH 15/32] feat: i18next tests --- .../i18next/in/I18nextFormatProcessorTest.kt | 388 ++++++++++++++++++ .../json/in/JsonFormatProcessorTest.kt | 30 -- .../i18next.json => i18next/example.json} | 4 +- .../i18next2.json => i18next/example2.json} | 0 .../import/i18next/example_params.json | 5 + .../test/resources/import/i18next/simple.json | 4 + 6 files changed, 399 insertions(+), 32 deletions(-) create mode 100644 backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt rename backend/data/src/test/resources/import/{json/i18next.json => i18next/example.json} (85%) rename backend/data/src/test/resources/import/{json/i18next2.json => i18next/example2.json} (100%) create mode 100644 backend/data/src/test/resources/import/i18next/example_params.json create mode 100644 backend/data/src/test/resources/import/i18next/simple.json diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt new file mode 100644 index 0000000000..d617671eb9 --- /dev/null +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt @@ -0,0 +1,388 @@ +package io.tolgee.unit.formats.i18next.`in` + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.tolgee.dtos.request.ImportFileMapping +import io.tolgee.dtos.request.SingleStepImportRequest +import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY +import io.tolgee.formats.importCommon.ImportFormat +import io.tolgee.formats.json.`in`.JsonFileProcessor +import io.tolgee.testing.assert +import io.tolgee.unit.formats.PlaceholderConversionTestHelper +import io.tolgee.util.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class I18nextFormatProcessorTest { + lateinit var mockUtil: FileProcessorContextMockUtil + + @BeforeEach + fun setup() { + mockUtil = FileProcessorContextMockUtil() + } + + // This is how to generate the test: + // 1. run the test in debug mode + // 2. copy the result of calling: + // io.tolgee.unit.util.generateTestsForImportResult(mockUtil.fileProcessorContext) + // from the debug window + @Test + fun `returns correct parsed result`() { + mockUtil.mockIt("example.json", "src/test/resources/import/i18next/example.json") + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(1) + mockUtil.fileProcessorContext.assertTranslations("example", "key") + .assertSingle { + hasText("value") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyDeep.inner") + .assertSingle { + hasText("value") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyNesting") + .assertSingle { + hasText("reuse ${'$'}t(keyDeep.inner) (is not supported)") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolate") + .assertSingle { + hasText("replace this {value}") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescaped") + .assertSingle { + hasText("replace this {value} (we save the - into metadata)") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateWithFormatting") + .assertSingle { + hasText("replace this {value, number} (only number is supported)") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyContext_male") + .assertSingle { + hasText("the male variant (is parsed as normal key and context is ignored)") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyContext_female") + .assertSingle { + hasText("the female variant (is parsed as normal key and context is ignored)") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyWithArrayValue[0]") + .assertSingle { + hasText("multipe") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyWithArrayValue[1]") + .assertSingle { + hasText("things") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyWithObjectValue.valueA") + .assertSingle { + hasText("valueA") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyWithObjectValue.valueB") + .assertSingle { + hasText("more text") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyPluralSimple") + .assertSinglePlural { + hasText( + """ + {value, plural, + one {the singular (is parsed as plural under one key - keyPluralSimple)} + other {the plural (is parsed as plural under one key - keyPluralSimple)} + } + """.trimIndent() + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyPluralMultipleEgArabic") + .assertSinglePlural { + hasText( + """ + {value, plural, + one {the plural form 1} + two {the plural form 2} + few {the plural form 3} + many {the plural form 4} + other {the plural form 5} + } + """.trimIndent() + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescaped"){ + customEquals( + """ + { + "_i18nextUnescapedPlaceholders" : [ "value" ] + } + """.trimIndent() + ) + description.assert.isNull() + } + mockUtil.fileProcessorContext.assertKey("keyPluralSimple"){ + custom.assert.isNull() + description.assert.isNull() + } + mockUtil.fileProcessorContext.assertKey("keyPluralMultipleEgArabic"){ + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `returns correct parsed result for more complex file`() { + mockUtil.mockIt("example.json", "src/test/resources/import/i18next/example2.json") + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(1) + mockUtil.fileProcessorContext.assertTranslations("example", "note") + .assertSingle { + hasText("Most features in this example are not supported by the tolgee yet") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.key") + .assertSingle { + hasText("Hello World") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.interpolation_example") + .assertSingle { + hasText("Hello {name}") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.plural_example") + .assertSinglePlural { + hasText( + """ + {value, plural, + one {You have one message} + other {You have {count} messages} + } + """.trimIndent() + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.context_example.male") + .assertSingle { + hasText("He is a teacher") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.context_example.female") + .assertSingle { + hasText("She is a teacher") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.nested_example") + .assertSingle { + hasText("This is a {type} message") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.type") + .assertSingle { + hasText("nested") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.formatted_value") + .assertSingle { + hasText("The price is '{{'value, currency'}}'") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.array_example[0]") + .assertSingle { + hasText("Apples") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.array_example[1]") + .assertSingle { + hasText("Oranges") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.array_example[2]") + .assertSingle { + hasText("Bananas") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.select_example.morning") + .assertSingle { + hasText("Good morning") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.select_example.afternoon") + .assertSingle { + hasText("Good afternoon") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.select_example.evening") + .assertSingle { + hasText("Good evening") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.multiline_example") + .assertSingle { + hasText("This is line one.\nThis is line two.") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.gender_with_plural.male") + .assertSinglePlural { + hasText( + """ + {value, plural, + one {He has one cat} + other {He has {count} cats} + } + """.trimIndent() + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.gender_with_plural.female") + .assertSinglePlural { + hasText( + """ + {value, plural, + one {She has one cat} + other {She has {count} cats} + } + """.trimIndent() + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.rich_text_example") + .assertSingle { + hasText("Welcome to our application!") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.json_value_example.key") + .assertSingle { + hasText("This is a value inside a JSON object") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.conditional_translations") + .assertSingle { + hasText("'{{'isLoggedIn, select, true '{'Welcome back, '{{'name'}}'!'}' false '{'Please log in'}}}'") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.language_switch.en") + .assertSingle { + hasText("English") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.language_switch.es") + .assertSingle { + hasText("Spanish") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.language_switch.fr") + .assertSingle { + hasText("French") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.missing_key_fallback") + .assertSingle { + hasText("This is the default value if the key is missing.") + } + mockUtil.fileProcessorContext.assertKey("translation.plural_example"){ + custom.assert.isNull() + description.assert.isNull() + } + mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural.male"){ + custom.assert.isNull() + description.assert.isNull() + } + mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural.female"){ + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `import with placeholder conversion (disabled ICU)`() { + mockPlaceholderConversionTestFile(convertPlaceholders = false, projectIcuPlaceholdersEnabled = false) + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(1) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("Hello {{icuPara}}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "plural") + .assertSinglePlural { + hasText( + """ + {value, plural, + one {Hello one '#' '{{'icuParam'}}'} + other {Hello other '{{'icuParam'}}'} + } + """.trimIndent() + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("plural"){ + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `import with placeholder conversion (no conversion)`() { + mockPlaceholderConversionTestFile(convertPlaceholders = false, projectIcuPlaceholdersEnabled = true) + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(1) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("Hello '{{'icuPara'}}'") + } + mockUtil.fileProcessorContext.assertTranslations("en", "plural") + .assertSinglePlural { + hasText( + """ + {value, plural, + one {Hello one '#' '{{'icuParam'}}'} + other {Hello other '{{'icuParam'}}'} + } + """.trimIndent() + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("plural"){ + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `import with placeholder conversion (with conversion)`() { + mockPlaceholderConversionTestFile(convertPlaceholders = true, projectIcuPlaceholdersEnabled = true) + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(1) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("Hello {icuPara}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "plural") + .assertSinglePlural { + hasText( + """ + {value, plural, + one {Hello one '#' {icuParam}} + other {Hello other {icuParam}} + } + """.trimIndent() + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("plural"){ + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `placeholder conversion setting application works`() { + PlaceholderConversionTestHelper.testFile( + "en.json", + "src/test/resources/import/i18next/simple.json", + assertBeforeSettingsApplication = + listOf( + "'{{'value, currency'}}' this is i18next {count, number}", + "'{{'value, currency'}}' this is i18next", + ), + assertAfterDisablingConversion = + listOf( + "'{{'value, currency'}}' this is i18next '{{'count, number'}}'", + ), + assertAfterReEnablingConversion = + listOf( + "'{{'value, currency'}}' this is i18next {count, number}", + ), + ) + } + + private fun mockPlaceholderConversionTestFile( + convertPlaceholders: Boolean, + projectIcuPlaceholdersEnabled: Boolean, + ) { + mockUtil.mockIt( + "en.json", + "src/test/resources/import/i18next/example_params.json", + convertPlaceholders, + projectIcuPlaceholdersEnabled, + ) + } + + private fun processFile() { + JsonFileProcessor(mockUtil.fileProcessorContext, jacksonObjectMapper()).process() + } +} diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt index 42796f3716..983573f926 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonFormatProcessorTest.kt @@ -177,36 +177,6 @@ class JsonFormatProcessorTest { ) } - @Test - fun `returns correct parsed result for i18next`() { - mockUtil.mockIt("i18next.json", "src/test/resources/import/json/i18next.json") - processFile() - mockUtil.fileProcessorContext.assertLanguagesCount(1) - mockUtil.fileProcessorContext.assertTranslations("i18next", "keyDeep.inner") - mockUtil.fileProcessorContext.assertTranslations("i18next", "keyWithArrayValue[0]") - mockUtil.fileProcessorContext.assertTranslations("i18next", "keyWithArrayValue[1]") - .assertSingle { - hasText("things") - } - mockUtil.fileProcessorContext.assertTranslations("i18next", "keyNesting") - .assertSingle { - hasText("reuse ${'$'}t(keyDeep.inner) (is not supported)") - } - mockUtil.fileProcessorContext.assertTranslations("i18next", "keyContext_male") - .assertSingle { - hasText("the male variant (is parsed as normal key and context is ignored)") - } - mockUtil.fileProcessorContext.assertTranslations("i18next", "keyPluralSimple") - .assertSingle { - hasText( - "{value, plural,\n" + - "one {the singular (is parsed as plural under one key - keyPluralSimple)}\n" + - "other {the plural (is parsed as plural under one key - keyPluralSimple)}\n" + - "}", - ) - } - } - @Test fun `respects provided format`() { mockUtil.mockIt("en.json", "src/test/resources/import/json/icu.json") diff --git a/backend/data/src/test/resources/import/json/i18next.json b/backend/data/src/test/resources/import/i18next/example.json similarity index 85% rename from backend/data/src/test/resources/import/json/i18next.json rename to backend/data/src/test/resources/import/i18next/example.json index c890a06331..4cc1144652 100644 --- a/backend/data/src/test/resources/import/json/i18next.json +++ b/backend/data/src/test/resources/import/i18next/example.json @@ -5,7 +5,7 @@ }, "keyNesting": "reuse $t(keyDeep.inner) (is not supported)", "keyInterpolate": "replace this {{value}}", - "keyInterpolateUnescaped": "replace this {{- value}} (we ignore the -)", + "keyInterpolateUnescaped": "replace this {{- value}} (we save the - into metadata)", "keyInterpolateWithFormatting": "replace this {{value, number}} (only number is supported)", "keyContext_male": "the male variant (is parsed as normal key and context is ignored)", "keyContext_female": "the female variant (is parsed as normal key and context is ignored)", @@ -17,5 +17,5 @@ "keyPluralMultipleEgArabic_many": "the plural form 4", "keyPluralMultipleEgArabic_other": "the plural form 5", "keyWithArrayValue": ["multipe", "things"], - "keyWithObjectValue": { "valueA": "return this with valueB", "valueB": "more text" } + "keyWithObjectValue": { "valueA": "valueA", "valueB": "more text" } } \ No newline at end of file diff --git a/backend/data/src/test/resources/import/json/i18next2.json b/backend/data/src/test/resources/import/i18next/example2.json similarity index 100% rename from backend/data/src/test/resources/import/json/i18next2.json rename to backend/data/src/test/resources/import/i18next/example2.json diff --git a/backend/data/src/test/resources/import/i18next/example_params.json b/backend/data/src/test/resources/import/i18next/example_params.json new file mode 100644 index 0000000000..e136f8d313 --- /dev/null +++ b/backend/data/src/test/resources/import/i18next/example_params.json @@ -0,0 +1,5 @@ +{ + "key": "Hello {{icuPara}}", + "plural_one": "Hello one # {{icuParam}}", + "plural_other": "Hello other {{icuParam}}" +} diff --git a/backend/data/src/test/resources/import/i18next/simple.json b/backend/data/src/test/resources/import/i18next/simple.json new file mode 100644 index 0000000000..03a3a5f6bd --- /dev/null +++ b/backend/data/src/test/resources/import/i18next/simple.json @@ -0,0 +1,4 @@ +{ + "key": "{{value, currency}} this is i18next {{count, number}}", + "key2": "{{value, currency}} this is i18next" +} From 7fd6091415b238601788810ec333f4fea86aeb44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 26 Sep 2024 12:15:01 +0200 Subject: [PATCH 16/32] fix: lint --- .../i18next/in/I18nextFormatProcessorTest.kt | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt index d617671eb9..758e7dd853 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt @@ -1,10 +1,6 @@ package io.tolgee.unit.formats.i18next.`in` import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import io.tolgee.dtos.request.ImportFileMapping -import io.tolgee.dtos.request.SingleStepImportRequest -import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY -import io.tolgee.formats.importCommon.ImportFormat import io.tolgee.formats.json.`in`.JsonFileProcessor import io.tolgee.testing.assert import io.tolgee.unit.formats.PlaceholderConversionTestHelper @@ -86,7 +82,7 @@ class I18nextFormatProcessorTest { one {the singular (is parsed as plural under one key - keyPluralSimple)} other {the plural (is parsed as plural under one key - keyPluralSimple)} } - """.trimIndent() + """.trimIndent(), ) isPluralOptimized() } @@ -101,25 +97,25 @@ class I18nextFormatProcessorTest { many {the plural form 4} other {the plural form 5} } - """.trimIndent() + """.trimIndent(), ) isPluralOptimized() } - mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescaped"){ + mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescaped") { customEquals( """ { "_i18nextUnescapedPlaceholders" : [ "value" ] } - """.trimIndent() + """.trimIndent(), ) description.assert.isNull() } - mockUtil.fileProcessorContext.assertKey("keyPluralSimple"){ + mockUtil.fileProcessorContext.assertKey("keyPluralSimple") { custom.assert.isNull() description.assert.isNull() } - mockUtil.fileProcessorContext.assertKey("keyPluralMultipleEgArabic"){ + mockUtil.fileProcessorContext.assertKey("keyPluralMultipleEgArabic") { custom.assert.isNull() description.assert.isNull() } @@ -150,7 +146,7 @@ class I18nextFormatProcessorTest { one {You have one message} other {You have {count} messages} } - """.trimIndent() + """.trimIndent(), ) isPluralOptimized() } @@ -210,7 +206,7 @@ class I18nextFormatProcessorTest { one {He has one cat} other {He has {count} cats} } - """.trimIndent() + """.trimIndent(), ) isPluralOptimized() } @@ -222,7 +218,7 @@ class I18nextFormatProcessorTest { one {She has one cat} other {She has {count} cats} } - """.trimIndent() + """.trimIndent(), ) isPluralOptimized() } @@ -254,15 +250,15 @@ class I18nextFormatProcessorTest { .assertSingle { hasText("This is the default value if the key is missing.") } - mockUtil.fileProcessorContext.assertKey("translation.plural_example"){ + mockUtil.fileProcessorContext.assertKey("translation.plural_example") { custom.assert.isNull() description.assert.isNull() } - mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural.male"){ + mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural.male") { custom.assert.isNull() description.assert.isNull() } - mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural.female"){ + mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural.female") { custom.assert.isNull() description.assert.isNull() } @@ -285,11 +281,11 @@ class I18nextFormatProcessorTest { one {Hello one '#' '{{'icuParam'}}'} other {Hello other '{{'icuParam'}}'} } - """.trimIndent() + """.trimIndent(), ) isPluralOptimized() } - mockUtil.fileProcessorContext.assertKey("plural"){ + mockUtil.fileProcessorContext.assertKey("plural") { custom.assert.isNull() description.assert.isNull() } @@ -312,11 +308,11 @@ class I18nextFormatProcessorTest { one {Hello one '#' '{{'icuParam'}}'} other {Hello other '{{'icuParam'}}'} } - """.trimIndent() + """.trimIndent(), ) isPluralOptimized() } - mockUtil.fileProcessorContext.assertKey("plural"){ + mockUtil.fileProcessorContext.assertKey("plural") { custom.assert.isNull() description.assert.isNull() } @@ -339,11 +335,11 @@ class I18nextFormatProcessorTest { one {Hello one '#' {icuParam}} other {Hello other {icuParam}} } - """.trimIndent() + """.trimIndent(), ) isPluralOptimized() } - mockUtil.fileProcessorContext.assertKey("plural"){ + mockUtil.fileProcessorContext.assertKey("plural") { custom.assert.isNull() description.assert.isNull() } From 239f5c6bfc2bbf4bbbf80c1243e2f98eda143f56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 26 Sep 2024 12:18:15 +0200 Subject: [PATCH 17/32] fix: path to resource in tests --- .../tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt index 1a5dd7bfb9..c3c4684625 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt @@ -21,7 +21,7 @@ class JsonImportFormatDetectorTest { @Test fun `detected i18next`() { - "src/test/resources/import/json/i18next.json".assertDetected(ImportFormat.JSON_I18NEXT) + "src/test/resources/import/i18next/example.json".assertDetected(ImportFormat.JSON_I18NEXT) } @Test From 04cee0c6103f85315139417d5e6711f09c8daa99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 26 Sep 2024 12:36:10 +0200 Subject: [PATCH 18/32] feat: add i18next export test --- .../i18next/out/I18nextFileExporterTest.kt | 139 ++++++++++++++++++ .../formats/json/out/JsonFileExporterTest.kt | 25 +--- 2 files changed, 141 insertions(+), 23 deletions(-) create mode 100644 backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt new file mode 100644 index 0000000000..dffb09749f --- /dev/null +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt @@ -0,0 +1,139 @@ +package io.tolgee.unit.formats.i18next.out + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.tolgee.dtos.request.export.ExportParams +import io.tolgee.formats.ExportMessageFormat +import io.tolgee.formats.genericStructuredFile.out.CustomPrettyPrinter +import io.tolgee.formats.json.out.JsonFileExporter +import io.tolgee.service.export.dataProvider.ExportTranslationView +import io.tolgee.unit.util.assertFile +import io.tolgee.unit.util.getExported +import io.tolgee.util.buildExportTranslationList +import org.junit.jupiter.api.Test + +class I18nextFileExporterTest { + @Test + fun `exports with placeholders (ICU placeholders disabled)`() { + // FIXME: Is this one correct? (I'm not sure) + val exporter = getIcuPlaceholdersDisabledExporter() + val data = getExported(exporter) + data.assertFile( + "cs.json", + """ + |{ + | "key3_one": "# den {icuParam}", + | "key3_few": "# dny", + | "key3_other": "# dní", + | "item": "I will be first {{icuParam, number}}" + |} + """.trimMargin(), + ) + } + + private fun getIcuPlaceholdersDisabledExporter(): JsonFileExporter { + val built = + buildExportTranslationList { + add( + languageTag = "cs", + keyName = "key3", + text = "{count, plural, one {'#' den '{'icuParam'}'} few {'#' dny} other {'#' dní}}", + ) { + key.isPlural = true + } + add( + languageTag = "cs", + keyName = "item", + text = "I will be first {icuParam, number}", + ) + } + return getExporter( + built.translations, + false, + exportParams = ExportParams(messageFormat = ExportMessageFormat.I18NEXT), + ) + } + + @Test + fun `exports with placeholders (ICU placeholders enabled)`() { + val exporter = getIcuPlaceholdersEnabledExporter() + val data = getExported(exporter) + data.assertFile( + "cs.json", + """ + |{ + | "key3_one": "{{count, number}} den {{icuParam, number}}", + | "key3_few": "{{count, number}} dny", + | "key3_other": "{{count, number}} dní", + | "item": "I will be first '{'icuParam'}' {{hello, number}}" + |} + """.trimMargin(), + ) + } + + @Test + fun `correct exports translation with colon`() { + val exporter = + getExporter( + getTranslationWithColon(), + exportParams = ExportParams(messageFormat = ExportMessageFormat.I18NEXT), + ) + val data = getExported(exporter) + data.assertFile( + "cs.json", + """ + |{ + | "item": "name : {{name}}" + |} + """.trimMargin(), + ) + } + + private fun getTranslationWithColon(): MutableList { + val built = + buildExportTranslationList { + add( + languageTag = "cs", + keyName = "item", + text = "name : {name}", + ) + } + return built.translations + } + + private fun getIcuPlaceholdersEnabledExporter(): JsonFileExporter { + val built = + buildExportTranslationList { + add( + languageTag = "cs", + keyName = "key3", + text = "{count, plural, one {# den {icuParam, number}} few {# dny} other {# dní}}", + ) { + key.isPlural = true + } + add( + languageTag = "cs", + keyName = "item", + text = "I will be first '{'icuParam'}' {hello, number}", + ) + } + return getExporter( + built.translations, + true, + exportParams = ExportParams(messageFormat = ExportMessageFormat.I18NEXT), + ) + } + + private fun getExporter( + translations: List, + isProjectIcuPlaceholdersEnabled: Boolean = true, + exportParams: ExportParams = ExportParams(), + ): JsonFileExporter { + return JsonFileExporter( + translations = translations, + exportParams = exportParams, + projectIcuPlaceholdersSupport = isProjectIcuPlaceholdersEnabled, + objectMapper = jacksonObjectMapper(), + customPrettyPrinter = CustomPrettyPrinter(), + ) + } +} diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/out/JsonFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/out/JsonFileExporterTest.kt index b3a0dac38b..56ecb39744 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/out/JsonFileExporterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/out/JsonFileExporterTest.kt @@ -164,23 +164,6 @@ class JsonFileExporterTest { ) } - @Test - fun `exports i18next correctly`() { - val exporter = getIcuPlaceholdersEnabledExporter(ExportMessageFormat.I18NEXT) - val data = getExported(exporter) - data.assertFile( - "cs.json", - """ - |{ - | "key3_one": "{{count, number}} den {{icuParam, number}}", - | "key3_few": "{{count, number}} dny", - | "key3_other": "{{count, number}} dní", - | "item": "I will be first '{'icuParam'}' {{hello, number}}" - |} - """.trimMargin(), - ) - } - @Test fun `correct exports translation with colon`() { val exporter = getExporter(getTranslationWithColon()) @@ -207,7 +190,7 @@ class JsonFileExporterTest { return built.translations } - private fun getIcuPlaceholdersEnabledExporter(messageFormat: ExportMessageFormat? = null): JsonFileExporter { + private fun getIcuPlaceholdersEnabledExporter(): JsonFileExporter { val built = buildExportTranslationList { add( @@ -223,11 +206,7 @@ class JsonFileExporterTest { text = "I will be first '{'icuParam'}' {hello, number}", ) } - return getExporter( - built.translations, - true, - exportParams = ExportParams(messageFormat = messageFormat), - ) + return getExporter(built.translations, true) } @Test From 7f942e431ee8917b18cc405722d7098740a6f9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 26 Sep 2024 13:25:19 +0200 Subject: [PATCH 19/32] feat: test unecaped placeholder export --- .../unit/formats/i18next/out/I18nextFileExporterTest.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt index dffb09749f..7c168082b4 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.tolgee.dtos.request.export.ExportParams import io.tolgee.formats.ExportMessageFormat import io.tolgee.formats.genericStructuredFile.out.CustomPrettyPrinter +import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY import io.tolgee.formats.json.out.JsonFileExporter import io.tolgee.service.export.dataProvider.ExportTranslationView import io.tolgee.unit.util.assertFile @@ -65,6 +66,7 @@ class I18nextFileExporterTest { | "key3_few": "{{count, number}} dny", | "key3_other": "{{count, number}} dní", | "item": "I will be first '{'icuParam'}' {{hello, number}}" + | "unescaped": "Unescaped {{- value}} with another text {{text}}" |} """.trimMargin(), ) @@ -115,6 +117,13 @@ class I18nextFileExporterTest { keyName = "item", text = "I will be first '{'icuParam'}' {hello, number}", ) + add( + languageTag = "cs", + keyName = "unescaped", + text = "Unescaped {value} with another text {text}", + ) { + key.custom = mapOf(I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY to listOf("value")) + } } return getExporter( built.translations, From 201ec64d3bb2d8304c14f34a87d4c1611e1453ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 26 Sep 2024 13:25:54 +0200 Subject: [PATCH 20/32] fix: typo in icu to i18 placeholder convertor --- .../paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt index 9c4974732d..46d267cc7c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt @@ -19,7 +19,7 @@ class IcuToI18nextPlaceholderConvertor : FromIcuPlaceholderConvertor { } if (customValues.hasUnescapedFlag(node.name)) { - return "{{- ${node.name}]}" + return "{{- ${node.name}}}" } return "{{${node.name}}}" From 107cbb15b8142b5c8b2fda9a783b19256a181ed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Thu, 26 Sep 2024 13:27:11 +0200 Subject: [PATCH 21/32] fix: missing comma --- .../tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt index 7c168082b4..cf3daf65f2 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt @@ -65,7 +65,7 @@ class I18nextFileExporterTest { | "key3_one": "{{count, number}} den {{icuParam, number}}", | "key3_few": "{{count, number}} dny", | "key3_other": "{{count, number}} dní", - | "item": "I will be first '{'icuParam'}' {{hello, number}}" + | "item": "I will be first '{'icuParam'}' {{hello, number}}", | "unescaped": "Unescaped {{- value}} with another text {{text}}" |} """.trimMargin(), From 05d73f237c5bae8ae2493de70a2053998dfa1575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 27 Sep 2024 13:11:39 +0200 Subject: [PATCH 22/32] fix: tests and breakage after rebase --- .../out/IcuToI18nextPlaceholderConvertor.kt | 7 +++++-- .../unit/formats/i18next/out/I18nextFileExporterTest.kt | 5 ++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt index 46d267cc7c..9aadd2f61a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt @@ -40,10 +40,13 @@ class IcuToI18nextPlaceholderConvertor : FromIcuPlaceholderConvertor { ?: false } - override fun convertText(string: String): String { + override fun convertText( + node: MessagePatternUtil.TextNode, + keepEscaping: Boolean, + ): String { // We should escape {{ and $t, but there doesn't seem to be a documented // way how to escape either {{ or $t in i18next - return string + return node.getText(keepEscaping) } override fun convertReplaceNumber( diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt index cf3daf65f2..fc64326654 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt @@ -15,7 +15,6 @@ import org.junit.jupiter.api.Test class I18nextFileExporterTest { @Test fun `exports with placeholders (ICU placeholders disabled)`() { - // FIXME: Is this one correct? (I'm not sure) val exporter = getIcuPlaceholdersDisabledExporter() val data = getExported(exporter) data.assertFile( @@ -25,7 +24,7 @@ class I18nextFileExporterTest { | "key3_one": "# den {icuParam}", | "key3_few": "# dny", | "key3_other": "# dní", - | "item": "I will be first {{icuParam, number}}" + | "item": "I will be first {icuParam, number}" |} """.trimMargin(), ) @@ -65,7 +64,7 @@ class I18nextFileExporterTest { | "key3_one": "{{count, number}} den {{icuParam, number}}", | "key3_few": "{{count, number}} dny", | "key3_other": "{{count, number}} dní", - | "item": "I will be first '{'icuParam'}' {{hello, number}}", + | "item": "I will be first {icuParam} {{hello, number}}", | "unescaped": "Unescaped {{- value}} with another text {{text}}" |} """.trimMargin(), From e1feeeda00a963ac4c25508b0ca1089ee53d36a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 27 Sep 2024 15:44:41 +0200 Subject: [PATCH 23/32] feat: add i18next logo to import screen --- webapp/src/svgs/logos/i18next.svg | 10 ++++++++++ .../import/component/ImportSupportedFormats.tsx | 2 ++ 2 files changed, 12 insertions(+) create mode 100644 webapp/src/svgs/logos/i18next.svg diff --git a/webapp/src/svgs/logos/i18next.svg b/webapp/src/svgs/logos/i18next.svg new file mode 100644 index 0000000000..30ad61184f --- /dev/null +++ b/webapp/src/svgs/logos/i18next.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx b/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx index da9337f159..7a265a1495 100644 --- a/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx +++ b/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx @@ -10,6 +10,7 @@ import AppleLogo from 'tg.svgs/logos/apple.svg?react'; import AndroidLogo from 'tg.svgs/logos/android.svg?react'; import FluttrerLogo from 'tg.svgs/logos/flutter.svg?react'; import RailsLogo from 'tg.svgs/logos/rails.svg?react'; +import I18nextLogo from 'tg.svgs/logos/i18next.svg?react'; const TechLogo = ({ svg, @@ -53,6 +54,7 @@ const FORMATS = [ { name: 'Android XML', logo: }, { name: 'Flutter ARB', logo: }, { name: 'Ruby YAML', logo: }, + { name: 'i18next', logo: }, ]; export const ImportSupportedFormats = () => { From 171d250e57bf23895ce517921b642e57057dba59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 27 Sep 2024 18:40:08 +0200 Subject: [PATCH 24/32] feat: placeholder convertors now use customValuesModifier function to report changes to be done to customValues --- .../tolgee/formats/MessageConvertorResult.kt | 2 +- .../formats/ToIcuPlaceholderConvertor.kt | 2 +- .../in/GenericStructuredProcessor.kt | 17 +--- .../BaseImportRawDataConverter.kt | 2 +- .../GenericMapPluralImportRawDataConvertor.kt | 10 ++- .../io/tolgee/formats/paramConversionUtil.kt | 6 +- .../in/I18nextToIcuPlaceholderConvertor.kt | 80 ++++++++++++++++--- .../kotlin/io/tolgee/model/key/KeyMeta.kt | 19 ----- .../processors/FileProcessorContext.kt | 24 +++--- .../i18next/in/I18nextFormatProcessorTest.kt | 22 ++++- .../resources/import/i18next/example.json | 5 +- 11 files changed, 121 insertions(+), 68 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt index 2da1889342..1f97fb5145 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt @@ -3,5 +3,5 @@ package io.tolgee.formats data class MessageConvertorResult( val message: String?, val pluralArgName: String?, - val customValues: Map? = null, + val customValuesModifier: ((customValues: MutableMap, memory: MutableMap) -> Unit)? = null, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt index 569b2f809f..3a3febb0b1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt @@ -10,6 +10,6 @@ interface ToIcuPlaceholderConvertor { val pluralArgName: String? - val customValues: Map? + val customValuesModifier: ((customValues: MutableMap, memory: MutableMap) -> Unit)? get() = null } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index 0953243436..b67bfb0cb8 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -42,9 +42,6 @@ class GenericStructuredProcessor( it.parseMap(key) return } - - // FIXME: I believe when this line is reached the convert function will always return null, am I missing something? - convert(this)?.firstOrNull()?.apply(key, this@import) } private fun convert(data: Any?): List? { @@ -78,15 +75,6 @@ class GenericStructuredProcessor( } } - private fun FileProcessorContext.mergeCustomAll( - key: String, - customValues: Map?, - ) { - customValues?.forEach { (cKey, cValue) -> - mergeCustom(key, cKey, cValue) - } - } - private fun MessageConvertorResult.apply( key: String, rawData: Any?, @@ -99,7 +87,10 @@ class GenericStructuredProcessor( convertedBy = format, pluralArgName = pluralArgName, ) - context.mergeCustomAll(key, customValues) + val customValues = context.getCustom(key)?.toMutableMap() ?: mutableMapOf() + customValuesModifier?.invoke(customValues, mutableMapOf()) + val customValuesNonEmpty = customValues.takeIf { it.isNotEmpty() } + context.setCustom(key, customValuesNonEmpty) } private fun List.applyAll( diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt index 08f3779bcd..8a29e0ca52 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt @@ -35,7 +35,7 @@ class BaseImportRawDataConverter( } val converted = convertMessage(stringValue, false) - return MessageConvertorResult(converted.message, null, converted.customValues) + return MessageConvertorResult(converted.message, null, converted.customValuesModifier) } private val doesNotNeedConversion = diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt index b38549e434..f8ec8e00f0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt @@ -51,7 +51,7 @@ class GenericMapPluralImportRawDataConvertor( baseImportRawDataConverter: BaseImportRawDataConverter, ): MessageConvertorResult? { var pluralArgName = DEFAULT_PLURAL_ARGUMENT_NAME - val customValues = mutableMapOf() + val customValuesModifiers = mutableListOf<(customValues: MutableMap, memory: MutableMap) -> Unit>() val converted = rawData.mapNotNull { (key, value) -> if (key !is String || value !is String?) { @@ -65,12 +65,14 @@ class GenericMapPluralImportRawDataConvertor( convertedMessage.pluralArgName?.let { pluralArgName = it } - convertedMessage.customValues?.let { - customValues.putAll(it) + convertedMessage.customValuesModifier?.let { + customValuesModifiers.add(it) } key to message }.toMap().toIcuPluralString(optimize = optimizePlurals, argName = pluralArgName) - return MessageConvertorResult(converted, pluralArgName, customValues) + return MessageConvertorResult(converted, pluralArgName) { customValues, memory -> + customValuesModifiers.forEach { it(customValues, memory) } + } } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt index cf6c4f3301..f75cc5fc4a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt @@ -59,14 +59,14 @@ fun convertMessage( ) val pluralArgName = if (isInPlural) convertor.pluralArgName ?: DEFAULT_PLURAL_ARGUMENT_NAME else null - return converted.toConvertorResult(pluralArgName, convertor.customValues) + return converted.toConvertorResult(pluralArgName, convertor.customValuesModifier) } private fun String?.toConvertorResult( pluralArgName: String? = null, - customValues: Map? = null, + customValuesModifier: ((customValues: MutableMap, memory: MutableMap) -> Unit)? = null, ) = MessageConvertorResult( this, pluralArgName, - customValues, + customValuesModifier, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index 693c4b0ac5..c99f92b941 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -4,6 +4,7 @@ import io.tolgee.formats.ToIcuPlaceholderConvertor import io.tolgee.formats.escapeIcu import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY import io.tolgee.formats.i18next.`in`.I18nextParameterParser +import io.tolgee.formats.i18next.`in`.ParsedI18nextParam import io.tolgee.formats.i18next.`in`.PluralsI18nextKeyParser class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { @@ -14,17 +15,78 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { override val pluralArgName: String? = null - private val unescapedKeys = mutableListOf() + private var unescapedKeys = mutableListOf() + private var escapedKeys = mutableListOf() - private val _customValues = mapOf(I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY to unescapedKeys) + override val customValuesModifier: ((MutableMap, MutableMap) -> Unit)? = modifier@{ customValues, memory -> + if (unescapedKeys.isEmpty() && escapedKeys.isEmpty()) { + // Optimization + return@modifier + } - override val customValues: Map? - get() { - if (unescapedKeys.isEmpty()) { - return null + customValues.modifyList(I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY) { allUnescapedKeys -> + memory.modifyList(I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY) { allEscapedKeys -> + unescapedKeys.forEach { unescapedKey -> + handleUnescapedModifier(allUnescapedKeys, allEscapedKeys, unescapedKey) + } + escapedKeys.forEach { escapedKey -> + handleEscapedModifier(allUnescapedKeys, allEscapedKeys, escapedKey) + } } - return _customValues } + } + + @Suppress("UNCHECKED_CAST") + private fun MutableMap.modifyList(key: String, modifier: (MutableList) -> Unit) { + val value = this[key] ?: mutableListOf() + val list = value as? MutableList ?: return + + modifier(list) + + if (list.isEmpty()) { + remove(key) + return + } + this[key] = list + } + + private fun handleUnescapedModifier( + unescapedKeys: MutableList, + escapedKeys: MutableList, + unescapedKey: String, + ) { + if (unescapedKey in escapedKeys || unescapedKey in unescapedKeys) { + return + } + unescapedKeys.add(unescapedKey) + } + + private fun handleEscapedModifier( + unescapedKeys: MutableList, + escapedKeys: MutableList, + escapedKey: String, + ) { + if (escapedKey in escapedKeys) { + return + } + escapedKeys.add(escapedKey) + + if (escapedKey !in unescapedKeys) { + return + } + unescapedKeys.remove(escapedKey) + } + + private fun ParsedI18nextParam.applyUnescapedFlag() { + if (key == null) { + return + } + if (keepUnescaped) { + unescapedKeys.add(key) + return + } + escapedKeys.add(key) + } override fun convert( matchResult: MatchResult, @@ -37,9 +99,7 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { return matchResult.value.escapeIcu(isInPlural) } - if (parsed.key != null && parsed.keepUnescaped) { - unescapedKeys.add(parsed.key) - } + parsed.applyUnescapedFlag() return when (parsed.format) { null -> "{${parsed.key}}" diff --git a/backend/data/src/main/kotlin/io/tolgee/model/key/KeyMeta.kt b/backend/data/src/main/kotlin/io/tolgee/model/key/KeyMeta.kt index e4c49d12c7..4305e19e03 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/key/KeyMeta.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/key/KeyMeta.kt @@ -85,25 +85,6 @@ class KeyMeta( custom[key] = value } - fun mergeCustom( - key: String, - value: Any, - ) { - val custom = - custom ?: mutableMapOf() - .also { - custom = it - } - - val currentValue = custom[key] - when { - currentValue == null -> custom[key] = value - currentValue is List<*> && value is List<*> -> custom[key] = currentValue + value - currentValue is Map<*, *> && value is Map<*, *> -> custom[key] = currentValue + value - else -> custom[key] = value - } - } - companion object { class KeyMetaListener { @PrePersist diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt index c5a572fa8d..ecf10458a6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt @@ -152,23 +152,19 @@ data class FileProcessorContext( } } - fun mergeCustom( + fun getCustom(translationKey: String): Map? { + return getOrCreateKeyMeta(translationKey).custom + } + + fun setCustom( translationKey: String, - customMapKey: String, - value: Any, + customMap: Map?, ) { - val keyMeta = getOrCreateKeyMeta(translationKey) - val previousValue = keyMeta.custom?.get(customMapKey) - keyMeta.mergeCustom(customMapKey, value) - keyMeta.custom?.let { - if (!customValuesValidator.isValid(it)) { - keyMeta.custom?.remove(customMapKey) - if (previousValue != null) { - keyMeta.custom?.put(customMapKey, previousValue) - } - fileEntity.addIssue(FileIssueType.INVALID_CUSTOM_VALUES, mapOf(FileIssueParamType.KEY_NAME to translationKey)) - } + if (customMap != null && !customValuesValidator.isValid(customMap)) { + fileEntity.addIssue(FileIssueType.INVALID_CUSTOM_VALUES, mapOf(FileIssueParamType.KEY_NAME to translationKey)) } + val keyMeta = getOrCreateKeyMeta(translationKey) + keyMeta.custom = customMap?.toMutableMap() } fun setCustom( diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt index 758e7dd853..fadfee171a 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt @@ -44,7 +44,19 @@ class I18nextFormatProcessorTest { } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescaped") .assertSingle { - hasText("replace this {value} (we save the - into metadata)") + hasText("replace this {value} (we save the flag into metadata)") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescapedMultiple") + .assertSingle { + hasText("replace this {value} and also don't escape {value} (we save the flag into metadata)") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescapedConflict") + .assertSingle { + hasText("replace this {value} but escape this {value} (we default to escaping both occurrences)") + } + mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescapedConflictInverted") + .assertSingle { + hasText("replace this {value} but don't escape this {value} (we default to escaping both occurrences)") } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateWithFormatting") .assertSingle { @@ -111,6 +123,14 @@ class I18nextFormatProcessorTest { ) description.assert.isNull() } + mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescapedConflict") { + custom.assert.isNull() + description.assert.isNull() + } + mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescapedConflictInverted") { + custom.assert.isNull() + description.assert.isNull() + } mockUtil.fileProcessorContext.assertKey("keyPluralSimple") { custom.assert.isNull() description.assert.isNull() diff --git a/backend/data/src/test/resources/import/i18next/example.json b/backend/data/src/test/resources/import/i18next/example.json index 4cc1144652..ba01616b3b 100644 --- a/backend/data/src/test/resources/import/i18next/example.json +++ b/backend/data/src/test/resources/import/i18next/example.json @@ -5,7 +5,10 @@ }, "keyNesting": "reuse $t(keyDeep.inner) (is not supported)", "keyInterpolate": "replace this {{value}}", - "keyInterpolateUnescaped": "replace this {{- value}} (we save the - into metadata)", + "keyInterpolateUnescaped": "replace this {{- value}} (we save the flag into metadata)", + "keyInterpolateUnescapedMultiple": "replace this {{- value}} and also don't escape {{- value}} (we save the flag into metadata)", + "keyInterpolateUnescapedConflict": "replace this {{- value}} but escape this {{value}} (we default to escaping both occurrences)", + "keyInterpolateUnescapedConflictInverted": "replace this {{value}} but don't escape this {{- value}} (we default to escaping both occurrences)", "keyInterpolateWithFormatting": "replace this {{value, number}} (only number is supported)", "keyContext_male": "the male variant (is parsed as normal key and context is ignored)", "keyContext_female": "the female variant (is parsed as normal key and context is ignored)", From aa01871a0fa0dd228d61aa08e006bbdaea5b6c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 27 Sep 2024 18:46:38 +0200 Subject: [PATCH 25/32] fix: lint --- .../io/tolgee/formats/MessageConvertorResult.kt | 7 ++++++- .../GenericMapPluralImportRawDataConvertor.kt | 8 +++++++- .../in/I18nextToIcuPlaceholderConvertor.kt | 11 +++++++---- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt index 1f97fb5145..5442d5a49c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt @@ -3,5 +3,10 @@ package io.tolgee.formats data class MessageConvertorResult( val message: String?, val pluralArgName: String?, - val customValuesModifier: ((customValues: MutableMap, memory: MutableMap) -> Unit)? = null, + val customValuesModifier: ( + ( + customValues: MutableMap, + memory: MutableMap, + ) -> Unit + )? = null, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt index f8ec8e00f0..33948a3124 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt @@ -51,7 +51,13 @@ class GenericMapPluralImportRawDataConvertor( baseImportRawDataConverter: BaseImportRawDataConverter, ): MessageConvertorResult? { var pluralArgName = DEFAULT_PLURAL_ARGUMENT_NAME - val customValuesModifiers = mutableListOf<(customValues: MutableMap, memory: MutableMap) -> Unit>() + val customValuesModifiers = + mutableListOf< + ( + customValues: MutableMap, + memory: MutableMap, + ) -> Unit, + >() val converted = rawData.mapNotNull { (key, value) -> if (key !is String || value !is String?) { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index c99f92b941..cb7fb9e093 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -37,7 +37,10 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { } @Suppress("UNCHECKED_CAST") - private fun MutableMap.modifyList(key: String, modifier: (MutableList) -> Unit) { + private fun MutableMap.modifyList( + key: String, + modifier: (MutableList) -> Unit, + ) { val value = this[key] ?: mutableListOf() val list = value as? MutableList ?: return @@ -51,9 +54,9 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { } private fun handleUnescapedModifier( - unescapedKeys: MutableList, - escapedKeys: MutableList, - unescapedKey: String, + unescapedKeys: MutableList, + escapedKeys: MutableList, + unescapedKey: String, ) { if (unescapedKey in escapedKeys || unescapedKey in unescapedKeys) { return From 2901e1aff4bf2ca7180eea54e5af7cb256fccd4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 27 Sep 2024 18:51:15 +0200 Subject: [PATCH 26/32] fix: lint --- .../paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index cb7fb9e093..ed5bd143a1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -18,7 +18,9 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { private var unescapedKeys = mutableListOf() private var escapedKeys = mutableListOf() - override val customValuesModifier: ((MutableMap, MutableMap) -> Unit)? = modifier@{ customValues, memory -> + override val customValuesModifier: ( + (MutableMap, MutableMap) -> Unit + )? = modifier@{ customValues, memory -> if (unescapedKeys.isEmpty() && escapedKeys.isEmpty()) { // Optimization return@modifier From 16a2459684aeda2320912ee6ccbaef35a14c1a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Fri, 27 Sep 2024 19:37:52 +0200 Subject: [PATCH 27/32] fix: only modify custom values when modifier function was returned --- .../in/GenericStructuredProcessor.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index b67bfb0cb8..763d667230 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -87,10 +87,12 @@ class GenericStructuredProcessor( convertedBy = format, pluralArgName = pluralArgName, ) - val customValues = context.getCustom(key)?.toMutableMap() ?: mutableMapOf() - customValuesModifier?.invoke(customValues, mutableMapOf()) - val customValuesNonEmpty = customValues.takeIf { it.isNotEmpty() } - context.setCustom(key, customValuesNonEmpty) + customValuesModifier?.let { + val customValues = context.getCustom(key)?.toMutableMap() ?: mutableMapOf() + it(customValues, mutableMapOf()) + val customValuesNonEmpty = customValues.takeIf { it.isNotEmpty() } + context.setCustom(key, customValuesNonEmpty) + } } private fun List.applyAll( From 16fbfa2ec070661907eff155478897c8f9a8bc58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Mon, 30 Sep 2024 15:00:37 +0200 Subject: [PATCH 28/32] fix: bugs found while writing documentation --- .../in/GenericSuffixedPluralsPreprocessor.kt | 2 +- .../in/I18nextToIcuPlaceholderConvertor.kt | 11 +++++- .../i18next/in/I18nextFormatProcessorTest.kt | 38 ++++++++----------- .../resources/import/i18next/example2.json | 32 +++++----------- 4 files changed, 34 insertions(+), 49 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt index 484e6d619d..992b965a4c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt @@ -53,7 +53,7 @@ class GenericSuffixedPluralsPreprocessor( private fun List>.useOriginalKey(): List> { return map { (parsedKey, value) -> - parsedKey.originalKey to value + parsedKey.originalKey to value.preprocess() } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index ed5bd143a1..8d8ab267dd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -13,7 +13,7 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { override val regex: Regex get() = I18NEXT_PLACEHOLDER_REGEX - override val pluralArgName: String? = null + override val pluralArgName: String? = I18NEXT_PLURAL_ARG_NAME private var unescapedKeys = mutableListOf() private var escapedKeys = mutableListOf() @@ -108,7 +108,12 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { return when (parsed.format) { null -> "{${parsed.key}}" - "number" -> "{${parsed.key}, number}" + "number" -> { + if (isInPlural && parsed.key == I18NEXT_PLURAL_ARG_NAME) { + return "#" + } + "{${parsed.key}, number}" + } else -> matchResult.value.escapeIcu(isInPlural) } } @@ -145,6 +150,8 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { ) """.trimIndent().toRegex() + val I18NEXT_PLURAL_ARG_NAME = "count" + val I18NEXT_PLURAL_SUFFIX_REGEX = """^(?\w+)_(?\w+)$""".toRegex() val I18NEXT_PLURAL_SUFFIX_KEY_PARSER = PluralsI18nextKeyParser(I18NEXT_PLURAL_SUFFIX_REGEX) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt index fadfee171a..86be09aaf3 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt @@ -90,7 +90,7 @@ class I18nextFormatProcessorTest { .assertSinglePlural { hasText( """ - {value, plural, + {count, plural, one {the singular (is parsed as plural under one key - keyPluralSimple)} other {the plural (is parsed as plural under one key - keyPluralSimple)} } @@ -102,7 +102,7 @@ class I18nextFormatProcessorTest { .assertSinglePlural { hasText( """ - {value, plural, + {count, plural, one {the plural form 1} two {the plural form 2} few {the plural form 3} @@ -162,19 +162,19 @@ class I18nextFormatProcessorTest { .assertSinglePlural { hasText( """ - {value, plural, + {count, plural, one {You have one message} - other {You have {count} messages} + other {You have # messages} } """.trimIndent(), ) isPluralOptimized() } - mockUtil.fileProcessorContext.assertTranslations("example", "translation.context_example.male") + mockUtil.fileProcessorContext.assertTranslations("example", "translation.context_example_male") .assertSingle { hasText("He is a teacher") } - mockUtil.fileProcessorContext.assertTranslations("example", "translation.context_example.female") + mockUtil.fileProcessorContext.assertTranslations("example", "translation.context_example_female") .assertSingle { hasText("She is a teacher") } @@ -218,25 +218,25 @@ class I18nextFormatProcessorTest { .assertSingle { hasText("This is line one.\nThis is line two.") } - mockUtil.fileProcessorContext.assertTranslations("example", "translation.gender_with_plural.male") + mockUtil.fileProcessorContext.assertTranslations("example", "translation.gender_with_plural_male") .assertSinglePlural { hasText( """ - {value, plural, + {count, plural, one {He has one cat} - other {He has {count} cats} + other {He has # cats} } """.trimIndent(), ) isPluralOptimized() } - mockUtil.fileProcessorContext.assertTranslations("example", "translation.gender_with_plural.female") + mockUtil.fileProcessorContext.assertTranslations("example", "translation.gender_with_plural_female") .assertSinglePlural { hasText( """ - {value, plural, + {count, plural, one {She has one cat} - other {She has {count} cats} + other {She has # cats} } """.trimIndent(), ) @@ -250,10 +250,6 @@ class I18nextFormatProcessorTest { .assertSingle { hasText("This is a value inside a JSON object") } - mockUtil.fileProcessorContext.assertTranslations("example", "translation.conditional_translations") - .assertSingle { - hasText("'{{'isLoggedIn, select, true '{'Welcome back, '{{'name'}}'!'}' false '{'Please log in'}}}'") - } mockUtil.fileProcessorContext.assertTranslations("example", "translation.language_switch.en") .assertSingle { hasText("English") @@ -266,19 +262,15 @@ class I18nextFormatProcessorTest { .assertSingle { hasText("French") } - mockUtil.fileProcessorContext.assertTranslations("example", "translation.missing_key_fallback") - .assertSingle { - hasText("This is the default value if the key is missing.") - } mockUtil.fileProcessorContext.assertKey("translation.plural_example") { custom.assert.isNull() description.assert.isNull() } - mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural.male") { + mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural_male") { custom.assert.isNull() description.assert.isNull() } - mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural.female") { + mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural_female") { custom.assert.isNull() description.assert.isNull() } @@ -351,7 +343,7 @@ class I18nextFormatProcessorTest { .assertSinglePlural { hasText( """ - {value, plural, + {count, plural, one {Hello one '#' {icuParam}} other {Hello other {icuParam}} } diff --git a/backend/data/src/test/resources/import/i18next/example2.json b/backend/data/src/test/resources/import/i18next/example2.json index 7955a04bac..950a4a7d7e 100644 --- a/backend/data/src/test/resources/import/i18next/example2.json +++ b/backend/data/src/test/resources/import/i18next/example2.json @@ -5,15 +5,11 @@ "interpolation_example": "Hello {{name}}", - "plural_example": { - "one": "You have one message", - "other": "You have {{count}} messages" - }, + "plural_example_one": "You have one message", + "plural_example_other": "You have {{count, number}} messages", - "context_example": { - "male": "He is a teacher", - "female": "She is a teacher" - }, + "context_example_male": "He is a teacher", + "context_example_female": "She is a teacher", "nested_example": "This is a {{type}} message", "type": "nested", @@ -34,16 +30,10 @@ "multiline_example": "This is line one.\nThis is line two.", - "gender_with_plural": { - "male": { - "one": "He has one cat", - "other": "He has {{count}} cats" - }, - "female": { - "one": "She has one cat", - "other": "She has {{count}} cats" - } - }, + "gender_with_plural_male_one": "He has one cat", + "gender_with_plural_male_other": "He has {{count, number}} cats", + "gender_with_plural_female_one": "She has one cat", + "gender_with_plural_female_other": "She has {{count, number}} cats", "rich_text_example": "Welcome to our application!", @@ -51,15 +41,11 @@ "key": "This is a value inside a JSON object" }, - "conditional_translations": "{{isLoggedIn, select, true {Welcome back, {{name}}!} false {Please log in}}}", - "language_switch": { "en": "English", "es": "Spanish", "fr": "French" - }, - - "missing_key_fallback": "This is the default value if the key is missing." + } } } From e2a257f3bddc22bfac6743baf13ffc29b63779df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Mon, 30 Sep 2024 16:11:04 +0200 Subject: [PATCH 29/32] fix: handling of missing other + convert to '#' only placeholders without format --- .../tolgee/formats/FormsToIcuPluralConvertor.kt | 7 +++++++ .../in/I18nextToIcuPlaceholderConvertor.kt | 11 +++++------ .../i18next/in/I18nextFormatProcessorTest.kt | 17 +++++++++++++++++ .../test/resources/import/i18next/example2.json | 9 ++++++--- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/FormsToIcuPluralConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/FormsToIcuPluralConvertor.kt index 6736ad0d77..9010c97bf8 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/FormsToIcuPluralConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/FormsToIcuPluralConvertor.kt @@ -1,5 +1,7 @@ package io.tolgee.formats +import com.ibm.icu.text.PluralRules + class FormsToIcuPluralConvertor( val forms: Map, val argName: String = DEFAULT_PLURAL_ARGUMENT_NAME, @@ -10,6 +12,11 @@ class FormsToIcuPluralConvertor( val newLineStringInit = if (addNewLines) "\n" else " " val icuMsg = StringBuffer("{$argName, plural,$newLineStringInit") forms.let { + if (PluralRules.KEYWORD_OTHER !in it) { + return@let it + (PluralRules.KEYWORD_OTHER to "") + } + return@let it + }.let { if (optimize) { return@let optimizePluralForms(it) } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index 8d8ab267dd..b99e952c2f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -106,14 +106,13 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { parsed.applyUnescapedFlag() + if (isInPlural && parsed.key == I18NEXT_PLURAL_ARG_NAME && parsed.format == null) { + return "#" + } + return when (parsed.format) { null -> "{${parsed.key}}" - "number" -> { - if (isInPlural && parsed.key == I18NEXT_PLURAL_ARG_NAME) { - return "#" - } - "{${parsed.key}, number}" - } + "number" -> "{${parsed.key}, number}" else -> matchResult.value.escapeIcu(isInPlural) } } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt index 86be09aaf3..e072b24cea 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt @@ -170,6 +170,19 @@ class I18nextFormatProcessorTest { ) isPluralOptimized() } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.invalid_plural_example") + .assertSinglePlural { + hasText( + """ + {count, plural, + one {You have one message} + many {You have # messages (translation is missing 'other' form)} + other {} + } + """.trimIndent(), + ) + isPluralOptimized() + } mockUtil.fileProcessorContext.assertTranslations("example", "translation.context_example_male") .assertSingle { hasText("He is a teacher") @@ -266,6 +279,10 @@ class I18nextFormatProcessorTest { custom.assert.isNull() description.assert.isNull() } + mockUtil.fileProcessorContext.assertKey("translation.invalid_plural_example") { + custom.assert.isNull() + description.assert.isNull() + } mockUtil.fileProcessorContext.assertKey("translation.gender_with_plural_male") { custom.assert.isNull() description.assert.isNull() diff --git a/backend/data/src/test/resources/import/i18next/example2.json b/backend/data/src/test/resources/import/i18next/example2.json index 950a4a7d7e..3bff27e6f2 100644 --- a/backend/data/src/test/resources/import/i18next/example2.json +++ b/backend/data/src/test/resources/import/i18next/example2.json @@ -6,7 +6,10 @@ "interpolation_example": "Hello {{name}}", "plural_example_one": "You have one message", - "plural_example_other": "You have {{count, number}} messages", + "plural_example_other": "You have {{count}} messages", + + "invalid_plural_example_one": "You have one message", + "invalid_plural_example_many": "You have {{count}} messages (translation is missing 'other' form)", "context_example_male": "He is a teacher", "context_example_female": "She is a teacher", @@ -31,9 +34,9 @@ "multiline_example": "This is line one.\nThis is line two.", "gender_with_plural_male_one": "He has one cat", - "gender_with_plural_male_other": "He has {{count, number}} cats", + "gender_with_plural_male_other": "He has {{count}} cats", "gender_with_plural_female_one": "She has one cat", - "gender_with_plural_female_other": "She has {{count, number}} cats", + "gender_with_plural_female_other": "She has {{count}} cats", "rich_text_example": "Welcome to our application!", From 6bb9d3f5fea506791c7920186db0658ca25975c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 1 Oct 2024 12:40:20 +0200 Subject: [PATCH 30/32] feat: remove unescaped flag support - keep it non converted --- .../tolgee/formats/BaseIcuMessageConvertor.kt | 7 +- .../formats/FromIcuPlaceholderConvertor.kt | 23 ----- .../tolgee/formats/MessageConvertorFactory.kt | 2 - .../tolgee/formats/MessageConvertorResult.kt | 11 +-- .../formats/ToIcuPlaceholderConvertor.kt | 3 - .../IcuToGenericFormatMessageConvertor.kt | 2 - .../in/GenericStructuredProcessor.kt | 6 -- .../out/GenericStructuredFileExporter.kt | 18 ++-- .../BaseImportRawDataConverter.kt | 2 +- .../GenericMapPluralImportRawDataConvertor.kt | 14 +-- .../io/tolgee/formats/paramConversionUtil.kt | 11 +-- .../in/I18nextToIcuPlaceholderConvertor.kt | 85 +------------------ .../out/IcuToI18nextPlaceholderConvertor.kt | 25 +----- .../properties/out/PropertiesFileExporter.kt | 4 +- .../formats/xliff/out/XliffFileExporter.kt | 5 +- .../processors/FileProcessorContext.kt | 15 ---- .../i18next/in/I18nextFormatProcessorTest.kt | 16 ++-- .../i18next/out/I18nextFileExporterTest.kt | 11 +-- .../resources/import/i18next/example.json | 8 +- 19 files changed, 31 insertions(+), 237 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/BaseIcuMessageConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/BaseIcuMessageConvertor.kt index abcf5aa0fd..f7106fdad1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/BaseIcuMessageConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/BaseIcuMessageConvertor.kt @@ -6,7 +6,6 @@ import io.tolgee.constants.Message class BaseIcuMessageConvertor( private val message: String, private val argumentConvertorFactory: () -> FromIcuPlaceholderConvertor, - private val customValues: Map? = null, private val keepEscaping: Boolean = false, private val forceIsPlural: Boolean? = null, ) { @@ -138,7 +137,7 @@ class BaseIcuMessageConvertor( is MessagePatternUtil.MessageContentsNode -> { if (node.type == MessagePatternUtil.MessageContentsNode.Type.REPLACE_NUMBER) { - addToResult(getFormPlaceholderConvertor(form).convertReplaceNumber(node, customValues, pluralArgName), form) + addToResult(getFormPlaceholderConvertor(form).convertReplaceNumber(node, pluralArgName), form) } } @@ -152,7 +151,7 @@ class BaseIcuMessageConvertor( form: String?, ) { val formPlaceholderConvertor = getFormPlaceholderConvertor(form) - val convertedText = formPlaceholderConvertor.convertText(node, keepEscaping, customValues) + val convertedText = formPlaceholderConvertor.convertText(node, keepEscaping) addToResult(convertedText, form) } @@ -165,7 +164,7 @@ class BaseIcuMessageConvertor( } when (node.argType) { MessagePattern.ArgType.SIMPLE, MessagePattern.ArgType.NONE -> { - addToResult(getFormPlaceholderConvertor(form).convert(node, customValues), form) + addToResult(getFormPlaceholderConvertor(form).convert(node), form) } MessagePattern.ArgType.PLURAL -> { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/FromIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/FromIcuPlaceholderConvertor.kt index d15ba553ea..88a81f1bc5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/FromIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/FromIcuPlaceholderConvertor.kt @@ -1,26 +1,11 @@ package io.tolgee.formats interface FromIcuPlaceholderConvertor { - fun convert( - node: MessagePatternUtil.ArgNode, - customValues: Map?, - ): String { - return convert(node) - } - fun convert(node: MessagePatternUtil.ArgNode): String /** * This method is called on the text parts (not argument parts) of the message */ - fun convertText( - node: MessagePatternUtil.TextNode, - keepEscaping: Boolean, - customValues: Map?, - ): String { - return convertText(node, keepEscaping) - } - fun convertText( node: MessagePatternUtil.TextNode, keepEscaping: Boolean, @@ -29,14 +14,6 @@ interface FromIcuPlaceholderConvertor { /** * How to # in ICU plural form */ - fun convertReplaceNumber( - node: MessagePatternUtil.MessageContentsNode, - customValues: Map?, - argName: String? = null, - ): String { - return convertReplaceNumber(node, argName) - } - fun convertReplaceNumber( node: MessagePatternUtil.MessageContentsNode, argName: String? = null, diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorFactory.kt b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorFactory.kt index b0d6a3abae..c8a365071f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorFactory.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorFactory.kt @@ -4,7 +4,6 @@ class MessageConvertorFactory( private val message: String, private val forceIsPlural: Boolean, private val isProjectIcuPlaceholdersEnabled: Boolean = false, - private val customValues: Map? = null, private val paramConvertorFactory: () -> FromIcuPlaceholderConvertor, ) { fun create(): BaseIcuMessageConvertor { @@ -13,7 +12,6 @@ class MessageConvertorFactory( argumentConvertorFactory = getParamConvertorFactory(), forceIsPlural = forceIsPlural, keepEscaping = keepEscaping, - customValues = customValues, ) } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt index 5442d5a49c..2f27e85f83 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/MessageConvertorResult.kt @@ -1,12 +1,3 @@ package io.tolgee.formats -data class MessageConvertorResult( - val message: String?, - val pluralArgName: String?, - val customValuesModifier: ( - ( - customValues: MutableMap, - memory: MutableMap, - ) -> Unit - )? = null, -) +data class MessageConvertorResult(val message: String?, val pluralArgName: String?) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt index 3a3febb0b1..4729e3dd71 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/ToIcuPlaceholderConvertor.kt @@ -9,7 +9,4 @@ interface ToIcuPlaceholderConvertor { val regex: Regex val pluralArgName: String? - - val customValuesModifier: ((customValues: MutableMap, memory: MutableMap) -> Unit)? - get() = null } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/generic/IcuToGenericFormatMessageConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/generic/IcuToGenericFormatMessageConvertor.kt index fbe0d1341a..8462867fec 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/generic/IcuToGenericFormatMessageConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/generic/IcuToGenericFormatMessageConvertor.kt @@ -16,7 +16,6 @@ class IcuToGenericFormatMessageConvertor( private val message: String?, private val forceIsPlural: Boolean, private val isProjectIcuPlaceholdersEnabled: Boolean, - private val customValues: Map?, private val paramConvertorFactory: () -> FromIcuPlaceholderConvertor, ) { fun convert(): String? { @@ -46,7 +45,6 @@ class IcuToGenericFormatMessageConvertor( message = message, forceIsPlural = forceIsPlural, isProjectIcuPlaceholdersEnabled = isProjectIcuPlaceholdersEnabled, - customValues = customValues, paramConvertorFactory = paramConvertorFactory, ).create().convert() } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt index 763d667230..d0b410f73d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt @@ -87,12 +87,6 @@ class GenericStructuredProcessor( convertedBy = format, pluralArgName = pluralArgName, ) - customValuesModifier?.let { - val customValues = context.getCustom(key)?.toMutableMap() ?: mutableMapOf() - it(customValues, mutableMapOf()) - val customValuesNonEmpty = customValues.takeIf { it.isNotEmpty() } - context.setCustom(key, customValuesNonEmpty) - } } private fun List.applyAll( diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt index 6eb3b7e3b4..34999a1b37 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt @@ -51,7 +51,7 @@ class GenericStructuredFileExporter( builder.addValue( translation.languageTag, translation.key.name, - convertMessage(translation.text, translation.key.isPlural, translation.key.custom), + convertMessage(translation.text, translation.key.isPlural), ) } @@ -76,7 +76,7 @@ class GenericStructuredFileExporter( private fun addNestedPlural(translation: ExportTranslationView) { val pluralForms = - convertMessageForNestedPlural(translation.text, translation.key.custom) ?: let { + convertMessageForNestedPlural(translation.text) ?: let { // this should never happen, but if it does, it's better to add a null key then crash or ignore it addNullValue(translation) return @@ -92,7 +92,7 @@ class GenericStructuredFileExporter( private fun addSuffixedPlural(translation: ExportTranslationView) { val pluralForms = - convertMessageForNestedPlural(translation.text, translation.key.custom) ?: let { + convertMessageForNestedPlural(translation.text) ?: let { // this should never happen, but if it does, it's better to add a null key then crash or ignore it addNullValue(translation) return @@ -120,28 +120,22 @@ class GenericStructuredFileExporter( private fun convertMessage( text: String?, isPlural: Boolean, - customValues: Map?, ): String? { - return getMessageConvertor(text, isPlural, customValues).convert() + return getMessageConvertor(text, isPlural).convert() } private fun getMessageConvertor( text: String?, isPlural: Boolean, - customValues: Map?, ) = IcuToGenericFormatMessageConvertor( text, isPlural, isProjectIcuPlaceholdersEnabled = projectIcuPlaceholdersSupport, - customValues = customValues, paramConvertorFactory = placeholderConvertorFactory, ) - private fun convertMessageForNestedPlural( - text: String?, - customValues: Map?, - ): Map? { - return getMessageConvertor(text, true, customValues).getForcedPluralForms() + private fun convertMessageForNestedPlural(text: String?): Map? { + return getMessageConvertor(text, true).getForcedPluralForms() } private fun getFileContentResultBuilder(translation: ExportTranslationView): StructureModelBuilder { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt index 8a29e0ca52..a910df031e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/BaseImportRawDataConverter.kt @@ -35,7 +35,7 @@ class BaseImportRawDataConverter( } val converted = convertMessage(stringValue, false) - return MessageConvertorResult(converted.message, null, converted.customValuesModifier) + return MessageConvertorResult(converted.message, null) } private val doesNotNeedConversion = diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt index 33948a3124..2572b002bc 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/GenericMapPluralImportRawDataConvertor.kt @@ -51,13 +51,6 @@ class GenericMapPluralImportRawDataConvertor( baseImportRawDataConverter: BaseImportRawDataConverter, ): MessageConvertorResult? { var pluralArgName = DEFAULT_PLURAL_ARGUMENT_NAME - val customValuesModifiers = - mutableListOf< - ( - customValues: MutableMap, - memory: MutableMap, - ) -> Unit, - >() val converted = rawData.mapNotNull { (key, value) -> if (key !is String || value !is String?) { @@ -71,14 +64,9 @@ class GenericMapPluralImportRawDataConvertor( convertedMessage.pluralArgName?.let { pluralArgName = it } - convertedMessage.customValuesModifier?.let { - customValuesModifiers.add(it) - } key to message }.toMap().toIcuPluralString(optimize = optimizePlurals, argName = pluralArgName) - return MessageConvertorResult(converted, pluralArgName) { customValues, memory -> - customValuesModifiers.forEach { it(customValues, memory) } - } + return MessageConvertorResult(converted, pluralArgName) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt index f75cc5fc4a..407454d1d0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConversionUtil.kt @@ -59,14 +59,7 @@ fun convertMessage( ) val pluralArgName = if (isInPlural) convertor.pluralArgName ?: DEFAULT_PLURAL_ARGUMENT_NAME else null - return converted.toConvertorResult(pluralArgName, convertor.customValuesModifier) + return converted.toConvertorResult(pluralArgName) } -private fun String?.toConvertorResult( - pluralArgName: String? = null, - customValuesModifier: ((customValues: MutableMap, memory: MutableMap) -> Unit)? = null, -) = MessageConvertorResult( - this, - pluralArgName, - customValuesModifier, -) +private fun String?.toConvertorResult(pluralArgName: String? = null) = MessageConvertorResult(this, pluralArgName) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt index b99e952c2f..3843030395 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/I18nextToIcuPlaceholderConvertor.kt @@ -2,9 +2,7 @@ package io.tolgee.formats.paramConvertors.`in` import io.tolgee.formats.ToIcuPlaceholderConvertor import io.tolgee.formats.escapeIcu -import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY import io.tolgee.formats.i18next.`in`.I18nextParameterParser -import io.tolgee.formats.i18next.`in`.ParsedI18nextParam import io.tolgee.formats.i18next.`in`.PluralsI18nextKeyParser class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { @@ -15,84 +13,6 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { override val pluralArgName: String? = I18NEXT_PLURAL_ARG_NAME - private var unescapedKeys = mutableListOf() - private var escapedKeys = mutableListOf() - - override val customValuesModifier: ( - (MutableMap, MutableMap) -> Unit - )? = modifier@{ customValues, memory -> - if (unescapedKeys.isEmpty() && escapedKeys.isEmpty()) { - // Optimization - return@modifier - } - - customValues.modifyList(I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY) { allUnescapedKeys -> - memory.modifyList(I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY) { allEscapedKeys -> - unescapedKeys.forEach { unescapedKey -> - handleUnescapedModifier(allUnescapedKeys, allEscapedKeys, unescapedKey) - } - escapedKeys.forEach { escapedKey -> - handleEscapedModifier(allUnescapedKeys, allEscapedKeys, escapedKey) - } - } - } - } - - @Suppress("UNCHECKED_CAST") - private fun MutableMap.modifyList( - key: String, - modifier: (MutableList) -> Unit, - ) { - val value = this[key] ?: mutableListOf() - val list = value as? MutableList ?: return - - modifier(list) - - if (list.isEmpty()) { - remove(key) - return - } - this[key] = list - } - - private fun handleUnescapedModifier( - unescapedKeys: MutableList, - escapedKeys: MutableList, - unescapedKey: String, - ) { - if (unescapedKey in escapedKeys || unescapedKey in unescapedKeys) { - return - } - unescapedKeys.add(unescapedKey) - } - - private fun handleEscapedModifier( - unescapedKeys: MutableList, - escapedKeys: MutableList, - escapedKey: String, - ) { - if (escapedKey in escapedKeys) { - return - } - escapedKeys.add(escapedKey) - - if (escapedKey !in unescapedKeys) { - return - } - unescapedKeys.remove(escapedKey) - } - - private fun ParsedI18nextParam.applyUnescapedFlag() { - if (key == null) { - return - } - if (keepUnescaped) { - unescapedKeys.add(key) - return - } - escapedKeys.add(key) - } - override fun convert( matchResult: MatchResult, isInPlural: Boolean, @@ -104,7 +24,10 @@ class I18nextToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor { return matchResult.value.escapeIcu(isInPlural) } - parsed.applyUnescapedFlag() + if (parsed.keepUnescaped) { + // Keys with unescaped flag are not supported + return matchResult.value.escapeIcu(isInPlural) + } if (isInPlural && parsed.key == I18NEXT_PLURAL_ARG_NAME && parsed.format == null) { return "#" diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt index 9aadd2f61a..4b8e02f67a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToI18nextPlaceholderConvertor.kt @@ -3,13 +3,9 @@ package io.tolgee.formats.paramConvertors.out import com.ibm.icu.text.MessagePattern import io.tolgee.formats.FromIcuPlaceholderConvertor import io.tolgee.formats.MessagePatternUtil -import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY class IcuToI18nextPlaceholderConvertor : FromIcuPlaceholderConvertor { - override fun convert( - node: MessagePatternUtil.ArgNode, - customValues: Map?, - ): String { + override fun convert(node: MessagePatternUtil.ArgNode): String { val type = node.argType if (type == MessagePattern.ArgType.SIMPLE) { @@ -18,28 +14,9 @@ class IcuToI18nextPlaceholderConvertor : FromIcuPlaceholderConvertor { } } - if (customValues.hasUnescapedFlag(node.name)) { - return "{{- ${node.name}}}" - } - return "{{${node.name}}}" } - override fun convert(node: MessagePatternUtil.ArgNode): String { - return convert(node, null) - } - - private fun Map?.hasUnescapedFlag(name: String?): Boolean { - if (this == null || name == null) { - return false - } - return this[I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY] - ?.let { it as? List<*> } - ?.mapNotNull { it as? String } - ?.contains(name) - ?: false - } - override fun convertText( node: MessagePatternUtil.TextNode, keepEscaping: Boolean, diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/properties/out/PropertiesFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/properties/out/PropertiesFileExporter.kt index 2d4afe6b4b..29b7f6fdda 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/properties/out/PropertiesFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/properties/out/PropertiesFileExporter.kt @@ -32,7 +32,7 @@ class PropertiesFileExporter( translations.forEach { translation -> val fileName = computeFileName(translation) val keyName = translation.key.name - val value = convertMessage(translation.text, translation.key.isPlural, translation.key.custom) + val value = convertMessage(translation.text, translation.key.isPlural) val properties = result.getOrPut(fileName) { PropertiesConfiguration() } properties.setProperty(keyName, value) properties.layout.setComment(keyName, translation.key.description) @@ -42,13 +42,11 @@ class PropertiesFileExporter( private fun convertMessage( text: String?, plural: Boolean, - customValues: Map?, ): String? { return IcuToGenericFormatMessageConvertor( text, plural, projectIcuPlaceholdersSupport, - customValues = customValues, paramConvertorFactory = messageFormat.paramConvertorFactory, ).convert() } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/xliff/out/XliffFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/xliff/out/XliffFileExporter.kt index 949e5c1b49..e9b271f15c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/xliff/out/XliffFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/xliff/out/XliffFileExporter.kt @@ -55,9 +55,8 @@ class XliffFileExporter( convertMessage( baseTranslations[translation.key.namespace to translation.key.name]?.text, translation.key.isPlural, - translation.key.custom, ) - this.target = convertMessage(translation.text, translation.key.isPlural, translation.key.custom) + this.target = convertMessage(translation.text, translation.key.isPlural) this.note = translation.key.description }, ) @@ -66,13 +65,11 @@ class XliffFileExporter( private fun convertMessage( text: String?, plural: Boolean, - customValues: Map?, ): String? { return IcuToGenericFormatMessageConvertor( text, plural, projectIcuPlaceholdersSupport, - customValues = customValues, paramConvertorFactory = messageFormat.paramConvertorFactory, ).convert() } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt index ecf10458a6..030471f26f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt @@ -152,21 +152,6 @@ data class FileProcessorContext( } } - fun getCustom(translationKey: String): Map? { - return getOrCreateKeyMeta(translationKey).custom - } - - fun setCustom( - translationKey: String, - customMap: Map?, - ) { - if (customMap != null && !customValuesValidator.isValid(customMap)) { - fileEntity.addIssue(FileIssueType.INVALID_CUSTOM_VALUES, mapOf(FileIssueParamType.KEY_NAME to translationKey)) - } - val keyMeta = getOrCreateKeyMeta(translationKey) - keyMeta.custom = customMap?.toMutableMap() - } - fun setCustom( translationKey: String, customMapKey: String, diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt index e072b24cea..42a846102a 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt @@ -44,19 +44,19 @@ class I18nextFormatProcessorTest { } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescaped") .assertSingle { - hasText("replace this {value} (we save the flag into metadata)") + hasText("replace this '{{'- value'}}' (we don't support the unescaped flag)") } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescapedMultiple") .assertSingle { - hasText("replace this {value} and also don't escape {value} (we save the flag into metadata)") + hasText("replace this '{{'- value'}}' and also don't escape '{{'- value'}}'") } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescapedConflict") .assertSingle { - hasText("replace this {value} but escape this {value} (we default to escaping both occurrences)") + hasText("replace this '{{'- value'}}' but escape this {value}") } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescapedConflictInverted") .assertSingle { - hasText("replace this {value} but don't escape this {value} (we default to escaping both occurrences)") + hasText("replace this {value} but don't escape this '{{'- value'}}'") } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateWithFormatting") .assertSingle { @@ -114,13 +114,7 @@ class I18nextFormatProcessorTest { isPluralOptimized() } mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescaped") { - customEquals( - """ - { - "_i18nextUnescapedPlaceholders" : [ "value" ] - } - """.trimIndent(), - ) + custom.assert.isNull() description.assert.isNull() } mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescapedConflict") { diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt index fc64326654..3ded959d31 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/out/I18nextFileExporterTest.kt @@ -4,7 +4,6 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.tolgee.dtos.request.export.ExportParams import io.tolgee.formats.ExportMessageFormat import io.tolgee.formats.genericStructuredFile.out.CustomPrettyPrinter -import io.tolgee.formats.i18next.I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY import io.tolgee.formats.json.out.JsonFileExporter import io.tolgee.service.export.dataProvider.ExportTranslationView import io.tolgee.unit.util.assertFile @@ -64,8 +63,7 @@ class I18nextFileExporterTest { | "key3_one": "{{count, number}} den {{icuParam, number}}", | "key3_few": "{{count, number}} dny", | "key3_other": "{{count, number}} dní", - | "item": "I will be first {icuParam} {{hello, number}}", - | "unescaped": "Unescaped {{- value}} with another text {{text}}" + | "item": "I will be first {icuParam} {{hello, number}}" |} """.trimMargin(), ) @@ -116,13 +114,6 @@ class I18nextFileExporterTest { keyName = "item", text = "I will be first '{'icuParam'}' {hello, number}", ) - add( - languageTag = "cs", - keyName = "unescaped", - text = "Unescaped {value} with another text {text}", - ) { - key.custom = mapOf(I18NEXT_UNESCAPED_FLAG_CUSTOM_KEY to listOf("value")) - } } return getExporter( built.translations, diff --git a/backend/data/src/test/resources/import/i18next/example.json b/backend/data/src/test/resources/import/i18next/example.json index ba01616b3b..203f2a3dcc 100644 --- a/backend/data/src/test/resources/import/i18next/example.json +++ b/backend/data/src/test/resources/import/i18next/example.json @@ -5,10 +5,10 @@ }, "keyNesting": "reuse $t(keyDeep.inner) (is not supported)", "keyInterpolate": "replace this {{value}}", - "keyInterpolateUnescaped": "replace this {{- value}} (we save the flag into metadata)", - "keyInterpolateUnescapedMultiple": "replace this {{- value}} and also don't escape {{- value}} (we save the flag into metadata)", - "keyInterpolateUnescapedConflict": "replace this {{- value}} but escape this {{value}} (we default to escaping both occurrences)", - "keyInterpolateUnescapedConflictInverted": "replace this {{value}} but don't escape this {{- value}} (we default to escaping both occurrences)", + "keyInterpolateUnescaped": "replace this {{- value}} (we don't support unescaped flag)", + "keyInterpolateUnescapedMultiple": "replace this {{- value}} and also don't escape {{- value}}", + "keyInterpolateUnescapedConflict": "replace this {{- value}} but escape this {{value}}", + "keyInterpolateUnescapedConflictInverted": "replace this {{value}} but don't escape this {{- value}}", "keyInterpolateWithFormatting": "replace this {{value, number}} (only number is supported)", "keyContext_male": "the male variant (is parsed as normal key and context is ignored)", "keyContext_female": "the female variant (is parsed as normal key and context is ignored)", From d459f9bab9cc439aef6f35793164e658af581df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 1 Oct 2024 12:47:10 +0200 Subject: [PATCH 31/32] fix: tests --- .../i18next/in/I18nextFormatProcessorTest.kt | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt index 42a846102a..a4347ea7ed 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt @@ -44,7 +44,7 @@ class I18nextFormatProcessorTest { } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescaped") .assertSingle { - hasText("replace this '{{'- value'}}' (we don't support the unescaped flag)") + hasText("replace this '{{'- value'}}' (we don't support unescaped flag)") } mockUtil.fileProcessorContext.assertTranslations("example", "keyInterpolateUnescapedMultiple") .assertSingle { @@ -113,18 +113,6 @@ class I18nextFormatProcessorTest { ) isPluralOptimized() } - mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescaped") { - custom.assert.isNull() - description.assert.isNull() - } - mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescapedConflict") { - custom.assert.isNull() - description.assert.isNull() - } - mockUtil.fileProcessorContext.assertKey("keyInterpolateUnescapedConflictInverted") { - custom.assert.isNull() - description.assert.isNull() - } mockUtil.fileProcessorContext.assertKey("keyPluralSimple") { custom.assert.isNull() description.assert.isNull() From d6bbbe00950a6ab892317fb8ed20a8a5045a2ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 1 Oct 2024 13:03:50 +0200 Subject: [PATCH 32/32] feat: add more tests --- .../formats/i18next/in/I18nextFormatProcessorTest.kt | 12 ++++++++++++ .../src/test/resources/import/i18next/example2.json | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt index a4347ea7ed..d561b3765c 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/i18next/in/I18nextFormatProcessorTest.kt @@ -185,6 +185,18 @@ class I18nextFormatProcessorTest { .assertSingle { hasText("The price is '{{'value, currency'}}'") } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.formatted_value_with_params") + .assertSingle { + hasText("The price is '{{'value, currency(USD)'}}'") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.intlNumberWithOptions") + .assertSingle { + hasText("Some '{{'val, number(minimumFractionDigits: 2)'}}'") + } + mockUtil.fileProcessorContext.assertTranslations("example", "translation.intlList") + .assertSingle { + hasText("A list of '{{'val, list'}}'") + } mockUtil.fileProcessorContext.assertTranslations("example", "translation.array_example[0]") .assertSingle { hasText("Apples") diff --git a/backend/data/src/test/resources/import/i18next/example2.json b/backend/data/src/test/resources/import/i18next/example2.json index 3bff27e6f2..338bcc8e05 100644 --- a/backend/data/src/test/resources/import/i18next/example2.json +++ b/backend/data/src/test/resources/import/i18next/example2.json @@ -18,7 +18,10 @@ "type": "nested", "formatted_value": "The price is {{value, currency}}", - + "formatted_value_with_params": "The price is {{value, currency(USD)}}", + "intlNumberWithOptions": "Some {{val, number(minimumFractionDigits: 2)}}", + "intlList": "A list of {{val, list}}", + "array_example": [ "Apples", "Oranges",