feat: import support for i18next

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt

@@ -26,7 +26,7 @@ class ExportInfoControllerTest : AbstractControllerTest() {
  node("defaultFileStructureTemplate")
  .isString.isEqualTo("{namespace}/{languageTag}.{extension}")
  }
- node("[4]") {
+ node("[5]") {
  node("extension").isEqualTo("")
  node("mediaType").isEqualTo("")
  node("defaultFileStructureTemplate")

backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt

@@ -9,6 +9,7 @@ enum class ExportFormat(
 ) {
  JSON("json", "application/json"),
  JSON_TOLGEE("json", "application/json"),
+ JSON_I18NEXT("json", "application/json"),
  XLIFF("xliff", "application/x-xliff+xml"),
  PO("po", "text/x-gettext-translation"),
  APPLE_STRINGS_STRINGSDICT(

backend/data/src/main/kotlin/io/tolgee/formats/ExportMessageFormat.kt

@@ -2,6 +2,7 @@ package io.tolgee.formats
 
 import io.tolgee.formats.paramConvertors.out.IcuToApplePlaceholderConvertor
 import io.tolgee.formats.paramConvertors.out.IcuToCPlaceholderConvertor
+import io.tolgee.formats.paramConvertors.out.IcuToI18nextPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.out.IcuToJavaPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.out.IcuToPhpPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.out.IcuToRubyPlaceholderConvertor
@@ -13,6 +14,7 @@ enum class ExportMessageFormat(val paramConvertorFactory: () -> FromIcuPlacehold
  JAVA_STRING_FORMAT(paramConvertorFactory = { IcuToJavaPlaceholderConvertor() }),
  APPLE_SPRINTF(paramConvertorFactory = { IcuToApplePlaceholderConvertor() }),
  RUBY_SPRINTF(paramConvertorFactory = { IcuToRubyPlaceholderConvertor() }),
+ I18NEXT(paramConvertorFactory = { IcuToI18nextPlaceholderConvertor() }),
  ICU(paramConvertorFactory = { NoOpFromIcuPlaceholderConvertor() }),
 // PYTHON_SPRINTF,
 }

backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredProcessor.kt

@@ -13,7 +13,16 @@ class GenericStructuredProcessor(
  private val format: ImportFormat,
 ) : ImportFileProcessor() {
  override fun process() {
- data.import("")
+ var processedData = data
+ if (format.pluralsViaSuffixesParser != null) {
+ processedData =
+ GenericSuffixedPluralsPreprocessor(
+ context = context,
+ data = data,
+ pluralsViaSuffixesParser = format.pluralsViaSuffixesParser,
+ ).preprocess()
+ }
+ processedData.import("")
  }
 
  private fun Any?.import(key: String) {

backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericStructuredRawDataToTextConvertor.kt

@@ -1,20 +1,14 @@
 package io.tolgee.formats.genericStructuredFile.`in`
 
-import com.ibm.icu.text.PluralRules
 import io.tolgee.formats.MessageConvertorResult
+import io.tolgee.formats.allPluralKeywords
 import io.tolgee.formats.importCommon.ImportFormat
 import io.tolgee.formats.importCommon.unwrapString
-import java.util.*
 
 class GenericStructuredRawDataToTextConvertor(
  private val format: ImportFormat,
  private val languageTag: String,
 ) : StructuredRawDataConvertor {
- private val availablePluralKeywords by lazy {
- val locale = Locale.forLanguageTag(languageTag)
- PluralRules.forLocale(locale).keywords.toSet()
- }
-
  override fun convert(
  rawData: Any?,
  projectIcuPlaceholdersEnabled: Boolean,
@@ -77,11 +71,15 @@ class GenericStructuredRawDataToTextConvertor(
  ): List<MessageConvertorResult>? {
  val map = rawData as? Map<*, *> ?: return null
 
- if (!format.pluralsViaNesting) {
+ if (!format.pluralsViaNesting && format.pluralsViaSuffixesParser == null) {
+ return null
+ }
+
+ if (!map.keys.all { it in allPluralKeywords }) {
  return null
  }
 
- if (!map.keys.all { it in availablePluralKeywords }) {
+ if (map.size < 2) {
  return null
  }
 

backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/GenericSuffixedPluralsPreprocessor.kt

@@ -0,0 +1,77 @@
+package io.tolgee.formats.genericStructuredFile.`in`
+
+import io.tolgee.formats.allPluralKeywords
+import io.tolgee.formats.importCommon.ParsedPluralsKey
+import io.tolgee.formats.importCommon.PluralsKeyParser
+import io.tolgee.service.dataImport.processors.FileProcessorContext
+
+class GenericSuffixedPluralsPreprocessor(
+ val context: FileProcessorContext,
+ private val data: Any?,
+ private val pluralsViaSuffixesParser: PluralsKeyParser,
+) {
+ fun preprocess(): Any? {
+ return data.preprocess()
+ }
+
+ private fun Any?.preprocess(): Any? {
+ if (this == null) {
+ return null
+ }
+
+ (this as? List<*>)?.let {
+ return it.preprocessList()
+ }
+
+ (this as? Map<*, *>)?.let {
+ return it.preprocessMap()
+ }
+
+ return this
+ }
+
+ private fun List<*>.preprocessList(): List<*> {
+ return this.map { it.preprocess() }
+ }
+
+ private fun Any?.parsePluralsKey(keyParser: PluralsKeyParser): ParsedPluralsKey? {
+ val key = this as? String ?: return null
+ return keyParser.parse(key).takeIf {
+ it.key != null && it.plural in allPluralKeywords
+ } ?: ParsedPluralsKey(null, null, key)
+ }
+
+ private fun Map<*, *>.groupByPlurals(keyParser: PluralsKeyParser): Map<String?, List<Pair<ParsedPluralsKey, Any?>>> {
+ return this.entries.mapIndexedNotNull { idx, (key, value) ->
+ key.parsePluralsKey(keyParser)?.let { it to value }.also {
+ if (it == null) {
+ context.fileEntity.addKeyIsNotStringIssue(key.toString(), idx)
+ }
+ }
+ }.groupBy { (parsedKey, _) -> parsedKey.key }.toMap()
+ }
+
+ private fun List<Pair<ParsedPluralsKey, Any?>>.useOriginalKey(): List<Pair<String, Any?>> {
+ return map { (parsedKey, value) ->
+ parsedKey.originalKey to value
+ }
+ }
+
+ private fun List<Pair<ParsedPluralsKey, Any?>>.usePluralsKey(commonKey: String): List<Pair<String, Any?>> {
+ return listOf(
+ commonKey to
+ this.associate { (parsedKey, value) ->
+ parsedKey.plural to value
+ },
+ )
+ }
+
+ private fun Map<*, *>.preprocessMap(): Map<*, *> {
+ return this.groupByPlurals(pluralsViaSuffixesParser).flatMap { (commonKey, values) ->
+ if (commonKey == null || values.size < 2) {
+ return@flatMap values.useOriginalKey()
+ }
+ return@flatMap values.usePluralsKey(commonKey)
+ }.toMap()
+ }
+}

backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/out/GenericStructuredFileExporter.kt

@@ -55,8 +55,11 @@ class GenericStructuredFileExporter(
  )
  }
 
+ private val pluralsViaSuffixes
+ get() = messageFormat == ExportMessageFormat.I18NEXT
+
  private val pluralsViaNesting
- get() = messageFormat != ExportMessageFormat.ICU
+ get() = !pluralsViaSuffixes && messageFormat != ExportMessageFormat.ICU
 
  private val placeholderConvertorFactory
  get() = messageFormat.paramConvertorFactory
@@ -65,6 +68,9 @@ class GenericStructuredFileExporter(
  if (pluralsViaNesting) {
  return addNestedPlural(translation)
  }
+ if (pluralsViaSuffixes) {
+ return addSuffixedPlural(translation)
+ }
  return addSingularTranslation(translation)
  }
 
@@ -84,6 +90,24 @@ class GenericStructuredFileExporter(
  )
  }
 
+ private fun addSuffixedPlural(translation: ExportTranslationView) {
+ val pluralForms =
+ convertMessageForNestedPlural(translation.text) ?: let {
+ // this should never happen, but if it does, it's better to add a null key then crash or ignore it
+ addNullValue(translation)
+ return
+ }
+
+ val builder = getFileContentResultBuilder(translation)
+ pluralForms.forEach { (keyword, form) ->
+ builder.addValue(
+ translation.languageTag,
+ "${translation.key.name}_$keyword",
+ form,
+ )
+ }
+ }
+
  private fun addNullValue(translation: ExportTranslationView) {
  val builder = getFileContentResultBuilder(translation)
  builder.addValue(

backend/data/src/main/kotlin/io/tolgee/formats/getGroupOrNull.kt

@@ -0,0 +1,12 @@
+package io.tolgee.formats
+
+fun MatchGroupCollection.getGroupOrNull(name: String): MatchGroup? {
+ try {
+ return this[name]
+ } catch (e: IllegalArgumentException) {
+ if (e.message?.contains("No group with name") != true) {
+ throw e
+ }
+ return null
+ }
+}

backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/I18nextParameterParser.kt

@@ -0,0 +1,15 @@
+package io.tolgee.formats.i18next.`in`
+
+import io.tolgee.formats.getGroupOrNull
+
+class I18nextParameterParser {
+ fun parse(match: MatchResult): ParsedI18nextParam? {
+ return ParsedI18nextParam(
+ key = match.groups.getGroupOrNull("key")?.value,
+ nestedKey = match.groups.getGroupOrNull("nestedKey")?.value,
+ format = match.groups.getGroupOrNull("format")?.value,
+ keepUnescaped = (match.groups.getGroupOrNull("unescapedflag")?.value?.length ?: 0) > 0,
+ fullMatch = match.value,
+ )
+ }
+}

backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/ParsedI18nextParam.kt

@@ -0,0 +1,9 @@
+package io.tolgee.formats.i18next.`in`
+
+data class ParsedI18nextParam(
+ val key: String? = null,
+ val nestedKey: String? = null,
+ val format: String? = null,
+ val keepUnescaped: Boolean = false,
+ val fullMatch: String,
+)

backend/data/src/main/kotlin/io/tolgee/formats/i18next/in/PluralsI18nextKeyParser.kt

@@ -0,0 +1,16 @@
+package io.tolgee.formats.i18next.`in`
+
+import io.tolgee.formats.getGroupOrNull
+import io.tolgee.formats.importCommon.ParsedPluralsKey
+import io.tolgee.formats.importCommon.PluralsKeyParser
+
+class PluralsI18nextKeyParser(private val keyRegex: Regex) : PluralsKeyParser {
+ override fun parse(key: String): ParsedPluralsKey {
+ val match = keyRegex.find(key)
+ return ParsedPluralsKey(
+ key = match?.groups?.getGroupOrNull("key")?.value,
+ plural = match?.groups?.getGroupOrNull("plural")?.value,
+ originalKey = key,
+ )
+ }
+}

backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt

@@ -2,6 +2,7 @@ package io.tolgee.formats.importCommon
 
 import io.tolgee.formats.paramConvertors.`in`.AppleToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.CToIcuPlaceholderConvertor
+import io.tolgee.formats.paramConvertors.`in`.I18nextToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor
@@ -10,9 +11,15 @@ import io.tolgee.formats.po.`in`.PoToIcuMessageConvertor
 enum class ImportFormat(
  val fileFormat: ImportFileFormat,
  val pluralsViaNesting: Boolean = false,
+ val pluralsViaSuffixesParser: PluralsKeyParser? = null,
  val messageConvertorOrNull: ImportMessageConvertor? = null,
  val rootKeyIsLanguageTag: Boolean = false,
 ) {
+ JSON_I18NEXT(
+ ImportFileFormat.JSON,
+ messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { I18nextToIcuPlaceholderConvertor() },
+ pluralsViaSuffixesParser = I18nextToIcuPlaceholderConvertor.I18NEXT_PLURAL_SUFFIX_KEY_PARSER,
+ ),
  JSON_ICU(
  ImportFileFormat.JSON,
  messageConvertorOrNull =

backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ParsedPluralsKey.kt

@@ -0,0 +1,7 @@
+package io.tolgee.formats.importCommon
+
+data class ParsedPluralsKey(
+ val key: String? = null,
+ val plural: String? = null,
+ val originalKey: String,
+)

backend/data/src/main/kotlin/io/tolgee/formats/importCommon/PluralsKeyParser.kt

@@ -0,0 +1,5 @@
+package io.tolgee.formats.importCommon
+
+interface PluralsKeyParser {
+ fun parse(key: String): ParsedPluralsKey
+}

backend/data/src/main/kotlin/io/tolgee/formats/json/in/JsonImportFormatDetector.kt

@@ -5,6 +5,7 @@ import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.ICU_DETE
 import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.detectFromPossibleFormats
 import io.tolgee.formats.importCommon.ImportFormat
 import io.tolgee.formats.paramConvertors.`in`.CToIcuPlaceholderConvertor
+import io.tolgee.formats.paramConvertors.`in`.I18nextToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor
@@ -45,6 +46,12 @@ class JsonImportFormatDetector {
  0.6,
  ),
  ),
+ ImportFormat.JSON_I18NEXT to
+ arrayOf(
+ FormatDetectionUtil.regexFactor(
+ I18nextToIcuPlaceholderConvertor.I18NEXT_DETECTION_REGEX,
+ ),
+ ),
  )
  }
 

backend/data/src/main/kotlin/io/tolgee/formats/json/out/JsonFileExporter.kt

@@ -23,6 +23,7 @@ class JsonFileExporter(
  private val messageFormat =
  when (exportParams.format) {
  ExportFormat.JSON_TOLGEE -> ExportMessageFormat.ICU
+ ExportFormat.JSON_I18NEXT -> ExportMessageFormat.I18NEXT
  else -> exportParams.messageFormat ?: ExportMessageFormat.ICU
  }