feat: allow import CSV

backend/data/build.gradle

@@ -171,6 +171,7 @@ dependencies {
  implementation libs.jacksonKotlin
  implementation("org.apache.commons:commons-configuration2:2.10.1")
  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$jacksonVersion"
+ implementation("com.opencsv:opencsv")
 
  /**
  * Google translation API

backend/data/src/main/kotlin/io/tolgee/formats/ImportFileProcessorFactory.kt

@@ -6,6 +6,7 @@ import io.tolgee.dtos.dataImport.ImportFileDto
 import io.tolgee.exceptions.ImportCannotParseFileException
 import io.tolgee.formats.android.`in`.AndroidStringsXmlProcessor
 import io.tolgee.formats.apple.`in`.strings.StringsFileProcessor
+import io.tolgee.formats.csv.`in`.CsvFileProcessor
 import io.tolgee.formats.flutter.`in`.FlutterArbFileProcessor
 import io.tolgee.formats.importCommon.ImportFileFormat
 import io.tolgee.formats.json.`in`.JsonFileProcessor
@@ -60,6 +61,7 @@ class ImportFileProcessorFactory(
  ImportFileFormat.XML -> AndroidStringsXmlProcessor(context)
  ImportFileFormat.ARB -> FlutterArbFileProcessor(context, objectMapper)
  ImportFileFormat.YAML -> YamlFileProcessor(context, yamlObjectMapper)
+ ImportFileFormat.CSV -> CsvFileProcessor(context)
  }
  }
 

backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt

@@ -0,0 +1,46 @@
+package io.tolgee.formats.csv.`in`
+
+import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil
+import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.ICU_DETECTION_REGEX
+import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.detectFromPossibleFormats
+import io.tolgee.formats.importCommon.ImportFormat
+import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor
+import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor
+import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor
+
+class CSVImportFormatDetector {
+ companion object {
+ private val possibleFormats =
+ mapOf(
+ ImportFormat.CSV_ICU to
+ arrayOf(
+ FormatDetectionUtil.regexFactor(
+ ICU_DETECTION_REGEX,
+ ),
+ ),
+ ImportFormat.CSV_PHP to
+ arrayOf(
+ FormatDetectionUtil.regexFactor(
+ PhpToIcuPlaceholderConvertor.PHP_DETECTION_REGEX,
+ ),
+ ),
+ ImportFormat.CSV_JAVA to
+ arrayOf(
+ FormatDetectionUtil.regexFactor(
+ JavaToIcuPlaceholderConvertor.JAVA_DETECTION_REGEX,
+ ),
+ ),
+ ImportFormat.CSV_RUBY to
+ arrayOf(
+ FormatDetectionUtil.regexFactor(
+ RubyToIcuPlaceholderConvertor.RUBY_DETECTION_REGEX,
+ 0.95,
+ ),
+ ),
+ )
+ }
+
+ fun detectFormat(data: List<Array<String>>): ImportFormat {
+ return detectFromPossibleFormats(possibleFormats, data) ?: ImportFormat.CSV_ICU
+ }
+}

backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt

@@ -0,0 +1,67 @@
+package io.tolgee.formats.csv.`in`
+
+import com.opencsv.CSVParserBuilder
+import com.opencsv.CSVReaderBuilder
+import io.tolgee.exceptions.ImportCannotParseFileException
+import io.tolgee.formats.ImportFileProcessor
+import io.tolgee.formats.importCommon.ImportFormat
+import io.tolgee.service.dataImport.processors.FileProcessorContext
+
+class CsvFileProcessor(
+ override val context: FileProcessorContext,
+) : ImportFileProcessor() {
+
+ override fun process() {
+ val inputStream = context.file.data.inputStream()
+ val reader = inputStream.reader()
+ val data: List<Array<String>>
+
+ CSVReaderBuilder(reader)
+ .withCSVParser(
+ CSVParserBuilder()
+ .withSeparator(';') // TODO make delimiter parametrizable
+ .build()
+ ).build().use { csvReader -> data = csvReader.readAll() }
+
+
+ val format = getFormat(data)
+
+ // Read the first line to extract headers (language keys) - "key_name", ...languages
+ val headers = data.firstOrNull()
+ val languages = headers?.drop(1) ?: emptyList()
+
+ // Parse body - key_name, ...translations
+ for ((idx, row) in data.drop(1).withIndex()) {
+ val keyName = row.getOrNull(0) ?: throw ImportCannotParseFileException(context.file.name, "empty row $idx")
+
+ for (i in 1 until row.size) {
+ val languageTag = languages.getOrNull(i - 1) ?: throw ImportCannotParseFileException(
+ context.file.name, "more translations than defined languages in row $idx"
+ )
+ val translation = row.getOrNull(i) ?: throw ImportCannotParseFileException(
+ context.file.name, "missing translation in row $idx"
+ )
+
+ val converted = format.messageConvertor.convert(
+ translation, languageTag,
+ convertPlaceholders = context.importSettings.convertPlaceholdersToIcu,
+ isProjectIcuEnabled = context.projectIcuPlaceholdersEnabled
+ )
+ context.addTranslation(
+ keyName,
+ languageTag,
+ converted.message,
+ idx,
+ pluralArgName = converted.pluralArgName,
+ rawData = row[i],
+ convertedBy = format,
+ )
+ }
+ }
+ }
+
+ private fun getFormat(data: List<Array<String>>): ImportFormat {
+ return context.mapping?.format ?: CSVImportFormatDetector().detectFormat(data)
+ }
+
+}

backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/FormatDetectionUtil.kt

@@ -36,6 +36,7 @@ object FormatDetectionUtil {
  when (data) {
  is Map<*, *> -> data.forEach { (_, value) -> processMapRecursive(value, regex, hits, total) }
  is List<*> -> data.forEach { item -> processMapRecursive(item, regex, hits, total) }
+ is Array<*> -> data.forEach { item -> processMapRecursive(item, regex, hits, total) }
  else -> {
  if (data is String) {
  val count = regex.findAll(data).count()

backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFileFormat.kt

@@ -10,6 +10,7 @@ enum class ImportFileFormat(val extensions: Array<String>) {
  XML(arrayOf("xml")),
  ARB(arrayOf("arb")),
  YAML(arrayOf("yaml", "yml")),
+ CSV(arrayOf("csv")),
  ;
 
  companion object {

backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt

@@ -13,6 +13,27 @@ enum class ImportFormat(
  val messageConvertorOrNull: ImportMessageConvertor? = null,
  val rootKeyIsLanguageTag: Boolean = false,
 ) {
+ CSV_ICU(
+ ImportFileFormat.CSV,
+ messageConvertorOrNull =
+ GenericMapPluralImportRawDataConvertor(
+ canContainIcu = true,
+ toIcuPlaceholderConvertorFactory = null
+ ),
+ ),
+ CSV_JAVA(
+ ImportFileFormat.CSV,
+ messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { JavaToIcuPlaceholderConvertor() },
+ ),
+ CSV_PHP(
+ ImportFileFormat.CSV,
+ messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { PhpToIcuPlaceholderConvertor() },
+ ),
+ CSV_RUBY(
+ ImportFileFormat.CSV,
+ messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { RubyToIcuPlaceholderConvertor() },
+ ),
+
  JSON_ICU(
  ImportFileFormat.JSON,
  messageConvertorOrNull =

webapp/src/views/projects/import/component/ImportSupportedFormats.tsx

@@ -53,6 +53,12 @@ const FORMATS = [
  { name: 'Android XML', logo: <AndroidLogo /> },
  { name: 'Flutter ARB', logo: <FluttrerLogo /> },
  { name: 'Ruby YAML', logo: <RailsLogo /> },
+ {
+ name: 'CSV',
+ logo: <TolgeeLogo />,
+ logoHeight: '24px',
+ logoWidth: '24px',
+ },
 ];
 
 export const ImportSupportedFormats = () => {