From 3596c37b50eb37c377f59b71dfb9f4d8ce5706b2 Mon Sep 17 00:00:00 2001 From: xReav3r Date: Mon, 26 Aug 2024 19:31:06 +0200 Subject: [PATCH 01/18] feat: allow import CSV --- backend/data/build.gradle | 1 + .../formats/ImportFileProcessorFactory.kt | 2 + .../formats/csv/in/CSVImportFormatDetector.kt | 46 +++++++++++++ .../tolgee/formats/csv/in/CsvFileProcessor.kt | 67 +++++++++++++++++++ .../in/FormatDetectionUtil.kt | 1 + .../formats/importCommon/ImportFileFormat.kt | 1 + .../formats/importCommon/ImportFormat.kt | 21 ++++++ .../component/ImportSupportedFormats.tsx | 6 ++ 8 files changed, 145 insertions(+) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt diff --git a/backend/data/build.gradle b/backend/data/build.gradle index 1fecf34701..7a678eac74 100644 --- a/backend/data/build.gradle +++ b/backend/data/build.gradle @@ -171,6 +171,7 @@ dependencies { implementation libs.jacksonKotlin implementation("org.apache.commons:commons-configuration2:2.10.1") implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$jacksonVersion" + implementation("com.opencsv:opencsv") /** * Google translation API diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/ImportFileProcessorFactory.kt b/backend/data/src/main/kotlin/io/tolgee/formats/ImportFileProcessorFactory.kt index edf8d26877..9272a1c622 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/ImportFileProcessorFactory.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/ImportFileProcessorFactory.kt @@ -6,6 +6,7 @@ import io.tolgee.dtos.dataImport.ImportFileDto import io.tolgee.exceptions.ImportCannotParseFileException import io.tolgee.formats.android.`in`.AndroidStringsXmlProcessor import io.tolgee.formats.apple.`in`.strings.StringsFileProcessor +import io.tolgee.formats.csv.`in`.CsvFileProcessor import io.tolgee.formats.flutter.`in`.FlutterArbFileProcessor import io.tolgee.formats.importCommon.ImportFileFormat import io.tolgee.formats.json.`in`.JsonFileProcessor @@ -60,6 +61,7 @@ class ImportFileProcessorFactory( ImportFileFormat.XML -> AndroidStringsXmlProcessor(context) ImportFileFormat.ARB -> FlutterArbFileProcessor(context, objectMapper) ImportFileFormat.YAML -> YamlFileProcessor(context, yamlObjectMapper) + ImportFileFormat.CSV -> CsvFileProcessor(context) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt new file mode 100644 index 0000000000..e294e59018 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt @@ -0,0 +1,46 @@ +package io.tolgee.formats.csv.`in` + +import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil +import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.ICU_DETECTION_REGEX +import io.tolgee.formats.genericStructuredFile.`in`.FormatDetectionUtil.detectFromPossibleFormats +import io.tolgee.formats.importCommon.ImportFormat +import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor +import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor +import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor + +class CSVImportFormatDetector { + companion object { + private val possibleFormats = + mapOf( + ImportFormat.CSV_ICU to + arrayOf( + FormatDetectionUtil.regexFactor( + ICU_DETECTION_REGEX, + ), + ), + ImportFormat.CSV_PHP to + arrayOf( + FormatDetectionUtil.regexFactor( + PhpToIcuPlaceholderConvertor.PHP_DETECTION_REGEX, + ), + ), + ImportFormat.CSV_JAVA to + arrayOf( + FormatDetectionUtil.regexFactor( + JavaToIcuPlaceholderConvertor.JAVA_DETECTION_REGEX, + ), + ), + ImportFormat.CSV_RUBY to + arrayOf( + FormatDetectionUtil.regexFactor( + RubyToIcuPlaceholderConvertor.RUBY_DETECTION_REGEX, + 0.95, + ), + ), + ) + } + + fun detectFormat(data: List>): ImportFormat { + return detectFromPossibleFormats(possibleFormats, data) ?: ImportFormat.CSV_ICU + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt new file mode 100644 index 0000000000..af4836fddb --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt @@ -0,0 +1,67 @@ +package io.tolgee.formats.csv.`in` + +import com.opencsv.CSVParserBuilder +import com.opencsv.CSVReaderBuilder +import io.tolgee.exceptions.ImportCannotParseFileException +import io.tolgee.formats.ImportFileProcessor +import io.tolgee.formats.importCommon.ImportFormat +import io.tolgee.service.dataImport.processors.FileProcessorContext + +class CsvFileProcessor( + override val context: FileProcessorContext, +) : ImportFileProcessor() { + + override fun process() { + val inputStream = context.file.data.inputStream() + val reader = inputStream.reader() + val data: List> + + CSVReaderBuilder(reader) + .withCSVParser( + CSVParserBuilder() + .withSeparator(';') // TODO make delimiter parametrizable + .build() + ).build().use { csvReader -> data = csvReader.readAll() } + + + val format = getFormat(data) + + // Read the first line to extract headers (language keys) - "key_name", ...languages + val headers = data.firstOrNull() + val languages = headers?.drop(1) ?: emptyList() + + // Parse body - key_name, ...translations + for ((idx, row) in data.drop(1).withIndex()) { + val keyName = row.getOrNull(0) ?: throw ImportCannotParseFileException(context.file.name, "empty row $idx") + + for (i in 1 until row.size) { + val languageTag = languages.getOrNull(i - 1) ?: throw ImportCannotParseFileException( + context.file.name, "more translations than defined languages in row $idx" + ) + val translation = row.getOrNull(i) ?: throw ImportCannotParseFileException( + context.file.name, "missing translation in row $idx" + ) + + val converted = format.messageConvertor.convert( + translation, languageTag, + convertPlaceholders = context.importSettings.convertPlaceholdersToIcu, + isProjectIcuEnabled = context.projectIcuPlaceholdersEnabled + ) + context.addTranslation( + keyName, + languageTag, + converted.message, + idx, + pluralArgName = converted.pluralArgName, + rawData = row[i], + convertedBy = format, + ) + } + } + } + + private fun getFormat(data: List>): ImportFormat { + return context.mapping?.format ?: CSVImportFormatDetector().detectFormat(data) + } + +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/FormatDetectionUtil.kt b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/FormatDetectionUtil.kt index 2930ef431e..47a6c08a5f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/FormatDetectionUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/genericStructuredFile/in/FormatDetectionUtil.kt @@ -36,6 +36,7 @@ object FormatDetectionUtil { when (data) { is Map<*, *> -> data.forEach { (_, value) -> processMapRecursive(value, regex, hits, total) } is List<*> -> data.forEach { item -> processMapRecursive(item, regex, hits, total) } + is Array<*> -> data.forEach { item -> processMapRecursive(item, regex, hits, total) } else -> { if (data is String) { val count = regex.findAll(data).count() diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFileFormat.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFileFormat.kt index 2fdaefd882..ca1d737d33 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFileFormat.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFileFormat.kt @@ -10,6 +10,7 @@ enum class ImportFileFormat(val extensions: Array) { XML(arrayOf("xml")), ARB(arrayOf("arb")), YAML(arrayOf("yaml", "yml")), + CSV(arrayOf("csv")), ; companion object { diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt index c59c828578..c9e533c2ba 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt @@ -15,6 +15,27 @@ enum class ImportFormat( val messageConvertorOrNull: ImportMessageConvertor? = null, val rootKeyIsLanguageTag: Boolean = false, ) { + CSV_ICU( + ImportFileFormat.CSV, + messageConvertorOrNull = + GenericMapPluralImportRawDataConvertor( + canContainIcu = true, + toIcuPlaceholderConvertorFactory = null + ), + ), + CSV_JAVA( + ImportFileFormat.CSV, + messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { JavaToIcuPlaceholderConvertor() }, + ), + CSV_PHP( + ImportFileFormat.CSV, + messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { PhpToIcuPlaceholderConvertor() }, + ), + CSV_RUBY( + ImportFileFormat.CSV, + messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { RubyToIcuPlaceholderConvertor() }, + ), + JSON_I18NEXT( ImportFileFormat.JSON, messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { I18nextToIcuPlaceholderConvertor() }, diff --git a/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx b/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx index 7a265a1495..064fb44f0c 100644 --- a/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx +++ b/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx @@ -55,6 +55,12 @@ const FORMATS = [ { name: 'Flutter ARB', logo: }, { name: 'Ruby YAML', logo: }, { name: 'i18next', logo: }, + { + name: 'CSV', + logo: , + logoHeight: '24px', + logoWidth: '24px', + }, ]; export const ImportSupportedFormats = () => { From 091f2d2c3ee4e0a85380c6f466f8e78ee1e23193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Sun, 6 Oct 2024 19:48:20 +0200 Subject: [PATCH 02/18] feat: csv import code fixups --- backend/data/build.gradle | 2 +- .../kotlin/io/tolgee/formats/csv/CsvModel.kt | 7 ++ .../formats/csv/in/CSVImportFormatDetector.kt | 2 +- .../io/tolgee/formats/csv/in/CsvFileParser.kt | 60 ++++++++++++++ .../tolgee/formats/csv/in/CsvFileProcessor.kt | 83 ++++++++----------- .../formats/importCommon/ImportFormat.kt | 2 +- .../component/ImportSupportedFormats.tsx | 2 +- 7 files changed, 107 insertions(+), 51 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/csv/CsvModel.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt diff --git a/backend/data/build.gradle b/backend/data/build.gradle index 7a678eac74..c6df704bda 100644 --- a/backend/data/build.gradle +++ b/backend/data/build.gradle @@ -171,7 +171,7 @@ dependencies { implementation libs.jacksonKotlin implementation("org.apache.commons:commons-configuration2:2.10.1") implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$jacksonVersion" - implementation("com.opencsv:opencsv") + implementation("com.opencsv:opencsv:5.9") /** * Google translation API diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/CsvModel.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/CsvModel.kt new file mode 100644 index 0000000000..ee906b3e69 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/CsvModel.kt @@ -0,0 +1,7 @@ +package io.tolgee.formats.csv + +data class CsvEntry( + val key: String, + val language: String?, + val value: String?, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt index e294e59018..be153fcaef 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CSVImportFormatDetector.kt @@ -40,7 +40,7 @@ class CSVImportFormatDetector { ) } - fun detectFormat(data: List>): ImportFormat { + fun detectFormat(data: Any?): ImportFormat { return detectFromPossibleFormats(possibleFormats, data) ?: ImportFormat.CSV_ICU } } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt new file mode 100644 index 0000000000..116d72d3f1 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt @@ -0,0 +1,60 @@ +package io.tolgee.formats.csv.`in` + +import com.opencsv.CSVParserBuilder +import com.opencsv.CSVReaderBuilder +import io.tolgee.formats.csv.CsvEntry +import java.io.ByteArrayInputStream + +class CsvFileParser( + private val inputStream: ByteArrayInputStream, + private val delimiter: Char, +) { + val rawData: List> by lazy { + val inputReader = inputStream.reader() + val parser = CSVParserBuilder().withSeparator(delimiter).build() + val reader = CSVReaderBuilder(inputReader).withCSVParser(parser).build() + + return@lazy reader.readAll() + } + + val headers: Array? by lazy { + rawData.firstOrNull() + } + + val languages: List by lazy { + headers?.takeIf { it.size > 1 }?.drop(1) ?: emptyList() + } + + val languagesWithFallback: Sequence + get() = languages.asSequence().plus(generateSequence { null }) + + val rows: List> by lazy { + rawData.takeIf { it.size > 1 }?.drop(1) ?: emptyList() + } + + fun Array.rowToCsvEntries(): Sequence { + if (isEmpty()) { + return emptySequence() + } + val keyName = getOrNull(0) ?: "" + if (size == 1) { + return sequenceOf(CsvEntry(keyName, null, null)) + } + val translations = drop(1).asSequence() + return translations + .zip(languagesWithFallback) + .map { (translation, languageTag) -> + CsvEntry( + keyName, + languageTag, + translation, + ) + } + } + + fun parse(): List { + return rows.flatMap { + it.rowToCsvEntries() + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt index af4836fddb..35854e4476 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt @@ -1,67 +1,56 @@ package io.tolgee.formats.csv.`in` -import com.opencsv.CSVParserBuilder -import com.opencsv.CSVReaderBuilder import io.tolgee.exceptions.ImportCannotParseFileException import io.tolgee.formats.ImportFileProcessor +import io.tolgee.formats.csv.CsvEntry import io.tolgee.formats.importCommon.ImportFormat import io.tolgee.service.dataImport.processors.FileProcessorContext class CsvFileProcessor( override val context: FileProcessorContext, ) : ImportFileProcessor() { - override fun process() { - val inputStream = context.file.data.inputStream() - val reader = inputStream.reader() - val data: List> - - CSVReaderBuilder(reader) - .withCSVParser( - CSVParserBuilder() - .withSeparator(';') // TODO make delimiter parametrizable - .build() - ).build().use { csvReader -> data = csvReader.readAll() } - - + val data = parse() val format = getFormat(data) + data.importAll(format) + } - // Read the first line to extract headers (language keys) - "key_name", ...languages - val headers = data.firstOrNull() - val languages = headers?.drop(1) ?: emptyList() - - // Parse body - key_name, ...translations - for ((idx, row) in data.drop(1).withIndex()) { - val keyName = row.getOrNull(0) ?: throw ImportCannotParseFileException(context.file.name, "empty row $idx") + fun Iterable.importAll(format: ImportFormat) { + forEachIndexed { idx, it -> it.import(idx, format) } + } - for (i in 1 until row.size) { - val languageTag = languages.getOrNull(i - 1) ?: throw ImportCannotParseFileException( - context.file.name, "more translations than defined languages in row $idx" - ) - val translation = row.getOrNull(i) ?: throw ImportCannotParseFileException( - context.file.name, "missing translation in row $idx" - ) + fun CsvEntry.import( + index: Int, + format: ImportFormat, + ) { + val selectedLanguage = language ?: firstLanguageTagGuessOrUnknown + val converted = + format.messageConvertor.convert( + value, + selectedLanguage, + convertPlaceholders = context.importSettings.convertPlaceholdersToIcu, + isProjectIcuEnabled = context.projectIcuPlaceholdersEnabled, + ) + context.addTranslation( + key, + selectedLanguage, + converted.message, + index, + pluralArgName = converted.pluralArgName, + rawData = value, + convertedBy = format, + ) + } - val converted = format.messageConvertor.convert( - translation, languageTag, - convertPlaceholders = context.importSettings.convertPlaceholdersToIcu, - isProjectIcuEnabled = context.projectIcuPlaceholdersEnabled - ) - context.addTranslation( - keyName, - languageTag, - converted.message, - idx, - pluralArgName = converted.pluralArgName, - rawData = row[i], - convertedBy = format, - ) - } + private fun parse() = + try { + // TODO: make delimiter configurable + CsvFileParser(context.file.data.inputStream(), ';').parse() + } catch (e: Exception) { + throw ImportCannotParseFileException(context.file.name, e.message ?: "", e) } - } - private fun getFormat(data: List>): ImportFormat { + private fun getFormat(data: Any?): ImportFormat { return context.mapping?.format ?: CSVImportFormatDetector().detectFormat(data) } - } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt index c9e533c2ba..936f7ad259 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt @@ -20,7 +20,7 @@ enum class ImportFormat( messageConvertorOrNull = GenericMapPluralImportRawDataConvertor( canContainIcu = true, - toIcuPlaceholderConvertorFactory = null + toIcuPlaceholderConvertorFactory = null, ), ), CSV_JAVA( diff --git a/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx b/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx index 064fb44f0c..1e38b7e6d7 100644 --- a/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx +++ b/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx @@ -57,7 +57,7 @@ const FORMATS = [ { name: 'i18next', logo: }, { name: 'CSV', - logo: , + logo: , // TODO: replace with proper icon? logoHeight: '24px', logoWidth: '24px', }, From 66f50fc5096b562150e469382ec65fa4412ce315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 8 Oct 2024 02:31:55 +0200 Subject: [PATCH 03/18] feat: csv export initial implementation --- .../kotlin/io/tolgee/formats/ExportFormat.kt | 1 + .../kotlin/io/tolgee/formats/csv/CsvModel.kt | 2 +- .../io/tolgee/formats/csv/in/CsvFileParser.kt | 7 +- .../tolgee/formats/csv/in/CsvFileProcessor.kt | 7 +- .../tolgee/formats/csv/out/CsvFileExporter.kt | 66 +++++++++++++++++++ .../tolgee/formats/csv/out/CsvFileWriter.kt | 35 ++++++++++ .../service/export/FileExporterFactory.kt | 8 +++ e2e/cypress/common/export.ts | 7 ++ .../export/components/formatGroups.tsx | 13 ++++ 9 files changed, 138 insertions(+), 8 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt b/backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt index ba14b1d3b4..3e2ffe6fe3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/ExportFormat.kt @@ -7,6 +7,7 @@ enum class ExportFormat( val mediaType: String, val defaultFileStructureTemplate: String = ExportFilePathProvider.DEFAULT_TEMPLATE, ) { + CSV("csv", "text/csv"), JSON("json", "application/json"), JSON_TOLGEE("json", "application/json"), JSON_I18NEXT("json", "application/json"), diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/CsvModel.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/CsvModel.kt index ee906b3e69..5231a56d82 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/CsvModel.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/CsvModel.kt @@ -2,6 +2,6 @@ package io.tolgee.formats.csv data class CsvEntry( val key: String, - val language: String?, + val language: String, val value: String?, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt index 116d72d3f1..b9d9476311 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt @@ -8,6 +8,7 @@ import java.io.ByteArrayInputStream class CsvFileParser( private val inputStream: ByteArrayInputStream, private val delimiter: Char, + private val languageFallback: String, ) { val rawData: List> by lazy { val inputReader = inputStream.reader() @@ -25,8 +26,8 @@ class CsvFileParser( headers?.takeIf { it.size > 1 }?.drop(1) ?: emptyList() } - val languagesWithFallback: Sequence - get() = languages.asSequence().plus(generateSequence { null }) + val languagesWithFallback: Sequence + get() = languages.asSequence().plus(generateSequence { languageFallback }) val rows: List> by lazy { rawData.takeIf { it.size > 1 }?.drop(1) ?: emptyList() @@ -38,7 +39,7 @@ class CsvFileParser( } val keyName = getOrNull(0) ?: "" if (size == 1) { - return sequenceOf(CsvEntry(keyName, null, null)) + return sequenceOf(CsvEntry(keyName, languageFallback, null)) } val translations = drop(1).asSequence() return translations diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt index 35854e4476..c9c81ac26a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt @@ -23,17 +23,16 @@ class CsvFileProcessor( index: Int, format: ImportFormat, ) { - val selectedLanguage = language ?: firstLanguageTagGuessOrUnknown val converted = format.messageConvertor.convert( value, - selectedLanguage, + language, convertPlaceholders = context.importSettings.convertPlaceholdersToIcu, isProjectIcuEnabled = context.projectIcuPlaceholdersEnabled, ) context.addTranslation( key, - selectedLanguage, + language, converted.message, index, pluralArgName = converted.pluralArgName, @@ -45,7 +44,7 @@ class CsvFileProcessor( private fun parse() = try { // TODO: make delimiter configurable - CsvFileParser(context.file.data.inputStream(), ';').parse() + CsvFileParser(context.file.data.inputStream(), ';', firstLanguageTagGuessOrUnknown).parse() } catch (e: Exception) { throw ImportCannotParseFileException(context.file.name, e.message ?: "", e) } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt new file mode 100644 index 0000000000..6440729dcc --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt @@ -0,0 +1,66 @@ +package io.tolgee.formats.csv.out + +import io.tolgee.dtos.IExportParams +import io.tolgee.formats.ExportMessageFormat +import io.tolgee.formats.csv.CsvEntry +import io.tolgee.formats.generic.IcuToGenericFormatMessageConvertor +import io.tolgee.service.export.dataProvider.ExportTranslationView +import io.tolgee.service.export.exporters.FileExporter +import java.io.InputStream + +class CsvFileExporter( + val translations: List, + val exportParams: IExportParams, + private val isProjectIcuPlaceholdersEnabled: Boolean = true, +) : FileExporter { + private val fileName + get() = exportParams.languages?.takeIf { it.size == 1 }?.iterator()?.next() ?: "exported" + + private val messageFormat + get() = exportParams.messageFormat ?: ExportMessageFormat.ICU + + private val placeholderConvertorFactory + get() = messageFormat.paramConvertorFactory + + val entries = + translations.map { + val converted = convertMessage(it.text, it.key.isPlural) + CsvEntry( + key = it.key.name, + language = it.languageTag, + value = converted, + ) + } + + private fun convertMessage( + text: String?, + isPlural: Boolean, + ): String? { + return getMessageConvertor(text, isPlural).convert() + } + + private fun getMessageConvertor( + text: String?, + isPlural: Boolean, + ) = IcuToGenericFormatMessageConvertor( + text, + isPlural, + isProjectIcuPlaceholdersEnabled = isProjectIcuPlaceholdersEnabled, + paramConvertorFactory = placeholderConvertorFactory, + ) + + override fun produceFiles(): Map { + return mapOf( + "$fileName.csv" to entries.toCsv(), + ) + } + + private fun List.toCsv(): InputStream { + // TODO: make delimiter configurable + return CsvFileWriter( + languageTags = exportParams.languages?.sorted()?.toTypedArray() ?: emptyArray(), + data = entries, + delimiter = ';', + ).produceFiles() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt new file mode 100644 index 0000000000..d03ce57990 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt @@ -0,0 +1,35 @@ +package io.tolgee.formats.csv.out + +import com.opencsv.CSVWriterBuilder +import io.tolgee.formats.csv.CsvEntry +import java.io.InputStream +import java.io.StringWriter + +class CsvFileWriter( + private val languageTags: Array, + private val data: List, + private val delimiter: Char, +) { + val translations: Map> by lazy { + data.groupBy { it.key }.mapValues { (_, values) -> + values.associate { it.language to it.value } + } + } + + fun produceFiles(): InputStream { + val output = StringWriter() + val writer = CSVWriterBuilder(output).withSeparator(delimiter).build() + writer.writeNext( + arrayOf("key") + languageTags, + ) + translations.forEach { + writer.writeNext( + arrayOf(it.key) + + languageTags.map { languageTag -> + it.value.getOrDefault(languageTag, null) ?: "" + }.toTypedArray(), + ) + } + return writer.toString().byteInputStream() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/export/FileExporterFactory.kt b/backend/data/src/main/kotlin/io/tolgee/service/export/FileExporterFactory.kt index e52863ef78..c7eb1d5c92 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/export/FileExporterFactory.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/export/FileExporterFactory.kt @@ -7,6 +7,7 @@ import io.tolgee.formats.ExportFormat import io.tolgee.formats.android.out.AndroidStringsXmlExporter import io.tolgee.formats.apple.out.AppleStringsStringsdictExporter import io.tolgee.formats.apple.out.AppleXliffExporter +import io.tolgee.formats.csv.out.CsvFileExporter import io.tolgee.formats.flutter.out.FlutterArbFileExporter import io.tolgee.formats.genericStructuredFile.out.CustomPrettyPrinter import io.tolgee.formats.json.out.JsonFileExporter @@ -34,6 +35,13 @@ class FileExporterFactory( projectIcuPlaceholdersSupport: Boolean, ): FileExporter { return when (exportParams.format) { + ExportFormat.CSV -> + CsvFileExporter( + data, + exportParams, + projectIcuPlaceholdersSupport, + ) + ExportFormat.JSON, ExportFormat.JSON_TOLGEE, ExportFormat.JSON_I18NEXT -> JsonFileExporter( data, diff --git a/e2e/cypress/common/export.ts b/e2e/cypress/common/export.ts index 788e6bc8ea..a14cab0142 100644 --- a/e2e/cypress/common/export.ts +++ b/e2e/cypress/common/export.ts @@ -224,6 +224,13 @@ export const testExportFormats = ( structureDelimiter: '.', }, }); + + testFormat(interceptFn, submitFn, clearCheckboxesAfter, afterFn, { + format: 'CSV', + expectedParams: { + format: 'CSV', + }, + }); }; const testFormat = ( diff --git a/webapp/src/views/projects/export/components/formatGroups.tsx b/webapp/src/views/projects/export/components/formatGroups.tsx index bb4bd5f897..bc1d8d8afc 100644 --- a/webapp/src/views/projects/export/components/formatGroups.tsx +++ b/webapp/src/views/projects/export/components/formatGroups.tsx @@ -143,6 +143,19 @@ export const formatGroups: FormatGroup[] = [ 'C_SPRINTF', ], }, + { + id: 'generic_csv', + extension: 'csv', + name: , + format: 'CSV', + supportedMessageFormats: [ + 'ICU', + 'JAVA_STRING_FORMAT', + 'PHP_SPRINTF', + 'C_SPRINTF', + 'RUBY_SPRINTF', + ], + }, ], }, { From b0b842e63efb027ecbde1245f43616fdcfd07d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 8 Oct 2024 03:08:47 +0200 Subject: [PATCH 04/18] fead: csv import - delimiter detector --- .../formats/csv/in/CsvDelimiterDetector.kt | 23 +++++++++++++++++++ .../tolgee/formats/csv/in/CsvFileProcessor.kt | 8 +++++-- .../tolgee/formats/csv/out/CsvFileExporter.kt | 3 +-- 3 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt new file mode 100644 index 0000000000..69e9e81643 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt @@ -0,0 +1,23 @@ +package io.tolgee.formats.csv.`in` + +import java.io.ByteArrayInputStream + +class CsvDelimiterDetector(private val inputStream: ByteArrayInputStream) { + companion object { + val DELIMITERS = listOf(',', ';', '\t') + } + + val delimiter by lazy { + val headerLine = inputStream.reader().buffered().lineSequence().firstOrNull() ?: "" + val counts = + DELIMITERS.map { delimiter -> + headerLine.count { it == delimiter } + } + val bestIndex = + counts.foldIndexed(0) { index, maxIndex, value -> + val maxValue = counts[maxIndex] + index.takeIf { value > maxValue } ?: maxIndex + } + DELIMITERS[bestIndex] + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt index c9c81ac26a..9fab52679c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt @@ -43,8 +43,12 @@ class CsvFileProcessor( private fun parse() = try { - // TODO: make delimiter configurable - CsvFileParser(context.file.data.inputStream(), ';', firstLanguageTagGuessOrUnknown).parse() + val detector = CsvDelimiterDetector(context.file.data.inputStream()) + CsvFileParser( + inputStream = context.file.data.inputStream(), + delimiter = detector.delimiter, + languageFallback = firstLanguageTagGuessOrUnknown, + ).parse() } catch (e: Exception) { throw ImportCannotParseFileException(context.file.name, e.message ?: "", e) } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt index 6440729dcc..51be48b137 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt @@ -56,11 +56,10 @@ class CsvFileExporter( } private fun List.toCsv(): InputStream { - // TODO: make delimiter configurable return CsvFileWriter( languageTags = exportParams.languages?.sorted()?.toTypedArray() ?: emptyArray(), data = entries, - delimiter = ';', + delimiter = ',', ).produceFiles() } } From bd52e7b3386fd0db64b4f253bd7f25441ec36b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 8 Oct 2024 03:09:27 +0200 Subject: [PATCH 05/18] fix: formats test --- .../tolgee/api/v2/controllers/ExportInfoControllerTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt index 09afb1bdec..e7ce89dc62 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ExportInfoControllerTest.kt @@ -14,19 +14,19 @@ class ExportInfoControllerTest : AbstractControllerTest() { .andAssertThatJson { node("_embedded.exportFormats") { isArray.hasSizeGreaterThan(5) - node("[0]") { + node("[1]") { node("extension").isEqualTo("json") node("mediaType").isEqualTo("application/json") node("defaultFileStructureTemplate") .isString.isEqualTo("{namespace}/{languageTag}.{extension}") } - node("[1]") { + node("[2]") { node("extension").isEqualTo("json") node("mediaType").isEqualTo("application/json") node("defaultFileStructureTemplate") .isString.isEqualTo("{namespace}/{languageTag}.{extension}") } - node("[5]") { + node("[6]") { node("extension").isEqualTo("") node("mediaType").isEqualTo("") node("defaultFileStructureTemplate") From 3b256172eb9d9fc8a244ca11a5d5ef3f296ebc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 8 Oct 2024 03:24:17 +0200 Subject: [PATCH 06/18] fix: typo --- .../src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt index d03ce57990..01fce234fd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileWriter.kt @@ -30,6 +30,6 @@ class CsvFileWriter( }.toTypedArray(), ) } - return writer.toString().byteInputStream() + return output.toString().byteInputStream() } } From 5eff2fbd9dad898d0eb333214adac3af3afe3e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 8 Oct 2024 04:28:14 +0200 Subject: [PATCH 07/18] feat: add csv import/export tests --- .../tolgee/formats/csv/out/CsvFileExporter.kt | 7 +- .../formats/csv/in/CsvFormatProcessorTest.kt | 254 ++++++++++++++++++ .../formats/csv/out/CsvFileExporterTest.kt | 117 ++++++++ .../src/test/resources/import/csv/example.csv | 6 + .../resources/import/csv/example_params.csv | 3 + .../src/test/resources/import/csv/simple.csv | 3 + 6 files changed, 388 insertions(+), 2 deletions(-) create mode 100644 backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt create mode 100644 backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt create mode 100644 backend/data/src/test/resources/import/csv/example.csv create mode 100644 backend/data/src/test/resources/import/csv/example_params.csv create mode 100644 backend/data/src/test/resources/import/csv/simple.csv diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt index 51be48b137..fbba94f1e5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/out/CsvFileExporter.kt @@ -56,9 +56,12 @@ class CsvFileExporter( } private fun List.toCsv(): InputStream { + val languageTags = + exportParams.languages?.sorted()?.toTypedArray() + ?: this.map { it.language }.distinct().sorted().toTypedArray() return CsvFileWriter( - languageTags = exportParams.languages?.sorted()?.toTypedArray() ?: emptyArray(), - data = entries, + languageTags = languageTags, + data = this, delimiter = ',', ).produceFiles() } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt new file mode 100644 index 0000000000..c05363e47b --- /dev/null +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt @@ -0,0 +1,254 @@ +package io.tolgee.unit.formats.csv.`in` + +import io.tolgee.formats.csv.`in`.CsvFileProcessor +import io.tolgee.testing.assert +import io.tolgee.unit.formats.PlaceholderConversionTestHelper +import io.tolgee.util.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class CsvFormatProcessorTest { + lateinit var mockUtil: FileProcessorContextMockUtil + + @BeforeEach + fun setup() { + mockUtil = FileProcessorContextMockUtil() + } + + // This is how to generate the test: + // 1. run the test in debug mode + // 2. copy the result of calling: + // io.tolgee.unit.util.generateTestsForImportResult(mockUtil.fileProcessorContext) + // from the debug window + @Test + fun `returns correct parsed result`() { + mockUtil.mockIt("example.csv", "src/test/resources/import/csv/example.csv") + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(2) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("value") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "key") + .assertSingle { + hasText("hodnota") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyDeep.inner") + .assertSingle { + hasText("value") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyDeep.inner") + .assertSingle { + hasText("hodnota") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyInterpolate") + .assertSingle { + hasText("replace this {value}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyInterpolate") + .assertSingle { + hasText("nahradit toto {value}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyInterpolateWithFormatting") + .assertSingle { + hasText("replace this {value, number}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyInterpolateWithFormatting") + .assertSingle { + hasText("nahradit toto {value, number}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyPluralSimple") + .assertSinglePlural { + hasText( + """ + {value, plural, + one { the singular} + other { the plural {value}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyPluralSimple") + .assertSinglePlural { + hasText( + """ + {value, plural, + one { jednotné číslo} + other { množné číslo {value}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("keyPluralSimple") { + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `import with placeholder conversion (disabled ICU)`() { + mockPlaceholderConversionTestFile(convertPlaceholders = false, projectIcuPlaceholdersEnabled = false) + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(2) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("Hello {icuPara}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "key") + .assertSingle { + hasText("Ahoj {icuPara}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "plural") + .assertSinglePlural { + hasText( + """ + {icuPara, plural, + one {Hello one '#' '{'icuParam'}'} + other {Hello other '{'icuParam'}'} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("cs", "plural") + .assertSinglePlural { + hasText( + """ + {icuPara, plural, + one {Ahoj jedno '#' '{'icuParam'}'} + other {Ahoj jiné '{'icuParam'}'} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("plural") { + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `import with placeholder conversion (no conversion)`() { + mockPlaceholderConversionTestFile(convertPlaceholders = false, projectIcuPlaceholdersEnabled = true) + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(2) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("Hello {icuPara}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "key") + .assertSingle { + hasText("Ahoj {icuPara}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "plural") + .assertSinglePlural { + hasText( + """ + {icuPara, plural, + one {Hello one # {icuParam}} + other {Hello other {icuParam}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("cs", "plural") + .assertSinglePlural { + hasText( + """ + {icuPara, plural, + one {Ahoj jedno # {icuParam}} + other {Ahoj jiné {icuParam}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("plural") { + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `import with placeholder conversion (with conversion)`() { + mockPlaceholderConversionTestFile(convertPlaceholders = true, projectIcuPlaceholdersEnabled = true) + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(2) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("Hello {icuPara}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "key") + .assertSingle { + hasText("Ahoj {icuPara}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "plural") + .assertSinglePlural { + hasText( + """ + {icuPara, plural, + one {Hello one # {icuParam}} + other {Hello other {icuParam}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("cs", "plural") + .assertSinglePlural { + hasText( + """ + {icuPara, plural, + one {Ahoj jedno # {icuParam}} + other {Ahoj jiné {icuParam}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("plural") { + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `placeholder conversion setting application works`() { + // FIXME: Is this correct? + PlaceholderConversionTestHelper.testFile( + "import.csv", + "src/test/resources/import/csv/simple.csv", + assertBeforeSettingsApplication = + listOf( + "this is csv {count}", + "this is csv", + "toto je csv {count}", + "toto je csv", + ), + assertAfterDisablingConversion = + listOf(), + assertAfterReEnablingConversion = + listOf(), + ) + } + + private fun mockPlaceholderConversionTestFile( + convertPlaceholders: Boolean, + projectIcuPlaceholdersEnabled: Boolean, + ) { + mockUtil.mockIt( + "import.csv", + "src/test/resources/import/csv/example_params.csv", + convertPlaceholders, + projectIcuPlaceholdersEnabled, + ) + } + + private fun processFile() { + CsvFileProcessor(mockUtil.fileProcessorContext).process() + } +} diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt new file mode 100644 index 0000000000..0a5f947fde --- /dev/null +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt @@ -0,0 +1,117 @@ +package io.tolgee.unit.formats.csv.out + +import io.tolgee.dtos.request.export.ExportParams +import io.tolgee.formats.csv.out.CsvFileExporter +import io.tolgee.service.export.dataProvider.ExportTranslationView +import io.tolgee.unit.util.assertFile +import io.tolgee.unit.util.getExported +import io.tolgee.util.buildExportTranslationList +import org.junit.jupiter.api.Test + +class CsvFileExporterTest { + @Test + fun `exports with placeholders (ICU placeholders disabled)`() { + val exporter = getIcuPlaceholdersDisabledExporter() + val data = getExported(exporter) + data.assertFile( + "exported.csv", + """ + |"key","cs" + |"key3","{count, plural, one {# den {icuParam}} few {# dny} other {# dní}}" + |"item","I will be first {icuParam, number}" + | + """.trimMargin(), + ) + } + + private fun getIcuPlaceholdersDisabledExporter(): CsvFileExporter { + val built = + buildExportTranslationList { + add( + languageTag = "cs", + keyName = "key3", + text = "{count, plural, one {'#' den '{'icuParam'}'} few {'#' dny} other {'#' dní}}", + ) { + key.isPlural = true + } + add( + languageTag = "cs", + keyName = "item", + text = "I will be first {icuParam, number}", + ) + } + return getExporter(built.translations, false) + } + + @Test + fun `exports with placeholders (ICU placeholders enabled)`() { + val exporter = getIcuPlaceholdersEnabledExporter() + val data = getExported(exporter) + data.assertFile( + "exported.csv", + """ + |"key","cs" + |"key3","{count, plural, one {# den {icuParam, number}} few {# dny} other {# dní}}" + |"item","I will be first '{'icuParam'}' {hello, number}" + | + """.trimMargin(), + ) + } + + @Test + fun `correct exports translation with colon`() { + val exporter = getExporter(getTranslationWithColon()) + val data = getExported(exporter) + data.assertFile( + "exported.csv", + """ + |"key","cs" + |"item","name : {name}" + | + """.trimMargin(), + ) + } + + private fun getTranslationWithColon(): MutableList { + val built = + buildExportTranslationList { + add( + languageTag = "cs", + keyName = "item", + text = "name : {name}", + ) + } + return built.translations + } + + private fun getIcuPlaceholdersEnabledExporter(): CsvFileExporter { + val built = + buildExportTranslationList { + add( + languageTag = "cs", + keyName = "key3", + text = "{count, plural, one {# den {icuParam, number}} few {# dny} other {# dní}}", + ) { + key.isPlural = true + } + add( + languageTag = "cs", + keyName = "item", + text = "I will be first '{'icuParam'}' {hello, number}", + ) + } + return getExporter(built.translations, true) + } + + private fun getExporter( + translations: List, + isProjectIcuPlaceholdersEnabled: Boolean = true, + exportParams: ExportParams = ExportParams(), + ): CsvFileExporter { + return CsvFileExporter( + translations = translations, + exportParams = exportParams, + isProjectIcuPlaceholdersEnabled = isProjectIcuPlaceholdersEnabled, + ) + } +} diff --git a/backend/data/src/test/resources/import/csv/example.csv b/backend/data/src/test/resources/import/csv/example.csv new file mode 100644 index 0000000000..9e488b8865 --- /dev/null +++ b/backend/data/src/test/resources/import/csv/example.csv @@ -0,0 +1,6 @@ +"key","en","cs" +"key","value","hodnota" +"keyDeep.inner","value","hodnota" +"keyInterpolate","replace this {value}","nahradit toto {value}" +"keyInterpolateWithFormatting","replace this {value, number}","nahradit toto {value, number}" +"keyPluralSimple"," {value, plural, one {the singular} other {the plural {value}}}"," {value, plural, one {jednotné číslo} other {množné číslo {value}}}" diff --git a/backend/data/src/test/resources/import/csv/example_params.csv b/backend/data/src/test/resources/import/csv/example_params.csv new file mode 100644 index 0000000000..1f794d5e63 --- /dev/null +++ b/backend/data/src/test/resources/import/csv/example_params.csv @@ -0,0 +1,3 @@ +"key","en","cs" +"key","Hello {icuPara}","Ahoj {icuPara}" +"plural","{icuPara, plural, one {Hello one # {icuParam}} other {Hello other {icuParam}}}","{icuPara, plural, one {Ahoj jedno # {icuParam}} other {Ahoj jiné {icuParam}}}" diff --git a/backend/data/src/test/resources/import/csv/simple.csv b/backend/data/src/test/resources/import/csv/simple.csv new file mode 100644 index 0000000000..c17d53b4f4 --- /dev/null +++ b/backend/data/src/test/resources/import/csv/simple.csv @@ -0,0 +1,3 @@ +"key","en","cs" +"key","this is csv {count}","toto je csv {count}" +"key2","this is csv","toto je csv" From df842af98cdc05c42509d2e6352dade4f1f15bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 8 Oct 2024 04:29:12 +0200 Subject: [PATCH 08/18] fix: export e2e csv test --- e2e/cypress/common/export.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/e2e/cypress/common/export.ts b/e2e/cypress/common/export.ts index a14cab0142..a0622d3999 100644 --- a/e2e/cypress/common/export.ts +++ b/e2e/cypress/common/export.ts @@ -225,12 +225,15 @@ export const testExportFormats = ( }, }); - testFormat(interceptFn, submitFn, clearCheckboxesAfter, afterFn, { - format: 'CSV', - expectedParams: { + testFormatWithMessageFormats( + ['ICU', 'PHP Sprintf', 'C Sprintf', 'Java String.format'], + { format: 'CSV', - }, - }); + expectedParams: { + format: 'CSV', + }, + } + ); }; const testFormat = ( From c157e8666aa47d78911bece15d262a61471e3929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 8 Oct 2024 04:38:38 +0200 Subject: [PATCH 09/18] feat: add tests for non-standard csv delimiters --- .../formats/csv/in/CsvFormatProcessorTest.kt | 134 ++++++++++++++++++ .../import/csv/example_semicolon.csv | 6 + .../test/resources/import/csv/example_tab.csv | 6 + 3 files changed, 146 insertions(+) create mode 100644 backend/data/src/test/resources/import/csv/example_semicolon.csv create mode 100644 backend/data/src/test/resources/import/csv/example_tab.csv diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt index c05363e47b..faa5274507 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt @@ -87,6 +87,140 @@ class CsvFormatProcessorTest { } } + @Test + fun `returns correct parsed result (semicolon delimiter)`() { + mockUtil.mockIt("example.csv", "src/test/resources/import/csv/example_semicolon.csv") + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(2) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("value") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "key") + .assertSingle { + hasText("hodnota") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyDeep.inner") + .assertSingle { + hasText("value") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyDeep.inner") + .assertSingle { + hasText("hodnota") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyInterpolate") + .assertSingle { + hasText("replace this {value}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyInterpolate") + .assertSingle { + hasText("nahradit toto {value}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyInterpolateWithFormatting") + .assertSingle { + hasText("replace this {value, number}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyInterpolateWithFormatting") + .assertSingle { + hasText("nahradit toto {value, number}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyPluralSimple") + .assertSinglePlural { + hasText( + """ + {value, plural, + one { the singular} + other { the plural {value}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyPluralSimple") + .assertSinglePlural { + hasText( + """ + {value, plural, + one { jednotné číslo} + other { množné číslo {value}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("keyPluralSimple") { + custom.assert.isNull() + description.assert.isNull() + } + } + + @Test + fun `returns correct parsed result (tab delimiter)`() { + mockUtil.mockIt("example.csv", "src/test/resources/import/csv/example_tab.csv") + processFile() + mockUtil.fileProcessorContext.assertLanguagesCount(2) + mockUtil.fileProcessorContext.assertTranslations("en", "key") + .assertSingle { + hasText("value") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "key") + .assertSingle { + hasText("hodnota") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyDeep.inner") + .assertSingle { + hasText("value") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyDeep.inner") + .assertSingle { + hasText("hodnota") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyInterpolate") + .assertSingle { + hasText("replace this {value}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyInterpolate") + .assertSingle { + hasText("nahradit toto {value}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyInterpolateWithFormatting") + .assertSingle { + hasText("replace this {value, number}") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyInterpolateWithFormatting") + .assertSingle { + hasText("nahradit toto {value, number}") + } + mockUtil.fileProcessorContext.assertTranslations("en", "keyPluralSimple") + .assertSinglePlural { + hasText( + """ + {value, plural, + one { the singular} + other { the plural {value}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertTranslations("cs", "keyPluralSimple") + .assertSinglePlural { + hasText( + """ + {value, plural, + one { jednotné číslo} + other { množné číslo {value}} + } + """.trimIndent(), + ) + isPluralOptimized() + } + mockUtil.fileProcessorContext.assertKey("keyPluralSimple") { + custom.assert.isNull() + description.assert.isNull() + } + } + @Test fun `import with placeholder conversion (disabled ICU)`() { mockPlaceholderConversionTestFile(convertPlaceholders = false, projectIcuPlaceholdersEnabled = false) diff --git a/backend/data/src/test/resources/import/csv/example_semicolon.csv b/backend/data/src/test/resources/import/csv/example_semicolon.csv new file mode 100644 index 0000000000..34cfe6099b --- /dev/null +++ b/backend/data/src/test/resources/import/csv/example_semicolon.csv @@ -0,0 +1,6 @@ +"key";"en";"cs" +"key";"value";"hodnota" +"keyDeep.inner";"value";"hodnota" +"keyInterpolate";"replace this {value}";"nahradit toto {value}" +"keyInterpolateWithFormatting";"replace this {value, number}";"nahradit toto {value, number}" +"keyPluralSimple";" {value, plural, one {the singular} other {the plural {value}}}";" {value, plural, one {jednotné číslo} other {množné číslo {value}}}" diff --git a/backend/data/src/test/resources/import/csv/example_tab.csv b/backend/data/src/test/resources/import/csv/example_tab.csv new file mode 100644 index 0000000000..7ab151c2c0 --- /dev/null +++ b/backend/data/src/test/resources/import/csv/example_tab.csv @@ -0,0 +1,6 @@ +"key" "en" "cs" +"key" "value" "hodnota" +"keyDeep.inner" "value" "hodnota" +"keyInterpolate" "replace this {value}" "nahradit toto {value}" +"keyInterpolateWithFormatting" "replace this {value, number}" "nahradit toto {value, number}" +"keyPluralSimple" " {value, plural, one {the singular} other {the plural {value}}}" " {value, plural, one {jednotné číslo} other {množné číslo {value}}}" From 3a76eed630f13c7384c9e52cd497f6664c18e053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Tue, 8 Oct 2024 04:45:17 +0200 Subject: [PATCH 10/18] fix: missing format in e2e test --- e2e/cypress/common/export.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cypress/common/export.ts b/e2e/cypress/common/export.ts index a0622d3999..5dfb86a6a7 100644 --- a/e2e/cypress/common/export.ts +++ b/e2e/cypress/common/export.ts @@ -226,7 +226,7 @@ export const testExportFormats = ( }); testFormatWithMessageFormats( - ['ICU', 'PHP Sprintf', 'C Sprintf', 'Java String.format'], + ['ICU', 'PHP Sprintf', 'C Sprintf', 'Ruby Sprintf', 'Java String.format'], { format: 'CSV', expectedParams: { From 5abb2d56723cb7a93fd728b7956aa0c792c6f789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Wed, 9 Oct 2024 10:52:28 +0200 Subject: [PATCH 11/18] feat: proper CSV logo --- webapp/src/svgs/logos/csv.svg | 3 +++ .../projects/import/component/ImportSupportedFormats.tsx | 8 ++------ 2 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 webapp/src/svgs/logos/csv.svg diff --git a/webapp/src/svgs/logos/csv.svg b/webapp/src/svgs/logos/csv.svg new file mode 100644 index 0000000000..92167f0266 --- /dev/null +++ b/webapp/src/svgs/logos/csv.svg @@ -0,0 +1,3 @@ + + + diff --git a/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx b/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx index 1e38b7e6d7..24ca9877ec 100644 --- a/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx +++ b/webapp/src/views/projects/import/component/ImportSupportedFormats.tsx @@ -11,6 +11,7 @@ import AndroidLogo from 'tg.svgs/logos/android.svg?react'; import FluttrerLogo from 'tg.svgs/logos/flutter.svg?react'; import RailsLogo from 'tg.svgs/logos/rails.svg?react'; import I18nextLogo from 'tg.svgs/logos/i18next.svg?react'; +import CsvLogo from 'tg.svgs/logos/csv.svg?react'; const TechLogo = ({ svg, @@ -55,12 +56,7 @@ const FORMATS = [ { name: 'Flutter ARB', logo: }, { name: 'Ruby YAML', logo: }, { name: 'i18next', logo: }, - { - name: 'CSV', - logo: , // TODO: replace with proper icon? - logoHeight: '24px', - logoWidth: '24px', - }, + { name: 'CSV', logo: }, ]; export const ImportSupportedFormats = () => { From d3d5aec912de9197d56540f687c2cf7e623c1d32 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Wed, 9 Oct 2024 13:48:21 +0200 Subject: [PATCH 12/18] chore: Break the tests --- .../formats/csv/in/CsvFormatProcessorTest.kt | 21 +++++++++++++++---- .../import/csv/placeholder_conversion.csv | 3 +++ 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 backend/data/src/test/resources/import/csv/placeholder_conversion.csv diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt index faa5274507..29e2f7a57d 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt @@ -3,7 +3,14 @@ package io.tolgee.unit.formats.csv.`in` import io.tolgee.formats.csv.`in`.CsvFileProcessor import io.tolgee.testing.assert import io.tolgee.unit.formats.PlaceholderConversionTestHelper -import io.tolgee.util.* +import io.tolgee.util.FileProcessorContextMockUtil +import io.tolgee.util.assertKey +import io.tolgee.util.assertLanguagesCount +import io.tolgee.util.assertSingle +import io.tolgee.util.assertSinglePlural +import io.tolgee.util.assertTranslations +import io.tolgee.util.custom +import io.tolgee.util.description import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -355,7 +362,7 @@ class CsvFormatProcessorTest { // FIXME: Is this correct? PlaceholderConversionTestHelper.testFile( "import.csv", - "src/test/resources/import/csv/simple.csv", + "src/test/resources/import/csv/placeholder_conversion.csv", assertBeforeSettingsApplication = listOf( "this is csv {count}", @@ -364,9 +371,15 @@ class CsvFormatProcessorTest { "toto je csv", ), assertAfterDisablingConversion = - listOf(), + listOf( + "this is csv %d", + "toto je csv %d", + ), assertAfterReEnablingConversion = - listOf(), + listOf( + "this is csv {count}", + "toto je csv {count}", + ), ) } diff --git a/backend/data/src/test/resources/import/csv/placeholder_conversion.csv b/backend/data/src/test/resources/import/csv/placeholder_conversion.csv new file mode 100644 index 0000000000..9ede1af17d --- /dev/null +++ b/backend/data/src/test/resources/import/csv/placeholder_conversion.csv @@ -0,0 +1,3 @@ +"key","en","cs" +"key","this is csv %d","toto je csv %d" +"key2","this is csv","toto je csv" From ee9523eaf30232dee23b9f387f7c3f5ab02b23e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Wed, 9 Oct 2024 14:57:39 +0200 Subject: [PATCH 13/18] fix: message format detecion --- .../io/tolgee/formats/csv/in/CsvFileProcessor.kt | 13 ++++++++----- .../unit/formats/csv/in/CsvFormatProcessorTest.kt | 9 ++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt index 9fab52679c..ce5f021be2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt @@ -10,8 +10,7 @@ class CsvFileProcessor( override val context: FileProcessorContext, ) : ImportFileProcessor() { override fun process() { - val data = parse() - val format = getFormat(data) + val (data, format) = parse() data.importAll(format) } @@ -41,17 +40,21 @@ class CsvFileProcessor( ) } - private fun parse() = + private fun parse(): Pair, ImportFormat> { try { val detector = CsvDelimiterDetector(context.file.data.inputStream()) - CsvFileParser( + val parser = CsvFileParser( inputStream = context.file.data.inputStream(), delimiter = detector.delimiter, languageFallback = firstLanguageTagGuessOrUnknown, - ).parse() + ) + val data = parser.parse() + val format = getFormat(parser.rows) + return data to format } catch (e: Exception) { throw ImportCannotParseFileException(context.file.name, e.message ?: "", e) } + } private fun getFormat(data: Any?): ImportFormat { return context.mapping?.format ?: CSVImportFormatDetector().detectFormat(data) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt index 29e2f7a57d..06234c4fb0 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt @@ -359,15 +359,14 @@ class CsvFormatProcessorTest { @Test fun `placeholder conversion setting application works`() { - // FIXME: Is this correct? PlaceholderConversionTestHelper.testFile( "import.csv", "src/test/resources/import/csv/placeholder_conversion.csv", assertBeforeSettingsApplication = listOf( - "this is csv {count}", + "this is csv {0, number}", "this is csv", - "toto je csv {count}", + "toto je csv {0, number}", "toto je csv", ), assertAfterDisablingConversion = @@ -377,8 +376,8 @@ class CsvFormatProcessorTest { ), assertAfterReEnablingConversion = listOf( - "this is csv {count}", - "toto je csv {count}", + "this is csv {0, number}", + "toto je csv {0, number}", ), ) } From 92fc95020b7f26e87f5ed33d2345a806ea6b3a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Wed, 9 Oct 2024 15:06:55 +0200 Subject: [PATCH 14/18] fix: lint --- .../io/tolgee/formats/csv/in/CsvFileProcessor.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt index ce5f021be2..3f251a556f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileProcessor.kt @@ -43,11 +43,12 @@ class CsvFileProcessor( private fun parse(): Pair, ImportFormat> { try { val detector = CsvDelimiterDetector(context.file.data.inputStream()) - val parser = CsvFileParser( - inputStream = context.file.data.inputStream(), - delimiter = detector.delimiter, - languageFallback = firstLanguageTagGuessOrUnknown, - ) + val parser = + CsvFileParser( + inputStream = context.file.data.inputStream(), + delimiter = detector.delimiter, + languageFallback = firstLanguageTagGuessOrUnknown, + ) val data = parser.parse() val format = getFormat(parser.rows) return data to format From 2222b445b251a5506d1f41bbe7c8dd9a57ae4e2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Wed, 9 Oct 2024 16:05:49 +0200 Subject: [PATCH 15/18] feat: more csv tests --- .../formats/csv/in/CsvDelimiterDetector.kt | 4 +- .../io/tolgee/formats/csv/in/CsvFileParser.kt | 4 +- .../formats/csv/in/CsvFormatProcessorTest.kt | 32 ++++++++++ .../csv/in/CsvImportFormatDetectorTest.kt | 58 +++++++++++++++++++ .../formats/csv/out/CsvFileExporterTest.kt | 7 +++ .../json/in/JsonImportFormatDetectorTest.kt | 3 +- .../io/tolgee/unit/util/testGenerationUtil.kt | 2 +- .../src/test/resources/import/csv/example.csv | 10 +++- .../src/test/resources/import/csv/icu.csv | 2 + .../src/test/resources/import/csv/java.csv | 3 + .../src/test/resources/import/csv/php.csv | 2 + .../src/test/resources/import/csv/unknown.csv | 2 + 12 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvImportFormatDetectorTest.kt create mode 100644 backend/data/src/test/resources/import/csv/icu.csv create mode 100644 backend/data/src/test/resources/import/csv/java.csv create mode 100644 backend/data/src/test/resources/import/csv/php.csv create mode 100644 backend/data/src/test/resources/import/csv/unknown.csv diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt index 69e9e81643..bea4aea80b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt @@ -1,8 +1,8 @@ package io.tolgee.formats.csv.`in` -import java.io.ByteArrayInputStream +import java.io.InputStream -class CsvDelimiterDetector(private val inputStream: ByteArrayInputStream) { +class CsvDelimiterDetector(private val inputStream: InputStream) { companion object { val DELIMITERS = listOf(',', ';', '\t') } diff --git a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt index b9d9476311..1a5de42bd2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt +++ b/backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt @@ -3,10 +3,10 @@ package io.tolgee.formats.csv.`in` import com.opencsv.CSVParserBuilder import com.opencsv.CSVReaderBuilder import io.tolgee.formats.csv.CsvEntry -import java.io.ByteArrayInputStream +import java.io.InputStream class CsvFileParser( - private val inputStream: ByteArrayInputStream, + private val inputStream: InputStream, private val delimiter: Char, private val languageFallback: String, ) { diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt index 06234c4fb0..88e085cfd5 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt @@ -88,6 +88,38 @@ class CsvFormatProcessorTest { ) isPluralOptimized() } + mockUtil.fileProcessorContext.assertTranslations("en", "escapedCharacters") + .assertSingle { + hasText("this is a \"quote\"") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "escapedCharacters") + .assertSingle { + hasText("toto je \"citace\"") + } + mockUtil.fileProcessorContext.assertTranslations("en", "escapedCharacters2") + .assertSingle { + hasText("this is a\nnew line") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "escapedCharacters2") + .assertSingle { + hasText("toto je\nnový řádek") + } + mockUtil.fileProcessorContext.assertTranslations("en", "escapedCharacters3") + .assertSingle { + hasText("this is a \\ backslash") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "escapedCharacters3") + .assertSingle { + hasText("toto je zpětné \\ lomítko") + } + mockUtil.fileProcessorContext.assertTranslations("en", "escapedCharacters4") + .assertSingle { + hasText("this is a , comma") + } + mockUtil.fileProcessorContext.assertTranslations("cs", "escapedCharacters4") + .assertSingle { + hasText("toto je , čárka") + } mockUtil.fileProcessorContext.assertKey("keyPluralSimple") { custom.assert.isNull() description.assert.isNull() diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvImportFormatDetectorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvImportFormatDetectorTest.kt new file mode 100644 index 0000000000..b83cda08e7 --- /dev/null +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvImportFormatDetectorTest.kt @@ -0,0 +1,58 @@ +package io.tolgee.unit.formats.csv.`in` + +import io.tolgee.formats.csv.`in`.CSVImportFormatDetector +import io.tolgee.formats.csv.`in`.CsvFileParser +import io.tolgee.formats.importCommon.ImportFormat +import io.tolgee.testing.assert +import io.tolgee.util.FileProcessorContextMockUtil +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.io.File + +class CsvImportFormatDetectorTest { + lateinit var mockUtil: FileProcessorContextMockUtil + + @BeforeEach + fun setup() { + mockUtil = FileProcessorContextMockUtil() + } + + @Test + fun `detected i18next`() { + "src/test/resources/import/csv/example.csv".assertDetected(ImportFormat.CSV_ICU) + } + + @Test + fun `detected icu`() { + "src/test/resources/import/csv/icu.csv".assertDetected(ImportFormat.CSV_ICU) + } + + @Test + fun `detected java`() { + "src/test/resources/import/csv/java.csv".assertDetected(ImportFormat.CSV_JAVA) + } + + @Test + fun `detected php`() { + "src/test/resources/import/csv/php.csv".assertDetected(ImportFormat.CSV_PHP) + } + + @Test + fun `fallbacks to icu`() { + "src/test/resources/import/csv/unknown.csv".assertDetected(ImportFormat.CSV_ICU) + } + + private fun parseFile(path: String): Any? { + val parser = + CsvFileParser( + inputStream = File(path).inputStream(), + delimiter = ',', + languageFallback = "unknown", + ) + return parser.rows + } + + private fun String.assertDetected(format: ImportFormat) { + CSVImportFormatDetector().detectFormat(parseFile(this)).assert.isEqualTo(format) + } +} diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt index 0a5f947fde..b3b0ad12cd 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt @@ -19,6 +19,8 @@ class CsvFileExporterTest { |"key","cs" |"key3","{count, plural, one {# den {icuParam}} few {# dny} other {# dní}}" |"item","I will be first {icuParam, number}" + |"key","Text with multiple lines + |and , commas and \"quotes\"" | """.trimMargin(), ) @@ -39,6 +41,11 @@ class CsvFileExporterTest { keyName = "item", text = "I will be first {icuParam, number}", ) + add( + languageTag = "cs", + keyName = "key", + text = "Text with multiple lines\nand , commas and \"quotes\"", + ) } return getExporter(built.translations, false) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt index c3c4684625..2100fd9ff5 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt @@ -2,6 +2,7 @@ package io.tolgee.unit.formats.json.`in` import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import io.tolgee.formats.importCommon.ImportFormat import io.tolgee.formats.json.`in`.JsonImportFormatDetector @@ -45,7 +46,7 @@ class JsonImportFormatDetectorTest { } private fun parseFile(path: String): Map<*, *> { - return ObjectMapper(YAMLFactory()).readValue>( + return jacksonObjectMapper().readValue>( File(path) .readBytes(), ) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/util/testGenerationUtil.kt b/backend/data/src/test/kotlin/io/tolgee/unit/util/testGenerationUtil.kt index d8de1c9f81..02d307477c 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/util/testGenerationUtil.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/util/testGenerationUtil.kt @@ -21,7 +21,7 @@ fun generateTestsForImportResult(fileProcessorContext: FileProcessorContext): St code.appendLine("""${i(indent)}${"\"\"\""}.trimIndent()""") } val escape = { str: String?, newLines: Boolean -> - str?.replace("\"", "\\\"").let { + str?.replace("\\", "\\\\")?.replace("\"", "\\\"").let { if (newLines) { return@let it?.replace("\n", "\\n") } diff --git a/backend/data/src/test/resources/import/csv/example.csv b/backend/data/src/test/resources/import/csv/example.csv index 9e488b8865..c8d7ac5ced 100644 --- a/backend/data/src/test/resources/import/csv/example.csv +++ b/backend/data/src/test/resources/import/csv/example.csv @@ -1,6 +1,12 @@ "key","en","cs" -"key","value","hodnota" +"key","value",hodnota "keyDeep.inner","value","hodnota" -"keyInterpolate","replace this {value}","nahradit toto {value}" +"keyInterpolate","replace this {value}",nahradit toto {value} "keyInterpolateWithFormatting","replace this {value, number}","nahradit toto {value, number}" "keyPluralSimple"," {value, plural, one {the singular} other {the plural {value}}}"," {value, plural, one {jednotné číslo} other {množné číslo {value}}}" +"escapedCharacters","this is a \"quote\"","toto je \"citace\"" +"escapedCharacters2","this is a +new line","toto je +nový řádek" +"escapedCharacters3","this is a \\ backslash","toto je zpětné \\ lomítko" +"escapedCharacters4","this is a , comma","toto je , čárka" \ No newline at end of file diff --git a/backend/data/src/test/resources/import/csv/icu.csv b/backend/data/src/test/resources/import/csv/icu.csv new file mode 100644 index 0000000000..4e5dbf8872 --- /dev/null +++ b/backend/data/src/test/resources/import/csv/icu.csv @@ -0,0 +1,2 @@ +key,en +key,{param} \ No newline at end of file diff --git a/backend/data/src/test/resources/import/csv/java.csv b/backend/data/src/test/resources/import/csv/java.csv new file mode 100644 index 0000000000..29799edbd3 --- /dev/null +++ b/backend/data/src/test/resources/import/csv/java.csv @@ -0,0 +1,3 @@ +key,en +"key","%D this is java %d" +"key2","%D this is java" \ No newline at end of file diff --git a/backend/data/src/test/resources/import/csv/php.csv b/backend/data/src/test/resources/import/csv/php.csv new file mode 100644 index 0000000000..ac696803bd --- /dev/null +++ b/backend/data/src/test/resources/import/csv/php.csv @@ -0,0 +1,2 @@ +key,en +key,"%'s with ' flag" \ No newline at end of file diff --git a/backend/data/src/test/resources/import/csv/unknown.csv b/backend/data/src/test/resources/import/csv/unknown.csv new file mode 100644 index 0000000000..9c711ab70c --- /dev/null +++ b/backend/data/src/test/resources/import/csv/unknown.csv @@ -0,0 +1,2 @@ +key,en +key,This suits php and java \ No newline at end of file From 1064508eaecb39119fc1279b522bda04656e0b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Wed, 9 Oct 2024 16:14:31 +0200 Subject: [PATCH 16/18] fix: lint --- .../tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt index 2100fd9ff5..6f60dda76b 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt @@ -1,7 +1,5 @@ package io.tolgee.unit.formats.json.`in` -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import io.tolgee.formats.importCommon.ImportFormat From efbfb14afdcdb2fb3c0f7c1b4f5fea5442ff40ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Wed, 9 Oct 2024 16:16:41 +0200 Subject: [PATCH 17/18] fix: update schema --- webapp/src/service/apiSchema.generated.ts | 121 ++++++++++++---------- 1 file changed, 65 insertions(+), 56 deletions(-) diff --git a/webapp/src/service/apiSchema.generated.ts b/webapp/src/service/apiSchema.generated.ts index 9a810ebd41..2a6febfd4e 100644 --- a/webapp/src/service/apiSchema.generated.ts +++ b/webapp/src/service/apiSchema.generated.ts @@ -1168,6 +1168,29 @@ export interface components { | "SERVER_ADMIN"; /** @description The user's permission type. This field is null if uses granular permissions */ type?: "NONE" | "VIEW" | "TRANSLATE" | "REVIEW" | "EDIT" | "MANAGE"; + /** + * @description List of languages user can translate to. If null, all languages editing is permitted. + * @example 200001,200004 + */ + translateLanguageIds?: number[]; + /** + * @description List of languages user can change state to. If null, changing state of all language values is permitted. + * @example 200001,200004 + */ + stateChangeLanguageIds?: number[]; + /** + * @deprecated + * @description Deprecated (use translateLanguageIds). + * + * List of languages current user has TRANSLATE permission to. If null, all languages edition is permitted. + * @example 200001,200004 + */ + permittedLanguageIds?: number[]; + /** + * @description List of languages user can view. If null, all languages view is permitted. + * @example 200001,200004 + */ + viewLanguageIds?: number[]; /** * @description Granted scopes to the user. When user has type permissions, this field contains permission scopes of the type. * @example KEYS_EDIT,TRANSLATIONS_VIEW @@ -1202,29 +1225,6 @@ export interface components { | "tasks.view" | "tasks.edit" )[]; - /** - * @description List of languages user can view. If null, all languages view is permitted. - * @example 200001,200004 - */ - viewLanguageIds?: number[]; - /** - * @description List of languages user can translate to. If null, all languages editing is permitted. - * @example 200001,200004 - */ - translateLanguageIds?: number[]; - /** - * @description List of languages user can change state to. If null, changing state of all language values is permitted. - * @example 200001,200004 - */ - stateChangeLanguageIds?: number[]; - /** - * @deprecated - * @description Deprecated (use translateLanguageIds). - * - * List of languages current user has TRANSLATE permission to. If null, all languages edition is permitted. - * @example 200001,200004 - */ - permittedLanguageIds?: number[]; }; LanguageModel: { /** Format: int64 */ @@ -1814,8 +1814,8 @@ export interface components { secretKey?: string; endpoint: string; signingRegion: string; - enabled?: boolean; contentStorageType?: "S3" | "AZURE"; + enabled?: boolean; }; AzureContentStorageConfigModel: { containerName?: string; @@ -1865,6 +1865,7 @@ export interface components { languages?: string[]; /** @description Format to export to */ format: + | "CSV" | "JSON" | "JSON_TOLGEE" | "JSON_I18NEXT" @@ -1964,6 +1965,7 @@ export interface components { languages?: string[]; /** @description Format to export to */ format: + | "CSV" | "JSON" | "JSON_TOLGEE" | "JSON_I18NEXT" @@ -2087,12 +2089,12 @@ export interface components { createNewKeys: boolean; }; ImportSettingsModel: { - /** @description If false, only updates keys, skipping the creation of new keys */ - createNewKeys: boolean; /** @description If true, placeholders from other formats will be converted to ICU when possible */ convertPlaceholdersToIcu: boolean; /** @description If true, key descriptions will be overridden by the import */ overrideKeyDescriptions: boolean; + /** @description If false, only updates keys, skipping the creation of new keys */ + createNewKeys: boolean; }; TranslationCommentModel: { /** @@ -2249,17 +2251,17 @@ export interface components { }; RevealedPatModel: { token: string; - description: string; /** Format: int64 */ id: number; - /** Format: int64 */ - lastUsedAt?: number; - /** Format: int64 */ - expiresAt?: number; + description: string; /** Format: int64 */ createdAt: number; /** Format: int64 */ updatedAt: number; + /** Format: int64 */ + expiresAt?: number; + /** Format: int64 */ + lastUsedAt?: number; }; SetOrganizationRoleDto: { roleType: "MEMBER" | "OWNER"; @@ -2396,19 +2398,19 @@ export interface components { RevealedApiKeyModel: { /** @description Resulting user's api key */ key: string; - description: string; /** Format: int64 */ id: number; + userFullName?: string; projectName: string; + description: string; username?: string; - /** Format: int64 */ - lastUsedAt?: number; scopes: string[]; /** Format: int64 */ + projectId: number; + /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - projectId: number; - userFullName?: string; + lastUsedAt?: number; }; SuperTokenRequest: { /** @description Has to be provided when TOTP enabled */ @@ -2951,6 +2953,10 @@ export interface components { * It is recommended to provide these values to prevent any issues with format detection. */ format?: + | "CSV_ICU" + | "CSV_JAVA" + | "CSV_PHP" + | "CSV_RUBY" | "JSON_I18NEXT" | "JSON_ICU" | "JSON_JAVA" @@ -3100,6 +3106,7 @@ export interface components { languages?: string[]; /** @description Format to export to */ format: + | "CSV" | "JSON" | "JSON_TOLGEE" | "JSON_I18NEXT" @@ -3571,15 +3578,10 @@ export interface components { | "TASKS" )[]; quickStart?: components["schemas"]["QuickStartModel"]; - /** @example This is a beautiful organization full of beautiful and clever people */ - description?: string; /** @example Beautiful organization */ name: string; /** Format: int64 */ id: number; - avatar?: components["schemas"]["Avatar"]; - /** @example btforg */ - slug: string; /** * @description The role of currently authorized user. * @@ -3587,6 +3589,11 @@ export interface components { */ currentUserRole?: "MEMBER" | "OWNER"; basePermissions: components["schemas"]["PermissionModel"]; + /** @example This is a beautiful organization full of beautiful and clever people */ + description?: string; + avatar?: components["schemas"]["Avatar"]; + /** @example btforg */ + slug: string; }; PublicBillingConfigurationDTO: { enabled: boolean; @@ -3632,6 +3639,7 @@ export interface components { }; ExportFormatModel: { format: + | "CSV" | "JSON" | "JSON_TOLGEE" | "JSON_I18NEXT" @@ -3649,9 +3657,9 @@ export interface components { defaultFileStructureTemplate: string; }; DocItem: { - description?: string; name: string; displayName?: string; + description?: string; }; PagedModelProjectModel: { _embedded?: { @@ -3746,23 +3754,23 @@ export interface components { formalitySupported: boolean; }; KeySearchResultView: { - description?: string; name: string; /** Format: int64 */ id: number; + baseTranslation?: string; namespace?: string; + description?: string; translation?: string; - baseTranslation?: string; }; KeySearchSearchResultModel: { view?: components["schemas"]["KeySearchResultView"]; - description?: string; name: string; /** Format: int64 */ id: number; + baseTranslation?: string; namespace?: string; + description?: string; translation?: string; - baseTranslation?: string; }; PagedModelKeySearchSearchResultModel: { _embedded?: { @@ -4329,17 +4337,17 @@ export interface components { }; PatWithUserModel: { user: components["schemas"]["SimpleUserAccountModel"]; - description: string; /** Format: int64 */ id: number; - /** Format: int64 */ - lastUsedAt?: number; - /** Format: int64 */ - expiresAt?: number; + description: string; /** Format: int64 */ createdAt: number; /** Format: int64 */ updatedAt: number; + /** Format: int64 */ + expiresAt?: number; + /** Format: int64 */ + lastUsedAt?: number; }; PagedModelOrganizationModel: { _embedded?: { @@ -4456,19 +4464,19 @@ export interface components { * @description Languages for which user has translate permission. */ permittedLanguageIds?: number[]; - description: string; /** Format: int64 */ id: number; + userFullName?: string; projectName: string; + description: string; username?: string; - /** Format: int64 */ - lastUsedAt?: number; scopes: string[]; /** Format: int64 */ + projectId: number; + /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - projectId: number; - userFullName?: string; + lastUsedAt?: number; }; PagedModelUserAccountModel: { _embedded?: { @@ -12257,6 +12265,7 @@ export interface operations { languages?: string[]; /** Format to export to */ format?: + | "CSV" | "JSON" | "JSON_TOLGEE" | "JSON_I18NEXT" From 6075858217fc245ab8d234c270dfa62bb410111f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Kuchy=C5=88ka=20=28Anty=29?= Date: Wed, 9 Oct 2024 16:31:16 +0200 Subject: [PATCH 18/18] fix: test --- .../io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt index b3b0ad12cd..dc87853699 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt @@ -20,7 +20,7 @@ class CsvFileExporterTest { |"key3","{count, plural, one {# den {icuParam}} few {# dny} other {# dní}}" |"item","I will be first {icuParam, number}" |"key","Text with multiple lines - |and , commas and \"quotes\"" + |and , commas and ""quotes"" " | """.trimMargin(), ) @@ -44,7 +44,7 @@ class CsvFileExporterTest { add( languageTag = "cs", keyName = "key", - text = "Text with multiple lines\nand , commas and \"quotes\"", + text = "Text with multiple lines\nand , commas and \"quotes\" ", ) } return getExporter(built.translations, false)