feat: more csv tests

backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvDelimiterDetector.kt

@@ -1,8 +1,8 @@
 package io.tolgee.formats.csv.`in`
 
-import java.io.ByteArrayInputStream
+import java.io.InputStream
 
-class CsvDelimiterDetector(private val inputStream: ByteArrayInputStream) {
+class CsvDelimiterDetector(private val inputStream: InputStream) {
   companion object {
     val DELIMITERS = listOf(',', ';', '\t')
   }

backend/data/src/main/kotlin/io/tolgee/formats/csv/in/CsvFileParser.kt

@@ -3,10 +3,10 @@ package io.tolgee.formats.csv.`in`
 import com.opencsv.CSVParserBuilder
 import com.opencsv.CSVReaderBuilder
 import io.tolgee.formats.csv.CsvEntry
-import java.io.ByteArrayInputStream
+import java.io.InputStream
 
 class CsvFileParser(
-  private val inputStream: ByteArrayInputStream,
+  private val inputStream: InputStream,
   private val delimiter: Char,
   private val languageFallback: String,
 ) {

backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvFormatProcessorTest.kt

@@ -88,6 +88,38 @@ class CsvFormatProcessorTest {
         )
         isPluralOptimized()
       }
+    mockUtil.fileProcessorContext.assertTranslations("en", "escapedCharacters")
+      .assertSingle {
+        hasText("this is a \"quote\"")
+      }
+    mockUtil.fileProcessorContext.assertTranslations("cs", "escapedCharacters")
+      .assertSingle {
+        hasText("toto je \"citace\"")
+      }
+    mockUtil.fileProcessorContext.assertTranslations("en", "escapedCharacters2")
+      .assertSingle {
+        hasText("this is a\nnew line")
+      }
+    mockUtil.fileProcessorContext.assertTranslations("cs", "escapedCharacters2")
+      .assertSingle {
+        hasText("toto je\nnový řádek")
+      }
+    mockUtil.fileProcessorContext.assertTranslations("en", "escapedCharacters3")
+      .assertSingle {
+        hasText("this is a \\ backslash")
+      }
+    mockUtil.fileProcessorContext.assertTranslations("cs", "escapedCharacters3")
+      .assertSingle {
+        hasText("toto je zpětné \\ lomítko")
+      }
+    mockUtil.fileProcessorContext.assertTranslations("en", "escapedCharacters4")
+      .assertSingle {
+        hasText("this is a , comma")
+      }
+    mockUtil.fileProcessorContext.assertTranslations("cs", "escapedCharacters4")
+      .assertSingle {
+        hasText("toto je , čárka")
+      }
     mockUtil.fileProcessorContext.assertKey("keyPluralSimple") {
       custom.assert.isNull()
       description.assert.isNull()

backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/in/CsvImportFormatDetectorTest.kt

@@ -0,0 +1,58 @@
+package io.tolgee.unit.formats.csv.`in`
+
+import io.tolgee.formats.csv.`in`.CSVImportFormatDetector
+import io.tolgee.formats.csv.`in`.CsvFileParser
+import io.tolgee.formats.importCommon.ImportFormat
+import io.tolgee.testing.assert
+import io.tolgee.util.FileProcessorContextMockUtil
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import java.io.File
+
+class CsvImportFormatDetectorTest {
+  lateinit var mockUtil: FileProcessorContextMockUtil
+
+  @BeforeEach
+  fun setup() {
+    mockUtil = FileProcessorContextMockUtil()
+  }
+
+  @Test
+  fun `detected i18next`() {
+    "src/test/resources/import/csv/example.csv".assertDetected(ImportFormat.CSV_ICU)
+  }
+
+  @Test
+  fun `detected icu`() {
+    "src/test/resources/import/csv/icu.csv".assertDetected(ImportFormat.CSV_ICU)
+  }
+
+  @Test
+  fun `detected java`() {
+    "src/test/resources/import/csv/java.csv".assertDetected(ImportFormat.CSV_JAVA)
+  }
+
+  @Test
+  fun `detected php`() {
+    "src/test/resources/import/csv/php.csv".assertDetected(ImportFormat.CSV_PHP)
+  }
+
+  @Test
+  fun `fallbacks to icu`() {
+    "src/test/resources/import/csv/unknown.csv".assertDetected(ImportFormat.CSV_ICU)
+  }
+
+  private fun parseFile(path: String): Any? {
+    val parser =
+      CsvFileParser(
+        inputStream = File(path).inputStream(),
+        delimiter = ',',
+        languageFallback = "unknown",
+      )
+    return parser.rows
+  }
+
+  private fun String.assertDetected(format: ImportFormat) {
+    CSVImportFormatDetector().detectFormat(parseFile(this)).assert.isEqualTo(format)
+  }
+}

backend/data/src/test/kotlin/io/tolgee/unit/formats/csv/out/CsvFileExporterTest.kt

@@ -19,6 +19,8 @@ class CsvFileExporterTest {
     |"key","cs"
     |"key3","{count, plural, one {# den {icuParam}} few {# dny} other {# dní}}"
     |"item","I will be first {icuParam, number}"
+    |"key","Text with multiple lines
+    |and , commas and \"quotes\""
     |
       """.trimMargin(),
     )
@@ -39,6 +41,11 @@ class CsvFileExporterTest {
           keyName = "item",
           text = "I will be first {icuParam, number}",
         )
+        add(
+          languageTag = "cs",
+          keyName = "key",
+          text = "Text with multiple lines\nand , commas and \"quotes\"",
+        )
       }
     return getExporter(built.translations, false)
   }

backend/data/src/test/kotlin/io/tolgee/unit/formats/json/in/JsonImportFormatDetectorTest.kt

@@ -2,6 +2,7 @@ package io.tolgee.unit.formats.json.`in`
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import io.tolgee.formats.importCommon.ImportFormat
 import io.tolgee.formats.json.`in`.JsonImportFormatDetector
@@ -45,7 +46,7 @@ class JsonImportFormatDetectorTest {
   }
 
   private fun parseFile(path: String): Map<*, *> {
-    return ObjectMapper(YAMLFactory()).readValue<Map<*, *>>(
+    return jacksonObjectMapper().readValue<Map<*, *>>(
       File(path)
         .readBytes(),
     )

backend/data/src/test/kotlin/io/tolgee/unit/util/testGenerationUtil.kt

@@ -21,7 +21,7 @@ fun generateTestsForImportResult(fileProcessorContext: FileProcessorContext): St
     code.appendLine("""${i(indent)}${"\"\"\""}.trimIndent()""")
   }
   val escape = { str: String?, newLines: Boolean ->
-    str?.replace("\"", "\\\"").let {
+    str?.replace("\\", "\\\\")?.replace("\"", "\\\"").let {
       if (newLines) {
         return@let it?.replace("\n", "\\n")
       }

backend/data/src/test/resources/import/csv/example.csv

@@ -1,6 +1,12 @@
 "key","en","cs"
-"key","value","hodnota"
+"key","value",hodnota
 "keyDeep.inner","value","hodnota"
-"keyInterpolate","replace this {value}","nahradit toto {value}"
+"keyInterpolate","replace this {value}",nahradit toto {value}
 "keyInterpolateWithFormatting","replace this {value, number}","nahradit toto {value, number}"
 "keyPluralSimple"," {value, plural, one {the singular} other {the plural {value}}}"," {value, plural, one {jednotné číslo} other {množné číslo {value}}}"
+"escapedCharacters","this is a \"quote\"","toto je \"citace\""
+"escapedCharacters2","this is a
+new line","toto je
+nový řádek"
+"escapedCharacters3","this is a \\ backslash","toto je zpětné \\ lomítko"
+"escapedCharacters4","this is a , comma","toto je , čárka"

backend/data/src/test/resources/import/csv/icu.csv

@@ -0,0 +1,2 @@
+key,en
+key,{param}

backend/data/src/test/resources/import/csv/java.csv

@@ -0,0 +1,3 @@
+key,en
+"key","%D this is java %d"
+"key2","%D this is java"

backend/data/src/test/resources/import/csv/php.csv

@@ -0,0 +1,2 @@
+key,en
+key,"%'s with ' flag"

backend/data/src/test/resources/import/csv/unknown.csv

@@ -0,0 +1,2 @@
+key,en
+key,This suits php and java