fix: Import improvements

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt

@@ -4,6 +4,7 @@
 
 package io.tolgee.api.v2.controllers
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import io.swagger.v3.oas.annotations.Operation
 import io.swagger.v3.oas.annotations.Parameter
 import io.swagger.v3.oas.annotations.tags.Tag
@@ -38,7 +39,10 @@ import io.tolgee.security.authorization.RequiresProjectPermissions
 import io.tolgee.service.LanguageService
 import io.tolgee.service.dataImport.ForceMode
 import io.tolgee.service.dataImport.ImportService
+import io.tolgee.service.dataImport.status.ImportApplicationStatus
+import io.tolgee.service.dataImport.status.ImportApplicationStatusItem
 import io.tolgee.service.key.NamespaceService
+import io.tolgee.util.StreamingResponseBodyProvider
 import jakarta.servlet.http.HttpServletRequest
 import org.springdoc.core.annotations.ParameterObject
 import org.springframework.data.domain.PageRequest
@@ -49,6 +53,7 @@ import org.springframework.hateoas.CollectionModel
 import org.springframework.hateoas.PagedModel
 import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport
 import org.springframework.http.MediaType
+import org.springframework.http.ResponseEntity
 import org.springframework.web.bind.annotation.CrossOrigin
 import org.springframework.web.bind.annotation.DeleteMapping
 import org.springframework.web.bind.annotation.GetMapping
@@ -61,6 +66,7 @@ import org.springframework.web.bind.annotation.RequestParam
 import org.springframework.web.bind.annotation.RequestPart
 import org.springframework.web.bind.annotation.RestController
 import org.springframework.web.multipart.MultipartFile
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody
 
 @Suppress("MVCPathVariableInspection")
 @RestController
@@ -85,6 +91,8 @@ class V2ImportController(
  private val languageService: LanguageService,
  private val namespaceService: NamespaceService,
  private val importFileIssueModelAssembler: ImportFileIssueModelAssembler,
+ private val streamingResponseBodyProvider: StreamingResponseBodyProvider,
+ private val objectMapper: ObjectMapper,
 ) {
  @PostMapping("", consumes = [MediaType.MULTIPART_FORM_DATA_VALUE])
  @Operation(description = "Prepares provided files to import.", summary = "Add files")
@@ -94,7 +102,7 @@ class V2ImportController(
  @RequestPart("files") files: Array<MultipartFile>,
  @ParameterObject params: ImportAddFilesParams,
  ): ImportAddFilesResultModel {
- val fileDtos = files.map { ImportFileDto(it.originalFilename ?: "", it.inputStream) }
+ val fileDtos = files.map { ImportFileDto(it.originalFilename ?: "", it.inputStream.readAllBytes()) }
  val errors =
  importService.addFiles(
  files = fileDtos,
@@ -116,7 +124,7 @@ class V2ImportController(
  }
 
  @PutMapping("/apply")
- @Operation(description = "Imports the data prepared in previous step", summary = "Apply")
+ @Operation(description = "Imports the data prepared in previous step")
  @RequestActivity(ActivityType.IMPORT)
  @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW])
  @AllowApiAccess
@@ -129,6 +137,27 @@ class V2ImportController(
  this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode)
  }
 
+ @PutMapping("/apply-streaming", produces = [MediaType.APPLICATION_NDJSON_VALUE])
+ @Operation(description = "Imports the data prepared in previous step. Streams current status.")
+ @RequestActivity(ActivityType.IMPORT)
+ @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW])
+ @AllowApiAccess
+ fun applyImportStreaming(
+ @Parameter(description = "Whether override or keep all translations with unresolved conflicts")
+ @RequestParam(defaultValue = "NO_FORCE")
+ forceMode: ForceMode,
+ ): ResponseEntity<StreamingResponseBody> {
+ val projectId = projectHolder.project.id
+
+ return streamingResponseBodyProvider.streamNdJson { write ->
+ val writeStatus = { status: ImportApplicationStatus ->
+ write(ImportApplicationStatusItem(status))
+ }
+
+ this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode, writeStatus)
+ }
+ }
+
  @GetMapping("/result")
  @Operation(description = "Returns the result of preparation.", summary = "Get result")
  @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW])

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt

@@ -18,6 +18,7 @@ import io.tolgee.service.LanguageService
 import io.tolgee.service.key.KeyService
 import io.tolgee.service.security.SecurityService
 import io.tolgee.service.translation.TranslationMemoryService
+import io.tolgee.util.disableAccelBuffering
 import jakarta.validation.Valid
 import org.springdoc.core.annotations.ParameterObject
 import org.springframework.data.domain.Pageable
@@ -49,7 +50,7 @@ class TranslationSuggestionController(
 ) {
  @PostMapping("/machine-translations")
  @Operation(summary = "Suggests machine translations from enabled services")
- @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ])
+ @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT])
  @AllowApiAccess
  fun suggestMachineTranslations(
  @RequestBody @Valid
@@ -65,15 +66,13 @@ class TranslationSuggestionController(
  "If an error occurs when any of the services is used," +
  " the error information is returned as a part of the result item, while the response has 200 status code.",
  )
- @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ])
+ @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT])
  @AllowApiAccess
  fun suggestMachineTranslationsStreaming(
  @RequestBody @Valid
  dto: SuggestRequestDto,
  ): ResponseEntity<StreamingResponseBody> {
- return ResponseEntity.ok().headers {
- it.add("X-Accel-Buffering", "no")
- }.body(
+ return ResponseEntity.ok().disableAccelBuffering().body(
  machineTranslationSuggestionFacade.suggestStreaming(dto),
  )
  }
@@ -84,7 +83,7 @@ class TranslationSuggestionController(
  "Suggests machine translations from translation memory." +
  "\n\nThe result is always sorted by similarity, so sorting is not supported.",
  )
- @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ])
+ @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT])
  @AllowApiAccess
  fun suggestTranslationMemory(
  @RequestBody @Valid

backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt

@@ -9,6 +9,7 @@ import io.tolgee.component.fileStorage.LocalFileStorage
 import io.tolgee.component.fileStorage.S3ClientProvider
 import io.tolgee.component.fileStorage.S3FileStorage
 import io.tolgee.configuration.tolgee.TolgeeProperties
+import io.tolgee.util.InMemoryFileStorage
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 
@@ -20,6 +21,9 @@ class FileStorageConfiguration(
 
  @Bean
  fun fileStorage(): FileStorage {
+ if (properties.internal.useInMemoryFileStorage) {
+ return InMemoryFileStorage()
+ }
  if (s3config.enabled) {
  val amazonS3 = S3ClientProvider(s3config).provide()
  val bucketName = properties.fileStorage.s3.bucketName ?: throw RuntimeException("Bucket name is not set")

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt

@@ -82,8 +82,6 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") {
  retryingOnCommonIssues {
  initBaseData()
  try {
- executeInNewTransaction {
- }
  performExport()
  performExport()
  waitForNotThrowing(pollTime = 50, timeout = 3000) {

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt

@@ -14,7 +14,6 @@ import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestInstance
 import org.springframework.boot.test.context.SpringBootTest
-import java.io.File
 import java.time.Duration
 import java.util.*
 
@@ -70,18 +69,16 @@ class SecuredV2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest()
  performAuthGet("/uploaded-images/${image.filename}.png?token=$token")
  .andIsOk.andReturn().response.contentAsByteArray
 
- val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + image.filename + ".png")
- assertThat(storedImage).isEqualTo(file.readBytes())
+ assertThat(storedImage).isEqualTo(fileStorage.readFile("uploadedImages/" + image.filename + ".png"))
  }
 
  @Test
  @ProjectJWTAuthTestMethod
  fun upload() {
  performStoreImage().andPrettyPrint.andIsCreated.andAssertThatJson {
  node("filename").isString.satisfies {
- val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + it + ".png")
- assertThat(file).exists()
- assertThat(file.readBytes().size).isCloseTo(5538, Offset.offset(500))
+ val path = "uploadedImages/" + it + ".png"
+ assertThat(fileStorage.readFile(path).size).isCloseTo(5538, Offset.offset(500))
  }
  node("requestFilename").isString.satisfies {
  val parts = it.split("?token=")

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt

@@ -11,6 +11,7 @@ import io.tolgee.fixtures.andIsCreated
 import io.tolgee.fixtures.andIsForbidden
 import io.tolgee.fixtures.andIsOk
 import io.tolgee.fixtures.andPrettyPrint
+import io.tolgee.testing.assert
 import io.tolgee.testing.assertions.Assertions.assertThat
 import org.assertj.core.data.Offset
 import org.junit.jupiter.api.AfterAll
@@ -22,7 +23,6 @@ import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.mock.mockito.MockBean
 import org.springframework.mock.web.MockMultipartFile
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header
-import java.io.File
 import java.util.stream.Collectors
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
@@ -50,9 +50,9 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() {
  performStoreImage().andPrettyPrint.andIsCreated.andAssertThatJson {
  node("fileUrl").isString.startsWith("http://").endsWith(".png")
  node("requestFilename").isString.satisfies {
- val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + it)
- assertThat(file).exists()
- assertThat(file.readBytes().size).isCloseTo(5538, Offset.offset(500))
+ val file = fileStorage.fileExists("uploadedImages/" + it).assert.isTrue()
+ fileStorage.readFile("uploadedImages/" + it)
+  .size.assert.isCloseTo(5538, Offset.offset(500))
  }
  }
  }
@@ -78,15 +78,16 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() {
  @Test
  fun `returns file`() {
  val image = imageUploadService.store(screenshotFile, userAccount!!, null)
-
- val file = File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${image.filenameWithExtension}""")
  val result =
  performAuthGet("/uploaded-images/${image.filenameWithExtension}").andIsOk
  .andExpect(
  header().string("Cache-Control", "max-age=365, must-revalidate, no-transform"),
  )
  .andReturn()
- assertThat(result.response.contentAsByteArray).isEqualTo(file.readBytes())
+ assertThat(result.response.contentAsByteArray)
+ .isEqualTo(
+ fileStorage.readFile("uploadedImages/${image.filenameWithExtension}"),
+ )
  }
 
  @Test
@@ -100,19 +101,15 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() {
  val idsToDelete = list.take(10).map { it.id }.joinToString(",")
 
  list.asSequence().take(10).forEach {
- assertThat(
- File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${it.filenameWithExtension}"""),
- ).exists()
+ fileStorage.fileExists("uploadedImages/${it.filenameWithExtension}").assert.isTrue()
  }
 
  performAuthDelete("/v2/image-upload/$idsToDelete", null).andIsOk
  val rest = imageUploadService.find(list.map { it.id }.toSet())
  assertThat(rest).isEqualTo(list.stream().skip(10).collect(Collectors.toList()))
 
  list.asSequence().take(10).forEach {
- assertThat(
- File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${it.filenameWithExtension}"""),
- ).doesNotExist()
+ fileStorage.fileExists("uploadedImages/${it.filenameWithExtension}").assert.isFalse()
  }
  }
 

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt

@@ -14,6 +14,7 @@ import io.tolgee.model.dataImport.issues.issueTypes.FileIssueType
 import io.tolgee.testing.AuthorizedControllerTest
 import io.tolgee.testing.assert
 import io.tolgee.testing.assertions.Assertions.assertThat
+import io.tolgee.util.InMemoryFileStorage
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Test
 import org.springframework.beans.factory.annotation.Value
@@ -67,16 +68,40 @@ class V2ImportControllerAddFilesTest : AuthorizedControllerTest() {
  @AfterEach
  fun resetProps() {
  tolgeeProperties.maxTranslationTextLength = 10000
+ (fileStorage as InMemoryFileStorage).clear()
+ tolgeeProperties.import.storeFilesForDebugging = false
  }
 
  @Test
- fun `it parses zip file and saves issues`() {
+ fun `it parses zip file stores it for debugging and saves issues`() {
+ tolgeeProperties.import.storeFilesForDebugging = true
  val base = dbPopulator.createBase(generateUniqueString())
  commitTransaction()
 
- performImport(projectId = base.project.id, mapOf(Pair("zipOfUnknown.zip", zipOfUnknown))).andAssertThatJson {
+ val fileName = "zipOfUnknown.zip"
+ performImport(projectId = base.project.id, mapOf(Pair(fileName, zipOfUnknown))).andAssertThatJson {
  node("errors[2].code").isEqualTo("cannot_parse_file")
  }
+
+ doesStoredFileExists(fileName, base.project.id).assert.isTrue()
+ }
+
+ @Test
+ fun `doesn't store file for when disabled`() {
+ val base = dbPopulator.createBase(generateUniqueString())
+ commitTransaction()
+
+ val fileName = "zipOfUnknown.zip"
+ performImport(projectId = base.project.id, mapOf(Pair(fileName, zipOfUnknown)))
+ doesStoredFileExists(fileName, base.project.id).assert.isFalse()
+ }
+
+ fun doesStoredFileExists(
+ fileName: String,
+ projectId: Long,
+ ): Boolean {
+ val import = importService.find(projectId, userAccount!!.id)
+ return fileStorage.fileExists("importFiles/${import!!.id}/$fileName")
  }
 
  @Test

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt

@@ -13,14 +13,14 @@ import io.tolgee.fixtures.andPrettyPrint
 import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod
 import io.tolgee.testing.assert
 import io.tolgee.testing.assertions.Assertions.assertThat
+import io.tolgee.util.InMemoryFileStorage
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestInstance
 import org.springframework.mock.web.MockMultipartFile
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status
-import java.io.File
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() {
@@ -34,6 +34,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() {
  @AfterAll
  fun after() {
  tolgeeProperties.fileStorageUrl = initialScreenshotUrl
+ (fileStorage as InMemoryFileStorage).clear()
  }
 
  @Test
@@ -63,8 +64,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() {
  val screenshots = screenshotService.findAll(key = key)
  assertThat(screenshots).hasSize(1)
  node("filename").isEqualTo(screenshots[0].filename)
- val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshots[0].filename)
- assertThat(file).exists()
+ fileStorage.fileExists("screenshots/" + screenshots[0].filename).assert.isTrue()
  val reference = screenshots[0].keyScreenshotReferences[0]
  reference.originalText.assert.isEqualTo(text)
  reference.positions!![0].x.assert.isEqualTo(200)
@@ -116,8 +116,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() {
  performProjectAuthGet("keys/${key.id}/screenshots").andIsOk.andPrettyPrint.andAssertThatJson {
  node("_embedded.screenshots").isArray.hasSize(2)
  node("_embedded.screenshots[0].filename").isString.satisfies {
- val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + it)
- assertThat(file.exists()).isTrue()
+ fileStorage.fileExists("screenshots/" + it).assert.isTrue()
  }
  }
 
@@ -151,15 +150,14 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() {
  val key = keyService.create(project, CreateKeyDto("test"))
  screenshotService.store(screenshotFile, key, null)
  }
- val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshot.filename)
  val result =
  performAuthGet("/screenshots/${screenshot.filename}").andIsOk
  .andExpect(
  header().string("Cache-Control", "max-age=365, must-revalidate, no-transform"),
  )
  .andReturn()
  performAuthGet("/screenshots/${screenshot.thumbnailFilename}").andIsOk
- assertThat(result.response.contentAsByteArray).isEqualTo(file.readBytes())
+ assertThat(result.response.contentAsByteArray).isEqualTo(fileStorage.readFile("screenshots/" + screenshot.filename))
  }
 
  @Test

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt

@@ -17,7 +17,6 @@ import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestInstance
 import org.springframework.boot.test.context.SpringBootTest
-import java.io.File
 import java.time.Duration
 import java.util.*
 
@@ -103,9 +102,8 @@ class SecuredKeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest()
  executeInNewTransaction {
  val screenshots = screenshotService.findAll(key = key)
  assertThat(screenshots).hasSize(1)
- val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshots[0].filename)
- assertThat(file).exists()
- assertThat(file.readBytes().size).isCloseTo(1070, Offset.offset(500))
+ val bytes = fileStorage.readFile("screenshots/" + screenshots[0].filename)
+ assertThat(bytes.size).isCloseTo(1070, Offset.offset(500))
  node("filename").isString.startsWith(screenshots[0].filename).satisfies {
  val parts = it.split("?token=")
  val auth = jwtService.validateTicket(parts[1], JwtService.TicketType.IMG_ACCESS)