diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt index bbe748a07d..21b4fe13a9 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt @@ -4,6 +4,7 @@ package io.tolgee.api.v2.controllers +import com.fasterxml.jackson.databind.ObjectMapper import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.tags.Tag @@ -38,7 +39,10 @@ import io.tolgee.security.authorization.RequiresProjectPermissions import io.tolgee.service.LanguageService import io.tolgee.service.dataImport.ForceMode import io.tolgee.service.dataImport.ImportService +import io.tolgee.service.dataImport.status.ImportApplicationStatus +import io.tolgee.service.dataImport.status.ImportApplicationStatusItem import io.tolgee.service.key.NamespaceService +import io.tolgee.util.StreamingResponseBodyProvider import jakarta.servlet.http.HttpServletRequest import org.springdoc.core.annotations.ParameterObject import org.springframework.data.domain.PageRequest @@ -49,6 +53,7 @@ import org.springframework.hateoas.CollectionModel import org.springframework.hateoas.PagedModel import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport import org.springframework.http.MediaType +import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.CrossOrigin import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping @@ -61,6 +66,7 @@ import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RequestPart import org.springframework.web.bind.annotation.RestController import org.springframework.web.multipart.MultipartFile +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody @Suppress("MVCPathVariableInspection") @RestController @@ -85,6 +91,8 @@ class V2ImportController( private val languageService: LanguageService, private val namespaceService: NamespaceService, private val importFileIssueModelAssembler: ImportFileIssueModelAssembler, + private val streamingResponseBodyProvider: StreamingResponseBodyProvider, + private val objectMapper: ObjectMapper, ) { @PostMapping("", consumes = [MediaType.MULTIPART_FORM_DATA_VALUE]) @Operation(description = "Prepares provided files to import.", summary = "Add files") @@ -94,7 +102,7 @@ class V2ImportController( @RequestPart("files") files: Array, @ParameterObject params: ImportAddFilesParams, ): ImportAddFilesResultModel { - val fileDtos = files.map { ImportFileDto(it.originalFilename ?: "", it.inputStream) } + val fileDtos = files.map { ImportFileDto(it.originalFilename ?: "", it.inputStream.readAllBytes()) } val errors = importService.addFiles( files = fileDtos, @@ -116,7 +124,7 @@ class V2ImportController( } @PutMapping("/apply") - @Operation(description = "Imports the data prepared in previous step", summary = "Apply") + @Operation(description = "Imports the data prepared in previous step") @RequestActivity(ActivityType.IMPORT) @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) @AllowApiAccess @@ -129,6 +137,27 @@ class V2ImportController( this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode) } + @PutMapping("/apply-streaming", produces = [MediaType.APPLICATION_NDJSON_VALUE]) + @Operation(description = "Imports the data prepared in previous step. Streams current status.") + @RequestActivity(ActivityType.IMPORT) + @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) + @AllowApiAccess + fun applyImportStreaming( + @Parameter(description = "Whether override or keep all translations with unresolved conflicts") + @RequestParam(defaultValue = "NO_FORCE") + forceMode: ForceMode, + ): ResponseEntity { + val projectId = projectHolder.project.id + + return streamingResponseBodyProvider.streamNdJson { write -> + val writeStatus = { status: ImportApplicationStatus -> + write(ImportApplicationStatusItem(status)) + } + + this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode, writeStatus) + } + } + @GetMapping("/result") @Operation(description = "Returns the result of preparation.", summary = "Get result") @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt index 07db3a5964..dce1e960d5 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt @@ -18,6 +18,7 @@ import io.tolgee.service.LanguageService import io.tolgee.service.key.KeyService import io.tolgee.service.security.SecurityService import io.tolgee.service.translation.TranslationMemoryService +import io.tolgee.util.disableAccelBuffering import jakarta.validation.Valid import org.springdoc.core.annotations.ParameterObject import org.springframework.data.domain.Pageable @@ -49,7 +50,7 @@ class TranslationSuggestionController( ) { @PostMapping("/machine-translations") @Operation(summary = "Suggests machine translations from enabled services") - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun suggestMachineTranslations( @RequestBody @Valid @@ -65,15 +66,13 @@ class TranslationSuggestionController( "If an error occurs when any of the services is used," + " the error information is returned as a part of the result item, while the response has 200 status code.", ) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun suggestMachineTranslationsStreaming( @RequestBody @Valid dto: SuggestRequestDto, ): ResponseEntity { - return ResponseEntity.ok().headers { - it.add("X-Accel-Buffering", "no") - }.body( + return ResponseEntity.ok().disableAccelBuffering().body( machineTranslationSuggestionFacade.suggestStreaming(dto), ) } @@ -84,7 +83,7 @@ class TranslationSuggestionController( "Suggests machine translations from translation memory." + "\n\nThe result is always sorted by similarity, so sorting is not supported.", ) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun suggestTranslationMemory( @RequestBody @Valid diff --git a/backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt b/backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt index a704b1f30d..1687b5d013 100644 --- a/backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt +++ b/backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt @@ -9,6 +9,7 @@ import io.tolgee.component.fileStorage.LocalFileStorage import io.tolgee.component.fileStorage.S3ClientProvider import io.tolgee.component.fileStorage.S3FileStorage import io.tolgee.configuration.tolgee.TolgeeProperties +import io.tolgee.util.InMemoryFileStorage import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -20,6 +21,9 @@ class FileStorageConfiguration( @Bean fun fileStorage(): FileStorage { + if (properties.internal.useInMemoryFileStorage) { + return InMemoryFileStorage() + } if (s3config.enabled) { val amazonS3 = S3ClientProvider(s3config).provide() val bucketName = properties.fileStorage.s3.bucketName ?: throw RuntimeException("Bucket name is not set") diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt index 2da74f7874..d66807faac 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt @@ -82,8 +82,6 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { retryingOnCommonIssues { initBaseData() try { - executeInNewTransaction { - } performExport() performExport() waitForNotThrowing(pollTime = 50, timeout = 3000) { diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt index e7f3203513..d27a75e50b 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt @@ -14,7 +14,6 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance import org.springframework.boot.test.context.SpringBootTest -import java.io.File import java.time.Duration import java.util.* @@ -70,8 +69,7 @@ class SecuredV2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() performAuthGet("/uploaded-images/${image.filename}.png?token=$token") .andIsOk.andReturn().response.contentAsByteArray - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + image.filename + ".png") - assertThat(storedImage).isEqualTo(file.readBytes()) + assertThat(storedImage).isEqualTo(fileStorage.readFile("uploadedImages/" + image.filename + ".png")) } @Test @@ -79,9 +77,8 @@ class SecuredV2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() fun upload() { performStoreImage().andPrettyPrint.andIsCreated.andAssertThatJson { node("filename").isString.satisfies { - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + it + ".png") - assertThat(file).exists() - assertThat(file.readBytes().size).isCloseTo(5538, Offset.offset(500)) + val path = "uploadedImages/" + it + ".png" + assertThat(fileStorage.readFile(path).size).isCloseTo(5538, Offset.offset(500)) } node("requestFilename").isString.satisfies { val parts = it.split("?token=") diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt index 4970fac2fa..a9905114c0 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt @@ -11,6 +11,7 @@ import io.tolgee.fixtures.andIsCreated import io.tolgee.fixtures.andIsForbidden import io.tolgee.fixtures.andIsOk import io.tolgee.fixtures.andPrettyPrint +import io.tolgee.testing.assert import io.tolgee.testing.assertions.Assertions.assertThat import org.assertj.core.data.Offset import org.junit.jupiter.api.AfterAll @@ -22,7 +23,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header -import java.io.File import java.util.stream.Collectors @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -50,9 +50,9 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { performStoreImage().andPrettyPrint.andIsCreated.andAssertThatJson { node("fileUrl").isString.startsWith("http://").endsWith(".png") node("requestFilename").isString.satisfies { - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + it) - assertThat(file).exists() - assertThat(file.readBytes().size).isCloseTo(5538, Offset.offset(500)) + val file = fileStorage.fileExists("uploadedImages/" + it).assert.isTrue() + fileStorage.readFile("uploadedImages/" + it) + .size.assert.isCloseTo(5538, Offset.offset(500)) } } } @@ -78,15 +78,16 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { @Test fun `returns file`() { val image = imageUploadService.store(screenshotFile, userAccount!!, null) - - val file = File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${image.filenameWithExtension}""") val result = performAuthGet("/uploaded-images/${image.filenameWithExtension}").andIsOk .andExpect( header().string("Cache-Control", "max-age=365, must-revalidate, no-transform"), ) .andReturn() - assertThat(result.response.contentAsByteArray).isEqualTo(file.readBytes()) + assertThat(result.response.contentAsByteArray) + .isEqualTo( + fileStorage.readFile("uploadedImages/${image.filenameWithExtension}"), + ) } @Test @@ -100,9 +101,7 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { val idsToDelete = list.take(10).map { it.id }.joinToString(",") list.asSequence().take(10).forEach { - assertThat( - File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${it.filenameWithExtension}"""), - ).exists() + fileStorage.fileExists("uploadedImages/${it.filenameWithExtension}").assert.isTrue() } performAuthDelete("/v2/image-upload/$idsToDelete", null).andIsOk @@ -110,9 +109,7 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { assertThat(rest).isEqualTo(list.stream().skip(10).collect(Collectors.toList())) list.asSequence().take(10).forEach { - assertThat( - File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${it.filenameWithExtension}"""), - ).doesNotExist() + fileStorage.fileExists("uploadedImages/${it.filenameWithExtension}").assert.isFalse() } } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt index 1d87a12a4f..dfef383d70 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt @@ -14,6 +14,7 @@ import io.tolgee.model.dataImport.issues.issueTypes.FileIssueType import io.tolgee.testing.AuthorizedControllerTest import io.tolgee.testing.assert import io.tolgee.testing.assertions.Assertions.assertThat +import io.tolgee.util.InMemoryFileStorage import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Value @@ -67,16 +68,40 @@ class V2ImportControllerAddFilesTest : AuthorizedControllerTest() { @AfterEach fun resetProps() { tolgeeProperties.maxTranslationTextLength = 10000 + (fileStorage as InMemoryFileStorage).clear() + tolgeeProperties.import.storeFilesForDebugging = false } @Test - fun `it parses zip file and saves issues`() { + fun `it parses zip file stores it for debugging and saves issues`() { + tolgeeProperties.import.storeFilesForDebugging = true val base = dbPopulator.createBase(generateUniqueString()) commitTransaction() - performImport(projectId = base.project.id, mapOf(Pair("zipOfUnknown.zip", zipOfUnknown))).andAssertThatJson { + val fileName = "zipOfUnknown.zip" + performImport(projectId = base.project.id, mapOf(Pair(fileName, zipOfUnknown))).andAssertThatJson { node("errors[2].code").isEqualTo("cannot_parse_file") } + + doesStoredFileExists(fileName, base.project.id).assert.isTrue() + } + + @Test + fun `doesn't store file for when disabled`() { + val base = dbPopulator.createBase(generateUniqueString()) + commitTransaction() + + val fileName = "zipOfUnknown.zip" + performImport(projectId = base.project.id, mapOf(Pair(fileName, zipOfUnknown))) + doesStoredFileExists(fileName, base.project.id).assert.isFalse() + } + + fun doesStoredFileExists( + fileName: String, + projectId: Long, + ): Boolean { + val import = importService.find(projectId, userAccount!!.id) + return fileStorage.fileExists("importFiles/${import!!.id}/$fileName") } @Test diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt index d3a9f87f4c..4b5365e7af 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt @@ -13,6 +13,7 @@ import io.tolgee.fixtures.andPrettyPrint import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert import io.tolgee.testing.assertions.Assertions.assertThat +import io.tolgee.util.InMemoryFileStorage import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -20,7 +21,6 @@ import org.junit.jupiter.api.TestInstance import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status -import java.io.File @TestInstance(TestInstance.Lifecycle.PER_CLASS) class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { @@ -34,6 +34,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { @AfterAll fun after() { tolgeeProperties.fileStorageUrl = initialScreenshotUrl + (fileStorage as InMemoryFileStorage).clear() } @Test @@ -63,8 +64,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { val screenshots = screenshotService.findAll(key = key) assertThat(screenshots).hasSize(1) node("filename").isEqualTo(screenshots[0].filename) - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshots[0].filename) - assertThat(file).exists() + fileStorage.fileExists("screenshots/" + screenshots[0].filename).assert.isTrue() val reference = screenshots[0].keyScreenshotReferences[0] reference.originalText.assert.isEqualTo(text) reference.positions!![0].x.assert.isEqualTo(200) @@ -116,8 +116,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { performProjectAuthGet("keys/${key.id}/screenshots").andIsOk.andPrettyPrint.andAssertThatJson { node("_embedded.screenshots").isArray.hasSize(2) node("_embedded.screenshots[0].filename").isString.satisfies { - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + it) - assertThat(file.exists()).isTrue() + fileStorage.fileExists("screenshots/" + it).assert.isTrue() } } @@ -151,7 +150,6 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { val key = keyService.create(project, CreateKeyDto("test")) screenshotService.store(screenshotFile, key, null) } - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshot.filename) val result = performAuthGet("/screenshots/${screenshot.filename}").andIsOk .andExpect( @@ -159,7 +157,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { ) .andReturn() performAuthGet("/screenshots/${screenshot.thumbnailFilename}").andIsOk - assertThat(result.response.contentAsByteArray).isEqualTo(file.readBytes()) + assertThat(result.response.contentAsByteArray).isEqualTo(fileStorage.readFile("screenshots/" + screenshot.filename)) } @Test diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt index a7ec67fb54..77284d42fc 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt @@ -17,7 +17,6 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance import org.springframework.boot.test.context.SpringBootTest -import java.io.File import java.time.Duration import java.util.* @@ -103,9 +102,8 @@ class SecuredKeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() executeInNewTransaction { val screenshots = screenshotService.findAll(key = key) assertThat(screenshots).hasSize(1) - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshots[0].filename) - assertThat(file).exists() - assertThat(file.readBytes().size).isCloseTo(1070, Offset.offset(500)) + val bytes = fileStorage.readFile("screenshots/" + screenshots[0].filename) + assertThat(bytes.size).isCloseTo(1070, Offset.offset(500)) node("filename").isString.startsWith(screenshots[0].filename).satisfies { val parts = it.split("?token=") val auth = jwtService.validateTicket(parts[1], JwtService.TicketType.IMG_ACCESS) diff --git a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt index 82363367f5..bee8fb3ded 100644 --- a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt @@ -10,7 +10,9 @@ import org.junit.jupiter.api.Test import org.springframework.boot.test.context.SpringBootTest import java.io.File -@SpringBootTest +@SpringBootTest( + properties = ["tolgee.internal.use-in-memory-file-storage=false"], +) class FileStorageFsTest : AbstractFileStorageServiceTest() { lateinit var file: File @@ -21,6 +23,11 @@ class FileStorageFsTest : AbstractFileStorageServiceTest() { file.writeText(testFileContent) } + @Test + fun `is LocalFileStorage`() { + assertThat(fileStorage is LocalFileStorage).isTrue() + } + @Test fun testReadFile() { val content = fileStorage.readFile(testFilePath).toString(charset("UTF-8")) diff --git a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt index 9c7f970e3a..9d5aa43cd9 100644 --- a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt +++ b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt @@ -29,6 +29,7 @@ import software.amazon.awssdk.services.s3.model.S3Exception "tolgee.file-storage.s3.signing-region=dummy_signing_region", "tolgee.file-storage.s3.bucket-name=$BUCKET_NAME", "tolgee.authentication.initial-password=hey password manager, please don't use the filesystem :3", + "tolgee.internal.use-in-memory-file-storage=false", ], ) @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -55,6 +56,11 @@ class FileStorageS3Test : AbstractFileStorageServiceTest() { s3Mock.stop() } + @Test + fun `is LocalFileStorage`() { + assertThat(fileStorage is S3FileStorage).isTrue() + } + @Test fun testGetFile() { s3.putObject({ req -> req.bucket(BUCKET_NAME).key(testFilePath) }, RequestBody.fromString(testFileContent)) diff --git a/backend/app/src/test/kotlin/io/tolgee/initialUserCreation/CreateEnabledTest.kt b/backend/app/src/test/kotlin/io/tolgee/initialUserCreation/CreateEnabledTest.kt index a32380d481..1fd712d98a 100644 --- a/backend/app/src/test/kotlin/io/tolgee/initialUserCreation/CreateEnabledTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/initialUserCreation/CreateEnabledTest.kt @@ -7,6 +7,7 @@ package io.tolgee.initialUserCreation import io.tolgee.Application import io.tolgee.CleanDbBeforeClass import io.tolgee.commandLineRunners.InitialUserCreatorCommandLineRunner +import io.tolgee.component.fileStorage.FileStorage import io.tolgee.configuration.tolgee.TolgeeProperties import io.tolgee.repository.UserAccountRepository import io.tolgee.security.InitialPasswordManager @@ -14,6 +15,7 @@ import io.tolgee.service.security.UserAccountService import io.tolgee.testing.AbstractTransactionalTest import io.tolgee.testing.ContextRecreatingTest import io.tolgee.testing.assertions.Assertions.assertThat +import io.tolgee.util.InMemoryFileStorage import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -22,13 +24,11 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.security.crypto.password.PasswordEncoder import org.springframework.transaction.annotation.Transactional -import java.io.File @ContextRecreatingTest @SpringBootTest( classes = [Application::class], properties = [ - "tolgee.file-storage.fs-data-path=./build/create-enabled-test-data/", "tolgee.authentication.initial-username=johny", "tolgee.internal.disable-initial-user-creation=false", ], @@ -52,7 +52,8 @@ class CreateEnabledTest : AbstractTransactionalTest() { @set:Autowired lateinit var initialPasswordManager: InitialPasswordManager - private val passwordFile = File("./build/create-enabled-test-data/initial.pwd") + @set:Autowired + lateinit var fileStorage: FileStorage @Autowired lateinit var initialUserCreatorCommandLineRunner: InitialUserCreatorCommandLineRunner @@ -64,14 +65,13 @@ class CreateEnabledTest : AbstractTransactionalTest() { @Test fun storesPassword() { - assertThat(passwordFile).exists() - assertThat(passwordFile.readText()).isNotBlank + assertThat(getPasswordFileContents().toString(Charsets.UTF_8)).isNotBlank } @Test fun passwordStoredInDb() { val johny = userAccountService.findActive("johny") - assertThat(passwordEncoder.matches(passwordFile.readText(), johny!!.password)).isTrue + assertThat(passwordEncoder.matches(getPasswordFileContents().toString(Charsets.UTF_8), johny!!.password)).isTrue } @Test @@ -107,7 +107,7 @@ class CreateEnabledTest : AbstractTransactionalTest() { @AfterAll fun cleanUp() { - passwordFile.delete() + (fileStorage as InMemoryFileStorage).clear() resetInitialPassword() val initial = userAccountService.findActive("johny")!! @@ -121,4 +121,12 @@ class CreateEnabledTest : AbstractTransactionalTest() { set(initialPasswordManager, null) } } + + private fun getPasswordFileContents(): ByteArray { + return fileStorage.readFile(FILE_NAME) + } + + companion object { + const val FILE_NAME = "initial.pwd" + } } diff --git a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt index e7318f42d2..b1301113c8 100644 --- a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt @@ -89,7 +89,7 @@ class StoredDataImporterTest : AbstractSpringTest() { fun `it force replaces translations`() { storedDataImporter = StoredDataImporter( - applicationContext!!, + applicationContext, importTestData.import, ForceMode.OVERRIDE, ) diff --git a/backend/app/src/test/resources/application.yaml b/backend/app/src/test/resources/application.yaml index 4404906561..8eb9fe92b4 100644 --- a/backend/app/src/test/resources/application.yaml +++ b/backend/app/src/test/resources/application.yaml @@ -4,7 +4,7 @@ spring: - org.redisson.spring.starter.RedissonAutoConfiguration - org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration jpa: -# show-sql: true + show-sql: true properties: hibernate: order_by: @@ -64,6 +64,7 @@ tolgee: fake-mt-providers: true mock-free-plan: true disable-initial-user-creation: true + use-in-memory-file-storage: true cache: caffeine-max-size: 1000 machine-translation: @@ -80,6 +81,7 @@ tolgee: server: http://localhost:8080 batch: concurrency: 10 + logging: level: io.tolgee.billing.api.v2.OrganizationInvoicesController: DEBUG diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt index 6c19c01b66..4d1639b682 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt @@ -6,14 +6,15 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "tolgee.import") @DocProperty( description = - "Bulk-imports exported json files in the database during startup. " + + "Properties for importing data to Tolgee and " + + "bulk-imports exported json files in the database during startup. " + "Useful to quickly provision a development server, and used for testing.", displayName = "Import", ) class ImportProperties { @DocProperty( description = - "File path of the directory where the files to import are located.\n" + + "File path of the directory where the files to import on startup are located.\n" + "\n" + ":::info\n" + "Your folder structure should look like:\n" + @@ -28,7 +29,7 @@ class ImportProperties { @DocProperty( description = - "Whether an implicit API key should be created.\n" + + "Whether an implicit API key should be created when importing data on startup.\n" + "\n" + "The key is built with a predictable format: " + "`\${lowercase filename (without extension)}-\${initial username}-imported-project-implicit`\n" + @@ -41,6 +42,17 @@ class ImportProperties { ) var createImplicitApiKey: Boolean = false - @DocProperty(description = "The language tag of the base language of the imported projects.") + @DocProperty( + description = + "The language tag of the base language of the imported " + + "project (for importing data on startup).", + ) var baseLanguageTag: String = "en" + + @DocProperty( + description = + "If true, uploaded files will be stored in configured file storage for future debugging. " + + "Such data is not automatically removed after successful import. You have to clean-up manually!", + ) + var storeFilesForDebugging: Boolean = false } diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/InternalProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/InternalProperties.kt index 5b61ac9e13..8564c0eb2e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/InternalProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/InternalProperties.kt @@ -28,4 +28,6 @@ class InternalProperties { var e3eContentStorageBypassOk: Boolean? = null var disableInitialUserCreation: Boolean = false + + var useInMemoryFileStorage: Boolean = false } diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt index d674e57ed8..2785ee5c7e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt @@ -9,4 +9,5 @@ class ImportAddFilesParams( "the delimiter which will be used in names of improted keys.", ) var structureDelimiter: Char? = '.', + var storeFilesToFileStorage: Boolean = true, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportFileDto.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportFileDto.kt index 4c83e9fe55..76e3b8e6db 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportFileDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportFileDto.kt @@ -1,8 +1,6 @@ package io.tolgee.dtos.dataImport -import java.io.InputStream - data class ImportFileDto( val name: String = "", - val inputStream: InputStream, + val data: ByteArray, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/request/validators/exceptions/ValidationException.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/request/validators/exceptions/ValidationException.kt index c0ab4cb01e..1eb36cbb22 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/request/validators/exceptions/ValidationException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/request/validators/exceptions/ValidationException.kt @@ -3,8 +3,9 @@ package io.tolgee.dtos.request.validators.exceptions import io.tolgee.constants.Message import io.tolgee.dtos.request.validators.ValidationError import io.tolgee.dtos.request.validators.ValidationErrorType +import io.tolgee.exceptions.ExpectedException -class ValidationException : RuntimeException { +class ValidationException : RuntimeException, ExpectedException { val validationErrors: MutableSet = LinkedHashSet() constructor(message: Message, vararg parameters: String) { diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt index ea7ed09623..ecea59de20 100644 --- a/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt @@ -4,7 +4,7 @@ import io.tolgee.constants.Message import org.springframework.http.HttpStatus import java.io.Serializable -abstract class ErrorException : ExceptionWithMessage { +abstract class ErrorException : ExceptionWithMessage, ExpectedException { constructor(message: Message, params: List? = null) : super(message, params) constructor(code: String, params: List? = null) : super(code, params) diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/ExpectedException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/ExpectedException.kt new file mode 100644 index 0000000000..48dd9d07f3 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/ExpectedException.kt @@ -0,0 +1,7 @@ +package io.tolgee.exceptions + +/** + * Exceptions, which are expected and handeled. + * All exceptions that lead to http response status >= 400 and status < 500 + */ +interface ExpectedException diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/InternalException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/InternalException.kt deleted file mode 100644 index 765838091b..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/exceptions/InternalException.kt +++ /dev/null @@ -1,12 +0,0 @@ -package io.tolgee.exceptions - -import org.springframework.http.HttpStatus -import java.io.Serializable - -class InternalException : ErrorException { - constructor(message: io.tolgee.constants.Message, params: List?) : super(message, params) - constructor(message: io.tolgee.constants.Message) : super(message) - - override val httpStatus: HttpStatus - get() = HttpStatus.INTERNAL_SERVER_ERROR -} diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt index 34be6899ac..feeb995dcf 100644 --- a/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt @@ -5,4 +5,4 @@ import org.springframework.http.HttpStatus import org.springframework.web.bind.annotation.ResponseStatus @ResponseStatus(value = HttpStatus.NOT_FOUND) -data class NotFoundException(val msg: Message? = Message.RESOURCE_NOT_FOUND) : RuntimeException() +data class NotFoundException(val msg: Message = Message.RESOURCE_NOT_FOUND) : RuntimeException(), ExpectedException diff --git a/backend/data/src/main/kotlin/io/tolgee/security/ProjectHolder.kt b/backend/data/src/main/kotlin/io/tolgee/security/ProjectHolder.kt index 207f90798c..238f62389b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/security/ProjectHolder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/security/ProjectHolder.kt @@ -1,5 +1,6 @@ package io.tolgee.security +import io.sentry.Sentry import io.tolgee.dtos.cacheable.ProjectDto import io.tolgee.model.Project import io.tolgee.service.project.ProjectService @@ -14,6 +15,7 @@ open class ProjectHolder( private var _project: ProjectDto? = null open var project: ProjectDto set(value) { + Sentry.addBreadcrumb("Project Id: ${value.id}") _project = value } get() { diff --git a/backend/data/src/main/kotlin/io/tolgee/security/ratelimit/RateLimitedException.kt b/backend/data/src/main/kotlin/io/tolgee/security/ratelimit/RateLimitedException.kt index fa09f78c29..ee2528a3dc 100644 --- a/backend/data/src/main/kotlin/io/tolgee/security/ratelimit/RateLimitedException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/security/ratelimit/RateLimitedException.kt @@ -18,8 +18,11 @@ package io.tolgee.security.ratelimit import io.tolgee.constants.Message import io.tolgee.exceptions.ExceptionWithMessage +import io.tolgee.exceptions.ExpectedException import org.springframework.http.HttpStatus import org.springframework.web.bind.annotation.ResponseStatus @ResponseStatus(HttpStatus.TOO_MANY_REQUESTS) -class RateLimitedException(val retryAfter: Long, val global: Boolean) : ExceptionWithMessage(Message.RATE_LIMITED) +class RateLimitedException(val retryAfter: Long, val global: Boolean) : + ExceptionWithMessage(Message.RATE_LIMITED), + ExpectedException diff --git a/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt index 2569232d7c..3f3cbdaeb1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt @@ -3,6 +3,7 @@ package io.tolgee.service import io.tolgee.configuration.tolgee.TolgeeProperties import io.tolgee.dtos.cacheable.ProjectDto import io.tolgee.dtos.cacheable.UserAccountDto +import io.tolgee.dtos.dataImport.ImportAddFilesParams import io.tolgee.dtos.dataImport.ImportFileDto import io.tolgee.dtos.request.LanguageDto import io.tolgee.dtos.request.project.CreateProjectDTO @@ -72,7 +73,7 @@ class StartupImportService( private fun getImportFileDtos(projectDir: File) = projectDir.walk().filter { !it.isDirectory }.map { val relativePath = it.path.replace(projectDir.path, "") - if (relativePath.isBlank()) null else ImportFileDto(relativePath, it.inputStream()) + if (relativePath.isBlank()) null else ImportFileDto(relativePath, it.readBytes()) }.filterNotNull().toList() private fun setAuthentication(userAccount: UserAccount) { @@ -104,7 +105,12 @@ class StartupImportService( project: Project, userAccount: UserAccount, ) { - importService.addFiles(fileDtos, project, userAccount) + importService.addFiles( + files = fileDtos, + project = project, + userAccount = userAccount, + params = ImportAddFilesParams(storeFilesToFileStorage = false), + ) entityManager.flush() entityManager.clear() val imports = importService.getAllByProject(project.id) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index ed62f78580..06592a9e25 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -1,8 +1,11 @@ package io.tolgee.service.dataImport +import io.sentry.Sentry import io.tolgee.component.CurrentDateProvider +import io.tolgee.component.fileStorage.FileStorage import io.tolgee.component.reporting.BusinessEventPublisher import io.tolgee.component.reporting.OnBusinessEventToCaptureEvent +import io.tolgee.configuration.tolgee.TolgeeProperties import io.tolgee.constants.Message import io.tolgee.dtos.dataImport.ImportAddFilesParams import io.tolgee.dtos.dataImport.ImportFileDto @@ -31,11 +34,14 @@ import io.tolgee.repository.dataImport.ImportRepository import io.tolgee.repository.dataImport.ImportTranslationRepository import io.tolgee.repository.dataImport.issues.ImportFileIssueParamRepository import io.tolgee.repository.dataImport.issues.ImportFileIssueRepository +import io.tolgee.service.dataImport.status.ImportApplicationStatus import io.tolgee.util.getSafeNamespace import jakarta.persistence.EntityManager import org.springframework.context.ApplicationContext +import org.springframework.context.annotation.Lazy import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable +import org.springframework.scheduling.annotation.Async import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.interceptor.TransactionInterceptor @@ -56,6 +62,10 @@ class ImportService( private val businessEventPublisher: BusinessEventPublisher, private val importDeleteService: ImportDeleteService, private val currentDateProvider: CurrentDateProvider, + @Suppress("SelfReferenceConstructorParameter") @Lazy + private val self: ImportService, + private val fileStorage: FileStorage, + private val tolgeeProperties: TolgeeProperties, ) { @Transactional fun addFiles( @@ -63,7 +73,7 @@ class ImportService( project: Project, userAccount: UserAccount, params: ImportAddFilesParams = ImportAddFilesParams(), - ): List { + ): MutableList { val import = findNotExpired(project.id, userAccount.id) ?: Import(project).also { it.author = userAccount @@ -76,6 +86,10 @@ class ImportService( } importRepository.save(import) + Sentry.addBreadcrumb("Import ID: ${import.id}") + + self.saveFilesToFileStorage(import.id, files) + val fileProcessor = CoreImportFilesProcessor( applicationContext = applicationContext, @@ -95,16 +109,19 @@ class ImportService( projectId: Long, authorId: Long, forceMode: ForceMode = ForceMode.NO_FORCE, + reportStatus: (ImportApplicationStatus) -> Unit = {}, ) { - import(getNotExpired(projectId, authorId), forceMode) + import(getNotExpired(projectId, authorId), forceMode, reportStatus) } @Transactional(noRollbackFor = [ImportConflictNotResolvedException::class]) fun import( import: Import, forceMode: ForceMode = ForceMode.NO_FORCE, + reportStatus: (ImportApplicationStatus) -> Unit = {}, ) { - StoredDataImporter(applicationContext, import, forceMode).doImport() + Sentry.addBreadcrumb("Import ID: ${import.id}") + StoredDataImporter(applicationContext, import, forceMode, reportStatus).doImport() deleteImport(import) businessEventPublisher.publish( OnBusinessEventToCaptureEvent( @@ -124,6 +141,7 @@ class ImportService( return } val import = importLanguage.file.import + Sentry.addBreadcrumb("Import ID: ${import.id}") val dataManager = ImportDataManager(applicationContext, import) existingLanguage?.let { val langAlreadySelectedInTheSameNS = @@ -145,6 +163,7 @@ class ImportService( namespace: String?, ) { val import = file.import + Sentry.addBreadcrumb("Import ID: ${import.id}") val dataManager = ImportDataManager(applicationContext, import) file.namespace = getSafeNamespace(namespace) importFileRepository.save(file) @@ -370,4 +389,30 @@ class ImportService( importFileIssueParamRepository.saveAll(params) fun getAllNamespaces(importId: Long) = importRepository.getAllNamespaces(importId) + + /** + * This function saves the files to file storage. + * When import fails, we need the files for future debugging + */ + @Async + fun saveFilesToFileStorage( + importId: Long, + files: List, + ) { + if (tolgeeProperties.import.storeFilesForDebugging) { + files.forEach { + fileStorage.storeFile(getFileStoragePath(importId, it.name), it.data) + } + } + } + + fun getFileStoragePath( + importId: Long, + fileName: String, + ): String { + val notBlankFilename = fileName.ifBlank { "blank_name" } + return "${getFileStorageImportRoot(importId)}/$notBlankFilename" + } + + private fun getFileStorageImportRoot(importId: Long) = "importFiles/$importId" } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt index aa61ca32cd..48fe3c79e6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt @@ -10,6 +10,7 @@ import io.tolgee.model.key.Key import io.tolgee.model.key.KeyMeta import io.tolgee.model.key.Namespace import io.tolgee.model.translation.Translation +import io.tolgee.service.dataImport.status.ImportApplicationStatus import io.tolgee.service.key.KeyMetaService import io.tolgee.service.key.KeyService import io.tolgee.service.key.NamespaceService @@ -23,6 +24,7 @@ class StoredDataImporter( applicationContext: ApplicationContext, private val import: Import, private val forceMode: ForceMode = ForceMode.NO_FORCE, + private val reportStatus: (ImportApplicationStatus) -> Unit = {}, ) { private val importDataManager = ImportDataManager(applicationContext, import) private val keyService = applicationContext.getBean(KeyService::class.java) @@ -75,21 +77,30 @@ class StoredDataImporter( } fun doImport() { + reportStatus(ImportApplicationStatus.PREPARING_AND_VALIDATING) importDataManager.storedLanguages.forEach { - it.doImport() + it.prepareImport() } addKeysAndCheckPermissions() handleKeyMetas() + reportStatus(ImportApplicationStatus.STORING_KEYS) + namespaceService.saveAll(namespacesToSave.values) val keyEntitiesToSave = saveKeys() + saveMetaData(keyEntitiesToSave) + + reportStatus(ImportApplicationStatus.STORING_TRANSLATIONS) + saveTranslations() - saveMetaData(keyEntitiesToSave) + reportStatus(ImportApplicationStatus.FINALIZING) + + entityManager.flush() translationService.setOutdatedBatch(outdatedFlagKeys) @@ -163,7 +174,7 @@ class StoredDataImporter( } } - private fun ImportLanguage.doImport() { + private fun ImportLanguage.prepareImport() { importDataManager.populateStoredTranslations(this) importDataManager.handleConflicts(true) importDataManager.getStoredTranslations(this).forEach { it.doImport() } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/JsonFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/JsonFileProcessor.kt index c7814c1c3c..0cb6628294 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/JsonFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/JsonFileProcessor.kt @@ -11,7 +11,7 @@ class JsonFileProcessor( ) : ImportFileProcessor() { override fun process() { try { - val data = jacksonObjectMapper().readValue>(context.file.inputStream) + val data = jacksonObjectMapper().readValue>(context.file.data) val parsed = data.parse() parsed.entries.forEachIndexed { index, it -> context.addTranslation(it.key, languageNameGuesses[0], it.value, index) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/PropertyFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/PropertyFileProcessor.kt index 3d0ec65118..cf5b0e7803 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/PropertyFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/PropertyFileProcessor.kt @@ -1,13 +1,13 @@ package io.tolgee.service.dataImport.processors -import java.util.Properties +import java.util.* class PropertyFileProcessor( override val context: FileProcessorContext, ) : ImportFileProcessor() { override fun process() { val props = Properties() - props.load(context.file.inputStream) + props.load(context.file.data.inputStream()) props.entries.forEachIndexed { idx, it -> context.addTranslation(it.key.toString(), languageNameGuesses[0], it.value, idx) } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt index 5a90c0e9dd..20f50db1c4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt @@ -10,7 +10,7 @@ class ZipTypeProcessor : ImportArchiveProcessor { } override fun process(file: ImportFileDto): Collection { - val zipInputStream = ZipInputStream(file.inputStream) + val zipInputStream = ZipInputStream(file.data.inputStream()) var nextEntry: ZipEntry? // .zip archives generated by MacOs compress feature return duplicities // I am removing them here by adding the files to map @@ -30,7 +30,7 @@ class ZipTypeProcessor : ImportArchiveProcessor { files[fileName] = ImportFileDto( name = fileName, - data.inputStream(), + data = data, ) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/po/PoParser.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/po/PoParser.kt index b3ee4e08b3..398093e82c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/po/PoParser.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/po/PoParser.kt @@ -93,7 +93,7 @@ class PoParser( } private fun processInputStream() { - context.file.inputStream.readAllBytes().decodeToString().forEach { + context.file.data.decodeToString().forEach { it.handle() } endTranslation() diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/xliff/XliffFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/xliff/XliffFileProcessor.kt index da9de913c6..e92b4766be 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/xliff/XliffFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/xliff/XliffFileProcessor.kt @@ -24,7 +24,7 @@ class XliffFileProcessor(override val context: FileProcessorContext) : ImportFil private val xmlEventReader: XMLEventReader by lazy { val inputFactory: XMLInputFactory = XMLInputFactory.newDefaultFactory() - inputFactory.createXMLEventReader(context.file.inputStream) + inputFactory.createXMLEventReader(context.file.data.inputStream()) } private val version: String by lazy { diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatus.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatus.kt new file mode 100644 index 0000000000..e0d74bebca --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatus.kt @@ -0,0 +1,8 @@ +package io.tolgee.service.dataImport.status + +enum class ImportApplicationStatus { + PREPARING_AND_VALIDATING, + STORING_KEYS, + STORING_TRANSLATIONS, + FINALIZING, +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatusItem.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatusItem.kt new file mode 100644 index 0000000000..62400956b3 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatusItem.kt @@ -0,0 +1,5 @@ +package io.tolgee.service.dataImport.status + +data class ImportApplicationStatusItem( + val status: ImportApplicationStatus, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt index 1a7bc7fb3f..87de639ea5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt @@ -16,15 +16,24 @@ package io.tolgee.util +import com.fasterxml.jackson.databind.ObjectMapper +import io.sentry.Sentry +import io.tolgee.exceptions.ErrorException +import io.tolgee.exceptions.ErrorResponseBody +import io.tolgee.exceptions.ExpectedException +import io.tolgee.exceptions.NotFoundException import jakarta.persistence.EntityManager import org.hibernate.Session +import org.springframework.http.ResponseEntity import org.springframework.stereotype.Component import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody import java.io.OutputStream +import java.io.OutputStreamWriter @Component class StreamingResponseBodyProvider( private val entityManager: EntityManager, + private val objectMapper: ObjectMapper, ) { fun createStreamingResponseBody(fn: (os: OutputStream) -> Unit): StreamingResponseBody { return StreamingResponseBody { @@ -40,4 +49,44 @@ class StreamingResponseBodyProvider( session.close() } } + + fun streamNdJson(stream: (write: (message: Any?) -> Unit) -> Unit): ResponseEntity { + return ResponseEntity.ok().disableAccelBuffering().body( + this.createStreamingResponseBody { outputStream -> + OutputStreamWriter(outputStream).use { writer -> + val write = + { message: Any? -> writer.writeJson(message) } + try { + stream(write) + } catch (e: Throwable) { + val message = getErrorMessage(e) + writer.writeJson(StreamedErrorMessage(message)) + if (e !is ExpectedException) { + Sentry.captureException(e) + } + } + } + }, + ) + } + + fun OutputStreamWriter.writeJson(message: Any?) { + this.write( + (objectMapper.writeValueAsString(message) + "\n"), + ) + this.flush() + } + + private fun getErrorMessage(e: Throwable) = + when (e) { + is NotFoundException -> ErrorResponseBody(e.msg.code, null) + is ErrorException -> e.errorResponseBody + else -> + ErrorResponseBody( + "unexpected_error_occurred", + listOf(e::class.java.name), + ) + } + + data class StreamedErrorMessage(val error: ErrorResponseBody) } diff --git a/backend/data/src/main/kotlin/io/tolgee/util/responseEntityExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/responseEntityExt.kt new file mode 100644 index 0000000000..88a173ef3e --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/util/responseEntityExt.kt @@ -0,0 +1,10 @@ +package io.tolgee.util + +import org.springframework.http.ResponseEntity + +fun ResponseEntity.BodyBuilder.disableAccelBuffering(): ResponseEntity.BodyBuilder { + this.headers { + it.add("X-Accel-Buffering", "no") + } + return this +} diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/CoreImportFileProcessorUnitTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/CoreImportFileProcessorUnitTest.kt index b192c7fd49..53f3bef5cc 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/CoreImportFileProcessorUnitTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/CoreImportFileProcessorUnitTest.kt @@ -64,7 +64,7 @@ class CoreImportFileProcessorUnitTest { tolgeePropertiesMock = mock() importFile = ImportFile("lgn.json", importMock) - importFileDto = ImportFileDto("lng.json", "".toByteArray().inputStream()) + importFileDto = ImportFileDto("lng.json", "".toByteArray()) existingLanguage = Language().also { it.name = "lng" } existingTranslation = Translation("helllo").also { it.key = Key(name = "colliding key") } processor = CoreImportFilesProcessor(applicationContextMock, importMock) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/PropertiesParserTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/PropertiesParserTest.kt index ffe2047d90..bb8a28738c 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/PropertiesParserTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/PropertiesParserTest.kt @@ -25,7 +25,7 @@ class PropertiesParserTest { ImportFileDto( "messages_en.properties", File("src/test/resources/import/example.properties") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/FormatDetectorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/FormatDetectorTest.kt index 95aa3572af..bc72c08ae6 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/FormatDetectorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/FormatDetectorTest.kt @@ -26,7 +26,7 @@ class FormatDetectorTest { ImportFileDto( "exmample.po", File("src/test/resources/import/po/example.po") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/ToICUConverterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/ToICUConverterTest.kt index dfd4925570..67ab4f0d8e 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/ToICUConverterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/ToICUConverterTest.kt @@ -27,7 +27,7 @@ class ToICUConverterTest { ImportFileDto( "exmample.po", File("src/test/resources/import/po/example.po") - .inputStream(), + .readBytes(), ) context = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt index 4ce4c4f4b2..613add3ca5 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt @@ -83,7 +83,7 @@ class PoFileProcessorTest { importFileDto = ImportFileDto( "exmample.po", - inputStream, + inputStream.readAllBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoParserTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoParserTest.kt index 05ef19e369..298bc21ae5 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoParserTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoParserTest.kt @@ -25,7 +25,7 @@ class PoParserTest { ImportFileDto( "exmample.po", File("src/test/resources/import/po/example.po") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt index 4cbc710b96..a88cf69572 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt @@ -26,7 +26,7 @@ class Xliff12FileProcessorTest { private val xmlEventReader: XMLEventReader get() { val inputFactory: XMLInputFactory = XMLInputFactory.newDefaultFactory() - return inputFactory.createXMLEventReader(importFileDto.inputStream) + return inputFactory.createXMLEventReader(importFileDto.data.inputStream()) } @BeforeEach @@ -37,7 +37,7 @@ class Xliff12FileProcessorTest { ImportFileDto( "exmample.xliff", File("src/test/resources/import/xliff/example.xliff") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } @@ -77,10 +77,10 @@ class Xliff12FileProcessorTest { ImportFileDto( "exmample.xliff", File("src/test/resources/import/xliff/larger.xlf") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) - xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.inputStream) + xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.data.inputStream()) val start = System.currentTimeMillis() Xliff12FileProcessor(fileProcessorContext, xmlEventReader).process() assertThat(System.currentTimeMillis() - start).isLessThan(4000) @@ -91,10 +91,9 @@ class Xliff12FileProcessorTest { importFileDto = ImportFileDto( "exmample.xliff", - File("src/test/resources/import/xliff/error_example.xliff") - .inputStream(), + File("src/test/resources/import/xliff/error_example.xliff").readBytes(), ) - xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.inputStream) + xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.data.inputStream()) fileProcessorContext = FileProcessorContext(importFileDto, importFile) Xliff12FileProcessor(fileProcessorContext, xmlEventReader).process() assertThat(fileProcessorContext.translations).hasSize(2) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/XliffFileProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/XliffFileProcessorTest.kt index 2ea25fb1f0..e45fdd52b5 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/XliffFileProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/XliffFileProcessorTest.kt @@ -25,7 +25,7 @@ class XliffFileProcessorTest { ImportFileDto( "exmample.xliff", File("src/test/resources/import/xliff/example.xliff") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/testing/src/main/kotlin/io/tolgee/testing/utils/InMemoryFileStorage.kt b/backend/development/src/main/kotlin/io/tolgee/util/InMemoryFileStorage.kt similarity index 95% rename from backend/testing/src/main/kotlin/io/tolgee/testing/utils/InMemoryFileStorage.kt rename to backend/development/src/main/kotlin/io/tolgee/util/InMemoryFileStorage.kt index c140c4726a..acdd6a583e 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/testing/utils/InMemoryFileStorage.kt +++ b/backend/development/src/main/kotlin/io/tolgee/util/InMemoryFileStorage.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.tolgee.testing.utils +package io.tolgee.util import io.tolgee.component.fileStorage.FileStorage import io.tolgee.exceptions.FileStoreException @@ -41,4 +41,8 @@ class InMemoryFileStorage : FileStorage { override fun fileExists(storageFilePath: String): Boolean { return files.contains(storageFilePath) } + + fun clear() { + files.clear() + } } diff --git a/e2e/cypress/e2e/import/importAddingFiles.cy.ts b/e2e/cypress/e2e/import/importAddingFiles.cy.ts index b0d4ad07d3..e12a3f44ad 100644 --- a/e2e/cypress/e2e/import/importAddingFiles.cy.ts +++ b/e2e/cypress/e2e/import/importAddingFiles.cy.ts @@ -115,19 +115,11 @@ describe('Import Adding files', () => { cy.get('[data-cy=dropzone]').trigger('dragenter', { dataTransfer: dt, }); - cy.get('[data-cy=dropzone-inner]').should( - 'have.css', - 'background-color', - 'rgb(232, 245, 233)' - ); + cy.get('[data-cy=dropzone-inner]').should('have.css', 'opacity', '1'); cy.get('[data-cy=dropzone]').trigger('dragleave', { dataTransfer: dt, }); - cy.get('[data-cy=dropzone-inner]').should( - 'have.css', - 'background-color', - 'rgba(0, 0, 0, 0)' - ); + cy.get('[data-cy=dropzone-inner]').should('have.css', 'opacity', '0'); }); }); @@ -144,19 +136,11 @@ describe('Import Adding files', () => { cy.get('[data-cy=dropzone]').trigger('dragenter', { dataTransfer: dt, }); - cy.get('[data-cy=dropzone-inner]').should( - 'have.css', - 'background-color', - 'rgb(255, 235, 238)' - ); + cy.get('[data-cy=dropzone-inner]').should('have.css', 'opacity', '1'); cy.get('[data-cy=dropzone]').trigger('dragleave', { dataTransfer: dt, }); - cy.get('[data-cy=dropzone-inner]').should( - 'have.css', - 'background-color', - 'rgba(0, 0, 0, 0)' - ); + cy.get('[data-cy=dropzone-inner]').should('have.css', 'opacity', '0'); }); }); diff --git a/e2e/cypress/e2e/import/importErrors.cy.ts b/e2e/cypress/e2e/import/importErrors.cy.ts index 36fa63b575..46c7ef110d 100644 --- a/e2e/cypress/e2e/import/importErrors.cy.ts +++ b/e2e/cypress/e2e/import/importErrors.cy.ts @@ -1,7 +1,6 @@ import 'cypress-file-upload'; import { assertMessage, gcy } from '../../common/shared'; import { visitImport } from '../../common/import'; -import { expectGlobalLoading } from '../../common/loading'; import { importTestData } from '../../common/apiCalls/testData/testData'; import { login } from '../../common/apiCalls/common'; @@ -67,7 +66,7 @@ describe('Import errors', () => { }); it('error shows more and less', { retries: { runMode: 3 } }, () => { - expectGlobalLoading(); + gcy('import-progress-overlay').should('be.visible'); gcy('import-file-error') .findDcy('import-file-error-more-less-button') .click(); diff --git a/e2e/cypress/support/dataCyType.d.ts b/e2e/cypress/support/dataCyType.d.ts index 734cb1427f..929944e6c4 100644 --- a/e2e/cypress/support/dataCyType.d.ts +++ b/e2e/cypress/support/dataCyType.d.ts @@ -184,6 +184,8 @@ declare namespace DataCy { "import-file-input" | "import-file-issues-button" | "import-file-issues-dialog" | + "import-progress" | + "import-progress-overlay" | "import-resolution-dialog-accept-imported-button" | "import-resolution-dialog-accept-old-button" | "import-resolution-dialog-close-button" | diff --git a/ee/backend/tests/src/test/resources/application.yaml b/ee/backend/tests/src/test/resources/application.yaml index 7b0f3ab515..e2a884a615 100644 --- a/ee/backend/tests/src/test/resources/application.yaml +++ b/ee/backend/tests/src/test/resources/application.yaml @@ -58,6 +58,7 @@ tolgee: fake-mt-providers: true mock-free-plan: true disable-initial-user-creation: true + use-in-memory-file-storage: true cache: caffeine-max-size: 1000 machine-translation: diff --git a/webapp/package-lock.json b/webapp/package-lock.json index fa07a79551..e7047b15b5 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -28,6 +28,7 @@ "copy-to-clipboard": "^3.3.1", "date-fns": "2.29.2", "diff": "^5.0.0", + "dotenv-flow": "4.0.1", "formik": "^2.2.9", "intl-messageformat": "^9.8.1", "node-fetch": "3.3.0", @@ -9590,6 +9591,28 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/dotenv-flow": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/dotenv-flow/-/dotenv-flow-4.0.1.tgz", + "integrity": "sha512-HuCQ487bSA43mtlxdWpyk5jt5Lljy+v1I8y/2l96gtjSve9p3OvJZCCOhQnz2hY4VhLogFfXpY20zBygMwaydA==", + "dependencies": { + "dotenv": "^16.0.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/dotenv-flow/node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, "node_modules/duplexer": { "version": "0.1.2", "dev": true, @@ -33438,6 +33461,21 @@ "version": "5.1.0", "dev": true }, + "dotenv-flow": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/dotenv-flow/-/dotenv-flow-4.0.1.tgz", + "integrity": "sha512-HuCQ487bSA43mtlxdWpyk5jt5Lljy+v1I8y/2l96gtjSve9p3OvJZCCOhQnz2hY4VhLogFfXpY20zBygMwaydA==", + "requires": { + "dotenv": "^16.0.0" + }, + "dependencies": { + "dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==" + } + } + }, "duplexer": { "version": "0.1.2", "dev": true diff --git a/webapp/package.json b/webapp/package.json index f7f8846a75..bc1dbef840 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -23,6 +23,7 @@ "copy-to-clipboard": "^3.3.1", "date-fns": "2.29.2", "diff": "^5.0.0", + "dotenv-flow": "4.0.1", "formik": "^2.2.9", "intl-messageformat": "^9.8.1", "node-fetch": "3.3.0", @@ -68,8 +69,8 @@ "startConcurrently": "NODE_OPTIONS=--openssl-legacy-provider concurrently \"npm run start\"", "build": "NODE_OPTIONS=--openssl-legacy-provider craco build", "test": "craco test", - "schema": "openapi-typescript http://localhost:8080/v3/api-docs/All%20Internal%20-%20for%20Tolgee%20Web%20application --output ./src/service/apiSchema.generated.ts", - "billing-schema": "openapi-typescript http://localhost:8080/v3/api-docs/V2%20Billing --output ./src/service/billingApiSchema.generated.ts", + "schema": "node ./scripts/generate-schemas.js public", + "billing-schema": "node ./scripts/generate-schemas.js billing", "prettier": "prettier --write ./src", "eslint": "eslint --ext .ts --ext .tsx --max-warnings 0 --report-unused-disable-directives .", "tsc": "tsc", diff --git a/webapp/scripts/generate-schemas.js b/webapp/scripts/generate-schemas.js new file mode 100644 index 0000000000..6199611ba5 --- /dev/null +++ b/webapp/scripts/generate-schemas.js @@ -0,0 +1,39 @@ +const path = require('path'); +// script.js +require('dotenv-flow').config({ + default_node_env: 'development', + path: path.resolve(__dirname, '../'), +}); + +const { exec } = require('child_process'); +const apiUrl = process.env.REACT_APP_API_URL || 'http://localhost:8080'; + +const definitions = { + public: { + schema: 'All%20Internal%20-%20for%20Tolgee%20Web%20application', + output: './src/service/apiSchema.generated.ts', + }, + billing: { + schema: 'V2%20Billing', + output: './src/service/billingApiSchema.generated.ts', + }, +}; + +const definition = definitions[process.argv[2]]; +if (!definition) { + throw new Error('Invalid definition'); +} + +const command = `openapi-typescript ${apiUrl}/v3/api-docs/${definition.schema} --output ${definition.output}`; + +exec(command, (error, stdout, stderr) => { + if (error) { + console.log(`error: ${error.message}`); + return; + } + if (stderr) { + console.log(`stderr: ${stderr}`); + return; + } + console.log(`stdout: ${stdout}`); +}); diff --git a/webapp/src/ThemeProvider.tsx b/webapp/src/ThemeProvider.tsx index 86e0603299..0d86a0c809 100644 --- a/webapp/src/ThemeProvider.tsx +++ b/webapp/src/ThemeProvider.tsx @@ -150,6 +150,7 @@ const getTheme = (mode: PaletteMode) => { marker: c.marker, topBanner: c.topBanner, quickStart: c.quickStart, + import: c.import, }, mixins: { toolbar: { diff --git a/webapp/src/colors.tsx b/webapp/src/colors.tsx index 8e6d121855..e436c03cae 100644 --- a/webapp/src/colors.tsx +++ b/webapp/src/colors.tsx @@ -159,6 +159,11 @@ export const colors = { progressBackground: '#bcbcbc70', itemBorder: '#EDF0F7', } as QuickStart, + import: { + progressDone: '#00B962', + progressWorking: '#EC407A', + progressBackground: '#D9D9D9', + }, }, dark: { white: '#dddddd', @@ -241,5 +246,10 @@ export const colors = { topBorder: '#2a384c', progressBackground: '#2c3c52', } as QuickStart, + import: { + progressDone: '#00B962', + progressWorking: '#EC407A', + progressBackground: '#D9D9D9', + }, }, } as const; diff --git a/webapp/src/component/CustomIcons.tsx b/webapp/src/component/CustomIcons.tsx index 39d0650e01..e22c279a1d 100644 --- a/webapp/src/component/CustomIcons.tsx +++ b/webapp/src/component/CustomIcons.tsx @@ -12,6 +12,7 @@ import { ReactComponent as TranslationMemorySvg } from '../svgs/icons/translatio import { ReactComponent as MachineTranslationSvg } from '../svgs/icons/machineTranslation.svg'; import { ReactComponent as TadaSvg } from '../svgs/icons/tada.svg'; import { ReactComponent as RocketSvg } from '../svgs/icons/rocket.svg'; +import { ReactComponent as DropZoneSvg } from '../svgs/icons/dropzone.svg'; type IconProps = ComponentProps; @@ -60,3 +61,7 @@ export const TadaIcon: React.FC = (props) => ( export const RocketIcon: React.FC = (props) => ( ); + +export const DropzoneIcon: React.FC = (props) => ( + +); diff --git a/webapp/src/component/layout/BaseView.tsx b/webapp/src/component/layout/BaseView.tsx index 54f268e40b..26cd47da76 100644 --- a/webapp/src/component/layout/BaseView.tsx +++ b/webapp/src/component/layout/BaseView.tsx @@ -47,6 +47,7 @@ export interface BaseViewProps { allCentered?: boolean; 'data-cy'?: string; initialSearch?: string; + overflow?: string; } export const BaseView = (props: BaseViewProps) => { @@ -144,6 +145,7 @@ export const BaseView = (props: BaseViewProps) => { display="grid" position="relative" maxWidth="100%" + sx={{ overflow: props.overflow }} > {typeof props.children === 'function' ? props.children() diff --git a/webapp/src/custom.d.ts b/webapp/src/custom.d.ts index b89d39253d..cd3639da53 100644 --- a/webapp/src/custom.d.ts +++ b/webapp/src/custom.d.ts @@ -18,6 +18,9 @@ declare module '*.svg' { const content: React.FunctionComponent>; export default content; } +import { colors } from './colors'; + +const all = { ...colors.light, ...colors.dark }; declare module '@mui/material/styles/createPalette' { interface Palette { @@ -36,6 +39,7 @@ declare module '@mui/material/styles/createPalette' { marker: Marker; topBanner: TopBanner; quickStart: QuickStart; + import: typeof all.import; } interface PaletteOptions { @@ -54,6 +58,7 @@ declare module '@mui/material/styles/createPalette' { marker: Marker; topBanner: TopBanner; quickStart: QuickStart; + import: typeof all.import; } } diff --git a/webapp/src/service/apiSchema.generated.ts b/webapp/src/service/apiSchema.generated.ts index 56b461c26d..917c326deb 100644 --- a/webapp/src/service/apiSchema.generated.ts +++ b/webapp/src/service/apiSchema.generated.ts @@ -141,6 +141,10 @@ export interface paths { /** Sets namespace for file to import. */ put: operations["selectNamespace"]; }; + "/v2/projects/{projectId}/import/apply-streaming": { + /** Imports the data prepared in previous step. Streams current status. */ + put: operations["applyImportStreaming"]; + }; "/v2/projects/{projectId}/import/apply": { /** Imports the data prepared in previous step */ put: operations["applyImport"]; @@ -260,9 +264,30 @@ export interface paths { "/v2/slug/generate-organization": { post: operations["generateOrganizationSlug"]; }; - "/v2/public/business-events/report": { + "/v2/public/telemetry/report": { post: operations["report"]; }; + "/v2/public/licensing/subscription": { + post: operations["getMySubscription"]; + }; + "/v2/public/licensing/set-key": { + post: operations["onLicenceSetKey"]; + }; + "/v2/public/licensing/report-usage": { + post: operations["reportUsage"]; + }; + "/v2/public/licensing/report-error": { + post: operations["reportError"]; + }; + "/v2/public/licensing/release-key": { + post: operations["releaseKey"]; + }; + "/v2/public/licensing/prepare-set-key": { + post: operations["prepareSetLicenseKey"]; + }; + "/v2/public/business-events/report": { + post: operations["report_1"]; + }; "/v2/public/business-events/identify": { post: operations["identify"]; }; @@ -397,7 +422,7 @@ export interface paths { post: operations["upload"]; }; "/v2/ee-license/prepare-set-license-key": { - post: operations["prepareSetLicenseKey"]; + post: operations["prepareSetLicenseKey_1"]; }; "/v2/api-keys": { get: operations["allByUser"]; @@ -708,24 +733,6 @@ export interface components { | "SERVER_ADMIN"; /** @description The user's permission type. This field is null if uses granular permissions */ type?: "NONE" | "VIEW" | "TRANSLATE" | "REVIEW" | "EDIT" | "MANAGE"; - /** - * @deprecated - * @description Deprecated (use translateLanguageIds). - * - * List of languages current user has TRANSLATE permission to. If null, all languages edition is permitted. - * @example 200001,200004 - */ - permittedLanguageIds?: number[]; - /** - * @description List of languages user can translate to. If null, all languages editing is permitted. - * @example 200001,200004 - */ - translateLanguageIds?: number[]; - /** - * @description List of languages user can change state to. If null, changing state of all language values is permitted. - * @example 200001,200004 - */ - stateChangeLanguageIds?: number[]; /** * @description List of languages user can view. If null, all languages view is permitted. * @example 200001,200004 @@ -763,6 +770,24 @@ export interface components { | "content-delivery.publish" | "webhooks.manage" )[]; + /** + * @description List of languages user can translate to. If null, all languages editing is permitted. + * @example 200001,200004 + */ + translateLanguageIds?: number[]; + /** + * @description List of languages user can change state to. If null, changing state of all language values is permitted. + * @example 200001,200004 + */ + stateChangeLanguageIds?: number[]; + /** + * @deprecated + * @description Deprecated (use translateLanguageIds). + * + * List of languages current user has TRANSLATE permission to. If null, all languages edition is permitted. + * @example 200001,200004 + */ + permittedLanguageIds?: number[]; }; LanguageModel: { /** Format: int64 */ @@ -1038,6 +1063,8 @@ export interface components { namespace?: string; /** @description Translations to update */ translations?: { [key: string]: string }; + /** @description Translation states to update, if not provided states won't be modified */ + states?: { [key: string]: "TRANSLATED" | "REVIEWED" }; /** @description Tags of the key. If not provided tags won't be modified */ tags?: string[]; /** @description IDs of screenshots to delete */ @@ -1045,6 +1072,8 @@ export interface components { /** @description Ids of screenshots uploaded with /v2/image-upload endpoint */ screenshotUploadedImageIds?: number[]; screenshotsToAdd?: components["schemas"]["KeyScreenshotDto"][]; + /** @description Keys in the document used as a context for machine translation. Keys in the same order as they appear in the document. The order is important! We are using it for graph distance calculation. */ + relatedKeysInOrder?: components["schemas"]["RelatedKeyDto"][]; }; KeyInScreenshotPositionDto: { /** Format: int32 */ @@ -1065,6 +1094,11 @@ export interface components { uploadedImageId: number; positions?: components["schemas"]["KeyInScreenshotPositionDto"][]; }; + /** @description Keys in the document used as a context for machine translation. Keys in the same order as they appear in the document. The order is important! We are using it for graph distance calculation. */ + RelatedKeyDto: { + namespace?: string; + keyName: string; + }; KeyInScreenshotModel: { /** Format: int64 */ keyId: number; @@ -1350,6 +1384,16 @@ export interface components { SetFileNamespaceRequest: { namespace?: string; }; + StreamingResponseBody: { [key: string]: unknown }; + /** @description User who created the comment */ + SimpleUserAccountModel: { + /** Format: int64 */ + id: number; + username: string; + name?: string; + avatar?: components["schemas"]["Avatar"]; + deleted: boolean; + }; TranslationCommentModel: { /** * Format: int64 @@ -1360,7 +1404,7 @@ export interface components { text: string; /** @description State of translation */ state: "RESOLUTION_NOT_NEEDED" | "NEEDS_RESOLUTION" | "RESOLVED"; - author: components["schemas"]["UserAccountModel"]; + author: components["schemas"]["SimpleUserAccountModel"]; /** * Format: date-time * @description Date when it was created @@ -1372,18 +1416,6 @@ export interface components { */ updatedAt: string; }; - /** @description User who created the comment */ - UserAccountModel: { - /** Format: int64 */ - id: number; - username: string; - name?: string; - emailAwaitingVerification?: string; - avatar?: components["schemas"]["Avatar"]; - globalServerRole: "USER" | "ADMIN"; - deleted: boolean; - disabled: boolean; - }; TranslationCommentDto: { text: string; state: "RESOLUTION_NOT_NEEDED" | "NEEDS_RESOLUTION" | "RESOLVED"; @@ -1487,13 +1519,13 @@ export interface components { id: number; description: string; /** Format: int64 */ - createdAt: number; - /** Format: int64 */ - updatedAt: number; + lastUsedAt?: number; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - lastUsedAt?: number; + createdAt: number; + /** Format: int64 */ + updatedAt: number; }; SetOrganizationRoleDto: { roleType: "MEMBER" | "OWNER"; @@ -1629,17 +1661,17 @@ export interface components { key: string; /** Format: int64 */ id: number; - userFullName?: string; projectName: string; - description: string; + userFullName?: string; username?: string; + description: string; /** Format: int64 */ - projectId: number; + lastUsedAt?: number; + scopes: string[]; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - lastUsedAt?: number; - scopes: string[]; + projectId: number; }; SuperTokenRequest: { /** @description Has to be provided when TOTP enabled */ @@ -1651,6 +1683,136 @@ export interface components { name: string; oldSlug?: string; }; + TelemetryReportRequest: { + instanceId: string; + /** Format: int64 */ + projectsCount: number; + /** Format: int64 */ + translationsCount: number; + /** Format: int64 */ + languagesCount: number; + /** Format: int64 */ + distinctLanguagesCount: number; + /** Format: int64 */ + usersCount: number; + }; + GetMySubscriptionDto: { + licenseKey: string; + instanceId: string; + }; + PlanIncludedUsageModel: { + /** Format: int64 */ + seats: number; + /** Format: int64 */ + translationSlots: number; + /** Format: int64 */ + translations: number; + /** Format: int64 */ + mtCredits: number; + }; + PlanPricesModel: { + perSeat: number; + perThousandTranslations?: number; + perThousandMtCredits?: number; + subscriptionMonthly: number; + subscriptionYearly: number; + }; + SelfHostedEePlanModel: { + /** Format: int64 */ + id: number; + name: string; + public: boolean; + enabledFeatures: ( + | "GRANULAR_PERMISSIONS" + | "PRIORITIZED_FEATURE_REQUESTS" + | "PREMIUM_SUPPORT" + | "DEDICATED_SLACK_CHANNEL" + | "ASSISTED_UPDATES" + | "DEPLOYMENT_ASSISTANCE" + | "BACKUP_CONFIGURATION" + | "TEAM_TRAINING" + | "ACCOUNT_MANAGER" + | "STANDARD_SUPPORT" + | "PROJECT_LEVEL_CONTENT_STORAGES" + | "WEBHOOKS" + | "MULTIPLE_CONTENT_DELIVERY_CONFIGS" + )[]; + prices: components["schemas"]["PlanPricesModel"]; + includedUsage: components["schemas"]["PlanIncludedUsageModel"]; + hasYearlyPrice: boolean; + }; + SelfHostedEeSubscriptionModel: { + /** Format: int64 */ + id: number; + /** Format: int64 */ + currentPeriodStart?: number; + /** Format: int64 */ + currentPeriodEnd?: number; + currentBillingPeriod: "MONTHLY" | "YEARLY"; + /** Format: int64 */ + createdAt: number; + plan: components["schemas"]["SelfHostedEePlanModel"]; + status: + | "ACTIVE" + | "CANCELED" + | "PAST_DUE" + | "UNPAID" + | "ERROR" + | "KEY_USED_BY_ANOTHER_INSTANCE"; + licenseKey?: string; + estimatedCosts?: number; + }; + SetLicenseKeyLicensingDto: { + licenseKey: string; + /** Format: int64 */ + seats: number; + instanceId: string; + }; + ReportUsageDto: { + licenseKey: string; + /** Format: int64 */ + seats: number; + }; + ReportErrorDto: { + stackTrace: string; + licenseKey: string; + }; + ReleaseKeyDto: { + licenseKey: string; + }; + PrepareSetLicenseKeyDto: { + licenseKey: string; + /** Format: int64 */ + seats: number; + }; + AverageProportionalUsageItemModel: { + total: number; + unusedQuantity: number; + usedQuantity: number; + usedQuantityOverPlan: number; + }; + PrepareSetEeLicenceKeyModel: { + plan: components["schemas"]["SelfHostedEePlanModel"]; + usage: components["schemas"]["UsageModel"]; + }; + SumUsageItemModel: { + total: number; + /** Format: int64 */ + unusedQuantity: number; + /** Format: int64 */ + usedQuantity: number; + /** Format: int64 */ + usedQuantityOverPlan: number; + }; + UsageModel: { + subscriptionPrice?: number; + /** @description Relevant for invoices only. When there are applied stripe credits, we need to reduce the total price by this amount. */ + appliedStripeCredits?: number; + seats: components["schemas"]["AverageProportionalUsageItemModel"]; + translations: components["schemas"]["AverageProportionalUsageItemModel"]; + credits?: components["schemas"]["SumUsageItemModel"]; + total: number; + }; BusinessEventReportRequest: { eventName: string; anonymousUserId?: string; @@ -1761,10 +1923,14 @@ export interface components { name: string; namespace?: string; translations?: { [key: string]: string }; + /** @description Translation states to update, if not provided states won't be modified */ + states?: { [key: string]: "TRANSLATED" | "REVIEWED" }; tags?: string[]; /** @description Ids of screenshots uploaded with /v2/image-upload endpoint */ screenshotUploadedImageIds?: number[]; screenshots?: components["schemas"]["KeyScreenshotDto"][]; + /** @description Keys in the document used as a context for machine translation. Keys in the same order as they appear in the document. The order is important! We are using it for graph distance calculation. */ + relatedKeysInOrder?: components["schemas"]["RelatedKeyDto"][]; }; StorageTestResult: { success: boolean; @@ -1947,7 +2113,9 @@ export interface components { | "UNEXPECTED_ERROR_WHILE_PUBLISHING_TO_CONTENT_STORAGE" | "WEBHOOK_RESPONDED_WITH_NON_200_STATUS" | "UNEXPECTED_ERROR_WHILE_EXECUTING_WEBHOOK" - | "CONTENT_STORAGE_IS_IN_USE"; + | "CONTENT_STORAGE_IS_IN_USE" + | "CANNOT_SET_STATE_FOR_MISSING_TRANSLATION" + | "NO_PROJECT_ID_PROVIDED"; params?: { [key: string]: unknown }[]; }; UntagKeysRequest: { @@ -2010,15 +2178,6 @@ export interface components { /** @description If the job failed, this is the error message */ errorMessage?: string; }; - /** @description The user who started the job */ - SimpleUserAccountModel: { - /** Format: int64 */ - id: number; - username: string; - name?: string; - avatar?: components["schemas"]["Avatar"]; - deleted: boolean; - }; TagKeysRequest: { keyIds: number[]; tags: string[]; @@ -2131,15 +2290,9 @@ export interface components { filterNamespace?: string[]; zip: boolean; }; - StreamingResponseBody: { [key: string]: unknown }; BigMetaDto: { - /** @description List of keys, visible, in order as they appear in the document. The order is important! We are using it for graph distance calculation. */ - relatedKeysInOrder: components["schemas"]["RelatedKeyDto"][]; - }; - /** @description List of keys, visible, in order as they appear in the document. The order is important! We are using it for graph distance calculation. */ - RelatedKeyDto: { - namespace?: string; - keyName: string; + /** @description Keys in the document used as a context for machine translation. Keys in the same order as they appear in the document. The order is important! We are using it for graph distance calculation. */ + relatedKeysInOrder?: components["schemas"]["RelatedKeyDto"][]; }; TranslationCommentWithLangKeyDto: { /** Format: int64 */ @@ -2234,75 +2387,6 @@ export interface components { createdAt: string; location?: string; }; - AverageProportionalUsageItemModel: { - total: number; - unusedQuantity: number; - usedQuantity: number; - usedQuantityOverPlan: number; - }; - PlanIncludedUsageModel: { - /** Format: int64 */ - seats: number; - /** Format: int64 */ - translationSlots: number; - /** Format: int64 */ - translations: number; - /** Format: int64 */ - mtCredits: number; - }; - PlanPricesModel: { - perSeat: number; - perThousandTranslations?: number; - perThousandMtCredits?: number; - subscriptionMonthly: number; - subscriptionYearly: number; - }; - PrepareSetEeLicenceKeyModel: { - plan: components["schemas"]["SelfHostedEePlanModel"]; - usage: components["schemas"]["UsageModel"]; - }; - SelfHostedEePlanModel: { - /** Format: int64 */ - id: number; - name: string; - public: boolean; - enabledFeatures: ( - | "GRANULAR_PERMISSIONS" - | "PRIORITIZED_FEATURE_REQUESTS" - | "PREMIUM_SUPPORT" - | "DEDICATED_SLACK_CHANNEL" - | "ASSISTED_UPDATES" - | "DEPLOYMENT_ASSISTANCE" - | "BACKUP_CONFIGURATION" - | "TEAM_TRAINING" - | "ACCOUNT_MANAGER" - | "STANDARD_SUPPORT" - | "PROJECT_LEVEL_CONTENT_STORAGES" - | "WEBHOOKS" - | "MULTIPLE_CONTENT_DELIVERY_CONFIGS" - )[]; - prices: components["schemas"]["PlanPricesModel"]; - includedUsage: components["schemas"]["PlanIncludedUsageModel"]; - hasYearlyPrice: boolean; - }; - SumUsageItemModel: { - total: number; - /** Format: int64 */ - unusedQuantity: number; - /** Format: int64 */ - usedQuantity: number; - /** Format: int64 */ - usedQuantityOverPlan: number; - }; - UsageModel: { - subscriptionPrice?: number; - /** @description Relevant for invoices only. When there are applied stripe credits, we need to reduce the total price by this amount. */ - appliedStripeCredits?: number; - seats: components["schemas"]["AverageProportionalUsageItemModel"]; - translations: components["schemas"]["AverageProportionalUsageItemModel"]; - credits?: components["schemas"]["SumUsageItemModel"]; - total: number; - }; CreateApiKeyDto: { /** Format: int64 */ projectId: number; @@ -2446,18 +2530,18 @@ export interface components { name: string; /** Format: int64 */ id: number; - basePermissions: components["schemas"]["PermissionModel"]; - /** @example This is a beautiful organization full of beautiful and clever people */ - description?: string; /** * @description The role of currently authorized user. * * Can be null when user has direct access to one of the projects owned by the organization. */ currentUserRole?: "MEMBER" | "OWNER"; + basePermissions: components["schemas"]["PermissionModel"]; + /** @example This is a beautiful organization full of beautiful and clever people */ + description?: string; + avatar?: components["schemas"]["Avatar"]; /** @example btforg */ slug: string; - avatar?: components["schemas"]["Avatar"]; }; PublicBillingConfigurationDTO: { enabled: boolean; @@ -2566,18 +2650,18 @@ export interface components { name: string; /** Format: int64 */ id: number; + baseTranslation?: string; namespace?: string; translation?: string; - baseTranslation?: string; }; KeySearchSearchResultModel: { view?: components["schemas"]["KeySearchResultView"]; name: string; /** Format: int64 */ id: number; + baseTranslation?: string; namespace?: string; translation?: string; - baseTranslation?: string; }; PagedModelKeySearchSearchResultModel: { _embedded?: { @@ -2767,7 +2851,7 @@ export interface components { }; page?: components["schemas"]["PageMetadata"]; }; - EntityModelImportFileIssueView: { + ImportFileIssueModel: { /** Format: int64 */ id: number; type: @@ -2781,10 +2865,9 @@ export interface components { | "TARGET_NOT_PROVIDED" | "TRANSLATION_TOO_LONG" | "KEY_IS_BLANK"; - params: components["schemas"]["ImportFileIssueParamView"][]; + params: components["schemas"]["ImportFileIssueParamModel"][]; }; - ImportFileIssueParamView: { - value?: string; + ImportFileIssueParamModel: { type: | "KEY_NAME" | "KEY_ID" @@ -2793,10 +2876,11 @@ export interface components { | "VALUE" | "LINE" | "FILE_NODE_ORIGINAL"; + value?: string; }; - PagedModelEntityModelImportFileIssueView: { + PagedModelImportFileIssueModel: { _embedded?: { - importFileIssueViews?: components["schemas"]["EntityModelImportFileIssueView"][]; + importFileIssues?: components["schemas"]["ImportFileIssueModel"][]; }; page?: components["schemas"]["PageMetadata"]; }; @@ -3079,13 +3163,13 @@ export interface components { id: number; description: string; /** Format: int64 */ - createdAt: number; - /** Format: int64 */ - updatedAt: number; + lastUsedAt?: number; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - lastUsedAt?: number; + createdAt: number; + /** Format: int64 */ + updatedAt: number; }; OrganizationRequestParamsDto: { filterCurrentUserOwner: boolean; @@ -3197,24 +3281,79 @@ export interface components { }; ApiKeyWithLanguagesModel: { /** + * @deprecated * @description Languages for which user has translate permission. - * - * If null, all languages are permitted. */ permittedLanguageIds?: number[]; /** Format: int64 */ id: number; - userFullName?: string; projectName: string; - description: string; + userFullName?: string; username?: string; + description: string; /** Format: int64 */ - projectId: number; + lastUsedAt?: number; + scopes: string[]; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - lastUsedAt?: number; - scopes: string[]; + projectId: number; + }; + ApiKeyPermissionsModel: { + /** + * Format: int64 + * @description The API key's project id or the one provided as query param + */ + projectId: number; + /** + * @description List of languages user can view. If null, all languages view is permitted. + * @example 200001,200004 + */ + viewLanguageIds?: number[]; + /** + * @description List of languages user can translate to. If null, all languages editing is permitted. + * @example 200001,200004 + */ + translateLanguageIds?: number[]; + /** + * @description List of languages user can change state to. If null, changing state of all language values is permitted. + * @example 200001,200004 + */ + stateChangeLanguageIds?: number[]; + /** + * @description Granted scopes to the user. When user has type permissions, this field contains permission scopes of the type. + * @example KEYS_EDIT,TRANSLATIONS_VIEW + */ + scopes: ( + | "translations.view" + | "translations.edit" + | "keys.edit" + | "screenshots.upload" + | "screenshots.delete" + | "screenshots.view" + | "activity.view" + | "languages.edit" + | "admin" + | "project.edit" + | "members.view" + | "members.edit" + | "translation-comments.add" + | "translation-comments.edit" + | "translation-comments.set-state" + | "translations.state-edit" + | "keys.view" + | "keys.delete" + | "keys.create" + | "batch-jobs.view" + | "batch-jobs.cancel" + | "translations.batch-by-tm" + | "translations.batch-machine" + | "content-delivery.manage" + | "content-delivery.publish" + | "webhooks.manage" + )[]; + /** @description The user's permission type. This field is null if user has assigned granular permissions or if returning API key's permissions */ + type?: "NONE" | "VIEW" | "TRANSLATE" | "REVIEW" | "EDIT" | "MANAGE"; }; PagedModelUserAccountModel: { _embedded?: { @@ -3222,6 +3361,17 @@ export interface components { }; page?: components["schemas"]["PageMetadata"]; }; + UserAccountModel: { + /** Format: int64 */ + id: number; + username: string; + name?: string; + emailAwaitingVerification?: string; + avatar?: components["schemas"]["Avatar"]; + globalServerRole: "USER" | "ADMIN"; + deleted: boolean; + disabled: boolean; + }; UserTotpDisableRequestDto: { password: string; }; @@ -4698,6 +4848,38 @@ export interface operations { }; }; }; + /** Imports the data prepared in previous step. Streams current status. */ + applyImportStreaming: { + parameters: { + query: { + /** Whether override or keep all translations with unresolved conflicts */ + forceMode?: "OVERRIDE" | "KEEP" | "NO_FORCE"; + }; + path: { + projectId: number; + }; + }; + responses: { + /** OK */ + 200: { + content: { + "application/x-ndjson": components["schemas"]["StreamingResponseBody"]; + }; + }; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + }; /** Imports the data prepared in previous step */ applyImport: { parameters: { @@ -6034,6 +6216,179 @@ export interface operations { }; }; report: { + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["TelemetryReportRequest"]; + }; + }; + }; + getMySubscription: { + responses: { + /** OK */ + 200: { + content: { + "*/*": components["schemas"]["SelfHostedEeSubscriptionModel"]; + }; + }; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GetMySubscriptionDto"]; + }; + }; + }; + onLicenceSetKey: { + responses: { + /** OK */ + 200: { + content: { + "*/*": components["schemas"]["SelfHostedEeSubscriptionModel"]; + }; + }; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SetLicenseKeyLicensingDto"]; + }; + }; + }; + reportUsage: { + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReportUsageDto"]; + }; + }; + }; + reportError: { + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReportErrorDto"]; + }; + }; + }; + releaseKey: { + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReleaseKeyDto"]; + }; + }; + }; + prepareSetLicenseKey: { + responses: { + /** OK */ + 200: { + content: { + "*/*": components["schemas"]["PrepareSetEeLicenceKeyModel"]; + }; + }; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PrepareSetLicenseKeyDto"]; + }; + }; + }; + report_1: { responses: { /** OK */ 200: unknown; @@ -7634,7 +7989,7 @@ export interface operations { }; }; }; - prepareSetLicenseKey: { + prepareSetLicenseKey_1: { responses: { /** OK */ 200: { @@ -8581,7 +8936,7 @@ export interface operations { /** OK */ 200: { content: { - "*/*": components["schemas"]["PagedModelEntityModelImportFileIssueView"]; + "*/*": components["schemas"]["PagedModelImportFileIssueModel"]; }; }; /** Bad Request */ diff --git a/webapp/src/service/http/useQueryApi.ts b/webapp/src/service/http/useQueryApi.ts index 416cf9355d..fbcb30185a 100644 --- a/webapp/src/service/http/useQueryApi.ts +++ b/webapp/src/service/http/useQueryApi.ts @@ -126,6 +126,45 @@ function autoErrorHandling( }; } +function getApiMutationOptions( + invalidatePrefix: string | undefined, + queryClient: QueryClient +) { + return (options: UseQueryOptions | undefined) => ({ + ...options, + onSuccess: (...params) => { + // @ts-ignore + options?.onSuccess?.(...params); + if (invalidatePrefix !== undefined) { + invalidateUrlPrefix(queryClient, invalidatePrefix); + } + }, + }); +} + +const getMutationCallback = < + MutationFn extends (variables: any, options: any) => any +>( + mutateFn: MutationFn, + customOptions: ReturnType, + fetchOptions: RequestOptions | undefined, + props: any +) => { + return useCallback( + ((variables, options) => { + return mutateFn( + variables, + autoErrorHandling( + customOptions(options as any), + Boolean( + fetchOptions?.disableAutoErrorHandle || props.options?.onError + ) + ) + ); + }) as any, + [mutateFn] + ) as MutationFn; +}; export const useApiMutation = < Url extends keyof Paths, Method extends keyof Paths[Url], @@ -137,18 +176,7 @@ export const useApiMutation = < const { url, method, fetchOptions, options, invalidatePrefix } = props; // inject custom onSuccess - const customOptions = ( - options: UseQueryOptions | undefined - ) => ({ - ...options, - onSuccess: (...params) => { - // @ts-ignore - options?.onSuccess?.(...params); - if (invalidatePrefix !== undefined) { - invalidateUrlPrefix(queryClient, invalidatePrefix); - } - }, - }); + const customOptions = getApiMutationOptions(invalidatePrefix, queryClient); const mutation = useMutation< ResponseContent, @@ -163,34 +191,18 @@ export const useApiMutation = < customOptions(options as any) as any ); - const mutate = useCallback( - (variables, options) => { - return mutation.mutate( - variables, - autoErrorHandling( - customOptions(options as any), - Boolean( - fetchOptions?.disableAutoErrorHandle || props.options?.onError - ) - ) - ); - }, - [mutation.mutate] + const mutate = getMutationCallback( + mutation.mutate, + customOptions, + fetchOptions, + props ); - const mutateAsync = useCallback( - (variables, options) => { - return mutation.mutateAsync( - variables, - autoErrorHandling( - customOptions(options as any), - Boolean( - fetchOptions?.disableAutoErrorHandle || props.options?.onError - ) - ) - ); - }, - [mutation.mutateAsync] + const mutateAsync = getMutationCallback( + mutation.mutateAsync, + customOptions, + fetchOptions, + props ); return { ...mutation, mutate, mutateAsync }; @@ -220,3 +232,81 @@ export const useBillingApiMutation = < >( props: MutationProps ) => useApiMutation(props); + +export const useNdJsonStreamedMutation = < + Url extends keyof Paths, + Method extends keyof Paths[Url], + Paths = paths +>( + props: MutationProps & { onData: (data: any) => void } +) => { + const queryClient = useQueryClient(); + const { url, method, fetchOptions, options, onData, invalidatePrefix } = + props; + + // inject custom onSuccess + const customOptions = getApiMutationOptions(invalidatePrefix, queryClient); + + const mutation = useMutation< + any[], + ApiError, + RequestParamsType + >(async (request) => { + const response = await apiHttpService.schemaRequestRaw( + url, + method, + { + ...fetchOptions, + } + )(request); + const reader = response.body?.getReader(); + const result: any[] = []; + while (reader) { + const { done, value } = await reader.read(); + const text = new TextDecoder().decode(value); + if (text) { + const parsed = getParsedJsonOrNull(text); + if (!parsed) { + continue; + } + if (parsed['error']) { + const error = parsed['error']; + throw new ApiError('Api error', error); + } + result.push(parsed); + onData(parsed); + } + if (done) { + break; + } + } + return result; + }, customOptions(options as any) as any); + + const mutate = getMutationCallback( + mutation.mutate, + customOptions, + fetchOptions, + props + ); + + const mutateAsync = getMutationCallback( + mutation.mutateAsync, + customOptions, + fetchOptions, + props + ); + return { ...mutation, mutate, mutateAsync }; +}; + +function getParsedJsonOrNull(json?: string): any { + if (!json) { + return null; + } + + try { + return JSON.parse(json); + } catch (e) { + return null; + } +} diff --git a/webapp/src/svgs/.DS_Store b/webapp/src/svgs/.DS_Store deleted file mode 100644 index 81dfd3a2db..0000000000 Binary files a/webapp/src/svgs/.DS_Store and /dev/null differ diff --git a/webapp/src/svgs/icons/dropzone.svg b/webapp/src/svgs/icons/dropzone.svg new file mode 100644 index 0000000000..faedf6c607 --- /dev/null +++ b/webapp/src/svgs/icons/dropzone.svg @@ -0,0 +1,4 @@ + + + diff --git a/webapp/src/translationTools/useErrorTranslation.ts b/webapp/src/translationTools/useErrorTranslation.ts index a50a12319b..346c8428ba 100644 --- a/webapp/src/translationTools/useErrorTranslation.ts +++ b/webapp/src/translationTools/useErrorTranslation.ts @@ -109,7 +109,8 @@ export function useErrorTranslation() { return t('webhook_responded_with_non_200_status'); case 'unexpected_error_while_executing_webhook': return t('unexpected_error_while_executing_webhook'); - + case 'resource_not_found': + return t('resource_not_found'); default: return code; } diff --git a/webapp/src/views/projects/import/ImportView.tsx b/webapp/src/views/projects/import/ImportView.tsx index 83adba1f68..bc7c8c2362 100644 --- a/webapp/src/views/projects/import/ImportView.tsx +++ b/webapp/src/views/projects/import/ImportView.tsx @@ -1,5 +1,5 @@ import { FunctionComponent, useEffect, useState } from 'react'; -import { Box } from '@mui/material'; +import { Box, Button } from '@mui/material'; import { T, useTranslate } from '@tolgee/react'; import { container } from 'tsyringe'; @@ -11,9 +11,8 @@ import { MessageService } from 'tg.service/MessageService'; import { components } from 'tg.service/apiSchema.generated'; import { useGlobalActions } from 'tg.globalContext/GlobalContext'; import { TranslatedError } from 'tg.translationTools/TranslatedError'; -import LoadingButton from 'tg.component/common/form/LoadingButton'; -import { ImportAlertError } from './ImportAlertError'; +import { ImportAlertError } from './component/ImportAlertError'; import { ImportConflictNotResolvedErrorDialog } from './component/ImportConflictNotResolvedErrorDialog'; import { ImportConflictResolutionDialog } from './component/ImportConflictResolutionDialog'; import ImportFileInput from './component/ImportFileInput'; @@ -21,6 +20,7 @@ import { ImportResult } from './component/ImportResult'; import { useApplyImportHelper } from './hooks/useApplyImportHelper'; import { useImportDataHelper } from './hooks/useImportDataHelper'; import { BaseProjectView } from '../BaseProjectView'; +import { ImportResultLoadingOverlay } from './component/ImportResultLoadingOverlay'; const messageService = container.resolve(MessageService); @@ -71,6 +71,11 @@ export const ImportView: FunctionComponent = () => { } }, [applyImportHelper.loading, applyImportHelper.loaded]); + const loading = + dataHelper.addFilesMutation.isLoading || applyImportHelper.loading; + + const [isProgressOverlayActive, setIsProgressOverlayActive] = useState(false); + return ( { ], ]} maxWidth="wide" + overflow="auto" > { { + applyImportHelper.clear(); + dataHelper.addFilesMutation.reset(); + }} + filesUploaded={dataHelper.addFilesMutation.isSuccess} + isProgressOverlayActive={isProgressOverlayActive} + onProgressOverlayActiveChange={(isActive) => + setIsProgressOverlayActive(isActive) + } /> {dataHelper.addFilesMutation.data?.errors?.map((e, idx) => ( @@ -101,44 +125,47 @@ export const ImportView: FunctionComponent = () => { addFilesMutation={dataHelper.addFilesMutation} /> ))} - - - {dataHelper.result && ( - - - { - confirmation({ - onConfirm: () => dataHelper.onCancel(), - title: , - message: , - }); - }} - > - - - - - - - - + + + + {dataHelper.result && ( + + + + + + + + + )} - )} + { resolveFirstUnresolved(); diff --git a/webapp/src/views/projects/import/ImportAlertError.tsx b/webapp/src/views/projects/import/component/ImportAlertError.tsx similarity index 97% rename from webapp/src/views/projects/import/ImportAlertError.tsx rename to webapp/src/views/projects/import/component/ImportAlertError.tsx index 77ebcebaae..be5bc03015 100644 --- a/webapp/src/views/projects/import/ImportAlertError.tsx +++ b/webapp/src/views/projects/import/component/ImportAlertError.tsx @@ -16,7 +16,7 @@ import CloseIcon from '@mui/icons-material/Close'; import { T } from '@tolgee/react'; import { components } from 'tg.service/apiSchema.generated'; -import { useImportDataHelper } from './hooks/useImportDataHelper'; +import { useImportDataHelper } from '../hooks/useImportDataHelper'; export const ImportAlertError: FunctionComponent<{ error: components['schemas']['ImportAddFilesResultModel']['errors'][0]; diff --git a/webapp/src/views/projects/import/component/ImportFileDropzone.tsx b/webapp/src/views/projects/import/component/ImportFileDropzone.tsx index 254b494357..0523698f18 100644 --- a/webapp/src/views/projects/import/component/ImportFileDropzone.tsx +++ b/webapp/src/views/projects/import/component/ImportFileDropzone.tsx @@ -1,52 +1,63 @@ import clsx from 'clsx'; import { Box, styled } from '@mui/material'; import { green, red } from '@mui/material/colors'; -import { Backup, HighlightOff } from '@mui/icons-material'; +import { HighlightOff } from '@mui/icons-material'; import React, { FunctionComponent, useState } from 'react'; import { FileUploadFixtures } from 'tg.fixtures/FileUploadFixtures'; import { MAX_FILE_COUNT } from './ImportFileInput'; +import { DropzoneIcon } from 'tg.component/CustomIcons'; export interface ScreenshotDropzoneProps { onNewFiles: (files: File[]) => void; + active: boolean; } const StyledWrapper = styled(Box)` pointer-events: none; opacity: 0; transition: opacity 0.2s; + background-color: ${({ theme }) => theme.palette.background.paper}; - &.valid { - backdrop-filter: blur(5px); - border: 1px solid ${green[200]}; - background-color: ${green[50]}; - opacity: 0.9; + &:before { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + border-radius: 4px; + height: 100%; } + &.valid, &.invalid { - border: 1px solid ${red[200]}; - opacity: 0.9; - background-color: ${red[50]}; + opacity: 1; + } + + &.valid:before, + &.invalid:before { backdrop-filter: blur(5px); + opacity: 0.3; + } + + &.valid:before { + background-color: ${green[200]}; + } + + &.invalid:before { + background-color: ${red[200]}; } `; -const StyledValidIcon = styled(Backup)` - filter: drop-shadow(1px 1px 0px ${green[200]}) - drop-shadow(-1px 1px 0px ${green[200]}) - drop-shadow(1px -1px 0px ${green[200]}) - drop-shadow(-1px -1px 0px ${green[200]}); +const StyledValidIcon = styled(DropzoneIcon)` font-size: 100px; - color: ${({ theme }) => theme.palette.common.white}; + color: ${({ theme }) => theme.palette.import.progressDone}; `; const StyledInvalidIcon = styled(HighlightOff)` - filter: drop-shadow(1px 1px 0px ${red[200]}) - drop-shadow(-1px 1px 0px ${red[200]}) drop-shadow(1px -1px 0px ${red[200]}) - drop-shadow(-1px -1px 0px ${red[200]}); font-size: 100px; - color: ${({ theme }) => theme.palette.common.white}; + fill: ${({ theme }) => theme.palette.error.main}; `; export const ImportFileDropzone: FunctionComponent = ( @@ -58,6 +69,9 @@ export const ImportFileDropzone: FunctionComponent = ( ); const onDragEnter = (e: React.DragEvent) => { + if (!props.active) { + return; + } e.stopPropagation(); e.preventDefault(); setDragEnterTarget(e.target); @@ -74,6 +88,9 @@ export const ImportFileDropzone: FunctionComponent = ( }; const onDragLeave = (e: React.DragEvent) => { + if (!props.active) { + return; + } e.stopPropagation(); e.preventDefault(); if (e.target === dragEnterTarget) { @@ -82,6 +99,9 @@ export const ImportFileDropzone: FunctionComponent = ( }; const onDrop = async (e: React.DragEvent) => { + if (!props.active) { + return; + } e.stopPropagation(); e.preventDefault(); if (e.dataTransfer.items) { diff --git a/webapp/src/views/projects/import/component/ImportFileInput.tsx b/webapp/src/views/projects/import/component/ImportFileInput.tsx index 2f9a9eb174..e691d8c99d 100644 --- a/webapp/src/views/projects/import/component/ImportFileInput.tsx +++ b/webapp/src/views/projects/import/component/ImportFileInput.tsx @@ -1,21 +1,43 @@ import React, { FunctionComponent, ReactNode, useState } from 'react'; import { QuickStartHighlight } from 'tg.component/layout/QuickStartGuide/QuickStartHighlight'; -import { Box, styled, Typography } from '@mui/material'; +import { Box, Button, styled, Typography } from '@mui/material'; import { T, useTranslate } from '@tolgee/react'; import { container } from 'tsyringe'; import { useConfig } from 'tg.globalContext/helpers'; import { MessageActions } from 'tg.store/global/MessageActions'; import { Message } from 'tg.store/global/types'; -import LoadingButton from 'tg.component/common/form/LoadingButton'; import { ImportFileDropzone } from './ImportFileDropzone'; +import { ImportProgressOverlay } from './ImportProgressOverlay'; +import { + ImportInputAreaLayout, + ImportInputAreaLayoutBottom, + ImportInputAreaLayoutCenter, + ImportInputAreaLayoutTitle, + ImportInputAreaLayoutTop, +} from './ImportInputAreaLayout'; export const MAX_FILE_COUNT = 20; +export type OperationType = 'addFiles' | 'apply'; + +export type OperationStatusType = + | 'PREPARING_AND_VALIDATING' + | 'STORING_KEYS' + | 'STORING_TRANSLATIONS' + | 'FINALIZING'; + type ImportFileInputProps = { onNewFiles: (files: File[]) => void; loading: boolean; + operation?: OperationType; + operationStatus?: OperationStatusType; + importDone: boolean; + onImportMore: () => void; + filesUploaded?: boolean; + onProgressOverlayActiveChange: (isActive: boolean) => void; + isProgressOverlayActive: boolean; }; export type ValidationResult = { @@ -25,10 +47,12 @@ export type ValidationResult = { const StyledRoot = styled(Box)(({ theme }) => ({ borderRadius: theme.shape.borderRadius, - border: `1px dashed ${theme.palette.emphasis[400]}`, - maxWidth: 600, + border: `1px dashed ${theme.palette.emphasis[100]}`, margin: '0px auto', width: '100%', + position: 'relative', + backgroundColor: theme.palette.background.paper, + marginTop: '16px', })); const messageActions = container.resolve(MessageActions); @@ -45,6 +69,7 @@ const ImportFileInput: FunctionComponent = (props) => { 'properties', ]; const [resetKey, setResetKey] = useState(0); + function resetInput() { setResetKey((key) => key + 1); } @@ -142,55 +167,63 @@ const ImportFileInput: FunctionComponent = (props) => { }; return ( - + - - onFileSelected(e)} - multiple - accept={ALLOWED_EXTENSIONS.join(',')} - /> - - - - - + + - fileRef.current?.dispatchEvent(new MouseEvent('click')) + onImportMore={props.onImportMore} + filesUploaded={props.filesUploaded} + operationStatus={props.operationStatus} + onActiveChange={(isActive) => + props.onProgressOverlayActiveChange(isActive) } - variant="outlined" - color="primary" - > - - - - - - + /> + + onFileSelected(e)} + multiple + accept={ALLOWED_EXTENSIONS.join(',')} + /> + + + + + + + + + + + + + ); }; - export default ImportFileInput; diff --git a/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx b/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx new file mode 100644 index 0000000000..aadf3c8fae --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx @@ -0,0 +1,43 @@ +import { Box, styled, Typography } from '@mui/material'; +import React, { FC, ReactNode } from 'react'; + +export const ImportInputAreaLayout = styled(Box)` + justify-content: center; + align-items: center; + flex-direction: column; + display: flex; + padding-top: 40px; + padding-bottom: 40px; + height: 100%; +`; + +export const ImportInputAreaLayoutCenter = styled(Box)` + height: 76px; + width: 100%; + display: flex; + justify-content: center; + align-items: center; +`; + +export const ImportInputAreaLayoutTop = styled(Box)` + display: flex; + justify-content: center; + align-items: center; +`; + +export const ImportInputAreaLayoutBottom = styled(Box)` + min-height: 24px; +`; + +export const ImportInputAreaLayoutTitle: FC<{ + icon?: ReactNode; +}> = (props) => { + return ( + <> + + {props.children} + + {props.icon} + + ); +}; diff --git a/webapp/src/views/projects/import/component/ImportOperationStatus.tsx b/webapp/src/views/projects/import/component/ImportOperationStatus.tsx new file mode 100644 index 0000000000..fdc6b35b1e --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportOperationStatus.tsx @@ -0,0 +1,22 @@ +import { T } from '@tolgee/react'; +import { OperationStatusType } from './ImportFileInput'; +import { useDebounce } from 'use-debounce/lib'; +import React from 'react'; + +export const ImportOperationStatus = (props: { + status?: OperationStatusType; +}) => { + const [debouncedStatus] = useDebounce(props.status, 1000, { leading: true }); + + switch (debouncedStatus) { + case 'PREPARING_AND_VALIDATING': + return ; + case 'STORING_KEYS': + return ; + case 'STORING_TRANSLATIONS': + return ; + case 'FINALIZING': + return ; + } + return null; +}; diff --git a/webapp/src/views/projects/import/component/ImportOperationTitle.tsx b/webapp/src/views/projects/import/component/ImportOperationTitle.tsx new file mode 100644 index 0000000000..d35bce5522 --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportOperationTitle.tsx @@ -0,0 +1,52 @@ +import { T } from '@tolgee/react'; +import { OperationType } from './ImportFileInput'; +import React from 'react'; +import { TadaIcon } from 'tg.component/CustomIcons'; +import { Box } from '@mui/material'; +import { ImportInputAreaLayoutTitle } from './ImportInputAreaLayout'; + +export const ImportOperationTitle = (props: { + operation?: OperationType; + filesUploaded?: boolean; + importDone?: boolean; +}) => { + const Message = () => { + if (props.importDone) { + return ( + }> + + + ); + } + + if (props.filesUploaded) { + return ( + + + + ); + } + switch (props.operation) { + case 'addFiles': + return ( + + + + ); + case 'apply': + return ( + + + + ); + } + + return null; + }; + + return ( + + + + ); +}; diff --git a/webapp/src/views/projects/import/component/ImportProgress.tsx b/webapp/src/views/projects/import/component/ImportProgress.tsx new file mode 100644 index 0000000000..5dff5b67a3 --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportProgress.tsx @@ -0,0 +1,64 @@ +import { Box, styled } from '@mui/material'; +import clsx from 'clsx'; +import React, { useEffect } from 'react'; +import { useLoadingRegister } from 'tg.component/GlobalLoading'; + +const StyledProgress = styled('div')<{ loading?: string; finish?: string }>` + height: 4px; + width: 100%; + border-radius: 2px; + background: ${({ theme }) => theme.palette.import.progressBackground}; + position: relative; + + &::before { + content: ''; + height: 100%; + width: 0%; + position: absolute; + top: 0; + left: 0; + background-color: ${({ theme }) => theme.palette.import.progressWorking}; + transition: width 1s steps(1, jump-end), + background-color 1s steps(1, jump-end); + } + + &.loading::before { + width: 99%; + transition: width 30s cubic-bezier(0.15, 0.735, 0.095, 1); + } + + &.finish::before { + width: 100%; + background-color: ${({ theme }) => theme.palette.import.progressDone}; + transition: width 0.2s ease-in-out, background-color 0.2s steps(1, jump-end); + } +`; + +export const ImportProgressBar = (props: { + loading: boolean; + loaded: boolean; +}) => { + const [transitionLoading, setTransitionLoading] = React.useState(false); + + useLoadingRegister(props.loading); + + useEffect(() => { + setTimeout(() => { + setTransitionLoading(true); + }, 10); + }, []); + + const classes = clsx({ + loading: transitionLoading && props.loading, + finish: props.loaded, + }); + + return ( + + + + ); +}; diff --git a/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx b/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx new file mode 100644 index 0000000000..9fb6cdd55c --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx @@ -0,0 +1,144 @@ +import { Box, Button, styled } from '@mui/material'; +import { OperationStatusType, OperationType } from './ImportFileInput'; +import React, { useEffect, useState } from 'react'; +import clsx from 'clsx'; +import { + ImportInputAreaLayout, + ImportInputAreaLayoutBottom, + ImportInputAreaLayoutCenter, + ImportInputAreaLayoutTop, +} from './ImportInputAreaLayout'; +import { ImportProgressBar } from './ImportProgress'; +import { ImportOperationStatus } from './ImportOperationStatus'; +import { ImportOperationTitle } from './ImportOperationTitle'; +import { Link } from 'react-router-dom'; +import { LINKS, PARAMS } from 'tg.constants/links'; +import { T } from '@tolgee/react'; +import { useProject } from 'tg.hooks/useProject'; + +const StyledRoot = styled(Box)` + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: ${({ theme }) => theme.palette.background.paper}; + z-index: 1; + opacity: 0; + transition: opacity 0.1s ease-in-out; + + &.visible { + opacity: 1; + } +`; + +export const ImportProgressOverlay = (props: { + operation?: OperationType; + filesUploaded?: boolean; + importDone: boolean; + loading: boolean; + operationStatus?: OperationStatusType; + onImportMore: () => void; + onActiveChange: (isActive: boolean) => void; +}) => { + const project = useProject(); + + const [{ visible, filesUploaded, operation }, setState] = useState({ + visible: false, + filesUploaded: false, + operation: undefined as OperationType | undefined, + }); + + const [previousOperation, setPreviousOperation] = useState(); + + useEffect(() => { + const localPreviousOperation = previousOperation; + setPreviousOperation(props.operation); + + if (localPreviousOperation == 'addFiles' && !props.loading) { + setState({ + visible: true, + filesUploaded: true, + operation: props.operation, + }); + const timeout = setTimeout(() => { + setState({ + visible: props.loading || props.importDone, + filesUploaded: false, + operation: props.operation, + }); + setPreviousOperation(undefined); + }, 1000); + return () => clearTimeout(timeout); + } + + setState({ + visible: props.loading || props.importDone, + filesUploaded: false, + operation: props.operation, + }); + }, [props.loading, props.operation, props.importDone]); + + useEffect(() => { + props.onActiveChange(visible); + }, [visible]); + + const showFilesUploaded = + filesUploaded || (props.filesUploaded && visible && !operation); + + const showImportDone = props.importDone; + + return ( + + + + + + + + + + {props.importDone ? ( + + + + + ) : ( + + )} + + + + ); +}; diff --git a/webapp/src/views/projects/import/component/ImportResultLoadingOverlay.tsx b/webapp/src/views/projects/import/component/ImportResultLoadingOverlay.tsx new file mode 100644 index 0000000000..03be73c0a7 --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportResultLoadingOverlay.tsx @@ -0,0 +1,28 @@ +import { Box, styled } from '@mui/material'; +import React from 'react'; +import clsx from 'clsx'; + +const StyledRoot = styled(Box)` + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + z-index: 1; + + opacity: 0; + pointer-events: none; + + transition: opacity 0.2s ease-in-out; + + background-color: ${({ theme }) => theme.palette.background.default}; + + &.visible { + pointer-events: all; + opacity: 0.6; + } +`; + +export const ImportResultLoadingOverlay = (props: { loading: boolean }) => { + return ; +}; diff --git a/webapp/src/views/projects/import/component/LanguageSelector.tsx b/webapp/src/views/projects/import/component/LanguageSelector.tsx index efde6b3adc..46ab369567 100644 --- a/webapp/src/views/projects/import/component/LanguageSelector.tsx +++ b/webapp/src/views/projects/import/component/LanguageSelector.tsx @@ -48,12 +48,13 @@ export const LanguageSelector: React.FC<{ const importData = useImportDataHelper(); const languageHelper = useImportLanguageHelper(props.row); - const usedLanguages = importData - .result!._embedded!.languages!.map((l) => ({ - existingId: l.existingLanguageId, - namespace: l.namespace, - })) - .filter((l) => !!l); + const usedLanguages = + importData.result?._embedded?.languages + ?.map((l) => ({ + existingId: l.existingLanguageId, + namespace: l.namespace, + })) + .filter((l) => !!l) || []; const state = useStateObject({ addNewLanguageDialogOpen: false }); diff --git a/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx b/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx index 2c1031833b..6c367ac3db 100644 --- a/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx +++ b/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx @@ -3,9 +3,10 @@ import { useEffect, useState } from 'react'; import { useProject } from 'tg.hooks/useProject'; import { useImportDataHelper } from './useImportDataHelper'; -import { useApiMutation } from 'tg.service/http/useQueryApi'; +import { useNdJsonStreamedMutation } from 'tg.service/http/useQueryApi'; import { useMessage } from 'tg.hooks/useSuccessMessage'; import { T } from '@tolgee/react'; +import { OperationStatusType } from '../component/ImportFileInput'; export const useApplyImportHelper = ( dataHelper: ReturnType @@ -13,17 +14,24 @@ export const useApplyImportHelper = ( const [conflictNotResolvedDialogOpen, setConflictNotResolvedDialogOpen] = useState(false); - const importApplyLoadable = useApiMutation({ - url: '/v2/projects/{projectId}/import/apply', + const [status, setStatus] = useState( + undefined as OperationStatusType | undefined + ); + + const importApplyMutation = useNdJsonStreamedMutation({ + url: '/v2/projects/{projectId}/import/apply-streaming', method: 'put', fetchOptions: { // error is displayed on the page disableErrorNotification: true, }, + onData(data) { + setStatus(data.status); + }, }); const project = useProject(); - const error = importApplyLoadable.error; + const error = importApplyMutation.error; const message = useMessage(); @@ -33,7 +41,7 @@ export const useApplyImportHelper = ( 0 ); if (unResolvedCount === 0) { - importApplyLoadable.mutate( + importApplyMutation.mutate( { path: { projectId: project.id, @@ -53,29 +61,36 @@ export const useApplyImportHelper = ( }; useEffect(() => { - const error = importApplyLoadable.error; + const error = importApplyMutation.error; if (error?.code == 'conflict_is_not_resolved') { setConflictNotResolvedDialogOpen(true); return; } - }, [importApplyLoadable.error]); + }, [importApplyMutation.error]); useEffect(() => { - if (importApplyLoadable.isSuccess) { + if (importApplyMutation.isSuccess) { dataHelper.refetchData(); } - }, [importApplyLoadable.isSuccess]); + }, [importApplyMutation.isSuccess]); const onDialogClose = () => { setConflictNotResolvedDialogOpen(false); }; + const clear = () => { + importApplyMutation.reset(); + setStatus(undefined); + }; + return { onDialogClose, onApplyImport, conflictNotResolvedDialogOpen, error, - loading: importApplyLoadable.isLoading, - loaded: importApplyLoadable.isSuccess, + loading: importApplyMutation.isLoading, + loaded: importApplyMutation.isSuccess, + status, + clear, }; };