From f91000aabf87839e3dced2df7b2d563075752cd5 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Tue, 19 Dec 2023 17:56:22 +0100 Subject: [PATCH 01/13] feat: Import status streaming & UI improvements --- .../api/v2/controllers/V2ImportController.kt | 29 +- .../TranslationSuggestionController.kt | 11 +- .../dataImport/StoredDataImporterTest.kt | 46 +- .../app/src/test/resources/application.yaml | 2 +- .../service/dataImport/ImportService.kt | 7 +- .../service/dataImport/StoredDataImporter.kt | 17 +- .../status/ImportApplicationStatus.kt | 8 + .../status/ImportApplicationStatusItem.kt | 5 + .../io/tolgee/util/responseEntityExt.kt | 10 + e2e/cypress/support/dataCyType.d.ts | 1 + webapp/package-lock.json | 38 ++ webapp/package.json | 5 +- webapp/scripts/generate-schemas.js | 39 ++ webapp/src/ThemeProvider.tsx | 1 + webapp/src/colors.tsx | 10 + webapp/src/component/layout/BaseView.tsx | 2 + webapp/src/custom.d.ts | 5 + webapp/src/service/apiSchema.generated.ts | 601 ++++++++++++++---- webapp/src/service/http/useQueryApi.ts | 151 +++-- .../src/views/projects/import/ImportView.tsx | 14 +- .../import/component/ImportFileInput.tsx | 80 ++- .../component/ImportOperationStatus.tsx | 20 + .../import/component/ImportOperationTitle.tsx | 11 + .../import/component/ImportProgress.tsx | 60 ++ .../import/hooks/useApplyImportHelper.tsx | 29 +- .../TranslationTools/useMTStreamed.tsx | 7 +- 26 files changed, 956 insertions(+), 253 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatus.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatusItem.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/util/responseEntityExt.kt create mode 100644 webapp/scripts/generate-schemas.js create mode 100644 webapp/src/views/projects/import/component/ImportOperationStatus.tsx create mode 100644 webapp/src/views/projects/import/component/ImportOperationTitle.tsx create mode 100644 webapp/src/views/projects/import/component/ImportProgress.tsx diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt index bbe748a07d..4f6d1e110c 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt @@ -4,6 +4,7 @@ package io.tolgee.api.v2.controllers +import com.fasterxml.jackson.databind.ObjectMapper import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.tags.Tag @@ -38,7 +39,11 @@ import io.tolgee.security.authorization.RequiresProjectPermissions import io.tolgee.service.LanguageService import io.tolgee.service.dataImport.ForceMode import io.tolgee.service.dataImport.ImportService +import io.tolgee.service.dataImport.status.ImportApplicationStatus +import io.tolgee.service.dataImport.status.ImportApplicationStatusItem import io.tolgee.service.key.NamespaceService +import io.tolgee.util.StreamingResponseBodyProvider +import io.tolgee.util.disableAccelBuffering import jakarta.servlet.http.HttpServletRequest import org.springdoc.core.annotations.ParameterObject import org.springframework.data.domain.PageRequest @@ -49,6 +54,7 @@ import org.springframework.hateoas.CollectionModel import org.springframework.hateoas.PagedModel import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport import org.springframework.http.MediaType +import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.CrossOrigin import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping @@ -61,6 +67,8 @@ import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RequestPart import org.springframework.web.bind.annotation.RestController import org.springframework.web.multipart.MultipartFile +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody +import java.io.OutputStreamWriter @Suppress("MVCPathVariableInspection") @RestController @@ -85,6 +93,8 @@ class V2ImportController( private val languageService: LanguageService, private val namespaceService: NamespaceService, private val importFileIssueModelAssembler: ImportFileIssueModelAssembler, + private val streamingResponseBodyProvider: StreamingResponseBodyProvider, + private val objectMapper: ObjectMapper ) { @PostMapping("", consumes = [MediaType.MULTIPART_FORM_DATA_VALUE]) @Operation(description = "Prepares provided files to import.", summary = "Add files") @@ -115,7 +125,7 @@ class V2ImportController( return ImportAddFilesResultModel(errors, result) } - @PutMapping("/apply") + @PutMapping("/apply", produces = [MediaType.APPLICATION_NDJSON_VALUE]) @Operation(description = "Imports the data prepared in previous step", summary = "Apply") @RequestActivity(ActivityType.IMPORT) @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) @@ -124,9 +134,22 @@ class V2ImportController( @Parameter(description = "Whether override or keep all translations with unresolved conflicts") @RequestParam(defaultValue = "NO_FORCE") forceMode: ForceMode, - ) { + ): ResponseEntity { val projectId = projectHolder.project.id - this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode) + return ResponseEntity.ok().disableAccelBuffering().body( + streamingResponseBodyProvider.createStreamingResponseBody { outputStream -> + val writer = OutputStreamWriter(outputStream) + val reportStatus = + { status: ImportApplicationStatus -> + writer.write( + (objectMapper.writeValueAsString(ImportApplicationStatusItem(status)) + "\n") + ) + writer.flush() + } + + this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode, reportStatus) + } + ) } @GetMapping("/result") diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt index 07db3a5964..dce1e960d5 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/suggestion/TranslationSuggestionController.kt @@ -18,6 +18,7 @@ import io.tolgee.service.LanguageService import io.tolgee.service.key.KeyService import io.tolgee.service.security.SecurityService import io.tolgee.service.translation.TranslationMemoryService +import io.tolgee.util.disableAccelBuffering import jakarta.validation.Valid import org.springdoc.core.annotations.ParameterObject import org.springframework.data.domain.Pageable @@ -49,7 +50,7 @@ class TranslationSuggestionController( ) { @PostMapping("/machine-translations") @Operation(summary = "Suggests machine translations from enabled services") - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun suggestMachineTranslations( @RequestBody @Valid @@ -65,15 +66,13 @@ class TranslationSuggestionController( "If an error occurs when any of the services is used," + " the error information is returned as a part of the result item, while the response has 200 status code.", ) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun suggestMachineTranslationsStreaming( @RequestBody @Valid dto: SuggestRequestDto, ): ResponseEntity { - return ResponseEntity.ok().headers { - it.add("X-Accel-Buffering", "no") - }.body( + return ResponseEntity.ok().disableAccelBuffering().body( machineTranslationSuggestionFacade.suggestStreaming(dto), ) } @@ -84,7 +83,7 @@ class TranslationSuggestionController( "Suggests machine translations from translation memory." + "\n\nThe result is always sorted by similarity, so sorting is not supported.", ) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun suggestTranslationMemory( @RequestBody @Valid diff --git a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt index e7318f42d2..327e37abfc 100644 --- a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt @@ -23,11 +23,11 @@ class StoredDataImporterTest : AbstractSpringTest() { @BeforeEach fun setup() { importTestData = ImportTestData() - storedDataImporter = - StoredDataImporter( - applicationContext, - importTestData.import, - ) + storedDataImporter = StoredDataImporter( + applicationContext, + importTestData.import, + reportStatus = reportStatus, + ) } fun login() { @@ -87,12 +87,12 @@ class StoredDataImporterTest : AbstractSpringTest() { @Test fun `it force replaces translations`() { - storedDataImporter = - StoredDataImporter( - applicationContext!!, - importTestData.import, - ForceMode.OVERRIDE, - ) + storedDataImporter = StoredDataImporter( + applicationContext!!, + importTestData.import, + ForceMode.OVERRIDE, + reportStatus, + ) testDataService.saveTestData(importTestData.root) login() storedDataImporter.doImport() @@ -107,12 +107,12 @@ class StoredDataImporterTest : AbstractSpringTest() { fun `it imports metadata`() { importTestData.addKeyMetadata() testDataService.saveTestData(importTestData.root) - storedDataImporter = - StoredDataImporter( - applicationContext, - importTestData.import, - ForceMode.OVERRIDE, - ) + storedDataImporter = StoredDataImporter( + applicationContext, + importTestData.import, + ForceMode.OVERRIDE, + reportStatus, + ) login() storedDataImporter.doImport() entityManager.flush() @@ -136,12 +136,12 @@ class StoredDataImporterTest : AbstractSpringTest() { fun `it force keeps translations`() { importTestData.translationWithConflict.override = true importTestData.translationWithConflict.resolve() - storedDataImporter = - StoredDataImporter( - applicationContext, - importTestData.import, - ForceMode.KEEP, - ) + storedDataImporter = StoredDataImporter( + applicationContext, + importTestData.import, + ForceMode.KEEP, + reportStatus, + ) testDataService.saveTestData(importTestData.root) login() diff --git a/backend/app/src/test/resources/application.yaml b/backend/app/src/test/resources/application.yaml index 4404906561..38d986af83 100644 --- a/backend/app/src/test/resources/application.yaml +++ b/backend/app/src/test/resources/application.yaml @@ -4,7 +4,7 @@ spring: - org.redisson.spring.starter.RedissonAutoConfiguration - org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration jpa: -# show-sql: true + show-sql: true properties: hibernate: order_by: diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index ed62f78580..f7f8ff8c37 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -31,6 +31,7 @@ import io.tolgee.repository.dataImport.ImportRepository import io.tolgee.repository.dataImport.ImportTranslationRepository import io.tolgee.repository.dataImport.issues.ImportFileIssueParamRepository import io.tolgee.repository.dataImport.issues.ImportFileIssueRepository +import io.tolgee.service.dataImport.status.ImportApplicationStatus import io.tolgee.util.getSafeNamespace import jakarta.persistence.EntityManager import org.springframework.context.ApplicationContext @@ -95,16 +96,18 @@ class ImportService( projectId: Long, authorId: Long, forceMode: ForceMode = ForceMode.NO_FORCE, + reportStatus: (ImportApplicationStatus) -> Unit = {} ) { - import(getNotExpired(projectId, authorId), forceMode) + import(getNotExpired(projectId, authorId), forceMode, reportStatus) } @Transactional(noRollbackFor = [ImportConflictNotResolvedException::class]) fun import( import: Import, forceMode: ForceMode = ForceMode.NO_FORCE, + reportStatus: (ImportApplicationStatus) -> Unit = {} ) { - StoredDataImporter(applicationContext, import, forceMode).doImport() + StoredDataImporter(applicationContext, import, forceMode, reportStatus).doImport() deleteImport(import) businessEventPublisher.publish( OnBusinessEventToCaptureEvent( diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt index aa61ca32cd..bd086037d6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt @@ -10,6 +10,7 @@ import io.tolgee.model.key.Key import io.tolgee.model.key.KeyMeta import io.tolgee.model.key.Namespace import io.tolgee.model.translation.Translation +import io.tolgee.service.dataImport.status.ImportApplicationStatus import io.tolgee.service.key.KeyMetaService import io.tolgee.service.key.KeyService import io.tolgee.service.key.NamespaceService @@ -23,6 +24,7 @@ class StoredDataImporter( applicationContext: ApplicationContext, private val import: Import, private val forceMode: ForceMode = ForceMode.NO_FORCE, + private val reportStatus: (ImportApplicationStatus) -> Unit, ) { private val importDataManager = ImportDataManager(applicationContext, import) private val keyService = applicationContext.getBean(KeyService::class.java) @@ -75,21 +77,30 @@ class StoredDataImporter( } fun doImport() { + reportStatus(ImportApplicationStatus.PREPARING_AND_VALIDATING) importDataManager.storedLanguages.forEach { - it.doImport() + it.prepareImport() } addKeysAndCheckPermissions() handleKeyMetas() + reportStatus(ImportApplicationStatus.STORING_KEYS) + namespaceService.saveAll(namespacesToSave.values) val keyEntitiesToSave = saveKeys() + saveMetaData(keyEntitiesToSave) + + reportStatus(ImportApplicationStatus.STORING_TRANSLATIONS) + saveTranslations() - saveMetaData(keyEntitiesToSave) + reportStatus(ImportApplicationStatus.FINALIZING) + + entityManager.flush() translationService.setOutdatedBatch(outdatedFlagKeys) @@ -163,7 +174,7 @@ class StoredDataImporter( } } - private fun ImportLanguage.doImport() { + private fun ImportLanguage.prepareImport() { importDataManager.populateStoredTranslations(this) importDataManager.handleConflicts(true) importDataManager.getStoredTranslations(this).forEach { it.doImport() } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatus.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatus.kt new file mode 100644 index 0000000000..e0d74bebca --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatus.kt @@ -0,0 +1,8 @@ +package io.tolgee.service.dataImport.status + +enum class ImportApplicationStatus { + PREPARING_AND_VALIDATING, + STORING_KEYS, + STORING_TRANSLATIONS, + FINALIZING, +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatusItem.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatusItem.kt new file mode 100644 index 0000000000..62400956b3 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/status/ImportApplicationStatusItem.kt @@ -0,0 +1,5 @@ +package io.tolgee.service.dataImport.status + +data class ImportApplicationStatusItem( + val status: ImportApplicationStatus, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/util/responseEntityExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/responseEntityExt.kt new file mode 100644 index 0000000000..88a173ef3e --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/util/responseEntityExt.kt @@ -0,0 +1,10 @@ +package io.tolgee.util + +import org.springframework.http.ResponseEntity + +fun ResponseEntity.BodyBuilder.disableAccelBuffering(): ResponseEntity.BodyBuilder { + this.headers { + it.add("X-Accel-Buffering", "no") + } + return this +} diff --git a/e2e/cypress/support/dataCyType.d.ts b/e2e/cypress/support/dataCyType.d.ts index 734cb1427f..887f0dc3fd 100644 --- a/e2e/cypress/support/dataCyType.d.ts +++ b/e2e/cypress/support/dataCyType.d.ts @@ -184,6 +184,7 @@ declare namespace DataCy { "import-file-input" | "import-file-issues-button" | "import-file-issues-dialog" | + "import-progress" | "import-resolution-dialog-accept-imported-button" | "import-resolution-dialog-accept-old-button" | "import-resolution-dialog-close-button" | diff --git a/webapp/package-lock.json b/webapp/package-lock.json index fa07a79551..e7047b15b5 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -28,6 +28,7 @@ "copy-to-clipboard": "^3.3.1", "date-fns": "2.29.2", "diff": "^5.0.0", + "dotenv-flow": "4.0.1", "formik": "^2.2.9", "intl-messageformat": "^9.8.1", "node-fetch": "3.3.0", @@ -9590,6 +9591,28 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/dotenv-flow": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/dotenv-flow/-/dotenv-flow-4.0.1.tgz", + "integrity": "sha512-HuCQ487bSA43mtlxdWpyk5jt5Lljy+v1I8y/2l96gtjSve9p3OvJZCCOhQnz2hY4VhLogFfXpY20zBygMwaydA==", + "dependencies": { + "dotenv": "^16.0.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/dotenv-flow/node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, "node_modules/duplexer": { "version": "0.1.2", "dev": true, @@ -33438,6 +33461,21 @@ "version": "5.1.0", "dev": true }, + "dotenv-flow": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/dotenv-flow/-/dotenv-flow-4.0.1.tgz", + "integrity": "sha512-HuCQ487bSA43mtlxdWpyk5jt5Lljy+v1I8y/2l96gtjSve9p3OvJZCCOhQnz2hY4VhLogFfXpY20zBygMwaydA==", + "requires": { + "dotenv": "^16.0.0" + }, + "dependencies": { + "dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==" + } + } + }, "duplexer": { "version": "0.1.2", "dev": true diff --git a/webapp/package.json b/webapp/package.json index f7f8846a75..bc1dbef840 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -23,6 +23,7 @@ "copy-to-clipboard": "^3.3.1", "date-fns": "2.29.2", "diff": "^5.0.0", + "dotenv-flow": "4.0.1", "formik": "^2.2.9", "intl-messageformat": "^9.8.1", "node-fetch": "3.3.0", @@ -68,8 +69,8 @@ "startConcurrently": "NODE_OPTIONS=--openssl-legacy-provider concurrently \"npm run start\"", "build": "NODE_OPTIONS=--openssl-legacy-provider craco build", "test": "craco test", - "schema": "openapi-typescript http://localhost:8080/v3/api-docs/All%20Internal%20-%20for%20Tolgee%20Web%20application --output ./src/service/apiSchema.generated.ts", - "billing-schema": "openapi-typescript http://localhost:8080/v3/api-docs/V2%20Billing --output ./src/service/billingApiSchema.generated.ts", + "schema": "node ./scripts/generate-schemas.js public", + "billing-schema": "node ./scripts/generate-schemas.js billing", "prettier": "prettier --write ./src", "eslint": "eslint --ext .ts --ext .tsx --max-warnings 0 --report-unused-disable-directives .", "tsc": "tsc", diff --git a/webapp/scripts/generate-schemas.js b/webapp/scripts/generate-schemas.js new file mode 100644 index 0000000000..6199611ba5 --- /dev/null +++ b/webapp/scripts/generate-schemas.js @@ -0,0 +1,39 @@ +const path = require('path'); +// script.js +require('dotenv-flow').config({ + default_node_env: 'development', + path: path.resolve(__dirname, '../'), +}); + +const { exec } = require('child_process'); +const apiUrl = process.env.REACT_APP_API_URL || 'http://localhost:8080'; + +const definitions = { + public: { + schema: 'All%20Internal%20-%20for%20Tolgee%20Web%20application', + output: './src/service/apiSchema.generated.ts', + }, + billing: { + schema: 'V2%20Billing', + output: './src/service/billingApiSchema.generated.ts', + }, +}; + +const definition = definitions[process.argv[2]]; +if (!definition) { + throw new Error('Invalid definition'); +} + +const command = `openapi-typescript ${apiUrl}/v3/api-docs/${definition.schema} --output ${definition.output}`; + +exec(command, (error, stdout, stderr) => { + if (error) { + console.log(`error: ${error.message}`); + return; + } + if (stderr) { + console.log(`stderr: ${stderr}`); + return; + } + console.log(`stdout: ${stdout}`); +}); diff --git a/webapp/src/ThemeProvider.tsx b/webapp/src/ThemeProvider.tsx index 86e0603299..0d86a0c809 100644 --- a/webapp/src/ThemeProvider.tsx +++ b/webapp/src/ThemeProvider.tsx @@ -150,6 +150,7 @@ const getTheme = (mode: PaletteMode) => { marker: c.marker, topBanner: c.topBanner, quickStart: c.quickStart, + import: c.import, }, mixins: { toolbar: { diff --git a/webapp/src/colors.tsx b/webapp/src/colors.tsx index 8e6d121855..e436c03cae 100644 --- a/webapp/src/colors.tsx +++ b/webapp/src/colors.tsx @@ -159,6 +159,11 @@ export const colors = { progressBackground: '#bcbcbc70', itemBorder: '#EDF0F7', } as QuickStart, + import: { + progressDone: '#00B962', + progressWorking: '#EC407A', + progressBackground: '#D9D9D9', + }, }, dark: { white: '#dddddd', @@ -241,5 +246,10 @@ export const colors = { topBorder: '#2a384c', progressBackground: '#2c3c52', } as QuickStart, + import: { + progressDone: '#00B962', + progressWorking: '#EC407A', + progressBackground: '#D9D9D9', + }, }, } as const; diff --git a/webapp/src/component/layout/BaseView.tsx b/webapp/src/component/layout/BaseView.tsx index 54f268e40b..26cd47da76 100644 --- a/webapp/src/component/layout/BaseView.tsx +++ b/webapp/src/component/layout/BaseView.tsx @@ -47,6 +47,7 @@ export interface BaseViewProps { allCentered?: boolean; 'data-cy'?: string; initialSearch?: string; + overflow?: string; } export const BaseView = (props: BaseViewProps) => { @@ -144,6 +145,7 @@ export const BaseView = (props: BaseViewProps) => { display="grid" position="relative" maxWidth="100%" + sx={{ overflow: props.overflow }} > {typeof props.children === 'function' ? props.children() diff --git a/webapp/src/custom.d.ts b/webapp/src/custom.d.ts index b89d39253d..cd3639da53 100644 --- a/webapp/src/custom.d.ts +++ b/webapp/src/custom.d.ts @@ -18,6 +18,9 @@ declare module '*.svg' { const content: React.FunctionComponent>; export default content; } +import { colors } from './colors'; + +const all = { ...colors.light, ...colors.dark }; declare module '@mui/material/styles/createPalette' { interface Palette { @@ -36,6 +39,7 @@ declare module '@mui/material/styles/createPalette' { marker: Marker; topBanner: TopBanner; quickStart: QuickStart; + import: typeof all.import; } interface PaletteOptions { @@ -54,6 +58,7 @@ declare module '@mui/material/styles/createPalette' { marker: Marker; topBanner: TopBanner; quickStart: QuickStart; + import: typeof all.import; } } diff --git a/webapp/src/service/apiSchema.generated.ts b/webapp/src/service/apiSchema.generated.ts index 56b461c26d..c6f337cb01 100644 --- a/webapp/src/service/apiSchema.generated.ts +++ b/webapp/src/service/apiSchema.generated.ts @@ -260,9 +260,30 @@ export interface paths { "/v2/slug/generate-organization": { post: operations["generateOrganizationSlug"]; }; - "/v2/public/business-events/report": { + "/v2/public/telemetry/report": { post: operations["report"]; }; + "/v2/public/licensing/subscription": { + post: operations["getMySubscription"]; + }; + "/v2/public/licensing/set-key": { + post: operations["onLicenceSetKey"]; + }; + "/v2/public/licensing/report-usage": { + post: operations["reportUsage"]; + }; + "/v2/public/licensing/report-error": { + post: operations["reportError"]; + }; + "/v2/public/licensing/release-key": { + post: operations["releaseKey"]; + }; + "/v2/public/licensing/prepare-set-key": { + post: operations["prepareSetLicenseKey"]; + }; + "/v2/public/business-events/report": { + post: operations["report_1"]; + }; "/v2/public/business-events/identify": { post: operations["identify"]; }; @@ -397,7 +418,7 @@ export interface paths { post: operations["upload"]; }; "/v2/ee-license/prepare-set-license-key": { - post: operations["prepareSetLicenseKey"]; + post: operations["prepareSetLicenseKey_1"]; }; "/v2/api-keys": { get: operations["allByUser"]; @@ -708,24 +729,6 @@ export interface components { | "SERVER_ADMIN"; /** @description The user's permission type. This field is null if uses granular permissions */ type?: "NONE" | "VIEW" | "TRANSLATE" | "REVIEW" | "EDIT" | "MANAGE"; - /** - * @deprecated - * @description Deprecated (use translateLanguageIds). - * - * List of languages current user has TRANSLATE permission to. If null, all languages edition is permitted. - * @example 200001,200004 - */ - permittedLanguageIds?: number[]; - /** - * @description List of languages user can translate to. If null, all languages editing is permitted. - * @example 200001,200004 - */ - translateLanguageIds?: number[]; - /** - * @description List of languages user can change state to. If null, changing state of all language values is permitted. - * @example 200001,200004 - */ - stateChangeLanguageIds?: number[]; /** * @description List of languages user can view. If null, all languages view is permitted. * @example 200001,200004 @@ -763,6 +766,24 @@ export interface components { | "content-delivery.publish" | "webhooks.manage" )[]; + /** + * @description List of languages user can translate to. If null, all languages editing is permitted. + * @example 200001,200004 + */ + translateLanguageIds?: number[]; + /** + * @description List of languages user can change state to. If null, changing state of all language values is permitted. + * @example 200001,200004 + */ + stateChangeLanguageIds?: number[]; + /** + * @deprecated + * @description Deprecated (use translateLanguageIds). + * + * List of languages current user has TRANSLATE permission to. If null, all languages edition is permitted. + * @example 200001,200004 + */ + permittedLanguageIds?: number[]; }; LanguageModel: { /** Format: int64 */ @@ -1038,6 +1059,8 @@ export interface components { namespace?: string; /** @description Translations to update */ translations?: { [key: string]: string }; + /** @description Translation states to update, if not provided states won't be modified */ + states?: { [key: string]: "TRANSLATED" | "REVIEWED" }; /** @description Tags of the key. If not provided tags won't be modified */ tags?: string[]; /** @description IDs of screenshots to delete */ @@ -1350,6 +1373,16 @@ export interface components { SetFileNamespaceRequest: { namespace?: string; }; + StreamingResponseBody: { [key: string]: unknown }; + /** @description User who created the comment */ + SimpleUserAccountModel: { + /** Format: int64 */ + id: number; + username: string; + name?: string; + avatar?: components["schemas"]["Avatar"]; + deleted: boolean; + }; TranslationCommentModel: { /** * Format: int64 @@ -1360,7 +1393,7 @@ export interface components { text: string; /** @description State of translation */ state: "RESOLUTION_NOT_NEEDED" | "NEEDS_RESOLUTION" | "RESOLVED"; - author: components["schemas"]["UserAccountModel"]; + author: components["schemas"]["SimpleUserAccountModel"]; /** * Format: date-time * @description Date when it was created @@ -1372,18 +1405,6 @@ export interface components { */ updatedAt: string; }; - /** @description User who created the comment */ - UserAccountModel: { - /** Format: int64 */ - id: number; - username: string; - name?: string; - emailAwaitingVerification?: string; - avatar?: components["schemas"]["Avatar"]; - globalServerRole: "USER" | "ADMIN"; - deleted: boolean; - disabled: boolean; - }; TranslationCommentDto: { text: string; state: "RESOLUTION_NOT_NEEDED" | "NEEDS_RESOLUTION" | "RESOLVED"; @@ -1485,7 +1506,6 @@ export interface components { token: string; /** Format: int64 */ id: number; - description: string; /** Format: int64 */ createdAt: number; /** Format: int64 */ @@ -1494,6 +1514,7 @@ export interface components { expiresAt?: number; /** Format: int64 */ lastUsedAt?: number; + description: string; }; SetOrganizationRoleDto: { roleType: "MEMBER" | "OWNER"; @@ -1631,15 +1652,15 @@ export interface components { id: number; userFullName?: string; projectName: string; - description: string; username?: string; + scopes: string[]; /** Format: int64 */ projectId: number; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ lastUsedAt?: number; - scopes: string[]; + description: string; }; SuperTokenRequest: { /** @description Has to be provided when TOTP enabled */ @@ -1651,6 +1672,136 @@ export interface components { name: string; oldSlug?: string; }; + TelemetryReportRequest: { + instanceId: string; + /** Format: int64 */ + projectsCount: number; + /** Format: int64 */ + translationsCount: number; + /** Format: int64 */ + languagesCount: number; + /** Format: int64 */ + distinctLanguagesCount: number; + /** Format: int64 */ + usersCount: number; + }; + GetMySubscriptionDto: { + licenseKey: string; + instanceId: string; + }; + PlanIncludedUsageModel: { + /** Format: int64 */ + seats: number; + /** Format: int64 */ + translationSlots: number; + /** Format: int64 */ + translations: number; + /** Format: int64 */ + mtCredits: number; + }; + PlanPricesModel: { + perSeat: number; + perThousandTranslations?: number; + perThousandMtCredits?: number; + subscriptionMonthly: number; + subscriptionYearly: number; + }; + SelfHostedEePlanModel: { + /** Format: int64 */ + id: number; + name: string; + public: boolean; + enabledFeatures: ( + | "GRANULAR_PERMISSIONS" + | "PRIORITIZED_FEATURE_REQUESTS" + | "PREMIUM_SUPPORT" + | "DEDICATED_SLACK_CHANNEL" + | "ASSISTED_UPDATES" + | "DEPLOYMENT_ASSISTANCE" + | "BACKUP_CONFIGURATION" + | "TEAM_TRAINING" + | "ACCOUNT_MANAGER" + | "STANDARD_SUPPORT" + | "PROJECT_LEVEL_CONTENT_STORAGES" + | "WEBHOOKS" + | "MULTIPLE_CONTENT_DELIVERY_CONFIGS" + )[]; + prices: components["schemas"]["PlanPricesModel"]; + includedUsage: components["schemas"]["PlanIncludedUsageModel"]; + hasYearlyPrice: boolean; + }; + SelfHostedEeSubscriptionModel: { + /** Format: int64 */ + id: number; + /** Format: int64 */ + currentPeriodStart?: number; + /** Format: int64 */ + currentPeriodEnd?: number; + currentBillingPeriod: "MONTHLY" | "YEARLY"; + /** Format: int64 */ + createdAt: number; + plan: components["schemas"]["SelfHostedEePlanModel"]; + status: + | "ACTIVE" + | "CANCELED" + | "PAST_DUE" + | "UNPAID" + | "ERROR" + | "KEY_USED_BY_ANOTHER_INSTANCE"; + licenseKey?: string; + estimatedCosts?: number; + }; + SetLicenseKeyLicensingDto: { + licenseKey: string; + /** Format: int64 */ + seats: number; + instanceId: string; + }; + ReportUsageDto: { + licenseKey: string; + /** Format: int64 */ + seats: number; + }; + ReportErrorDto: { + stackTrace: string; + licenseKey: string; + }; + ReleaseKeyDto: { + licenseKey: string; + }; + PrepareSetLicenseKeyDto: { + licenseKey: string; + /** Format: int64 */ + seats: number; + }; + AverageProportionalUsageItemModel: { + total: number; + unusedQuantity: number; + usedQuantity: number; + usedQuantityOverPlan: number; + }; + PrepareSetEeLicenceKeyModel: { + plan: components["schemas"]["SelfHostedEePlanModel"]; + usage: components["schemas"]["UsageModel"]; + }; + SumUsageItemModel: { + total: number; + /** Format: int64 */ + unusedQuantity: number; + /** Format: int64 */ + usedQuantity: number; + /** Format: int64 */ + usedQuantityOverPlan: number; + }; + UsageModel: { + subscriptionPrice?: number; + /** @description Relevant for invoices only. When there are applied stripe credits, we need to reduce the total price by this amount. */ + appliedStripeCredits?: number; + seats: components["schemas"]["AverageProportionalUsageItemModel"]; + translations: components["schemas"]["AverageProportionalUsageItemModel"]; + credits?: components["schemas"]["SumUsageItemModel"]; + total: number; + }; BusinessEventReportRequest: { eventName: string; anonymousUserId?: string; @@ -1761,6 +1912,8 @@ export interface components { name: string; namespace?: string; translations?: { [key: string]: string }; + /** @description Translation states to update, if not provided states won't be modified */ + states?: { [key: string]: "TRANSLATED" | "REVIEWED" }; tags?: string[]; /** @description Ids of screenshots uploaded with /v2/image-upload endpoint */ screenshotUploadedImageIds?: number[]; @@ -1947,7 +2100,9 @@ export interface components { | "UNEXPECTED_ERROR_WHILE_PUBLISHING_TO_CONTENT_STORAGE" | "WEBHOOK_RESPONDED_WITH_NON_200_STATUS" | "UNEXPECTED_ERROR_WHILE_EXECUTING_WEBHOOK" - | "CONTENT_STORAGE_IS_IN_USE"; + | "CONTENT_STORAGE_IS_IN_USE" + | "CANNOT_SET_STATE_FOR_MISSING_TRANSLATION" + | "NO_PROJECT_ID_PROVIDED"; params?: { [key: string]: unknown }[]; }; UntagKeysRequest: { @@ -2010,15 +2165,6 @@ export interface components { /** @description If the job failed, this is the error message */ errorMessage?: string; }; - /** @description The user who started the job */ - SimpleUserAccountModel: { - /** Format: int64 */ - id: number; - username: string; - name?: string; - avatar?: components["schemas"]["Avatar"]; - deleted: boolean; - }; TagKeysRequest: { keyIds: number[]; tags: string[]; @@ -2131,7 +2277,6 @@ export interface components { filterNamespace?: string[]; zip: boolean; }; - StreamingResponseBody: { [key: string]: unknown }; BigMetaDto: { /** @description List of keys, visible, in order as they appear in the document. The order is important! We are using it for graph distance calculation. */ relatedKeysInOrder: components["schemas"]["RelatedKeyDto"][]; @@ -2228,80 +2373,11 @@ export interface components { /** Format: int64 */ id: number; filename: string; - fileUrl: string; - requestFilename: string; - /** Format: date-time */ - createdAt: string; - location?: string; - }; - AverageProportionalUsageItemModel: { - total: number; - unusedQuantity: number; - usedQuantity: number; - usedQuantityOverPlan: number; - }; - PlanIncludedUsageModel: { - /** Format: int64 */ - seats: number; - /** Format: int64 */ - translationSlots: number; - /** Format: int64 */ - translations: number; - /** Format: int64 */ - mtCredits: number; - }; - PlanPricesModel: { - perSeat: number; - perThousandTranslations?: number; - perThousandMtCredits?: number; - subscriptionMonthly: number; - subscriptionYearly: number; - }; - PrepareSetEeLicenceKeyModel: { - plan: components["schemas"]["SelfHostedEePlanModel"]; - usage: components["schemas"]["UsageModel"]; - }; - SelfHostedEePlanModel: { - /** Format: int64 */ - id: number; - name: string; - public: boolean; - enabledFeatures: ( - | "GRANULAR_PERMISSIONS" - | "PRIORITIZED_FEATURE_REQUESTS" - | "PREMIUM_SUPPORT" - | "DEDICATED_SLACK_CHANNEL" - | "ASSISTED_UPDATES" - | "DEPLOYMENT_ASSISTANCE" - | "BACKUP_CONFIGURATION" - | "TEAM_TRAINING" - | "ACCOUNT_MANAGER" - | "STANDARD_SUPPORT" - | "PROJECT_LEVEL_CONTENT_STORAGES" - | "WEBHOOKS" - | "MULTIPLE_CONTENT_DELIVERY_CONFIGS" - )[]; - prices: components["schemas"]["PlanPricesModel"]; - includedUsage: components["schemas"]["PlanIncludedUsageModel"]; - hasYearlyPrice: boolean; - }; - SumUsageItemModel: { - total: number; - /** Format: int64 */ - unusedQuantity: number; - /** Format: int64 */ - usedQuantity: number; - /** Format: int64 */ - usedQuantityOverPlan: number; - }; - UsageModel: { - subscriptionPrice?: number; - /** @description Relevant for invoices only. When there are applied stripe credits, we need to reduce the total price by this amount. */ - appliedStripeCredits?: number; - seats: components["schemas"]["AverageProportionalUsageItemModel"]; - translations: components["schemas"]["AverageProportionalUsageItemModel"]; - credits?: components["schemas"]["SumUsageItemModel"]; - total: number; + fileUrl: string; + requestFilename: string; + /** Format: date-time */ + createdAt: string; + location?: string; }; CreateApiKeyDto: { /** Format: int64 */ @@ -2446,18 +2522,18 @@ export interface components { name: string; /** Format: int64 */ id: number; - basePermissions: components["schemas"]["PermissionModel"]; - /** @example This is a beautiful organization full of beautiful and clever people */ - description?: string; /** * @description The role of currently authorized user. * * Can be null when user has direct access to one of the projects owned by the organization. */ currentUserRole?: "MEMBER" | "OWNER"; + basePermissions: components["schemas"]["PermissionModel"]; + avatar?: components["schemas"]["Avatar"]; /** @example btforg */ slug: string; - avatar?: components["schemas"]["Avatar"]; + /** @example This is a beautiful organization full of beautiful and clever people */ + description?: string; }; PublicBillingConfigurationDTO: { enabled: boolean; @@ -2567,8 +2643,8 @@ export interface components { /** Format: int64 */ id: number; namespace?: string; - translation?: string; baseTranslation?: string; + translation?: string; }; KeySearchSearchResultModel: { view?: components["schemas"]["KeySearchResultView"]; @@ -2576,8 +2652,8 @@ export interface components { /** Format: int64 */ id: number; namespace?: string; - translation?: string; baseTranslation?: string; + translation?: string; }; PagedModelKeySearchSearchResultModel: { _embedded?: { @@ -2767,7 +2843,7 @@ export interface components { }; page?: components["schemas"]["PageMetadata"]; }; - EntityModelImportFileIssueView: { + ImportFileIssueModel: { /** Format: int64 */ id: number; type: @@ -2781,10 +2857,9 @@ export interface components { | "TARGET_NOT_PROVIDED" | "TRANSLATION_TOO_LONG" | "KEY_IS_BLANK"; - params: components["schemas"]["ImportFileIssueParamView"][]; + params: components["schemas"]["ImportFileIssueParamModel"][]; }; - ImportFileIssueParamView: { - value?: string; + ImportFileIssueParamModel: { type: | "KEY_NAME" | "KEY_ID" @@ -2793,10 +2868,11 @@ export interface components { | "VALUE" | "LINE" | "FILE_NODE_ORIGINAL"; + value?: string; }; - PagedModelEntityModelImportFileIssueView: { + PagedModelImportFileIssueModel: { _embedded?: { - importFileIssueViews?: components["schemas"]["EntityModelImportFileIssueView"][]; + importFileIssues?: components["schemas"]["ImportFileIssueModel"][]; }; page?: components["schemas"]["PageMetadata"]; }; @@ -3077,7 +3153,6 @@ export interface components { user: components["schemas"]["SimpleUserAccountModel"]; /** Format: int64 */ id: number; - description: string; /** Format: int64 */ createdAt: number; /** Format: int64 */ @@ -3086,6 +3161,7 @@ export interface components { expiresAt?: number; /** Format: int64 */ lastUsedAt?: number; + description: string; }; OrganizationRequestParamsDto: { filterCurrentUserOwner: boolean; @@ -3197,24 +3273,79 @@ export interface components { }; ApiKeyWithLanguagesModel: { /** + * @deprecated * @description Languages for which user has translate permission. - * - * If null, all languages are permitted. */ permittedLanguageIds?: number[]; /** Format: int64 */ id: number; userFullName?: string; projectName: string; - description: string; username?: string; + scopes: string[]; /** Format: int64 */ projectId: number; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ lastUsedAt?: number; - scopes: string[]; + description: string; + }; + ApiKeyPermissionsModel: { + /** + * Format: int64 + * @description The API key's project id or the one provided as query param + */ + projectId: number; + /** + * @description List of languages user can view. If null, all languages view is permitted. + * @example 200001,200004 + */ + viewLanguageIds?: number[]; + /** + * @description List of languages user can translate to. If null, all languages editing is permitted. + * @example 200001,200004 + */ + translateLanguageIds?: number[]; + /** + * @description List of languages user can change state to. If null, changing state of all language values is permitted. + * @example 200001,200004 + */ + stateChangeLanguageIds?: number[]; + /** + * @description Granted scopes to the user. When user has type permissions, this field contains permission scopes of the type. + * @example KEYS_EDIT,TRANSLATIONS_VIEW + */ + scopes: ( + | "translations.view" + | "translations.edit" + | "keys.edit" + | "screenshots.upload" + | "screenshots.delete" + | "screenshots.view" + | "activity.view" + | "languages.edit" + | "admin" + | "project.edit" + | "members.view" + | "members.edit" + | "translation-comments.add" + | "translation-comments.edit" + | "translation-comments.set-state" + | "translations.state-edit" + | "keys.view" + | "keys.delete" + | "keys.create" + | "batch-jobs.view" + | "batch-jobs.cancel" + | "translations.batch-by-tm" + | "translations.batch-machine" + | "content-delivery.manage" + | "content-delivery.publish" + | "webhooks.manage" + )[]; + /** @description The user's permission type. This field is null if user has assigned granular permissions or if returning API key's permissions */ + type?: "NONE" | "VIEW" | "TRANSLATE" | "REVIEW" | "EDIT" | "MANAGE"; }; PagedModelUserAccountModel: { _embedded?: { @@ -3222,6 +3353,17 @@ export interface components { }; page?: components["schemas"]["PageMetadata"]; }; + UserAccountModel: { + /** Format: int64 */ + id: number; + username: string; + name?: string; + emailAwaitingVerification?: string; + avatar?: components["schemas"]["Avatar"]; + globalServerRole: "USER" | "ADMIN"; + deleted: boolean; + disabled: boolean; + }; UserTotpDisableRequestDto: { password: string; }; @@ -4711,7 +4853,11 @@ export interface operations { }; responses: { /** OK */ - 200: unknown; + 200: { + content: { + "application/x-ndjson": components["schemas"]["StreamingResponseBody"]; + }; + }; /** Bad Request */ 400: { content: { @@ -6034,6 +6180,179 @@ export interface operations { }; }; report: { + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["TelemetryReportRequest"]; + }; + }; + }; + getMySubscription: { + responses: { + /** OK */ + 200: { + content: { + "*/*": components["schemas"]["SelfHostedEeSubscriptionModel"]; + }; + }; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GetMySubscriptionDto"]; + }; + }; + }; + onLicenceSetKey: { + responses: { + /** OK */ + 200: { + content: { + "*/*": components["schemas"]["SelfHostedEeSubscriptionModel"]; + }; + }; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SetLicenseKeyLicensingDto"]; + }; + }; + }; + reportUsage: { + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReportUsageDto"]; + }; + }; + }; + reportError: { + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReportErrorDto"]; + }; + }; + }; + releaseKey: { + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ReleaseKeyDto"]; + }; + }; + }; + prepareSetLicenseKey: { + responses: { + /** OK */ + 200: { + content: { + "*/*": components["schemas"]["PrepareSetEeLicenceKeyModel"]; + }; + }; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PrepareSetLicenseKeyDto"]; + }; + }; + }; + report_1: { responses: { /** OK */ 200: unknown; @@ -7634,7 +7953,7 @@ export interface operations { }; }; }; - prepareSetLicenseKey: { + prepareSetLicenseKey_1: { responses: { /** OK */ 200: { @@ -8581,7 +8900,7 @@ export interface operations { /** OK */ 200: { content: { - "*/*": components["schemas"]["PagedModelEntityModelImportFileIssueView"]; + "*/*": components["schemas"]["PagedModelImportFileIssueModel"]; }; }; /** Bad Request */ diff --git a/webapp/src/service/http/useQueryApi.ts b/webapp/src/service/http/useQueryApi.ts index 416cf9355d..d44e243265 100644 --- a/webapp/src/service/http/useQueryApi.ts +++ b/webapp/src/service/http/useQueryApi.ts @@ -126,6 +126,45 @@ function autoErrorHandling( }; } +function getApiMutationOptions( + invalidatePrefix: string | undefined, + queryClient: QueryClient +) { + return (options: UseQueryOptions | undefined) => ({ + ...options, + onSuccess: (...params) => { + // @ts-ignore + options?.onSuccess?.(...params); + if (invalidatePrefix !== undefined) { + invalidateUrlPrefix(queryClient, invalidatePrefix); + } + }, + }); +} + +const getMutationCallback = < + MutationFn extends (variables: any, options: any) => any +>( + mutateFn: MutationFn, + customOptions: ReturnType, + fetchOptions: RequestOptions | undefined, + props: any +) => { + return useCallback( + ((variables, options) => { + return mutateFn( + variables, + autoErrorHandling( + customOptions(options as any), + Boolean( + fetchOptions?.disableAutoErrorHandle || props.options?.onError + ) + ) + ); + }) as any, + [mutateFn] + ) as MutationFn; +}; export const useApiMutation = < Url extends keyof Paths, Method extends keyof Paths[Url], @@ -137,18 +176,7 @@ export const useApiMutation = < const { url, method, fetchOptions, options, invalidatePrefix } = props; // inject custom onSuccess - const customOptions = ( - options: UseQueryOptions | undefined - ) => ({ - ...options, - onSuccess: (...params) => { - // @ts-ignore - options?.onSuccess?.(...params); - if (invalidatePrefix !== undefined) { - invalidateUrlPrefix(queryClient, invalidatePrefix); - } - }, - }); + const customOptions = getApiMutationOptions(invalidatePrefix, queryClient); const mutation = useMutation< ResponseContent, @@ -163,34 +191,18 @@ export const useApiMutation = < customOptions(options as any) as any ); - const mutate = useCallback( - (variables, options) => { - return mutation.mutate( - variables, - autoErrorHandling( - customOptions(options as any), - Boolean( - fetchOptions?.disableAutoErrorHandle || props.options?.onError - ) - ) - ); - }, - [mutation.mutate] + const mutate = getMutationCallback( + mutation.mutate, + customOptions, + fetchOptions, + props ); - const mutateAsync = useCallback( - (variables, options) => { - return mutation.mutateAsync( - variables, - autoErrorHandling( - customOptions(options as any), - Boolean( - fetchOptions?.disableAutoErrorHandle || props.options?.onError - ) - ) - ); - }, - [mutation.mutateAsync] + const mutateAsync = getMutationCallback( + mutation.mutateAsync, + customOptions, + fetchOptions, + props ); return { ...mutation, mutate, mutateAsync }; @@ -220,3 +232,66 @@ export const useBillingApiMutation = < >( props: MutationProps ) => useApiMutation(props); + +export const useNdJsonStreamedMutation = < + Url extends keyof Paths, + Method extends keyof Paths[Url], + Paths = paths +>( + props: MutationProps & { onData: (data: any) => void } +) => { + const queryClient = useQueryClient(); + const { url, method, fetchOptions, options, onData, invalidatePrefix } = + props; + + // inject custom onSuccess + const customOptions = getApiMutationOptions(invalidatePrefix, queryClient); + + const mutation = useMutation< + any[], + ApiError, + RequestParamsType + >(async (request) => { + const response = await apiHttpService.schemaRequestRaw( + url, + method, + { + ...fetchOptions, + } + )(request); + const reader = response.body?.getReader(); + const result: any[] = []; + while (reader) { + const { done, value } = await reader.read(); + const text = new TextDecoder().decode(value); + if (text) { + try { + const parsed = JSON.parse(text); + result.push(parsed); + onData(parsed); + } catch (e) { + // ignore + } + } + if (done) { + break; + } + } + return result; + }, customOptions(options as any) as any); + + const mutate = getMutationCallback( + mutation.mutate, + customOptions, + fetchOptions, + props + ); + + const mutateAsync = getMutationCallback( + mutation.mutateAsync, + customOptions, + fetchOptions, + props + ); + return { ...mutation, mutate, mutateAsync }; +}; diff --git a/webapp/src/views/projects/import/ImportView.tsx b/webapp/src/views/projects/import/ImportView.tsx index 83adba1f68..eec96cebe0 100644 --- a/webapp/src/views/projects/import/ImportView.tsx +++ b/webapp/src/views/projects/import/ImportView.tsx @@ -83,6 +83,7 @@ export const ImportView: FunctionComponent = () => { ], ]} maxWidth="wide" + overflow="scroll" > { {dataHelper.addFilesMutation.data?.errors?.map((e, idx) => ( diff --git a/webapp/src/views/projects/import/component/ImportFileInput.tsx b/webapp/src/views/projects/import/component/ImportFileInput.tsx index 2f9a9eb174..192a5a962d 100644 --- a/webapp/src/views/projects/import/component/ImportFileInput.tsx +++ b/webapp/src/views/projects/import/component/ImportFileInput.tsx @@ -10,12 +10,26 @@ import { Message } from 'tg.store/global/types'; import LoadingButton from 'tg.component/common/form/LoadingButton'; import { ImportFileDropzone } from './ImportFileDropzone'; +import { ImportProgressBar } from './ImportProgress'; +import { ImportOperationStatus } from './ImportOperationStatus'; +import { ImportOperationTitle } from './ImportOperationTitle'; export const MAX_FILE_COUNT = 20; +export type OperationType = 'addFiles' | 'apply'; + +export type OperationStatusType = + | 'PREPARING_AND_VALIDATING' + | 'STORING_KEYS' + | 'STORING_TRANSLATIONS' + | 'FINALIZING'; + type ImportFileInputProps = { onNewFiles: (files: File[]) => void; loading: boolean; + operation?: OperationType; + operationStatus?: OperationStatusType; + importDone: boolean; }; export type ValidationResult = { @@ -25,8 +39,7 @@ export type ValidationResult = { const StyledRoot = styled(Box)(({ theme }) => ({ borderRadius: theme.shape.borderRadius, - border: `1px dashed ${theme.palette.emphasis[400]}`, - maxWidth: 600, + border: `1px dashed ${theme.palette.emphasis[100]}`, margin: '0px auto', width: '100%', })); @@ -45,6 +58,7 @@ const ImportFileInput: FunctionComponent = (props) => { 'properties', ]; const [resetKey, setResetKey] = useState(0); + function resetInput() { setResetKey((key) => key + 1); } @@ -149,7 +163,8 @@ const ImportFileInput: FunctionComponent = (props) => { message={t('quick_start_item_pick_import_file_hint')} > ({ + backgroundColor: theme.palette.background.paper, mt: 4, pt: 5, pb: 5, @@ -157,7 +172,7 @@ const ImportFileInput: FunctionComponent = (props) => { alignItems: 'center', flexDirection: 'column', display: 'flex', - }} + })} > = (props) => { multiple accept={ALLOWED_EXTENSIONS.join(',')} /> - - - - - - fileRef.current?.dispatchEvent(new MouseEvent('click')) - } - variant="outlined" - color="primary" - > - - + {props.operation ? ( + + ) : ( + + + + )} + + {props.loading || props.importDone ? ( + + ) : ( + + fileRef.current?.dispatchEvent(new MouseEvent('click')) + } + variant="outlined" + color="primary" + > + + + )} - - - + {props.operationStatus ? ( + + ) : ( + !props.loading && ( + + + + ) + )} diff --git a/webapp/src/views/projects/import/component/ImportOperationStatus.tsx b/webapp/src/views/projects/import/component/ImportOperationStatus.tsx new file mode 100644 index 0000000000..53c85cfec3 --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportOperationStatus.tsx @@ -0,0 +1,20 @@ +import { T } from '@tolgee/react'; +import { OperationStatusType, OperationType } from './ImportFileInput'; +import { useDebounce } from 'use-debounce/lib'; + +export const ImportOperationStatus = (props: { + status: OperationStatusType; +}) => { + const [debouncedStatus] = useDebounce(props.status, 1000); + switch (debouncedStatus) { + case 'PREPARING_AND_VALIDATING': + return ; + case 'STORING_KEYS': + return ; + case 'STORING_TRANSLATIONS': + return ; + case 'FINALIZING': + return ; + } + return null; +}; diff --git a/webapp/src/views/projects/import/component/ImportOperationTitle.tsx b/webapp/src/views/projects/import/component/ImportOperationTitle.tsx new file mode 100644 index 0000000000..3ab8f15b44 --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportOperationTitle.tsx @@ -0,0 +1,11 @@ +import { T } from '@tolgee/react'; +import { OperationType } from './ImportFileInput'; + +export const ImportOperationTitle = (props: { operation: OperationType }) => { + switch (props.operation) { + case 'addFiles': + return ; + case 'apply': + return ; + } +}; diff --git a/webapp/src/views/projects/import/component/ImportProgress.tsx b/webapp/src/views/projects/import/component/ImportProgress.tsx new file mode 100644 index 0000000000..531fbd59b1 --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportProgress.tsx @@ -0,0 +1,60 @@ +import { Box, styled } from '@mui/material'; +import clsx from 'clsx'; +import React, { useEffect } from 'react'; +import { useLoadingRegister } from 'tg.component/GlobalLoading'; + +const StyledProgress = styled('div')<{ loading?: string; finish?: string }>` + height: 4px; + width: 100%; + border-radius: 2px; + background: ${({ theme }) => theme.palette.import.progressBackground}; + position: relative; + + &::before { + content: ''; + height: 100%; + width: 0%; + position: absolute; + top: 0; + left: 0; + } + + &.loading::before { + width: 99%; + background: ${({ theme }) => theme.palette.import.progressWorking}; + transition: width 30s cubic-bezier(0.15, 0.735, 0.095, 1); + } + + &.finish::before { + width: 100%; + background: ${({ theme }) => theme.palette.import.progressDone}; + transition: width 0.2s ease-in-out; + } +`; + +export const ImportProgressBar = (props: { loading: boolean }) => { + const [transitionLoading, setTransitionLoading] = React.useState(false); + + useLoadingRegister(props.loading); + + useEffect(() => { + setTimeout(() => { + setTransitionLoading(true); + }, 100); + }, []); + + return ( + + + + ); +}; diff --git a/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx b/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx index 2c1031833b..3268321a11 100644 --- a/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx +++ b/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx @@ -3,9 +3,10 @@ import { useEffect, useState } from 'react'; import { useProject } from 'tg.hooks/useProject'; import { useImportDataHelper } from './useImportDataHelper'; -import { useApiMutation } from 'tg.service/http/useQueryApi'; +import { useNdJsonStreamedMutation } from 'tg.service/http/useQueryApi'; import { useMessage } from 'tg.hooks/useSuccessMessage'; import { T } from '@tolgee/react'; +import { OperationStatusType } from '../component/ImportFileInput'; export const useApplyImportHelper = ( dataHelper: ReturnType @@ -13,17 +14,24 @@ export const useApplyImportHelper = ( const [conflictNotResolvedDialogOpen, setConflictNotResolvedDialogOpen] = useState(false); - const importApplyLoadable = useApiMutation({ + const [status, setStatus] = useState( + undefined as OperationStatusType | undefined + ); + + const importApplyMutation = useNdJsonStreamedMutation({ url: '/v2/projects/{projectId}/import/apply', method: 'put', fetchOptions: { // error is displayed on the page disableErrorNotification: true, }, + onData(data) { + setStatus(data.status); + }, }); const project = useProject(); - const error = importApplyLoadable.error; + const error = importApplyMutation.error; const message = useMessage(); @@ -33,7 +41,7 @@ export const useApplyImportHelper = ( 0 ); if (unResolvedCount === 0) { - importApplyLoadable.mutate( + importApplyMutation.mutate( { path: { projectId: project.id, @@ -53,18 +61,18 @@ export const useApplyImportHelper = ( }; useEffect(() => { - const error = importApplyLoadable.error; + const error = importApplyMutation.error; if (error?.code == 'conflict_is_not_resolved') { setConflictNotResolvedDialogOpen(true); return; } - }, [importApplyLoadable.error]); + }, [importApplyMutation.error]); useEffect(() => { - if (importApplyLoadable.isSuccess) { + if (importApplyMutation.isSuccess) { dataHelper.refetchData(); } - }, [importApplyLoadable.isSuccess]); + }, [importApplyMutation.isSuccess]); const onDialogClose = () => { setConflictNotResolvedDialogOpen(false); @@ -75,7 +83,8 @@ export const useApplyImportHelper = ( onApplyImport, conflictNotResolvedDialogOpen, error, - loading: importApplyLoadable.isLoading, - loaded: importApplyLoadable.isSuccess, + loading: importApplyMutation.isLoading, + loaded: importApplyMutation.isSuccess, + status, }; }; diff --git a/webapp/src/views/projects/translations/TranslationTools/useMTStreamed.tsx b/webapp/src/views/projects/translations/TranslationTools/useMTStreamed.tsx index 3b43f8120f..15730456d6 100644 --- a/webapp/src/views/projects/translations/TranslationTools/useMTStreamed.tsx +++ b/webapp/src/views/projects/translations/TranslationTools/useMTStreamed.tsx @@ -1,10 +1,13 @@ -import { useQuery, UseQueryResult } from 'react-query'; +import { useQuery, UseQueryOptions, UseQueryResult } from 'react-query'; import { container } from 'tsyringe'; import { useState } from 'react'; import { paths } from 'tg.service/apiSchema.generated'; import { ApiError } from 'tg.service/http/ApiError'; -import { ApiSchemaHttpService } from 'tg.service/http/ApiSchemaHttpService'; +import { + ApiSchemaHttpService, + ResponseContent, +} from 'tg.service/http/ApiSchemaHttpService'; import { QueryProps } from 'tg.service/http/useQueryApi'; const apiHttpService = container.resolve(ApiSchemaHttpService); From a164cd4d819e7ad7c11f853a251a680ea6fe57dd Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sat, 23 Dec 2023 21:49:44 +0100 Subject: [PATCH 02/13] feat: Import enhancements > UI --- .../dataImport/StoredDataImporterTest.kt | 6 +- .../service/dataImport/StoredDataImporter.kt | 2 +- webapp/src/component/CustomIcons.tsx | 5 + webapp/src/svgs/icons/dropzone.svg | 4 + .../src/views/projects/import/ImportView.tsx | 5 + .../import/component/ImportFileDropzone.tsx | 46 +++--- .../import/component/ImportFileInput.tsx | 106 ++++++-------- .../component/ImportInputAreaLayout.tsx | 46 ++++++ .../component/ImportOperationStatus.tsx | 8 +- .../import/component/ImportOperationTitle.tsx | 47 +++++- .../import/component/ImportProgress.tsx | 26 ++-- .../component/ImportProgressOverlay.tsx | 137 ++++++++++++++++++ .../import/component/LanguageSelector.tsx | 13 +- .../import/hooks/useApplyImportHelper.tsx | 6 + 14 files changed, 347 insertions(+), 110 deletions(-) create mode 100644 webapp/src/svgs/icons/dropzone.svg create mode 100644 webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx create mode 100644 webapp/src/views/projects/import/component/ImportProgressOverlay.tsx diff --git a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt index 327e37abfc..926f8e00d4 100644 --- a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt @@ -26,7 +26,6 @@ class StoredDataImporterTest : AbstractSpringTest() { storedDataImporter = StoredDataImporter( applicationContext, importTestData.import, - reportStatus = reportStatus, ) } @@ -88,10 +87,9 @@ class StoredDataImporterTest : AbstractSpringTest() { @Test fun `it force replaces translations`() { storedDataImporter = StoredDataImporter( - applicationContext!!, + applicationContext, importTestData.import, ForceMode.OVERRIDE, - reportStatus, ) testDataService.saveTestData(importTestData.root) login() @@ -111,7 +109,6 @@ class StoredDataImporterTest : AbstractSpringTest() { applicationContext, importTestData.import, ForceMode.OVERRIDE, - reportStatus, ) login() storedDataImporter.doImport() @@ -140,7 +137,6 @@ class StoredDataImporterTest : AbstractSpringTest() { applicationContext, importTestData.import, ForceMode.KEEP, - reportStatus, ) testDataService.saveTestData(importTestData.root) login() diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt index bd086037d6..48fe3c79e6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt @@ -24,7 +24,7 @@ class StoredDataImporter( applicationContext: ApplicationContext, private val import: Import, private val forceMode: ForceMode = ForceMode.NO_FORCE, - private val reportStatus: (ImportApplicationStatus) -> Unit, + private val reportStatus: (ImportApplicationStatus) -> Unit = {}, ) { private val importDataManager = ImportDataManager(applicationContext, import) private val keyService = applicationContext.getBean(KeyService::class.java) diff --git a/webapp/src/component/CustomIcons.tsx b/webapp/src/component/CustomIcons.tsx index 39d0650e01..e22c279a1d 100644 --- a/webapp/src/component/CustomIcons.tsx +++ b/webapp/src/component/CustomIcons.tsx @@ -12,6 +12,7 @@ import { ReactComponent as TranslationMemorySvg } from '../svgs/icons/translatio import { ReactComponent as MachineTranslationSvg } from '../svgs/icons/machineTranslation.svg'; import { ReactComponent as TadaSvg } from '../svgs/icons/tada.svg'; import { ReactComponent as RocketSvg } from '../svgs/icons/rocket.svg'; +import { ReactComponent as DropZoneSvg } from '../svgs/icons/dropzone.svg'; type IconProps = ComponentProps; @@ -60,3 +61,7 @@ export const TadaIcon: React.FC = (props) => ( export const RocketIcon: React.FC = (props) => ( ); + +export const DropzoneIcon: React.FC = (props) => ( + +); diff --git a/webapp/src/svgs/icons/dropzone.svg b/webapp/src/svgs/icons/dropzone.svg new file mode 100644 index 0000000000..faedf6c607 --- /dev/null +++ b/webapp/src/svgs/icons/dropzone.svg @@ -0,0 +1,4 @@ + + + diff --git a/webapp/src/views/projects/import/ImportView.tsx b/webapp/src/views/projects/import/ImportView.tsx index eec96cebe0..e845bbcefc 100644 --- a/webapp/src/views/projects/import/ImportView.tsx +++ b/webapp/src/views/projects/import/ImportView.tsx @@ -104,6 +104,11 @@ export const ImportView: FunctionComponent = () => { ? 'addFiles' : undefined } + onImportMore={() => { + applyImportHelper.clear(); + dataHelper.addFilesMutation.reset(); + }} + filesUploaded={dataHelper.addFilesMutation.isSuccess} /> {dataHelper.addFilesMutation.data?.errors?.map((e, idx) => ( diff --git a/webapp/src/views/projects/import/component/ImportFileDropzone.tsx b/webapp/src/views/projects/import/component/ImportFileDropzone.tsx index 254b494357..b5bbe4766e 100644 --- a/webapp/src/views/projects/import/component/ImportFileDropzone.tsx +++ b/webapp/src/views/projects/import/component/ImportFileDropzone.tsx @@ -7,6 +7,7 @@ import React, { FunctionComponent, useState } from 'react'; import { FileUploadFixtures } from 'tg.fixtures/FileUploadFixtures'; import { MAX_FILE_COUNT } from './ImportFileInput'; +import { DropzoneIcon } from 'tg.component/CustomIcons'; export interface ScreenshotDropzoneProps { onNewFiles: (files: File[]) => void; @@ -16,37 +17,46 @@ const StyledWrapper = styled(Box)` pointer-events: none; opacity: 0; transition: opacity 0.2s; + background-color: ${({ theme }) => theme.palette.background.paper}; - &.valid { - backdrop-filter: blur(5px); - border: 1px solid ${green[200]}; - background-color: ${green[50]}; - opacity: 0.9; + &:before { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + border-radius: 4px; + height: 100%; } + &.valid, &.invalid { - border: 1px solid ${red[200]}; - opacity: 0.9; - background-color: ${red[50]}; + opacity: 1; + } + + &.valid:before, + &.invalid:before { backdrop-filter: blur(5px); + opacity: 0.3; + } + + &.valid:before { + background-color: ${green[200]}; + } + + &.invalid:before { + background-color: ${red[200]}; } `; -const StyledValidIcon = styled(Backup)` - filter: drop-shadow(1px 1px 0px ${green[200]}) - drop-shadow(-1px 1px 0px ${green[200]}) - drop-shadow(1px -1px 0px ${green[200]}) - drop-shadow(-1px -1px 0px ${green[200]}); +const StyledValidIcon = styled(DropzoneIcon)` font-size: 100px; - color: ${({ theme }) => theme.palette.common.white}; + color: ${({ theme }) => theme.palette.import.progressDone}; `; const StyledInvalidIcon = styled(HighlightOff)` - filter: drop-shadow(1px 1px 0px ${red[200]}) - drop-shadow(-1px 1px 0px ${red[200]}) drop-shadow(1px -1px 0px ${red[200]}) - drop-shadow(-1px -1px 0px ${red[200]}); font-size: 100px; - color: ${({ theme }) => theme.palette.common.white}; + fill: ${({ theme }) => theme.palette.error.main}; `; export const ImportFileDropzone: FunctionComponent = ( diff --git a/webapp/src/views/projects/import/component/ImportFileInput.tsx b/webapp/src/views/projects/import/component/ImportFileInput.tsx index 192a5a962d..78ef6fa884 100644 --- a/webapp/src/views/projects/import/component/ImportFileInput.tsx +++ b/webapp/src/views/projects/import/component/ImportFileInput.tsx @@ -1,18 +1,22 @@ import React, { FunctionComponent, ReactNode, useState } from 'react'; import { QuickStartHighlight } from 'tg.component/layout/QuickStartGuide/QuickStartHighlight'; -import { Box, styled, Typography } from '@mui/material'; +import { Box, Button, styled, Typography } from '@mui/material'; import { T, useTranslate } from '@tolgee/react'; import { container } from 'tsyringe'; import { useConfig } from 'tg.globalContext/helpers'; import { MessageActions } from 'tg.store/global/MessageActions'; import { Message } from 'tg.store/global/types'; -import LoadingButton from 'tg.component/common/form/LoadingButton'; import { ImportFileDropzone } from './ImportFileDropzone'; -import { ImportProgressBar } from './ImportProgress'; -import { ImportOperationStatus } from './ImportOperationStatus'; -import { ImportOperationTitle } from './ImportOperationTitle'; +import { ImportProgressOverlay } from './ImportProgressOverlay'; +import { + ImportInputAreaLayout, + ImportInputAreaLayoutBottom, + ImportInputAreaLayoutCenter, + ImportInputAreaLayoutTitle, + ImportInputAreaLayoutTop, +} from './ImportInputAreaLayout'; export const MAX_FILE_COUNT = 20; @@ -30,6 +34,8 @@ type ImportFileInputProps = { operation?: OperationType; operationStatus?: OperationStatusType; importDone: boolean; + onImportMore: () => void; + filesUploaded?: boolean; }; export type ValidationResult = { @@ -42,6 +48,10 @@ const StyledRoot = styled(Box)(({ theme }) => ({ border: `1px dashed ${theme.palette.emphasis[100]}`, margin: '0px auto', width: '100%', + position: 'relative', + backgroundColor: theme.palette.background.paper, + marginTop: '16px', + height: '240px', })); const messageActions = container.resolve(MessageActions); @@ -162,50 +172,33 @@ const ImportFileInput: FunctionComponent = (props) => { itemKey="pick_import_file" message={t('quick_start_item_pick_import_file_hint')} > - ({ - backgroundColor: theme.palette.background.paper, - mt: 4, - pt: 5, - pb: 5, - justifyContent: 'space-between', - alignItems: 'center', - flexDirection: 'column', - display: 'flex', - })} - > - onFileSelected(e)} - multiple - accept={ALLOWED_EXTENSIONS.join(',')} - /> - {props.operation ? ( - - ) : ( - - - - )} - - {props.loading || props.importDone ? ( - - ) : ( - + + + + onFileSelected(e)} + multiple + accept={ALLOWED_EXTENSIONS.join(',')} + /> + + + + + + + + - ) - )} + + ); }; - export default ImportFileInput; diff --git a/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx b/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx new file mode 100644 index 0000000000..350375f8b4 --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx @@ -0,0 +1,46 @@ +import { Box, styled, Typography } from '@mui/material'; +import React, { FC, ReactNode } from 'react'; + +export const ImportInputAreaLayout = styled(Box)` + justify-content: center; + align-items: center; + flex-direction: column; + display: flex; + padding-top: 20px; + padding-bottom: 20px; + height: 100%; +`; + +export const ImportInputAreaLayoutCenter = styled(Box)` + height: 50px; + width: 100%; + display: flex; + justify-content: center; + margin-top: 8px; + margin-bottom: 8px; + align-items: center; +`; + +export const ImportInputAreaLayoutTop = styled(Box)` + height: 40px; + display: flex; + justify-content: center; + align-items: center; +`; + +export const ImportInputAreaLayoutBottom = styled(Box)` + min-height: 40px; +`; + +export const ImportInputAreaLayoutTitle: FC<{ + icon?: ReactNode; +}> = (props) => { + return ( + <> + + {props.children} + + {props.icon} + + ); +}; diff --git a/webapp/src/views/projects/import/component/ImportOperationStatus.tsx b/webapp/src/views/projects/import/component/ImportOperationStatus.tsx index 53c85cfec3..fdc6b35b1e 100644 --- a/webapp/src/views/projects/import/component/ImportOperationStatus.tsx +++ b/webapp/src/views/projects/import/component/ImportOperationStatus.tsx @@ -1,11 +1,13 @@ import { T } from '@tolgee/react'; -import { OperationStatusType, OperationType } from './ImportFileInput'; +import { OperationStatusType } from './ImportFileInput'; import { useDebounce } from 'use-debounce/lib'; +import React from 'react'; export const ImportOperationStatus = (props: { - status: OperationStatusType; + status?: OperationStatusType; }) => { - const [debouncedStatus] = useDebounce(props.status, 1000); + const [debouncedStatus] = useDebounce(props.status, 1000, { leading: true }); + switch (debouncedStatus) { case 'PREPARING_AND_VALIDATING': return ; diff --git a/webapp/src/views/projects/import/component/ImportOperationTitle.tsx b/webapp/src/views/projects/import/component/ImportOperationTitle.tsx index 3ab8f15b44..1e0b462f1c 100644 --- a/webapp/src/views/projects/import/component/ImportOperationTitle.tsx +++ b/webapp/src/views/projects/import/component/ImportOperationTitle.tsx @@ -1,11 +1,44 @@ import { T } from '@tolgee/react'; import { OperationType } from './ImportFileInput'; +import React from 'react'; +import { TadaIcon } from 'tg.component/CustomIcons'; +import { Box } from '@mui/material'; +import { ImportInputAreaLayoutTitle } from './ImportInputAreaLayout'; -export const ImportOperationTitle = (props: { operation: OperationType }) => { - switch (props.operation) { - case 'addFiles': - return ; - case 'apply': - return ; - } +export const ImportOperationTitle = (props: { + operation?: OperationType; + filesUploaded?: boolean; + importDone?: boolean; +}) => { + const Message = () => { + if (props.importDone) { + return ( + }> + + + ); + } + + if (props.filesUploaded) { + return ( + + + + ); + } + switch (props.operation) { + case 'addFiles': + return ; + case 'apply': + return ; + } + + return null; + }; + + return ( + + + + ); }; diff --git a/webapp/src/views/projects/import/component/ImportProgress.tsx b/webapp/src/views/projects/import/component/ImportProgress.tsx index 531fbd59b1..8fd3999a45 100644 --- a/webapp/src/views/projects/import/component/ImportProgress.tsx +++ b/webapp/src/views/projects/import/component/ImportProgress.tsx @@ -17,22 +17,27 @@ const StyledProgress = styled('div')<{ loading?: string; finish?: string }>` position: absolute; top: 0; left: 0; + background-color: ${({ theme }) => theme.palette.import.progressWorking}; + transition: width 1s steps(1, jump-end), + background-color 1s steps(1, jump-end); } &.loading::before { width: 99%; - background: ${({ theme }) => theme.palette.import.progressWorking}; transition: width 30s cubic-bezier(0.15, 0.735, 0.095, 1); } &.finish::before { width: 100%; - background: ${({ theme }) => theme.palette.import.progressDone}; - transition: width 0.2s ease-in-out; + background-color: ${({ theme }) => theme.palette.import.progressDone}; + transition: width 0.2s ease-in-out, background-color 0.2s steps(1, jump-end); } `; -export const ImportProgressBar = (props: { loading: boolean }) => { +export const ImportProgressBar = (props: { + loading: boolean; + loaded: boolean; +}) => { const [transitionLoading, setTransitionLoading] = React.useState(false); useLoadingRegister(props.loading); @@ -43,18 +48,17 @@ export const ImportProgressBar = (props: { loading: boolean }) => { }, 100); }, []); + const classes = clsx({ + loading: transitionLoading && props.loading, + finish: props.loaded, + }); + return ( - + ); }; diff --git a/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx b/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx new file mode 100644 index 0000000000..9bb73b5f59 --- /dev/null +++ b/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx @@ -0,0 +1,137 @@ +import { Box, Button, styled } from '@mui/material'; +import { OperationStatusType, OperationType } from './ImportFileInput'; +import React, { useEffect, useState } from 'react'; +import clsx from 'clsx'; +import { + ImportInputAreaLayout, + ImportInputAreaLayoutBottom, + ImportInputAreaLayoutCenter, + ImportInputAreaLayoutTop, +} from './ImportInputAreaLayout'; +import { ImportProgressBar } from './ImportProgress'; +import { ImportOperationStatus } from './ImportOperationStatus'; +import { ImportOperationTitle } from './ImportOperationTitle'; +import { Link } from 'react-router-dom'; +import { LINKS, PARAMS } from 'tg.constants/links'; +import { T } from '@tolgee/react'; +import { useProject } from 'tg.hooks/useProject'; + +const StyledRoot = styled(Box)` + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: ${({ theme }) => theme.palette.background.paper}; + z-index: 1; + opacity: 0; + transition: opacity 0.1s ease-in-out; + + &.visible { + opacity: 1; + } +`; + +export const ImportProgressOverlay = (props: { + operation?: OperationType; + filesUploaded?: boolean; + importDone: boolean; + loading: boolean; + operationStatus?: OperationStatusType; + onImportMore: () => void; +}) => { + const project = useProject(); + + const [{ visible, filesUploaded, operation }, setState] = useState({ + visible: false, + filesUploaded: false, + operation: undefined as OperationType | undefined, + }); + + const [previousOperation, setPreviousOperation] = useState(); + + useEffect(() => { + const localPreviousOperation = previousOperation; + setPreviousOperation(props.operation); + + if (localPreviousOperation == 'addFiles' && !props.loading) { + setState({ + visible: true, + filesUploaded: true, + operation: props.operation, + }); + const timeout = setTimeout(() => { + setState({ + visible: props.loading || props.importDone, + filesUploaded: false, + operation: props.operation, + }); + setPreviousOperation(undefined); + }, 1000); + return () => clearTimeout(timeout); + } + + setState({ + visible: props.loading || props.importDone, + filesUploaded: false, + operation: props.operation, + }); + }, [props.loading, props.operation, props.importDone]); + + return ( + + + + + + + + + + {props.importDone ? ( + + + + + ) : ( + + )} + + + + ); +}; diff --git a/webapp/src/views/projects/import/component/LanguageSelector.tsx b/webapp/src/views/projects/import/component/LanguageSelector.tsx index efde6b3adc..46ab369567 100644 --- a/webapp/src/views/projects/import/component/LanguageSelector.tsx +++ b/webapp/src/views/projects/import/component/LanguageSelector.tsx @@ -48,12 +48,13 @@ export const LanguageSelector: React.FC<{ const importData = useImportDataHelper(); const languageHelper = useImportLanguageHelper(props.row); - const usedLanguages = importData - .result!._embedded!.languages!.map((l) => ({ - existingId: l.existingLanguageId, - namespace: l.namespace, - })) - .filter((l) => !!l); + const usedLanguages = + importData.result?._embedded?.languages + ?.map((l) => ({ + existingId: l.existingLanguageId, + namespace: l.namespace, + })) + .filter((l) => !!l) || []; const state = useStateObject({ addNewLanguageDialogOpen: false }); diff --git a/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx b/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx index 3268321a11..7c6860ff7b 100644 --- a/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx +++ b/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx @@ -78,6 +78,11 @@ export const useApplyImportHelper = ( setConflictNotResolvedDialogOpen(false); }; + const clear = () => { + importApplyMutation.reset(); + setStatus(undefined); + }; + return { onDialogClose, onApplyImport, @@ -86,5 +91,6 @@ export const useApplyImportHelper = ( loading: importApplyMutation.isLoading, loaded: importApplyMutation.isSuccess, status, + clear, }; }; From 3715b4d0dfc04393e59324c7366d1ebcd2b5ffd4 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 25 Dec 2023 13:05:00 +0100 Subject: [PATCH 03/13] fix: Dropzones UI, handle exceptions --- .../api/v2/controllers/V2ImportController.kt | 41 ++++--- .../io/tolgee/exceptions/NotFoundException.kt | 2 +- .../util/StreamingResponseBodyProvider.kt | 43 ++++++++ webapp/src/service/apiSchema.generated.ts | 102 ++++++++++++------ webapp/src/service/http/useQueryApi.ts | 27 +++-- .../translationTools/useErrorTranslation.ts | 3 +- .../import/component/ImportFileDropzone.tsx | 2 +- .../import/component/ImportOperationTitle.tsx | 12 ++- .../import/hooks/useApplyImportHelper.tsx | 2 +- .../TranslationTools/useMTStreamed.tsx | 7 +- 10 files changed, 174 insertions(+), 67 deletions(-) diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt index 4f6d1e110c..c8351ee2c4 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt @@ -10,6 +10,7 @@ import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.tags.Tag import io.tolgee.activity.RequestActivity import io.tolgee.activity.data.ActivityType +import io.tolgee.constants.Message import io.tolgee.dtos.dataImport.ImportAddFilesParams import io.tolgee.dtos.dataImport.ImportFileDto import io.tolgee.dtos.dataImport.SetFileNamespaceRequest @@ -43,7 +44,6 @@ import io.tolgee.service.dataImport.status.ImportApplicationStatus import io.tolgee.service.dataImport.status.ImportApplicationStatusItem import io.tolgee.service.key.NamespaceService import io.tolgee.util.StreamingResponseBodyProvider -import io.tolgee.util.disableAccelBuffering import jakarta.servlet.http.HttpServletRequest import org.springdoc.core.annotations.ParameterObject import org.springframework.data.domain.PageRequest @@ -68,7 +68,6 @@ import org.springframework.web.bind.annotation.RequestPart import org.springframework.web.bind.annotation.RestController import org.springframework.web.multipart.MultipartFile import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody -import java.io.OutputStreamWriter @Suppress("MVCPathVariableInspection") @RestController @@ -125,8 +124,8 @@ class V2ImportController( return ImportAddFilesResultModel(errors, result) } - @PutMapping("/apply", produces = [MediaType.APPLICATION_NDJSON_VALUE]) - @Operation(description = "Imports the data prepared in previous step", summary = "Apply") + @PutMapping("/apply") + @Operation(description = "Imports the data prepared in previous step") @RequestActivity(ActivityType.IMPORT) @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) @AllowApiAccess @@ -134,22 +133,30 @@ class V2ImportController( @Parameter(description = "Whether override or keep all translations with unresolved conflicts") @RequestParam(defaultValue = "NO_FORCE") forceMode: ForceMode, + ) { + val projectId = projectHolder.project.id + this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode) + } + + @PutMapping("/apply-streaming", produces = [MediaType.APPLICATION_NDJSON_VALUE]) + @Operation(description = "Imports the data prepared in previous step. Streams current status.") + @RequestActivity(ActivityType.IMPORT) + @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) + @AllowApiAccess + fun applyImportStreaming( + @Parameter(description = "Whether override or keep all translations with unresolved conflicts") + @RequestParam(defaultValue = "NO_FORCE") + forceMode: ForceMode, ): ResponseEntity { val projectId = projectHolder.project.id - return ResponseEntity.ok().disableAccelBuffering().body( - streamingResponseBodyProvider.createStreamingResponseBody { outputStream -> - val writer = OutputStreamWriter(outputStream) - val reportStatus = - { status: ImportApplicationStatus -> - writer.write( - (objectMapper.writeValueAsString(ImportApplicationStatusItem(status)) + "\n") - ) - writer.flush() - } - - this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode, reportStatus) + + return streamingResponseBodyProvider.streamNdJson { write -> + val writeStatus = { status: ImportApplicationStatus -> + write(ImportApplicationStatusItem(status)) } - ) + + this.importService.import(projectId, authenticationFacade.authenticatedUser.id, forceMode, writeStatus) + } } @GetMapping("/result") diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt index 34be6899ac..0362f4377f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt @@ -5,4 +5,4 @@ import org.springframework.http.HttpStatus import org.springframework.web.bind.annotation.ResponseStatus @ResponseStatus(value = HttpStatus.NOT_FOUND) -data class NotFoundException(val msg: Message? = Message.RESOURCE_NOT_FOUND) : RuntimeException() +data class NotFoundException(val msg: Message = Message.RESOURCE_NOT_FOUND) : RuntimeException() diff --git a/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt index 1a7bc7fb3f..c0f09ecd42 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt @@ -16,15 +16,22 @@ package io.tolgee.util +import com.fasterxml.jackson.databind.ObjectMapper +import io.tolgee.exceptions.ErrorException +import io.tolgee.exceptions.ErrorResponseBody +import io.tolgee.exceptions.NotFoundException import jakarta.persistence.EntityManager import org.hibernate.Session +import org.springframework.http.ResponseEntity import org.springframework.stereotype.Component import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody import java.io.OutputStream +import java.io.OutputStreamWriter @Component class StreamingResponseBodyProvider( private val entityManager: EntityManager, + private val objectMapper: ObjectMapper, ) { fun createStreamingResponseBody(fn: (os: OutputStream) -> Unit): StreamingResponseBody { return StreamingResponseBody { @@ -40,4 +47,40 @@ class StreamingResponseBodyProvider( session.close() } } + + fun streamNdJson(stream: (write: (message: Any?) -> Unit) -> Unit): ResponseEntity { + return ResponseEntity.ok().disableAccelBuffering().body( + this.createStreamingResponseBody { outputStream -> + OutputStreamWriter(outputStream).use { writer -> + val write = + { message: Any? -> writer.writeJson(message) } + try { + stream(write) + } catch (e: Throwable) { + val message = getErrorMessage(e) + writer.writeJson(StreamedErrorMessage(message)) + } + } + } + ) + } + + fun OutputStreamWriter.writeJson(message: Any?) { + this.write( + (objectMapper.writeValueAsString(message) + "\n") + ) + this.flush() + } + + private fun getErrorMessage(e: Throwable) = + when (e) { + is NotFoundException -> ErrorResponseBody(e.msg.code, null) + is ErrorException -> e.errorResponseBody + else -> ErrorResponseBody( + "unexpected_error_occurred", + listOf(e::class.java.name) + ) + } + + data class StreamedErrorMessage(val error: ErrorResponseBody) } diff --git a/webapp/src/service/apiSchema.generated.ts b/webapp/src/service/apiSchema.generated.ts index c6f337cb01..917c326deb 100644 --- a/webapp/src/service/apiSchema.generated.ts +++ b/webapp/src/service/apiSchema.generated.ts @@ -141,6 +141,10 @@ export interface paths { /** Sets namespace for file to import. */ put: operations["selectNamespace"]; }; + "/v2/projects/{projectId}/import/apply-streaming": { + /** Imports the data prepared in previous step. Streams current status. */ + put: operations["applyImportStreaming"]; + }; "/v2/projects/{projectId}/import/apply": { /** Imports the data prepared in previous step */ put: operations["applyImport"]; @@ -1068,6 +1072,8 @@ export interface components { /** @description Ids of screenshots uploaded with /v2/image-upload endpoint */ screenshotUploadedImageIds?: number[]; screenshotsToAdd?: components["schemas"]["KeyScreenshotDto"][]; + /** @description Keys in the document used as a context for machine translation. Keys in the same order as they appear in the document. The order is important! We are using it for graph distance calculation. */ + relatedKeysInOrder?: components["schemas"]["RelatedKeyDto"][]; }; KeyInScreenshotPositionDto: { /** Format: int32 */ @@ -1088,6 +1094,11 @@ export interface components { uploadedImageId: number; positions?: components["schemas"]["KeyInScreenshotPositionDto"][]; }; + /** @description Keys in the document used as a context for machine translation. Keys in the same order as they appear in the document. The order is important! We are using it for graph distance calculation. */ + RelatedKeyDto: { + namespace?: string; + keyName: string; + }; KeyInScreenshotModel: { /** Format: int64 */ keyId: number; @@ -1506,15 +1517,15 @@ export interface components { token: string; /** Format: int64 */ id: number; + description: string; /** Format: int64 */ - createdAt: number; - /** Format: int64 */ - updatedAt: number; + lastUsedAt?: number; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - lastUsedAt?: number; - description: string; + createdAt: number; + /** Format: int64 */ + updatedAt: number; }; SetOrganizationRoleDto: { roleType: "MEMBER" | "OWNER"; @@ -1650,17 +1661,17 @@ export interface components { key: string; /** Format: int64 */ id: number; - userFullName?: string; projectName: string; + userFullName?: string; username?: string; - scopes: string[]; + description: string; /** Format: int64 */ - projectId: number; + lastUsedAt?: number; + scopes: string[]; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - lastUsedAt?: number; - description: string; + projectId: number; }; SuperTokenRequest: { /** @description Has to be provided when TOTP enabled */ @@ -1918,6 +1929,8 @@ export interface components { /** @description Ids of screenshots uploaded with /v2/image-upload endpoint */ screenshotUploadedImageIds?: number[]; screenshots?: components["schemas"]["KeyScreenshotDto"][]; + /** @description Keys in the document used as a context for machine translation. Keys in the same order as they appear in the document. The order is important! We are using it for graph distance calculation. */ + relatedKeysInOrder?: components["schemas"]["RelatedKeyDto"][]; }; StorageTestResult: { success: boolean; @@ -2278,13 +2291,8 @@ export interface components { zip: boolean; }; BigMetaDto: { - /** @description List of keys, visible, in order as they appear in the document. The order is important! We are using it for graph distance calculation. */ - relatedKeysInOrder: components["schemas"]["RelatedKeyDto"][]; - }; - /** @description List of keys, visible, in order as they appear in the document. The order is important! We are using it for graph distance calculation. */ - RelatedKeyDto: { - namespace?: string; - keyName: string; + /** @description Keys in the document used as a context for machine translation. Keys in the same order as they appear in the document. The order is important! We are using it for graph distance calculation. */ + relatedKeysInOrder?: components["schemas"]["RelatedKeyDto"][]; }; TranslationCommentWithLangKeyDto: { /** Format: int64 */ @@ -2529,11 +2537,11 @@ export interface components { */ currentUserRole?: "MEMBER" | "OWNER"; basePermissions: components["schemas"]["PermissionModel"]; + /** @example This is a beautiful organization full of beautiful and clever people */ + description?: string; avatar?: components["schemas"]["Avatar"]; /** @example btforg */ slug: string; - /** @example This is a beautiful organization full of beautiful and clever people */ - description?: string; }; PublicBillingConfigurationDTO: { enabled: boolean; @@ -2642,8 +2650,8 @@ export interface components { name: string; /** Format: int64 */ id: number; - namespace?: string; baseTranslation?: string; + namespace?: string; translation?: string; }; KeySearchSearchResultModel: { @@ -2651,8 +2659,8 @@ export interface components { name: string; /** Format: int64 */ id: number; - namespace?: string; baseTranslation?: string; + namespace?: string; translation?: string; }; PagedModelKeySearchSearchResultModel: { @@ -3153,15 +3161,15 @@ export interface components { user: components["schemas"]["SimpleUserAccountModel"]; /** Format: int64 */ id: number; + description: string; /** Format: int64 */ - createdAt: number; - /** Format: int64 */ - updatedAt: number; + lastUsedAt?: number; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - lastUsedAt?: number; - description: string; + createdAt: number; + /** Format: int64 */ + updatedAt: number; }; OrganizationRequestParamsDto: { filterCurrentUserOwner: boolean; @@ -3279,17 +3287,17 @@ export interface components { permittedLanguageIds?: number[]; /** Format: int64 */ id: number; - userFullName?: string; projectName: string; + userFullName?: string; username?: string; - scopes: string[]; + description: string; /** Format: int64 */ - projectId: number; + lastUsedAt?: number; + scopes: string[]; /** Format: int64 */ expiresAt?: number; /** Format: int64 */ - lastUsedAt?: number; - description: string; + projectId: number; }; ApiKeyPermissionsModel: { /** @@ -4840,8 +4848,8 @@ export interface operations { }; }; }; - /** Imports the data prepared in previous step */ - applyImport: { + /** Imports the data prepared in previous step. Streams current status. */ + applyImportStreaming: { parameters: { query: { /** Whether override or keep all translations with unresolved conflicts */ @@ -4872,6 +4880,34 @@ export interface operations { }; }; }; + /** Imports the data prepared in previous step */ + applyImport: { + parameters: { + query: { + /** Whether override or keep all translations with unresolved conflicts */ + forceMode?: "OVERRIDE" | "KEEP" | "NO_FORCE"; + }; + path: { + projectId: number; + }; + }; + responses: { + /** OK */ + 200: unknown; + /** Bad Request */ + 400: { + content: { + "*/*": string; + }; + }; + /** Not Found */ + 404: { + content: { + "*/*": string; + }; + }; + }; + }; cancel: { parameters: { path: { diff --git a/webapp/src/service/http/useQueryApi.ts b/webapp/src/service/http/useQueryApi.ts index d44e243265..fbcb30185a 100644 --- a/webapp/src/service/http/useQueryApi.ts +++ b/webapp/src/service/http/useQueryApi.ts @@ -265,13 +265,16 @@ export const useNdJsonStreamedMutation = < const { done, value } = await reader.read(); const text = new TextDecoder().decode(value); if (text) { - try { - const parsed = JSON.parse(text); - result.push(parsed); - onData(parsed); - } catch (e) { - // ignore + const parsed = getParsedJsonOrNull(text); + if (!parsed) { + continue; } + if (parsed['error']) { + const error = parsed['error']; + throw new ApiError('Api error', error); + } + result.push(parsed); + onData(parsed); } if (done) { break; @@ -295,3 +298,15 @@ export const useNdJsonStreamedMutation = < ); return { ...mutation, mutate, mutateAsync }; }; + +function getParsedJsonOrNull(json?: string): any { + if (!json) { + return null; + } + + try { + return JSON.parse(json); + } catch (e) { + return null; + } +} diff --git a/webapp/src/translationTools/useErrorTranslation.ts b/webapp/src/translationTools/useErrorTranslation.ts index a50a12319b..346c8428ba 100644 --- a/webapp/src/translationTools/useErrorTranslation.ts +++ b/webapp/src/translationTools/useErrorTranslation.ts @@ -109,7 +109,8 @@ export function useErrorTranslation() { return t('webhook_responded_with_non_200_status'); case 'unexpected_error_while_executing_webhook': return t('unexpected_error_while_executing_webhook'); - + case 'resource_not_found': + return t('resource_not_found'); default: return code; } diff --git a/webapp/src/views/projects/import/component/ImportFileDropzone.tsx b/webapp/src/views/projects/import/component/ImportFileDropzone.tsx index b5bbe4766e..0d863098f8 100644 --- a/webapp/src/views/projects/import/component/ImportFileDropzone.tsx +++ b/webapp/src/views/projects/import/component/ImportFileDropzone.tsx @@ -1,7 +1,7 @@ import clsx from 'clsx'; import { Box, styled } from '@mui/material'; import { green, red } from '@mui/material/colors'; -import { Backup, HighlightOff } from '@mui/icons-material'; +import { HighlightOff } from '@mui/icons-material'; import React, { FunctionComponent, useState } from 'react'; import { FileUploadFixtures } from 'tg.fixtures/FileUploadFixtures'; diff --git a/webapp/src/views/projects/import/component/ImportOperationTitle.tsx b/webapp/src/views/projects/import/component/ImportOperationTitle.tsx index 1e0b462f1c..d35bce5522 100644 --- a/webapp/src/views/projects/import/component/ImportOperationTitle.tsx +++ b/webapp/src/views/projects/import/component/ImportOperationTitle.tsx @@ -28,9 +28,17 @@ export const ImportOperationTitle = (props: { } switch (props.operation) { case 'addFiles': - return ; + return ( + + + + ); case 'apply': - return ; + return ( + + + + ); } return null; diff --git a/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx b/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx index 7c6860ff7b..6c367ac3db 100644 --- a/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx +++ b/webapp/src/views/projects/import/hooks/useApplyImportHelper.tsx @@ -19,7 +19,7 @@ export const useApplyImportHelper = ( ); const importApplyMutation = useNdJsonStreamedMutation({ - url: '/v2/projects/{projectId}/import/apply', + url: '/v2/projects/{projectId}/import/apply-streaming', method: 'put', fetchOptions: { // error is displayed on the page diff --git a/webapp/src/views/projects/translations/TranslationTools/useMTStreamed.tsx b/webapp/src/views/projects/translations/TranslationTools/useMTStreamed.tsx index 15730456d6..3b43f8120f 100644 --- a/webapp/src/views/projects/translations/TranslationTools/useMTStreamed.tsx +++ b/webapp/src/views/projects/translations/TranslationTools/useMTStreamed.tsx @@ -1,13 +1,10 @@ -import { useQuery, UseQueryOptions, UseQueryResult } from 'react-query'; +import { useQuery, UseQueryResult } from 'react-query'; import { container } from 'tsyringe'; import { useState } from 'react'; import { paths } from 'tg.service/apiSchema.generated'; import { ApiError } from 'tg.service/http/ApiError'; -import { - ApiSchemaHttpService, - ResponseContent, -} from 'tg.service/http/ApiSchemaHttpService'; +import { ApiSchemaHttpService } from 'tg.service/http/ApiSchemaHttpService'; import { QueryProps } from 'tg.service/http/useQueryApi'; const apiHttpService = container.resolve(ApiSchemaHttpService); From a8e7fb3a0011946e4ca57e3f1e6dda2c1d9dac09 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Tue, 26 Dec 2023 20:43:10 +0100 Subject: [PATCH 04/13] feat: UI improvements, store files on upload --- .../api/v2/controllers/V2ImportController.kt | 3 +- .../configuration/FileStorageConfiguration.kt | 4 + .../V2ImportControllerAddFilesTest.kt | 26 +++++- .../fileStorage/FileStorageFsTest.kt | 9 +- .../fileStorage/FileStorageS3Test.kt | 6 ++ .../app/src/test/resources/application.yaml | 2 + .../configuration/tolgee/ImportProperties.kt | 16 +++- .../tolgee/InternalProperties.kt | 2 + .../dtos/dataImport/ImportAddFilesParams.kt | 2 + .../tolgee/dtos/dataImport/ImportFileDto.kt | 4 +- .../exceptions/ValidationException.kt | 3 +- .../io/tolgee/exceptions/ErrorException.kt | 2 +- .../io/tolgee/exceptions/ExpectedException.kt | 7 ++ .../io/tolgee/exceptions/InternalException.kt | 12 --- .../io/tolgee/exceptions/NotFoundException.kt | 2 +- .../io/tolgee/security/ProjectHolder.kt | 2 + .../ratelimit/RateLimitedException.kt | 5 +- .../io/tolgee/service/StartupImportService.kt | 10 ++- .../service/dataImport/ImportService.kt | 38 +++++++- .../processors/JsonFileProcessor.kt | 2 +- .../dataImport/processors/ZipTypeProcessor.kt | 4 +- .../dataImport/processors/po/PoParser.kt | 2 +- .../processors/xliff/XliffFileProcessor.kt | 2 +- .../util/StreamingResponseBodyProvider.kt | 5 ++ .../unit/CoreImportFileProcessorUnitTest.kt | 2 +- .../messageFormat/FormatDetectorTest.kt | 2 +- .../messageFormat/ToICUConverterTest.kt | 2 +- .../processors/po/PoFileProcessorTest.kt | 9 +- .../processors/processors/po/PoParserTest.kt | 2 +- .../xliff/Xliff12FileProcessorTest.kt | 18 ++-- .../xliff/XliffFileProcessorTest.kt | 2 +- .../io/tolgee/util}/InMemoryFileStorage.kt | 6 +- .../tests/src/test/resources/application.yaml | 1 + .../src/views/projects/import/ImportView.tsx | 90 ++++++++++--------- .../{ => component}/ImportAlertError.tsx | 2 +- .../import/component/ImportFileDropzone.tsx | 10 +++ .../import/component/ImportFileInput.tsx | 10 ++- .../component/ImportProgressOverlay.tsx | 5 ++ .../component/ImportResultLoadingOverlay.tsx | 28 ++++++ 39 files changed, 257 insertions(+), 102 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/exceptions/ExpectedException.kt delete mode 100644 backend/data/src/main/kotlin/io/tolgee/exceptions/InternalException.kt rename backend/{testing/src/main/kotlin/io/tolgee/testing/utils => development/src/main/kotlin/io/tolgee/util}/InMemoryFileStorage.kt (95%) rename webapp/src/views/projects/import/{ => component}/ImportAlertError.tsx (97%) create mode 100644 webapp/src/views/projects/import/component/ImportResultLoadingOverlay.tsx diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt index c8351ee2c4..b0302cd38f 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt @@ -10,7 +10,6 @@ import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.tags.Tag import io.tolgee.activity.RequestActivity import io.tolgee.activity.data.ActivityType -import io.tolgee.constants.Message import io.tolgee.dtos.dataImport.ImportAddFilesParams import io.tolgee.dtos.dataImport.ImportFileDto import io.tolgee.dtos.dataImport.SetFileNamespaceRequest @@ -103,7 +102,7 @@ class V2ImportController( @RequestPart("files") files: Array, @ParameterObject params: ImportAddFilesParams, ): ImportAddFilesResultModel { - val fileDtos = files.map { ImportFileDto(it.originalFilename ?: "", it.inputStream) } + val fileDtos = files.map { ImportFileDto(it.originalFilename ?: "", it.inputStream.readAllBytes()) } val errors = importService.addFiles( files = fileDtos, diff --git a/backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt b/backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt index a704b1f30d..1687b5d013 100644 --- a/backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt +++ b/backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt @@ -9,6 +9,7 @@ import io.tolgee.component.fileStorage.LocalFileStorage import io.tolgee.component.fileStorage.S3ClientProvider import io.tolgee.component.fileStorage.S3FileStorage import io.tolgee.configuration.tolgee.TolgeeProperties +import io.tolgee.util.InMemoryFileStorage import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -20,6 +21,9 @@ class FileStorageConfiguration( @Bean fun fileStorage(): FileStorage { + if (properties.internal.useInMemoryFileStorage) { + return InMemoryFileStorage() + } if (s3config.enabled) { val amazonS3 = S3ClientProvider(s3config).provide() val bucketName = properties.fileStorage.s3.bucketName ?: throw RuntimeException("Bucket name is not set") diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt index 1d87a12a4f..dd337ca3af 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt @@ -14,6 +14,7 @@ import io.tolgee.model.dataImport.issues.issueTypes.FileIssueType import io.tolgee.testing.AuthorizedControllerTest import io.tolgee.testing.assert import io.tolgee.testing.assertions.Assertions.assertThat +import io.tolgee.util.InMemoryFileStorage import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Value @@ -67,16 +68,37 @@ class V2ImportControllerAddFilesTest : AuthorizedControllerTest() { @AfterEach fun resetProps() { tolgeeProperties.maxTranslationTextLength = 10000 + (fileStorage as InMemoryFileStorage).clear() + tolgeeProperties.import.storeFilesForDebugging = false } @Test - fun `it parses zip file and saves issues`() { + fun `it parses zip file stores it for debugging and saves issues`() { + tolgeeProperties.import.storeFilesForDebugging = true val base = dbPopulator.createBase(generateUniqueString()) commitTransaction() - performImport(projectId = base.project.id, mapOf(Pair("zipOfUnknown.zip", zipOfUnknown))).andAssertThatJson { + val fileName = "zipOfUnknown.zip" + performImport(projectId = base.project.id, mapOf(Pair(fileName, zipOfUnknown))).andAssertThatJson { node("errors[2].code").isEqualTo("cannot_parse_file") } + + doesStoredFileExists(fileName, base.project.id).assert.isTrue() + } + + @Test + fun `doesn't store file for when disabled`() { + val base = dbPopulator.createBase(generateUniqueString()) + commitTransaction() + + val fileName = "zipOfUnknown.zip" + performImport(projectId = base.project.id, mapOf(Pair(fileName, zipOfUnknown))) + doesStoredFileExists(fileName, base.project.id).assert.isFalse() + } + + fun doesStoredFileExists(fileName: String, projectId: Long): Boolean { + val import = importService.find(projectId, userAccount!!.id) + return fileStorage.fileExists("importFiles/${import!!.id}/$fileName") } @Test diff --git a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt index 82363367f5..380bc3a4b8 100644 --- a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt @@ -10,7 +10,9 @@ import org.junit.jupiter.api.Test import org.springframework.boot.test.context.SpringBootTest import java.io.File -@SpringBootTest +@SpringBootTest( + properties = ["tolgee.internal.use-in-memory-file-storage=false"] +) class FileStorageFsTest : AbstractFileStorageServiceTest() { lateinit var file: File @@ -21,6 +23,11 @@ class FileStorageFsTest : AbstractFileStorageServiceTest() { file.writeText(testFileContent) } + @Test + fun `is LocalFileStorage`() { + assertThat(fileStorage is LocalFileStorage).isTrue() + } + @Test fun testReadFile() { val content = fileStorage.readFile(testFilePath).toString(charset("UTF-8")) diff --git a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt index 9c7f970e3a..ef65b15393 100644 --- a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt +++ b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt @@ -29,6 +29,7 @@ import software.amazon.awssdk.services.s3.model.S3Exception "tolgee.file-storage.s3.signing-region=dummy_signing_region", "tolgee.file-storage.s3.bucket-name=$BUCKET_NAME", "tolgee.authentication.initial-password=hey password manager, please don't use the filesystem :3", + "tolgee.internal.use-in-memory-file-storage=false" ], ) @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -55,6 +56,11 @@ class FileStorageS3Test : AbstractFileStorageServiceTest() { s3Mock.stop() } + @Test + fun `is LocalFileStorage`() { + assertThat(fileStorage is S3FileStorage).isTrue() + } + @Test fun testGetFile() { s3.putObject({ req -> req.bucket(BUCKET_NAME).key(testFilePath) }, RequestBody.fromString(testFileContent)) diff --git a/backend/app/src/test/resources/application.yaml b/backend/app/src/test/resources/application.yaml index 38d986af83..8eb9fe92b4 100644 --- a/backend/app/src/test/resources/application.yaml +++ b/backend/app/src/test/resources/application.yaml @@ -64,6 +64,7 @@ tolgee: fake-mt-providers: true mock-free-plan: true disable-initial-user-creation: true + use-in-memory-file-storage: true cache: caffeine-max-size: 1000 machine-translation: @@ -80,6 +81,7 @@ tolgee: server: http://localhost:8080 batch: concurrency: 10 + logging: level: io.tolgee.billing.api.v2.OrganizationInvoicesController: DEBUG diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt index 6c19c01b66..06de541297 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt @@ -6,14 +6,15 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "tolgee.import") @DocProperty( description = - "Bulk-imports exported json files in the database during startup. " + + "Properties for importing data to Tolgee and " + + "bulk-imports exported json files in the database during startup. " + "Useful to quickly provision a development server, and used for testing.", displayName = "Import", ) class ImportProperties { @DocProperty( description = - "File path of the directory where the files to import are located.\n" + + "File path of the directory where the files to import on startup are located.\n" + "\n" + ":::info\n" + "Your folder structure should look like:\n" + @@ -28,7 +29,7 @@ class ImportProperties { @DocProperty( description = - "Whether an implicit API key should be created.\n" + + "Whether an implicit API key should be created when importing data on startup.\n" + "\n" + "The key is built with a predictable format: " + "`\${lowercase filename (without extension)}-\${initial username}-imported-project-implicit`\n" + @@ -41,6 +42,13 @@ class ImportProperties { ) var createImplicitApiKey: Boolean = false - @DocProperty(description = "The language tag of the base language of the imported projects.") + @DocProperty(description = "The language tag of the base language of the imported projects " + + "(for importing data on startup).") var baseLanguageTag: String = "en" + + @DocProperty( + description = "If true, uploaded files will be stored in configured file storage for future debugging. " + + "Such data is not automatically removed after successful import. You have to clean-up manually!" + ) + var storeFilesForDebugging: Boolean = false } diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/InternalProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/InternalProperties.kt index 5b61ac9e13..8564c0eb2e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/InternalProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/InternalProperties.kt @@ -28,4 +28,6 @@ class InternalProperties { var e3eContentStorageBypassOk: Boolean? = null var disableInitialUserCreation: Boolean = false + + var useInMemoryFileStorage: Boolean = false } diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt index d674e57ed8..7f758529cf 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt @@ -9,4 +9,6 @@ class ImportAddFilesParams( "the delimiter which will be used in names of improted keys.", ) var structureDelimiter: Char? = '.', + + var storeFilesToFileStorage: Boolean = true ) diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportFileDto.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportFileDto.kt index 4c83e9fe55..76e3b8e6db 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportFileDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportFileDto.kt @@ -1,8 +1,6 @@ package io.tolgee.dtos.dataImport -import java.io.InputStream - data class ImportFileDto( val name: String = "", - val inputStream: InputStream, + val data: ByteArray, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/request/validators/exceptions/ValidationException.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/request/validators/exceptions/ValidationException.kt index c0ab4cb01e..1eb36cbb22 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/request/validators/exceptions/ValidationException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/request/validators/exceptions/ValidationException.kt @@ -3,8 +3,9 @@ package io.tolgee.dtos.request.validators.exceptions import io.tolgee.constants.Message import io.tolgee.dtos.request.validators.ValidationError import io.tolgee.dtos.request.validators.ValidationErrorType +import io.tolgee.exceptions.ExpectedException -class ValidationException : RuntimeException { +class ValidationException : RuntimeException, ExpectedException { val validationErrors: MutableSet = LinkedHashSet() constructor(message: Message, vararg parameters: String) { diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt index ea7ed09623..ecea59de20 100644 --- a/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt @@ -4,7 +4,7 @@ import io.tolgee.constants.Message import org.springframework.http.HttpStatus import java.io.Serializable -abstract class ErrorException : ExceptionWithMessage { +abstract class ErrorException : ExceptionWithMessage, ExpectedException { constructor(message: Message, params: List? = null) : super(message, params) constructor(code: String, params: List? = null) : super(code, params) diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/ExpectedException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/ExpectedException.kt new file mode 100644 index 0000000000..48dd9d07f3 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/ExpectedException.kt @@ -0,0 +1,7 @@ +package io.tolgee.exceptions + +/** + * Exceptions, which are expected and handeled. + * All exceptions that lead to http response status >= 400 and status < 500 + */ +interface ExpectedException diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/InternalException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/InternalException.kt deleted file mode 100644 index 765838091b..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/exceptions/InternalException.kt +++ /dev/null @@ -1,12 +0,0 @@ -package io.tolgee.exceptions - -import org.springframework.http.HttpStatus -import java.io.Serializable - -class InternalException : ErrorException { - constructor(message: io.tolgee.constants.Message, params: List?) : super(message, params) - constructor(message: io.tolgee.constants.Message) : super(message) - - override val httpStatus: HttpStatus - get() = HttpStatus.INTERNAL_SERVER_ERROR -} diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt index 0362f4377f..feeb995dcf 100644 --- a/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/NotFoundException.kt @@ -5,4 +5,4 @@ import org.springframework.http.HttpStatus import org.springframework.web.bind.annotation.ResponseStatus @ResponseStatus(value = HttpStatus.NOT_FOUND) -data class NotFoundException(val msg: Message = Message.RESOURCE_NOT_FOUND) : RuntimeException() +data class NotFoundException(val msg: Message = Message.RESOURCE_NOT_FOUND) : RuntimeException(), ExpectedException diff --git a/backend/data/src/main/kotlin/io/tolgee/security/ProjectHolder.kt b/backend/data/src/main/kotlin/io/tolgee/security/ProjectHolder.kt index 207f90798c..238f62389b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/security/ProjectHolder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/security/ProjectHolder.kt @@ -1,5 +1,6 @@ package io.tolgee.security +import io.sentry.Sentry import io.tolgee.dtos.cacheable.ProjectDto import io.tolgee.model.Project import io.tolgee.service.project.ProjectService @@ -14,6 +15,7 @@ open class ProjectHolder( private var _project: ProjectDto? = null open var project: ProjectDto set(value) { + Sentry.addBreadcrumb("Project Id: ${value.id}") _project = value } get() { diff --git a/backend/data/src/main/kotlin/io/tolgee/security/ratelimit/RateLimitedException.kt b/backend/data/src/main/kotlin/io/tolgee/security/ratelimit/RateLimitedException.kt index fa09f78c29..ee2528a3dc 100644 --- a/backend/data/src/main/kotlin/io/tolgee/security/ratelimit/RateLimitedException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/security/ratelimit/RateLimitedException.kt @@ -18,8 +18,11 @@ package io.tolgee.security.ratelimit import io.tolgee.constants.Message import io.tolgee.exceptions.ExceptionWithMessage +import io.tolgee.exceptions.ExpectedException import org.springframework.http.HttpStatus import org.springframework.web.bind.annotation.ResponseStatus @ResponseStatus(HttpStatus.TOO_MANY_REQUESTS) -class RateLimitedException(val retryAfter: Long, val global: Boolean) : ExceptionWithMessage(Message.RATE_LIMITED) +class RateLimitedException(val retryAfter: Long, val global: Boolean) : + ExceptionWithMessage(Message.RATE_LIMITED), + ExpectedException diff --git a/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt index 2569232d7c..1119b4244c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt @@ -3,6 +3,7 @@ package io.tolgee.service import io.tolgee.configuration.tolgee.TolgeeProperties import io.tolgee.dtos.cacheable.ProjectDto import io.tolgee.dtos.cacheable.UserAccountDto +import io.tolgee.dtos.dataImport.ImportAddFilesParams import io.tolgee.dtos.dataImport.ImportFileDto import io.tolgee.dtos.request.LanguageDto import io.tolgee.dtos.request.project.CreateProjectDTO @@ -72,7 +73,7 @@ class StartupImportService( private fun getImportFileDtos(projectDir: File) = projectDir.walk().filter { !it.isDirectory }.map { val relativePath = it.path.replace(projectDir.path, "") - if (relativePath.isBlank()) null else ImportFileDto(relativePath, it.inputStream()) + if (relativePath.isBlank()) null else ImportFileDto(relativePath, it.readBytes()) }.filterNotNull().toList() private fun setAuthentication(userAccount: UserAccount) { @@ -104,7 +105,12 @@ class StartupImportService( project: Project, userAccount: UserAccount, ) { - importService.addFiles(fileDtos, project, userAccount) + importService.addFiles( + files = fileDtos, + project = project, + userAccount = userAccount, + params = ImportAddFilesParams(storeFilesToFileStorage = false) + ) entityManager.flush() entityManager.clear() val imports = importService.getAllByProject(project.id) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index f7f8ff8c37..b8bcf787a0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -1,8 +1,11 @@ package io.tolgee.service.dataImport +import io.sentry.Sentry import io.tolgee.component.CurrentDateProvider +import io.tolgee.component.fileStorage.FileStorage import io.tolgee.component.reporting.BusinessEventPublisher import io.tolgee.component.reporting.OnBusinessEventToCaptureEvent +import io.tolgee.configuration.tolgee.TolgeeProperties import io.tolgee.constants.Message import io.tolgee.dtos.dataImport.ImportAddFilesParams import io.tolgee.dtos.dataImport.ImportFileDto @@ -35,8 +38,10 @@ import io.tolgee.service.dataImport.status.ImportApplicationStatus import io.tolgee.util.getSafeNamespace import jakarta.persistence.EntityManager import org.springframework.context.ApplicationContext +import org.springframework.context.annotation.Lazy import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable +import org.springframework.scheduling.annotation.Async import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.interceptor.TransactionInterceptor @@ -57,6 +62,10 @@ class ImportService( private val businessEventPublisher: BusinessEventPublisher, private val importDeleteService: ImportDeleteService, private val currentDateProvider: CurrentDateProvider, + @Suppress("SelfReferenceConstructorParameter") @Lazy + private val self: ImportService, + private val fileStorage: FileStorage, + private val tolgeeProperties: TolgeeProperties ) { @Transactional fun addFiles( @@ -64,7 +73,7 @@ class ImportService( project: Project, userAccount: UserAccount, params: ImportAddFilesParams = ImportAddFilesParams(), - ): List { + ): MutableList { val import = findNotExpired(project.id, userAccount.id) ?: Import(project).also { it.author = userAccount @@ -77,6 +86,10 @@ class ImportService( } importRepository.save(import) + Sentry.addBreadcrumb("Import ID: ${import.id}") + + self.saveFilesToFileStorage(import.id, files) + val fileProcessor = CoreImportFilesProcessor( applicationContext = applicationContext, @@ -107,6 +120,7 @@ class ImportService( forceMode: ForceMode = ForceMode.NO_FORCE, reportStatus: (ImportApplicationStatus) -> Unit = {} ) { + Sentry.addBreadcrumb("Import ID: ${import.id}") StoredDataImporter(applicationContext, import, forceMode, reportStatus).doImport() deleteImport(import) businessEventPublisher.publish( @@ -127,6 +141,7 @@ class ImportService( return } val import = importLanguage.file.import + Sentry.addBreadcrumb("Import ID: ${import.id}") val dataManager = ImportDataManager(applicationContext, import) existingLanguage?.let { val langAlreadySelectedInTheSameNS = @@ -148,6 +163,7 @@ class ImportService( namespace: String?, ) { val import = file.import + Sentry.addBreadcrumb("Import ID: ${import.id}") val dataManager = ImportDataManager(applicationContext, import) file.namespace = getSafeNamespace(namespace) importFileRepository.save(file) @@ -373,4 +389,24 @@ class ImportService( importFileIssueParamRepository.saveAll(params) fun getAllNamespaces(importId: Long) = importRepository.getAllNamespaces(importId) + + /** + * This function saves the files to file storage. + * When import fails, we need the files for future debugging + */ + @Async + fun saveFilesToFileStorage(importId: Long, files: List) { + if (tolgeeProperties.import.storeFilesForDebugging) { + files.forEach { + fileStorage.storeFile(getFileStoragePath(importId, it.name), it.data) + } + } + } + + fun getFileStoragePath(importId: Long, fileName: String): String { + val notBlankFilename = fileName.ifBlank { "blank_name" } + return "${getFileStorageImportRoot(importId)}/$notBlankFilename" + } + + private fun getFileStorageImportRoot(importId: Long) = "importFiles/$importId" } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/JsonFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/JsonFileProcessor.kt index c7814c1c3c..0cb6628294 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/JsonFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/JsonFileProcessor.kt @@ -11,7 +11,7 @@ class JsonFileProcessor( ) : ImportFileProcessor() { override fun process() { try { - val data = jacksonObjectMapper().readValue>(context.file.inputStream) + val data = jacksonObjectMapper().readValue>(context.file.data) val parsed = data.parse() parsed.entries.forEachIndexed { index, it -> context.addTranslation(it.key, languageNameGuesses[0], it.value, index) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt index 5a90c0e9dd..f9a5f1fa17 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt @@ -10,7 +10,7 @@ class ZipTypeProcessor : ImportArchiveProcessor { } override fun process(file: ImportFileDto): Collection { - val zipInputStream = ZipInputStream(file.inputStream) + val zipInputStream = ZipInputStream(file.data.inputStream()) var nextEntry: ZipEntry? // .zip archives generated by MacOs compress feature return duplicities // I am removing them here by adding the files to map @@ -30,7 +30,7 @@ class ZipTypeProcessor : ImportArchiveProcessor { files[fileName] = ImportFileDto( name = fileName, - data.inputStream(), + data = data ) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/po/PoParser.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/po/PoParser.kt index b3ee4e08b3..398093e82c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/po/PoParser.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/po/PoParser.kt @@ -93,7 +93,7 @@ class PoParser( } private fun processInputStream() { - context.file.inputStream.readAllBytes().decodeToString().forEach { + context.file.data.decodeToString().forEach { it.handle() } endTranslation() diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/xliff/XliffFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/xliff/XliffFileProcessor.kt index da9de913c6..e92b4766be 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/xliff/XliffFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/xliff/XliffFileProcessor.kt @@ -24,7 +24,7 @@ class XliffFileProcessor(override val context: FileProcessorContext) : ImportFil private val xmlEventReader: XMLEventReader by lazy { val inputFactory: XMLInputFactory = XMLInputFactory.newDefaultFactory() - inputFactory.createXMLEventReader(context.file.inputStream) + inputFactory.createXMLEventReader(context.file.data.inputStream()) } private val version: String by lazy { diff --git a/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt index c0f09ecd42..b08d035e6d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt @@ -17,8 +17,10 @@ package io.tolgee.util import com.fasterxml.jackson.databind.ObjectMapper +import io.sentry.Sentry import io.tolgee.exceptions.ErrorException import io.tolgee.exceptions.ErrorResponseBody +import io.tolgee.exceptions.ExpectedException import io.tolgee.exceptions.NotFoundException import jakarta.persistence.EntityManager import org.hibernate.Session @@ -59,6 +61,9 @@ class StreamingResponseBodyProvider( } catch (e: Throwable) { val message = getErrorMessage(e) writer.writeJson(StreamedErrorMessage(message)) + if (e !is ExpectedException) { + Sentry.captureException(e) + } } } } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/CoreImportFileProcessorUnitTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/CoreImportFileProcessorUnitTest.kt index b192c7fd49..53f3bef5cc 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/CoreImportFileProcessorUnitTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/CoreImportFileProcessorUnitTest.kt @@ -64,7 +64,7 @@ class CoreImportFileProcessorUnitTest { tolgeePropertiesMock = mock() importFile = ImportFile("lgn.json", importMock) - importFileDto = ImportFileDto("lng.json", "".toByteArray().inputStream()) + importFileDto = ImportFileDto("lng.json", "".toByteArray()) existingLanguage = Language().also { it.name = "lng" } existingTranslation = Translation("helllo").also { it.key = Key(name = "colliding key") } processor = CoreImportFilesProcessor(applicationContextMock, importMock) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/FormatDetectorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/FormatDetectorTest.kt index 95aa3572af..bc72c08ae6 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/FormatDetectorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/FormatDetectorTest.kt @@ -26,7 +26,7 @@ class FormatDetectorTest { ImportFileDto( "exmample.po", File("src/test/resources/import/po/example.po") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/ToICUConverterTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/ToICUConverterTest.kt index dfd4925570..67ab4f0d8e 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/ToICUConverterTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/messageFormat/ToICUConverterTest.kt @@ -27,7 +27,7 @@ class ToICUConverterTest { ImportFileDto( "exmample.po", File("src/test/resources/import/po/example.po") - .inputStream(), + .readBytes(), ) context = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt index 4ce4c4f4b2..d233489be4 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt @@ -80,11 +80,10 @@ class PoFileProcessorTest { private fun mockImportFile(inputStream: InputStream) { importMock = mock() importFile = ImportFile("exmample.po", importMock) - importFileDto = - ImportFileDto( - "exmample.po", - inputStream, - ) + importFileDto = ImportFileDto( + "exmample.po", + inputStream.readAllBytes() + ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoParserTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoParserTest.kt index 05ef19e369..298bc21ae5 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoParserTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoParserTest.kt @@ -25,7 +25,7 @@ class PoParserTest { ImportFileDto( "exmample.po", File("src/test/resources/import/po/example.po") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt index 4cbc710b96..028dc80fb6 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt @@ -26,7 +26,7 @@ class Xliff12FileProcessorTest { private val xmlEventReader: XMLEventReader get() { val inputFactory: XMLInputFactory = XMLInputFactory.newDefaultFactory() - return inputFactory.createXMLEventReader(importFileDto.inputStream) + return inputFactory.createXMLEventReader(importFileDto.data.inputStream()) } @BeforeEach @@ -37,7 +37,7 @@ class Xliff12FileProcessorTest { ImportFileDto( "exmample.xliff", File("src/test/resources/import/xliff/example.xliff") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } @@ -77,10 +77,10 @@ class Xliff12FileProcessorTest { ImportFileDto( "exmample.xliff", File("src/test/resources/import/xliff/larger.xlf") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) - xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.inputStream) + xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.data.inputStream()) val start = System.currentTimeMillis() Xliff12FileProcessor(fileProcessorContext, xmlEventReader).process() assertThat(System.currentTimeMillis() - start).isLessThan(4000) @@ -88,13 +88,11 @@ class Xliff12FileProcessorTest { @Test fun `handles errors correctly`() { - importFileDto = - ImportFileDto( - "exmample.xliff", - File("src/test/resources/import/xliff/error_example.xliff") - .inputStream(), + importFileDto = ImportFileDto( + "exmample.xliff", + File("src/test/resources/import/xliff/error_example.xliff").readBytes(), ) - xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.inputStream) + xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.data.inputStream()) fileProcessorContext = FileProcessorContext(importFileDto, importFile) Xliff12FileProcessor(fileProcessorContext, xmlEventReader).process() assertThat(fileProcessorContext.translations).hasSize(2) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/XliffFileProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/XliffFileProcessorTest.kt index 2ea25fb1f0..e45fdd52b5 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/XliffFileProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/XliffFileProcessorTest.kt @@ -25,7 +25,7 @@ class XliffFileProcessorTest { ImportFileDto( "exmample.xliff", File("src/test/resources/import/xliff/example.xliff") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } diff --git a/backend/testing/src/main/kotlin/io/tolgee/testing/utils/InMemoryFileStorage.kt b/backend/development/src/main/kotlin/io/tolgee/util/InMemoryFileStorage.kt similarity index 95% rename from backend/testing/src/main/kotlin/io/tolgee/testing/utils/InMemoryFileStorage.kt rename to backend/development/src/main/kotlin/io/tolgee/util/InMemoryFileStorage.kt index c140c4726a..acdd6a583e 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/testing/utils/InMemoryFileStorage.kt +++ b/backend/development/src/main/kotlin/io/tolgee/util/InMemoryFileStorage.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.tolgee.testing.utils +package io.tolgee.util import io.tolgee.component.fileStorage.FileStorage import io.tolgee.exceptions.FileStoreException @@ -41,4 +41,8 @@ class InMemoryFileStorage : FileStorage { override fun fileExists(storageFilePath: String): Boolean { return files.contains(storageFilePath) } + + fun clear() { + files.clear() + } } diff --git a/ee/backend/tests/src/test/resources/application.yaml b/ee/backend/tests/src/test/resources/application.yaml index 7b0f3ab515..e2a884a615 100644 --- a/ee/backend/tests/src/test/resources/application.yaml +++ b/ee/backend/tests/src/test/resources/application.yaml @@ -58,6 +58,7 @@ tolgee: fake-mt-providers: true mock-free-plan: true disable-initial-user-creation: true + use-in-memory-file-storage: true cache: caffeine-max-size: 1000 machine-translation: diff --git a/webapp/src/views/projects/import/ImportView.tsx b/webapp/src/views/projects/import/ImportView.tsx index e845bbcefc..51ae4eb502 100644 --- a/webapp/src/views/projects/import/ImportView.tsx +++ b/webapp/src/views/projects/import/ImportView.tsx @@ -1,5 +1,5 @@ import { FunctionComponent, useEffect, useState } from 'react'; -import { Box } from '@mui/material'; +import { Box, Button } from '@mui/material'; import { T, useTranslate } from '@tolgee/react'; import { container } from 'tsyringe'; @@ -11,9 +11,8 @@ import { MessageService } from 'tg.service/MessageService'; import { components } from 'tg.service/apiSchema.generated'; import { useGlobalActions } from 'tg.globalContext/GlobalContext'; import { TranslatedError } from 'tg.translationTools/TranslatedError'; -import LoadingButton from 'tg.component/common/form/LoadingButton'; -import { ImportAlertError } from './ImportAlertError'; +import { ImportAlertError } from './component/ImportAlertError'; import { ImportConflictNotResolvedErrorDialog } from './component/ImportConflictNotResolvedErrorDialog'; import { ImportConflictResolutionDialog } from './component/ImportConflictResolutionDialog'; import ImportFileInput from './component/ImportFileInput'; @@ -21,6 +20,7 @@ import { ImportResult } from './component/ImportResult'; import { useApplyImportHelper } from './hooks/useApplyImportHelper'; import { useImportDataHelper } from './hooks/useImportDataHelper'; import { BaseProjectView } from '../BaseProjectView'; +import { ImportResultLoadingOverlay } from './component/ImportResultLoadingOverlay'; const messageService = container.resolve(MessageService); @@ -71,6 +71,9 @@ export const ImportView: FunctionComponent = () => { } }, [applyImportHelper.loading, applyImportHelper.loaded]); + const loading = + dataHelper.addFilesMutation.isLoading || applyImportHelper.loading; + return ( { { addFilesMutation={dataHelper.addFilesMutation} /> ))} - - - {dataHelper.result && ( - - - { - confirmation({ - onConfirm: () => dataHelper.onCancel(), - title: , - message: , - }); - }} - > - - - - - - - - + + + + {dataHelper.result && ( + + + + + + + + + )} - )} + { resolveFirstUnresolved(); diff --git a/webapp/src/views/projects/import/ImportAlertError.tsx b/webapp/src/views/projects/import/component/ImportAlertError.tsx similarity index 97% rename from webapp/src/views/projects/import/ImportAlertError.tsx rename to webapp/src/views/projects/import/component/ImportAlertError.tsx index 77ebcebaae..be5bc03015 100644 --- a/webapp/src/views/projects/import/ImportAlertError.tsx +++ b/webapp/src/views/projects/import/component/ImportAlertError.tsx @@ -16,7 +16,7 @@ import CloseIcon from '@mui/icons-material/Close'; import { T } from '@tolgee/react'; import { components } from 'tg.service/apiSchema.generated'; -import { useImportDataHelper } from './hooks/useImportDataHelper'; +import { useImportDataHelper } from '../hooks/useImportDataHelper'; export const ImportAlertError: FunctionComponent<{ error: components['schemas']['ImportAddFilesResultModel']['errors'][0]; diff --git a/webapp/src/views/projects/import/component/ImportFileDropzone.tsx b/webapp/src/views/projects/import/component/ImportFileDropzone.tsx index 0d863098f8..0523698f18 100644 --- a/webapp/src/views/projects/import/component/ImportFileDropzone.tsx +++ b/webapp/src/views/projects/import/component/ImportFileDropzone.tsx @@ -11,6 +11,7 @@ import { DropzoneIcon } from 'tg.component/CustomIcons'; export interface ScreenshotDropzoneProps { onNewFiles: (files: File[]) => void; + active: boolean; } const StyledWrapper = styled(Box)` @@ -68,6 +69,9 @@ export const ImportFileDropzone: FunctionComponent = ( ); const onDragEnter = (e: React.DragEvent) => { + if (!props.active) { + return; + } e.stopPropagation(); e.preventDefault(); setDragEnterTarget(e.target); @@ -84,6 +88,9 @@ export const ImportFileDropzone: FunctionComponent = ( }; const onDragLeave = (e: React.DragEvent) => { + if (!props.active) { + return; + } e.stopPropagation(); e.preventDefault(); if (e.target === dragEnterTarget) { @@ -92,6 +99,9 @@ export const ImportFileDropzone: FunctionComponent = ( }; const onDrop = async (e: React.DragEvent) => { + if (!props.active) { + return; + } e.stopPropagation(); e.preventDefault(); if (e.dataTransfer.items) { diff --git a/webapp/src/views/projects/import/component/ImportFileInput.tsx b/webapp/src/views/projects/import/component/ImportFileInput.tsx index 78ef6fa884..8dc9a73ca3 100644 --- a/webapp/src/views/projects/import/component/ImportFileInput.tsx +++ b/webapp/src/views/projects/import/component/ImportFileInput.tsx @@ -165,8 +165,13 @@ const ImportFileInput: FunctionComponent = (props) => { return { ...result, valid }; }; + const [isProgressOverlayActive, setIsProgressOverlayActive] = useState(false); + return ( - + = (props) => { onImportMore={props.onImportMore} filesUploaded={props.filesUploaded} operationStatus={props.operationStatus} + onActiveChange={(isActive) => + setIsProgressOverlayActive(isActive) + } /> void; + onActiveChange: (isActive: boolean) => void; }) => { const project = useProject(); @@ -78,6 +79,10 @@ export const ImportProgressOverlay = (props: { }); }, [props.loading, props.operation, props.importDone]); + useEffect(() => { + props.onActiveChange(visible); + }, [visible]); + return ( theme.palette.background.default}; + + &.visible { + pointer-events: all; + opacity: 0.6; + } +`; + +export const ImportResultLoadingOverlay = (props: { loading: boolean }) => { + return ; +}; From 17d4ff4f6db75d450550019ad341b2e19ffde9d7 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Wed, 27 Dec 2023 09:22:11 +0100 Subject: [PATCH 05/13] fix: Finalize UI --- webapp/src/views/projects/import/ImportView.tsx | 8 +++++++- .../projects/import/component/ImportFileInput.tsx | 9 ++++----- .../import/component/ImportInputAreaLayout.tsx | 11 ++++------- .../projects/import/component/ImportProgress.tsx | 4 +++- .../import/component/ImportProgressOverlay.tsx | 11 ++++++----- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/webapp/src/views/projects/import/ImportView.tsx b/webapp/src/views/projects/import/ImportView.tsx index 51ae4eb502..3a29f5baf1 100644 --- a/webapp/src/views/projects/import/ImportView.tsx +++ b/webapp/src/views/projects/import/ImportView.tsx @@ -74,6 +74,8 @@ export const ImportView: FunctionComponent = () => { const loading = dataHelper.addFilesMutation.isLoading || applyImportHelper.loading; + const [isProgressOverlayActive, setIsProgressOverlayActive] = useState(false); + return ( { dataHelper.addFilesMutation.reset(); }} filesUploaded={dataHelper.addFilesMutation.isSuccess} + isProgressOverlayActive={isProgressOverlayActive} + onProgressOverlayActiveChange={(isActive) => + setIsProgressOverlayActive(isActive) + } /> {dataHelper.addFilesMutation.data?.errors?.map((e, idx) => ( @@ -120,7 +126,7 @@ export const ImportView: FunctionComponent = () => { /> ))} - + void; filesUploaded?: boolean; + onProgressOverlayActiveChange: (isActive: boolean) => void; + isProgressOverlayActive: boolean; }; export type ValidationResult = { @@ -51,7 +53,6 @@ const StyledRoot = styled(Box)(({ theme }) => ({ position: 'relative', backgroundColor: theme.palette.background.paper, marginTop: '16px', - height: '240px', })); const messageActions = container.resolve(MessageActions); @@ -165,12 +166,10 @@ const ImportFileInput: FunctionComponent = (props) => { return { ...result, valid }; }; - const [isProgressOverlayActive, setIsProgressOverlayActive] = useState(false); - return ( = (props) => { filesUploaded={props.filesUploaded} operationStatus={props.operationStatus} onActiveChange={(isActive) => - setIsProgressOverlayActive(isActive) + props.onProgressOverlayActiveChange(isActive) } /> diff --git a/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx b/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx index 350375f8b4..aadf3c8fae 100644 --- a/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx +++ b/webapp/src/views/projects/import/component/ImportInputAreaLayout.tsx @@ -6,30 +6,27 @@ export const ImportInputAreaLayout = styled(Box)` align-items: center; flex-direction: column; display: flex; - padding-top: 20px; - padding-bottom: 20px; + padding-top: 40px; + padding-bottom: 40px; height: 100%; `; export const ImportInputAreaLayoutCenter = styled(Box)` - height: 50px; + height: 76px; width: 100%; display: flex; justify-content: center; - margin-top: 8px; - margin-bottom: 8px; align-items: center; `; export const ImportInputAreaLayoutTop = styled(Box)` - height: 40px; display: flex; justify-content: center; align-items: center; `; export const ImportInputAreaLayoutBottom = styled(Box)` - min-height: 40px; + min-height: 24px; `; export const ImportInputAreaLayoutTitle: FC<{ diff --git a/webapp/src/views/projects/import/component/ImportProgress.tsx b/webapp/src/views/projects/import/component/ImportProgress.tsx index 8fd3999a45..403f069e16 100644 --- a/webapp/src/views/projects/import/component/ImportProgress.tsx +++ b/webapp/src/views/projects/import/component/ImportProgress.tsx @@ -45,7 +45,7 @@ export const ImportProgressBar = (props: { useEffect(() => { setTimeout(() => { setTransitionLoading(true); - }, 100); + }, 10); }, []); const classes = clsx({ @@ -53,6 +53,8 @@ export const ImportProgressBar = (props: { finish: props.loaded, }); + console.table({ classes, ...props }); + return ( From cf270d1b1ed4a8e53a3073f360f08b074526723e Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Wed, 27 Dec 2023 09:50:40 +0100 Subject: [PATCH 06/13] chore: Fix tests --- .../v2/controllers/V2ExportControllerTest.kt | 2 -- .../SecuredV2ImageUploadControllerTest.kt | 9 +++----- .../V2ImageUploadControllerTest.kt | 23 ++++++++----------- .../KeyScreenshotControllerTest.kt | 12 ++++------ .../SecuredKeyScreenshotControllerTest.kt | 6 ++--- .../initialUserCreation/CreateEnabledTest.kt | 22 ++++++++++++------ .../configuration/tolgee/ImportProperties.kt | 6 +++-- 7 files changed, 39 insertions(+), 41 deletions(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt index 2da74f7874..d66807faac 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt @@ -82,8 +82,6 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { retryingOnCommonIssues { initBaseData() try { - executeInNewTransaction { - } performExport() performExport() waitForNotThrowing(pollTime = 50, timeout = 3000) { diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt index e7f3203513..8952df215f 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt @@ -14,7 +14,6 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance import org.springframework.boot.test.context.SpringBootTest -import java.io.File import java.time.Duration import java.util.* @@ -70,8 +69,7 @@ class SecuredV2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() performAuthGet("/uploaded-images/${image.filename}.png?token=$token") .andIsOk.andReturn().response.contentAsByteArray - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + image.filename + ".png") - assertThat(storedImage).isEqualTo(file.readBytes()) + assertThat(storedImage).isEqualTo(fileStorage.readFile("uploadedImages/" + image.filename + ".png")) } @Test @@ -79,9 +77,8 @@ class SecuredV2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() fun upload() { performStoreImage().andPrettyPrint.andIsCreated.andAssertThatJson { node("filename").isString.satisfies { - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + it + ".png") - assertThat(file).exists() - assertThat(file.readBytes().size).isCloseTo(5538, Offset.offset(500)) + val path = "uploadedImages/" + it + ".png"; + assertThat(fileStorage.readFile(path).size).isCloseTo(5538, Offset.offset(500)) } node("requestFilename").isString.satisfies { val parts = it.split("?token=") diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt index 4970fac2fa..40467799ad 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt @@ -11,6 +11,7 @@ import io.tolgee.fixtures.andIsCreated import io.tolgee.fixtures.andIsForbidden import io.tolgee.fixtures.andIsOk import io.tolgee.fixtures.andPrettyPrint +import io.tolgee.testing.assert import io.tolgee.testing.assertions.Assertions.assertThat import org.assertj.core.data.Offset import org.junit.jupiter.api.AfterAll @@ -22,7 +23,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header -import java.io.File import java.util.stream.Collectors @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -50,9 +50,9 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { performStoreImage().andPrettyPrint.andIsCreated.andAssertThatJson { node("fileUrl").isString.startsWith("http://").endsWith(".png") node("requestFilename").isString.satisfies { - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/uploadedImages/" + it) - assertThat(file).exists() - assertThat(file.readBytes().size).isCloseTo(5538, Offset.offset(500)) + val file = fileStorage.fileExists("uploadedImages/" + it).assert.isTrue() + fileStorage.readFile("uploadedImages/" + it) + .size.assert.isCloseTo(5538, Offset.offset(500)) } } } @@ -78,15 +78,16 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { @Test fun `returns file`() { val image = imageUploadService.store(screenshotFile, userAccount!!, null) - - val file = File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${image.filenameWithExtension}""") val result = performAuthGet("/uploaded-images/${image.filenameWithExtension}").andIsOk .andExpect( header().string("Cache-Control", "max-age=365, must-revalidate, no-transform"), ) .andReturn() - assertThat(result.response.contentAsByteArray).isEqualTo(file.readBytes()) + assertThat(result.response.contentAsByteArray) + .isEqualTo( + fileStorage.readFile("uploadedImages/${image.filenameWithExtension}") + ) } @Test @@ -100,9 +101,7 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { val idsToDelete = list.take(10).map { it.id }.joinToString(",") list.asSequence().take(10).forEach { - assertThat( - File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${it.filenameWithExtension}"""), - ).exists() + fileStorage.fileExists("uploadedImages/${it.filenameWithExtension}").assert.isTrue() } performAuthDelete("/v2/image-upload/$idsToDelete", null).andIsOk @@ -110,9 +109,7 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { assertThat(rest).isEqualTo(list.stream().skip(10).collect(Collectors.toList())) list.asSequence().take(10).forEach { - assertThat( - File("""${tolgeeProperties.fileStorage.fsDataPath}/uploadedImages/${it.filenameWithExtension}"""), - ).doesNotExist() + fileStorage.fileExists("uploadedImages/${it.filenameWithExtension}").assert.isFalse() } } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt index d3a9f87f4c..4b5365e7af 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/KeyScreenshotControllerTest.kt @@ -13,6 +13,7 @@ import io.tolgee.fixtures.andPrettyPrint import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert import io.tolgee.testing.assertions.Assertions.assertThat +import io.tolgee.util.InMemoryFileStorage import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -20,7 +21,6 @@ import org.junit.jupiter.api.TestInstance import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status -import java.io.File @TestInstance(TestInstance.Lifecycle.PER_CLASS) class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { @@ -34,6 +34,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { @AfterAll fun after() { tolgeeProperties.fileStorageUrl = initialScreenshotUrl + (fileStorage as InMemoryFileStorage).clear() } @Test @@ -63,8 +64,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { val screenshots = screenshotService.findAll(key = key) assertThat(screenshots).hasSize(1) node("filename").isEqualTo(screenshots[0].filename) - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshots[0].filename) - assertThat(file).exists() + fileStorage.fileExists("screenshots/" + screenshots[0].filename).assert.isTrue() val reference = screenshots[0].keyScreenshotReferences[0] reference.originalText.assert.isEqualTo(text) reference.positions!![0].x.assert.isEqualTo(200) @@ -116,8 +116,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { performProjectAuthGet("keys/${key.id}/screenshots").andIsOk.andPrettyPrint.andAssertThatJson { node("_embedded.screenshots").isArray.hasSize(2) node("_embedded.screenshots[0].filename").isString.satisfies { - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + it) - assertThat(file.exists()).isTrue() + fileStorage.fileExists("screenshots/" + it).assert.isTrue() } } @@ -151,7 +150,6 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { val key = keyService.create(project, CreateKeyDto("test")) screenshotService.store(screenshotFile, key, null) } - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshot.filename) val result = performAuthGet("/screenshots/${screenshot.filename}").andIsOk .andExpect( @@ -159,7 +157,7 @@ class KeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() { ) .andReturn() performAuthGet("/screenshots/${screenshot.thumbnailFilename}").andIsOk - assertThat(result.response.contentAsByteArray).isEqualTo(file.readBytes()) + assertThat(result.response.contentAsByteArray).isEqualTo(fileStorage.readFile("screenshots/" + screenshot.filename)) } @Test diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt index a7ec67fb54..3c055744f1 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt @@ -17,7 +17,6 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance import org.springframework.boot.test.context.SpringBootTest -import java.io.File import java.time.Duration import java.util.* @@ -103,9 +102,8 @@ class SecuredKeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() executeInNewTransaction { val screenshots = screenshotService.findAll(key = key) assertThat(screenshots).hasSize(1) - val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshots[0].filename) - assertThat(file).exists() - assertThat(file.readBytes().size).isCloseTo(1070, Offset.offset(500)) + val bytes = fileStorage.readFile("/screenshots/" + screenshots[0].filename) + assertThat(bytes.size).isCloseTo(1070, Offset.offset(500)) node("filename").isString.startsWith(screenshots[0].filename).satisfies { val parts = it.split("?token=") val auth = jwtService.validateTicket(parts[1], JwtService.TicketType.IMG_ACCESS) diff --git a/backend/app/src/test/kotlin/io/tolgee/initialUserCreation/CreateEnabledTest.kt b/backend/app/src/test/kotlin/io/tolgee/initialUserCreation/CreateEnabledTest.kt index a32380d481..1fd712d98a 100644 --- a/backend/app/src/test/kotlin/io/tolgee/initialUserCreation/CreateEnabledTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/initialUserCreation/CreateEnabledTest.kt @@ -7,6 +7,7 @@ package io.tolgee.initialUserCreation import io.tolgee.Application import io.tolgee.CleanDbBeforeClass import io.tolgee.commandLineRunners.InitialUserCreatorCommandLineRunner +import io.tolgee.component.fileStorage.FileStorage import io.tolgee.configuration.tolgee.TolgeeProperties import io.tolgee.repository.UserAccountRepository import io.tolgee.security.InitialPasswordManager @@ -14,6 +15,7 @@ import io.tolgee.service.security.UserAccountService import io.tolgee.testing.AbstractTransactionalTest import io.tolgee.testing.ContextRecreatingTest import io.tolgee.testing.assertions.Assertions.assertThat +import io.tolgee.util.InMemoryFileStorage import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -22,13 +24,11 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.security.crypto.password.PasswordEncoder import org.springframework.transaction.annotation.Transactional -import java.io.File @ContextRecreatingTest @SpringBootTest( classes = [Application::class], properties = [ - "tolgee.file-storage.fs-data-path=./build/create-enabled-test-data/", "tolgee.authentication.initial-username=johny", "tolgee.internal.disable-initial-user-creation=false", ], @@ -52,7 +52,8 @@ class CreateEnabledTest : AbstractTransactionalTest() { @set:Autowired lateinit var initialPasswordManager: InitialPasswordManager - private val passwordFile = File("./build/create-enabled-test-data/initial.pwd") + @set:Autowired + lateinit var fileStorage: FileStorage @Autowired lateinit var initialUserCreatorCommandLineRunner: InitialUserCreatorCommandLineRunner @@ -64,14 +65,13 @@ class CreateEnabledTest : AbstractTransactionalTest() { @Test fun storesPassword() { - assertThat(passwordFile).exists() - assertThat(passwordFile.readText()).isNotBlank + assertThat(getPasswordFileContents().toString(Charsets.UTF_8)).isNotBlank } @Test fun passwordStoredInDb() { val johny = userAccountService.findActive("johny") - assertThat(passwordEncoder.matches(passwordFile.readText(), johny!!.password)).isTrue + assertThat(passwordEncoder.matches(getPasswordFileContents().toString(Charsets.UTF_8), johny!!.password)).isTrue } @Test @@ -107,7 +107,7 @@ class CreateEnabledTest : AbstractTransactionalTest() { @AfterAll fun cleanUp() { - passwordFile.delete() + (fileStorage as InMemoryFileStorage).clear() resetInitialPassword() val initial = userAccountService.findActive("johny")!! @@ -121,4 +121,12 @@ class CreateEnabledTest : AbstractTransactionalTest() { set(initialPasswordManager, null) } } + + private fun getPasswordFileContents(): ByteArray { + return fileStorage.readFile(FILE_NAME) + } + + companion object { + const val FILE_NAME = "initial.pwd" + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt index 06de541297..4dea981b31 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt @@ -42,8 +42,10 @@ class ImportProperties { ) var createImplicitApiKey: Boolean = false - @DocProperty(description = "The language tag of the base language of the imported projects " + - "(for importing data on startup).") + @DocProperty( + description = "The language tag of the base language of the imported " + + "project (for importing data on startup)." + ) var baseLanguageTag: String = "en" @DocProperty( From 266f3d99c8839bf7c3d42fb208d85211a1f88041 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Wed, 27 Dec 2023 09:51:04 +0100 Subject: [PATCH 07/13] chore: Fix tests --- .../SecuredV2ImageUploadControllerTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt index 8952df215f..d27a75e50b 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/SecuredV2ImageUploadControllerTest.kt @@ -77,7 +77,7 @@ class SecuredV2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() fun upload() { performStoreImage().andPrettyPrint.andIsCreated.andAssertThatJson { node("filename").isString.satisfies { - val path = "uploadedImages/" + it + ".png"; + val path = "uploadedImages/" + it + ".png" assertThat(fileStorage.readFile(path).size).isCloseTo(5538, Offset.offset(500)) } node("requestFilename").isString.satisfies { From 2dc87eccd4c7f96e022e00010dda8f27b3c6071f Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Wed, 27 Dec 2023 11:12:54 +0100 Subject: [PATCH 08/13] chore: Fix tests --- .../SecuredKeyScreenshotControllerTest.kt | 2 +- .../e2e/import/importAddingFiles.cy.ts | 24 ++++--------------- e2e/cypress/e2e/import/importErrors.cy.ts | 3 +-- e2e/cypress/support/dataCyType.d.ts | 1 + .../component/ImportProgressOverlay.tsx | 1 + 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt index 3c055744f1..77284d42fc 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt @@ -102,7 +102,7 @@ class SecuredKeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() executeInNewTransaction { val screenshots = screenshotService.findAll(key = key) assertThat(screenshots).hasSize(1) - val bytes = fileStorage.readFile("/screenshots/" + screenshots[0].filename) + val bytes = fileStorage.readFile("screenshots/" + screenshots[0].filename) assertThat(bytes.size).isCloseTo(1070, Offset.offset(500)) node("filename").isString.startsWith(screenshots[0].filename).satisfies { val parts = it.split("?token=") diff --git a/e2e/cypress/e2e/import/importAddingFiles.cy.ts b/e2e/cypress/e2e/import/importAddingFiles.cy.ts index b0d4ad07d3..e12a3f44ad 100644 --- a/e2e/cypress/e2e/import/importAddingFiles.cy.ts +++ b/e2e/cypress/e2e/import/importAddingFiles.cy.ts @@ -115,19 +115,11 @@ describe('Import Adding files', () => { cy.get('[data-cy=dropzone]').trigger('dragenter', { dataTransfer: dt, }); - cy.get('[data-cy=dropzone-inner]').should( - 'have.css', - 'background-color', - 'rgb(232, 245, 233)' - ); + cy.get('[data-cy=dropzone-inner]').should('have.css', 'opacity', '1'); cy.get('[data-cy=dropzone]').trigger('dragleave', { dataTransfer: dt, }); - cy.get('[data-cy=dropzone-inner]').should( - 'have.css', - 'background-color', - 'rgba(0, 0, 0, 0)' - ); + cy.get('[data-cy=dropzone-inner]').should('have.css', 'opacity', '0'); }); }); @@ -144,19 +136,11 @@ describe('Import Adding files', () => { cy.get('[data-cy=dropzone]').trigger('dragenter', { dataTransfer: dt, }); - cy.get('[data-cy=dropzone-inner]').should( - 'have.css', - 'background-color', - 'rgb(255, 235, 238)' - ); + cy.get('[data-cy=dropzone-inner]').should('have.css', 'opacity', '1'); cy.get('[data-cy=dropzone]').trigger('dragleave', { dataTransfer: dt, }); - cy.get('[data-cy=dropzone-inner]').should( - 'have.css', - 'background-color', - 'rgba(0, 0, 0, 0)' - ); + cy.get('[data-cy=dropzone-inner]').should('have.css', 'opacity', '0'); }); }); diff --git a/e2e/cypress/e2e/import/importErrors.cy.ts b/e2e/cypress/e2e/import/importErrors.cy.ts index 36fa63b575..46c7ef110d 100644 --- a/e2e/cypress/e2e/import/importErrors.cy.ts +++ b/e2e/cypress/e2e/import/importErrors.cy.ts @@ -1,7 +1,6 @@ import 'cypress-file-upload'; import { assertMessage, gcy } from '../../common/shared'; import { visitImport } from '../../common/import'; -import { expectGlobalLoading } from '../../common/loading'; import { importTestData } from '../../common/apiCalls/testData/testData'; import { login } from '../../common/apiCalls/common'; @@ -67,7 +66,7 @@ describe('Import errors', () => { }); it('error shows more and less', { retries: { runMode: 3 } }, () => { - expectGlobalLoading(); + gcy('import-progress-overlay').should('be.visible'); gcy('import-file-error') .findDcy('import-file-error-more-less-button') .click(); diff --git a/e2e/cypress/support/dataCyType.d.ts b/e2e/cypress/support/dataCyType.d.ts index 887f0dc3fd..929944e6c4 100644 --- a/e2e/cypress/support/dataCyType.d.ts +++ b/e2e/cypress/support/dataCyType.d.ts @@ -185,6 +185,7 @@ declare namespace DataCy { "import-file-issues-button" | "import-file-issues-dialog" | "import-progress" | + "import-progress-overlay" | "import-resolution-dialog-accept-imported-button" | "import-resolution-dialog-accept-old-button" | "import-resolution-dialog-close-button" | diff --git a/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx b/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx index d18d220e9d..9fb6cdd55c 100644 --- a/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx +++ b/webapp/src/views/projects/import/component/ImportProgressOverlay.tsx @@ -94,6 +94,7 @@ export const ImportProgressOverlay = (props: { sx={{ pointerEvents: props.importDone ? 'all' : 'none', }} + data-cy="import-progress-overlay" > From bd0d93aef53a0402f24407e5a441af18eb6fc3c9 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Wed, 27 Dec 2023 11:13:37 +0100 Subject: [PATCH 09/13] chore: Fix eslint --- webapp/src/views/projects/import/component/ImportProgress.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/webapp/src/views/projects/import/component/ImportProgress.tsx b/webapp/src/views/projects/import/component/ImportProgress.tsx index 403f069e16..5dff5b67a3 100644 --- a/webapp/src/views/projects/import/component/ImportProgress.tsx +++ b/webapp/src/views/projects/import/component/ImportProgress.tsx @@ -53,8 +53,6 @@ export const ImportProgressBar = (props: { finish: props.loaded, }); - console.table({ classes, ...props }); - return ( Date: Wed, 27 Dec 2023 14:40:49 +0100 Subject: [PATCH 10/13] chore: rm .DS_Store --- webapp/src/svgs/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 webapp/src/svgs/.DS_Store diff --git a/webapp/src/svgs/.DS_Store b/webapp/src/svgs/.DS_Store deleted file mode 100644 index 81dfd3a2dbaef8dcd9d5a9b14c4e0de44887cb08..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z>*NO({YSiXH=ATTG#%;3d@h0!H+pQWH}&7li-7Q|vQQwec2fsotlSVS@(%DISAmFsJV)wbGWdvCGmxm~B{F1ya6KNx_1 za(uL0wyoWR!_$lLbMhkOo92-N;Yzj*mhcYBNZhypskB@i8hp20#RctE&L1=OkB zJTbUV2fJh9JcETsozA#g8Rju7myZ{&RtLMI!Ws88QcnyJ1M3Xb_0Y!i{~Ugq%18cs z3XO;XV&I=Kz#AifG=!qe+4`+KJZl|j_s~!3~(RWS5EDBP=`3rV4)Fb TLAy!^q>F$eggRp27Z~^gbpJ}G From dd98925ab0fa6b7ea626e76838eecfa54d3c86ce Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sun, 31 Dec 2023 09:50:21 +0100 Subject: [PATCH 11/13] fix: Rebase issues --- .../service/dataImport/processors/PropertyFileProcessor.kt | 4 ++-- .../dataImport/processors/processors/PropertiesParserTest.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/PropertyFileProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/PropertyFileProcessor.kt index 3d0ec65118..cf5b0e7803 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/PropertyFileProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/PropertyFileProcessor.kt @@ -1,13 +1,13 @@ package io.tolgee.service.dataImport.processors -import java.util.Properties +import java.util.* class PropertyFileProcessor( override val context: FileProcessorContext, ) : ImportFileProcessor() { override fun process() { val props = Properties() - props.load(context.file.inputStream) + props.load(context.file.data.inputStream()) props.entries.forEachIndexed { idx, it -> context.addTranslation(it.key.toString(), languageNameGuesses[0], it.value, idx) } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/PropertiesParserTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/PropertiesParserTest.kt index ffe2047d90..bb8a28738c 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/PropertiesParserTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/PropertiesParserTest.kt @@ -25,7 +25,7 @@ class PropertiesParserTest { ImportFileDto( "messages_en.properties", File("src/test/resources/import/example.properties") - .inputStream(), + .readBytes(), ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } From 6c5a57f7ab5f9ba831b2d4b88b6b9de134e6a630 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sun, 31 Dec 2023 09:50:48 +0100 Subject: [PATCH 12/13] fix: ktLint --- .../api/v2/controllers/V2ImportController.kt | 2 +- .../V2ImageUploadControllerTest.kt | 2 +- .../V2ImportControllerAddFilesTest.kt | 5 ++- .../fileStorage/FileStorageFsTest.kt | 2 +- .../fileStorage/FileStorageS3Test.kt | 2 +- .../dataImport/StoredDataImporterTest.kt | 42 ++++++++++--------- .../configuration/tolgee/ImportProperties.kt | 12 +++--- .../dtos/dataImport/ImportAddFilesParams.kt | 3 +- .../io/tolgee/service/StartupImportService.kt | 2 +- .../service/dataImport/ImportService.kt | 16 ++++--- .../dataImport/processors/ZipTypeProcessor.kt | 2 +- .../util/StreamingResponseBodyProvider.kt | 13 +++--- .../processors/po/PoFileProcessorTest.kt | 9 ++-- .../xliff/Xliff12FileProcessorTest.kt | 7 ++-- 14 files changed, 68 insertions(+), 51 deletions(-) diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt index b0302cd38f..21b4fe13a9 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt @@ -92,7 +92,7 @@ class V2ImportController( private val namespaceService: NamespaceService, private val importFileIssueModelAssembler: ImportFileIssueModelAssembler, private val streamingResponseBodyProvider: StreamingResponseBodyProvider, - private val objectMapper: ObjectMapper + private val objectMapper: ObjectMapper, ) { @PostMapping("", consumes = [MediaType.MULTIPART_FORM_DATA_VALUE]) @Operation(description = "Prepares provided files to import.", summary = "Add files") diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt index 40467799ad..a9905114c0 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImageUploadController/V2ImageUploadControllerTest.kt @@ -86,7 +86,7 @@ class V2ImageUploadControllerTest : AbstractV2ImageUploadControllerTest() { .andReturn() assertThat(result.response.contentAsByteArray) .isEqualTo( - fileStorage.readFile("uploadedImages/${image.filenameWithExtension}") + fileStorage.readFile("uploadedImages/${image.filenameWithExtension}"), ) } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt index dd337ca3af..dfef383d70 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerAddFilesTest.kt @@ -96,7 +96,10 @@ class V2ImportControllerAddFilesTest : AuthorizedControllerTest() { doesStoredFileExists(fileName, base.project.id).assert.isFalse() } - fun doesStoredFileExists(fileName: String, projectId: Long): Boolean { + fun doesStoredFileExists( + fileName: String, + projectId: Long, + ): Boolean { val import = importService.find(projectId, userAccount!!.id) return fileStorage.fileExists("importFiles/${import!!.id}/$fileName") } diff --git a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt index 380bc3a4b8..bee8fb3ded 100644 --- a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageFsTest.kt @@ -11,7 +11,7 @@ import org.springframework.boot.test.context.SpringBootTest import java.io.File @SpringBootTest( - properties = ["tolgee.internal.use-in-memory-file-storage=false"] + properties = ["tolgee.internal.use-in-memory-file-storage=false"], ) class FileStorageFsTest : AbstractFileStorageServiceTest() { lateinit var file: File diff --git a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt index ef65b15393..9d5aa43cd9 100644 --- a/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt +++ b/backend/app/src/test/kotlin/io/tolgee/component/fileStorage/FileStorageS3Test.kt @@ -29,7 +29,7 @@ import software.amazon.awssdk.services.s3.model.S3Exception "tolgee.file-storage.s3.signing-region=dummy_signing_region", "tolgee.file-storage.s3.bucket-name=$BUCKET_NAME", "tolgee.authentication.initial-password=hey password manager, please don't use the filesystem :3", - "tolgee.internal.use-in-memory-file-storage=false" + "tolgee.internal.use-in-memory-file-storage=false", ], ) @TestInstance(TestInstance.Lifecycle.PER_CLASS) diff --git a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt index 926f8e00d4..b1301113c8 100644 --- a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt @@ -23,10 +23,11 @@ class StoredDataImporterTest : AbstractSpringTest() { @BeforeEach fun setup() { importTestData = ImportTestData() - storedDataImporter = StoredDataImporter( - applicationContext, - importTestData.import, - ) + storedDataImporter = + StoredDataImporter( + applicationContext, + importTestData.import, + ) } fun login() { @@ -86,11 +87,12 @@ class StoredDataImporterTest : AbstractSpringTest() { @Test fun `it force replaces translations`() { - storedDataImporter = StoredDataImporter( - applicationContext, - importTestData.import, - ForceMode.OVERRIDE, - ) + storedDataImporter = + StoredDataImporter( + applicationContext, + importTestData.import, + ForceMode.OVERRIDE, + ) testDataService.saveTestData(importTestData.root) login() storedDataImporter.doImport() @@ -105,11 +107,12 @@ class StoredDataImporterTest : AbstractSpringTest() { fun `it imports metadata`() { importTestData.addKeyMetadata() testDataService.saveTestData(importTestData.root) - storedDataImporter = StoredDataImporter( - applicationContext, - importTestData.import, - ForceMode.OVERRIDE, - ) + storedDataImporter = + StoredDataImporter( + applicationContext, + importTestData.import, + ForceMode.OVERRIDE, + ) login() storedDataImporter.doImport() entityManager.flush() @@ -133,11 +136,12 @@ class StoredDataImporterTest : AbstractSpringTest() { fun `it force keeps translations`() { importTestData.translationWithConflict.override = true importTestData.translationWithConflict.resolve() - storedDataImporter = StoredDataImporter( - applicationContext, - importTestData.import, - ForceMode.KEEP, - ) + storedDataImporter = + StoredDataImporter( + applicationContext, + importTestData.import, + ForceMode.KEEP, + ) testDataService.saveTestData(importTestData.root) login() diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt index 4dea981b31..4d1639b682 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ImportProperties.kt @@ -7,7 +7,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties @DocProperty( description = "Properties for importing data to Tolgee and " + - "bulk-imports exported json files in the database during startup. " + + "bulk-imports exported json files in the database during startup. " + "Useful to quickly provision a development server, and used for testing.", displayName = "Import", ) @@ -43,14 +43,16 @@ class ImportProperties { var createImplicitApiKey: Boolean = false @DocProperty( - description = "The language tag of the base language of the imported " + - "project (for importing data on startup)." + description = + "The language tag of the base language of the imported " + + "project (for importing data on startup).", ) var baseLanguageTag: String = "en" @DocProperty( - description = "If true, uploaded files will be stored in configured file storage for future debugging. " + - "Such data is not automatically removed after successful import. You have to clean-up manually!" + description = + "If true, uploaded files will be stored in configured file storage for future debugging. " + + "Such data is not automatically removed after successful import. You have to clean-up manually!", ) var storeFilesForDebugging: Boolean = false } diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt index 7f758529cf..2785ee5c7e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/dataImport/ImportAddFilesParams.kt @@ -9,6 +9,5 @@ class ImportAddFilesParams( "the delimiter which will be used in names of improted keys.", ) var structureDelimiter: Char? = '.', - - var storeFilesToFileStorage: Boolean = true + var storeFilesToFileStorage: Boolean = true, ) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt index 1119b4244c..3f3cbdaeb1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/StartupImportService.kt @@ -109,7 +109,7 @@ class StartupImportService( files = fileDtos, project = project, userAccount = userAccount, - params = ImportAddFilesParams(storeFilesToFileStorage = false) + params = ImportAddFilesParams(storeFilesToFileStorage = false), ) entityManager.flush() entityManager.clear() diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index b8bcf787a0..06592a9e25 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -65,7 +65,7 @@ class ImportService( @Suppress("SelfReferenceConstructorParameter") @Lazy private val self: ImportService, private val fileStorage: FileStorage, - private val tolgeeProperties: TolgeeProperties + private val tolgeeProperties: TolgeeProperties, ) { @Transactional fun addFiles( @@ -109,7 +109,7 @@ class ImportService( projectId: Long, authorId: Long, forceMode: ForceMode = ForceMode.NO_FORCE, - reportStatus: (ImportApplicationStatus) -> Unit = {} + reportStatus: (ImportApplicationStatus) -> Unit = {}, ) { import(getNotExpired(projectId, authorId), forceMode, reportStatus) } @@ -118,7 +118,7 @@ class ImportService( fun import( import: Import, forceMode: ForceMode = ForceMode.NO_FORCE, - reportStatus: (ImportApplicationStatus) -> Unit = {} + reportStatus: (ImportApplicationStatus) -> Unit = {}, ) { Sentry.addBreadcrumb("Import ID: ${import.id}") StoredDataImporter(applicationContext, import, forceMode, reportStatus).doImport() @@ -395,7 +395,10 @@ class ImportService( * When import fails, we need the files for future debugging */ @Async - fun saveFilesToFileStorage(importId: Long, files: List) { + fun saveFilesToFileStorage( + importId: Long, + files: List, + ) { if (tolgeeProperties.import.storeFilesForDebugging) { files.forEach { fileStorage.storeFile(getFileStoragePath(importId, it.name), it.data) @@ -403,7 +406,10 @@ class ImportService( } } - fun getFileStoragePath(importId: Long, fileName: String): String { + fun getFileStoragePath( + importId: Long, + fileName: String, + ): String { val notBlankFilename = fileName.ifBlank { "blank_name" } return "${getFileStorageImportRoot(importId)}/$notBlankFilename" } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt index f9a5f1fa17..20f50db1c4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/ZipTypeProcessor.kt @@ -30,7 +30,7 @@ class ZipTypeProcessor : ImportArchiveProcessor { files[fileName] = ImportFileDto( name = fileName, - data = data + data = data, ) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt index b08d035e6d..87de639ea5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/StreamingResponseBodyProvider.kt @@ -66,13 +66,13 @@ class StreamingResponseBodyProvider( } } } - } + }, ) } fun OutputStreamWriter.writeJson(message: Any?) { this.write( - (objectMapper.writeValueAsString(message) + "\n") + (objectMapper.writeValueAsString(message) + "\n"), ) this.flush() } @@ -81,10 +81,11 @@ class StreamingResponseBodyProvider( when (e) { is NotFoundException -> ErrorResponseBody(e.msg.code, null) is ErrorException -> e.errorResponseBody - else -> ErrorResponseBody( - "unexpected_error_occurred", - listOf(e::class.java.name) - ) + else -> + ErrorResponseBody( + "unexpected_error_occurred", + listOf(e::class.java.name), + ) } data class StreamedErrorMessage(val error: ErrorResponseBody) diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt index d233489be4..613add3ca5 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/po/PoFileProcessorTest.kt @@ -80,10 +80,11 @@ class PoFileProcessorTest { private fun mockImportFile(inputStream: InputStream) { importMock = mock() importFile = ImportFile("exmample.po", importMock) - importFileDto = ImportFileDto( - "exmample.po", - inputStream.readAllBytes() - ) + importFileDto = + ImportFileDto( + "exmample.po", + inputStream.readAllBytes(), + ) fileProcessorContext = FileProcessorContext(importFileDto, importFile) } } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt index 028dc80fb6..a88cf69572 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/service/dataImport/processors/processors/xliff/Xliff12FileProcessorTest.kt @@ -88,9 +88,10 @@ class Xliff12FileProcessorTest { @Test fun `handles errors correctly`() { - importFileDto = ImportFileDto( - "exmample.xliff", - File("src/test/resources/import/xliff/error_example.xliff").readBytes(), + importFileDto = + ImportFileDto( + "exmample.xliff", + File("src/test/resources/import/xliff/error_example.xliff").readBytes(), ) xmlStreamReader = inputFactory.createXMLEventReader(importFileDto.data.inputStream()) fileProcessorContext = FileProcessorContext(importFileDto, importFile) From 3f5190c322d9c09d414c2763157befaf665774bd Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sun, 31 Dec 2023 09:55:37 +0100 Subject: [PATCH 13/13] fix: Scroll: Auto --- webapp/src/views/projects/import/ImportView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/views/projects/import/ImportView.tsx b/webapp/src/views/projects/import/ImportView.tsx index 3a29f5baf1..bc7c8c2362 100644 --- a/webapp/src/views/projects/import/ImportView.tsx +++ b/webapp/src/views/projects/import/ImportView.tsx @@ -88,7 +88,7 @@ export const ImportView: FunctionComponent = () => { ], ]} maxWidth="wide" - overflow="scroll" + overflow="auto" >