From 5525cb32f2dc572af9f48da01e185a94c84e8dd6 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 20:03:30 +0100 Subject: [PATCH] fix: Import performance (#2025) Co-authored-by: Geert Zondervan --- .github/workflows/test.yml | 9 ++ .../TranslationCommentController.kt | 6 +- .../translation/TranslationsController.kt | 17 ++- .../comments/TranslationCommentModel.kt | 4 +- .../TranslationCommentModelAssembler.kt | 6 +- .../app/src/main/resources/application.yaml | 2 - .../controllers/V2LanguageControllerTest.kt | 29 +++-- .../SecuredKeyScreenshotControllerTest.kt | 6 +- .../service/dataImport/ImportServiceTest.kt | 38 +++++- backend/data/build.gradle | 2 +- .../io/tolgee/activity/ActivityHolder.kt | 21 +++ .../io/tolgee/activity/ActivityService.kt | 86 ++++++++---- .../iterceptor/ActivityDatabaseInterceptor.kt | 11 +- .../iterceptor/InterceptedEventsManager.kt | 24 ++-- .../TagsPropChangesProvider.kt | 4 + .../io/tolgee/batch/BatchJobActionService.kt | 4 - .../batch/BatchJobChunkExecutionQueue.kt | 22 +++- .../kotlin/io/tolgee/batch/BatchJobService.kt | 60 ++++++++- .../kotlin/io/tolgee/batch/ProgressManager.kt | 11 +- .../batch/data/BatchJobChunkExecutionDto.kt | 17 +++ .../component/ActivityHolderProvider.kt | 65 +++++++++ .../AutoTranslationEventHandler.kt} | 100 ++++++++------ .../AutoTranslationListener.kt | 20 +++ .../configuration/ActivityHolderConfig.kt | 18 +-- .../tolgee/configuration/HibernateSession.kt | 16 --- .../io/tolgee/events/OnImportSoftDeleted.kt | 5 + .../io/tolgee/model/StandardAuditModel.kt | 7 +- .../activity/ActivityDescribingEntity.kt | 5 - .../model/activity/ActivityModifiedEntity.kt | 2 +- .../io/tolgee/model/dataImport/Import.kt | 6 +- .../AutoTranslationConfigRepository.kt | 1 + .../repository/TranslationRepository.kt | 7 + .../repository/dataImport/ImportRepository.kt | 18 +++ .../TranslationCommentRepository.kt | 11 +- .../dataImport/AsyncImportHardDeleter.kt | 19 +++ .../service/dataImport/ImportDeleteService.kt | 123 ++++++++++++++++++ .../service/dataImport/ImportService.kt | 30 +++-- .../service/dataImport/StoredDataImporter.kt | 6 + .../io/tolgee/service/key/KeyMetaService.kt | 7 - .../tolgee/service/project/ProjectService.kt | 2 +- .../translation/AutoTranslationService.kt | 19 ++- .../translation/TranslationCommentService.kt | 12 ++ .../service/translation/TranslationService.kt | 4 + .../io/tolgee/util/SequenceIdProvider.kt | 34 +++++ .../kotlin/io/tolgee/util/entityManagerExt.kt | 12 ++ .../main/resources/db/changelog/schema.xml | 9 ++ gradle.properties | 2 +- 47 files changed, 741 insertions(+), 198 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt rename backend/data/src/main/kotlin/io/tolgee/component/{AutoTranslationListener.kt => autoTranslation/AutoTranslationEventHandler.kt} (62%) create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt delete mode 100644 backend/data/src/main/kotlin/io/tolgee/configuration/HibernateSession.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/events/OnImportSoftDeleted.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/dataImport/AsyncImportHardDeleter.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDeleteService.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 072a931f3b..bd10db573f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -139,6 +139,15 @@ jobs: path: | ./**/build/reports/**/* + - name: Test Report + uses: dorny/test-reporter@v1 + if: always() + with: + name: Backend Tests + path: "**/build/test-results/**/TEST-*.xml" + reporter: java-junit + + e2e: needs: [frontend-build, backend-build, e2e-install-deps] runs-on: ubuntu-latest diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt index 95ee2aff3d..86857432d4 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt @@ -95,7 +95,7 @@ class TranslationCommentController( @UseDefaultPermissions @AllowApiAccess fun get(@PathVariable translationId: Long, @PathVariable commentId: Long): TranslationCommentModel { - val comment = translationCommentService.get(commentId) + val comment = translationCommentService.getWithAuthorFetched(commentId) comment.checkFromProject() return translationCommentModelAssembler.toModel(comment) } @@ -106,7 +106,7 @@ class TranslationCommentController( @UseDefaultPermissions // Security: Permission check done inside; users should be able to edit their comments @AllowApiAccess fun update(@PathVariable commentId: Long, @RequestBody @Valid dto: TranslationCommentDto): TranslationCommentModel { - val comment = translationCommentService.get(commentId) + val comment = translationCommentService.getWithAuthorFetched(commentId) if (comment.author.id != authenticationFacade.authenticatedUser.id) { throw BadRequestException(io.tolgee.constants.Message.CAN_EDIT_ONLY_OWN_COMMENT) } @@ -123,7 +123,7 @@ class TranslationCommentController( @PathVariable commentId: Long, @PathVariable state: TranslationCommentState ): TranslationCommentModel { - val comment = translationCommentService.get(commentId) + val comment = translationCommentService.getWithAuthorFetched(commentId) comment.checkFromProject() translationCommentService.setState(comment, state) return translationCommentModelAssembler.toModel(comment) diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt index 783a3c74cf..50b99a9997 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt @@ -172,7 +172,7 @@ When null, resulting file will be a flat key-value object. @PutMapping("") @Operation(summary = "Sets translations for existing key") @RequestActivity(ActivityType.SET_TRANSLATIONS) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun setTranslations(@RequestBody @Valid dto: SetTranslationsWithKeyDto): SetTranslationsResponseModel { val key = keyService.get(projectHolder.project.id, dto.key, dto.namespace) @@ -182,8 +182,7 @@ When null, resulting file will be a flat key-value object. val translations = dto.languagesToReturn ?.let { languagesToReturn -> - key.translations - .filter { languagesToReturn.contains(it.language.tag) } + translationService.findForKeyByLanguages(key, languagesToReturn) .associateBy { it.language.tag } } ?: modifiedTranslations @@ -194,7 +193,7 @@ When null, resulting file will be a flat key-value object. @PostMapping("") @Operation(summary = "Sets translations for existing or not existing key.") @RequestActivity(ActivityType.SET_TRANSLATIONS) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun createOrUpdateTranslations(@RequestBody @Valid dto: SetTranslationsWithKeyDto): SetTranslationsResponseModel { val key = keyService.find(projectHolder.projectEntity.id, dto.key, dto.namespace)?.also { @@ -211,7 +210,7 @@ When null, resulting file will be a flat key-value object. @PutMapping("/{translationId}/set-state/{state}") @Operation(summary = "Sets translation state") @RequestActivity(ActivityType.SET_TRANSLATION_STATE) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_STATE_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_STATE_EDIT]) @AllowApiAccess fun setTranslationState( @PathVariable translationId: Long, @@ -266,7 +265,7 @@ When null, resulting file will be a flat key-value object. @GetMapping(value = ["select-all"]) @Operation(summary = "Get select all keys") - @RequiresProjectPermissions([ Scope.KEYS_VIEW ]) + @RequiresProjectPermissions([Scope.KEYS_VIEW]) @AllowApiAccess fun getSelectAllKeyIds( @ParameterObject @ModelAttribute("translationFilters") params: TranslationFilters, @@ -289,7 +288,7 @@ When null, resulting file will be a flat key-value object. @PutMapping(value = ["/{translationId:[0-9]+}/dismiss-auto-translated-state"]) @Operation(summary = """Removes "auto translated" indication""") @RequestActivity(ActivityType.DISMISS_AUTO_TRANSLATED_STATE) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_STATE_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_STATE_EDIT]) @AllowApiAccess fun dismissAutoTranslatedState( @PathVariable translationId: Long @@ -304,7 +303,7 @@ When null, resulting file will be a flat key-value object. @PutMapping(value = ["/{translationId:[0-9]+}/set-outdated-flag/{state}"]) @Operation(summary = """Set's "outdated" indication""") @RequestActivity(ActivityType.SET_OUTDATED_FLAG) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_STATE_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_STATE_EDIT]) @AllowApiAccess fun setOutdated( @PathVariable translationId: Long, @@ -322,7 +321,7 @@ When null, resulting file will be a flat key-value object. Sorting is not supported for supported. It is automatically sorted from newest to oldest.""" ) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_VIEW ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) @AllowApiAccess fun getTranslationHistory( @PathVariable translationId: Long, diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt index 48c57bc389..dfe6d0e11d 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt @@ -1,7 +1,7 @@ package io.tolgee.hateoas.translations.comments import io.swagger.v3.oas.annotations.media.Schema -import io.tolgee.hateoas.user_account.UserAccountModel +import io.tolgee.hateoas.user_account.SimpleUserAccountModel import io.tolgee.model.enums.TranslationCommentState import org.springframework.hateoas.RepresentationModel import org.springframework.hateoas.server.core.Relation @@ -20,7 +20,7 @@ open class TranslationCommentModel( val state: TranslationCommentState, @Schema(description = "User who created the comment") - val author: UserAccountModel, + val author: SimpleUserAccountModel, @Schema(description = "Date when it was created") val createdAt: Date, diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt index 94d63b9136..ae623a6dcb 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt @@ -1,7 +1,7 @@ package io.tolgee.hateoas.translations.comments import io.tolgee.api.v2.controllers.translation.TranslationCommentController -import io.tolgee.hateoas.user_account.UserAccountModelAssembler +import io.tolgee.hateoas.user_account.SimpleUserAccountModelAssembler import io.tolgee.model.translation.TranslationComment import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport import org.springframework.stereotype.Component @@ -9,7 +9,7 @@ import java.util.* @Component class TranslationCommentModelAssembler( - private val userAccountModelAssembler: UserAccountModelAssembler + private val simpleUserAccountModelAssembler: SimpleUserAccountModelAssembler ) : RepresentationModelAssemblerSupport( TranslationCommentController::class.java, TranslationCommentModel::class.java ) { @@ -18,7 +18,7 @@ class TranslationCommentModelAssembler( id = entity.id, text = entity.text, state = entity.state, - author = entity.author.let { userAccountModelAssembler.toModel(it) }, + author = entity.author.let { simpleUserAccountModelAssembler.toModel(it) }, createdAt = entity.createdAt ?: Date(), updatedAt = entity.updatedAt ?: Date() ) diff --git a/backend/app/src/main/resources/application.yaml b/backend/app/src/main/resources/application.yaml index 280c38e0ba..6e2b4d1c74 100644 --- a/backend/app/src/main/resources/application.yaml +++ b/backend/app/src/main/resources/application.yaml @@ -9,7 +9,6 @@ spring: pathmatch: matching-strategy: ant_path_matcher jpa: -# open-in-view: false properties: hibernate: order_by: @@ -25,7 +24,6 @@ spring: enhancer: enableLazyInitialization: true enableDirtyTracking: true - # open-in-view: false batch: job: enabled: false diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2LanguageControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2LanguageControllerTest.kt index 1ef17f93ed..033ef25669 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2LanguageControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2LanguageControllerTest.kt @@ -69,11 +69,11 @@ class V2LanguageControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Test fun deleteLanguage() { + val base = dbPopulator.createBase(generateUniqueString()) + val project = base.project + val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } + performDelete(project.id, deutsch.id).andExpect(MockMvcResultMatchers.status().isOk) executeInNewTransaction { - val base = dbPopulator.createBase(generateUniqueString()) - val project = base.project - val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } - performDelete(project.id, deutsch.id).andExpect(MockMvcResultMatchers.status().isOk) Assertions.assertThat(languageService.findById(deutsch.id)).isEmpty } } @@ -81,13 +81,14 @@ class V2LanguageControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Test @ProjectApiKeyAuthTestMethod(scopes = [Scope.LANGUAGES_EDIT]) fun `deletes language with API key`() { + val base = dbPopulator.createBase(generateUniqueString()) + this.userAccount = base.userAccount + this.projectSupplier = { base.project } + val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } + + performProjectAuthDelete("languages/${deutsch.id}", null) + .andExpect(MockMvcResultMatchers.status().isOk) executeInNewTransaction { - val base = dbPopulator.createBase(generateUniqueString()) - this.userAccount = base.userAccount - this.projectSupplier = { base.project } - val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } - performProjectAuthDelete("languages/${deutsch.id}", null) - .andExpect(MockMvcResultMatchers.status().isOk) Assertions.assertThat(languageService.findById(deutsch.id)).isEmpty } } @@ -95,11 +96,11 @@ class V2LanguageControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Test @ProjectApiKeyAuthTestMethod(scopes = [Scope.TRANSLATIONS_VIEW]) fun `does not delete language with API key (permissions)`() { + val base = dbPopulator.createBase(generateUniqueString()) + this.userAccount = base.userAccount + this.projectSupplier = { base.project } + val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } executeInNewTransaction { - val base = dbPopulator.createBase(generateUniqueString()) - this.userAccount = base.userAccount - this.projectSupplier = { base.project } - val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } performProjectAuthDelete("languages/${deutsch.id}", null).andIsForbidden } } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt index 25fa0bf65e..17551e5c92 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt @@ -95,10 +95,10 @@ class SecuredKeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() @Test @ProjectJWTAuthTestMethod fun uploadScreenshot() { - executeInNewTransaction { - val key = keyService.create(project, CreateKeyDto("test")) + val key = keyService.create(project, CreateKeyDto("test")) - performStoreScreenshot(project, key).andIsCreated.andAssertThatJson { + performStoreScreenshot(project, key).andIsCreated.andAssertThatJson { + executeInNewTransaction { val screenshots = screenshotService.findAll(key = key) assertThat(screenshots).hasSize(1) val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshots[0].filename) diff --git a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/ImportServiceTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/ImportServiceTest.kt index 67a86a9233..8fd91800cc 100644 --- a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/ImportServiceTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/ImportServiceTest.kt @@ -4,6 +4,9 @@ import io.tolgee.AbstractSpringTest import io.tolgee.development.testDataBuilder.data.dataImport.ImportNamespacesTestData import io.tolgee.development.testDataBuilder.data.dataImport.ImportTestData import io.tolgee.dtos.cacheable.UserAccountDto +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.dataImport.Import +import io.tolgee.model.dataImport.ImportTranslation import io.tolgee.security.authentication.TolgeeAuthentication import io.tolgee.security.authentication.TolgeeAuthenticationDetails import io.tolgee.testing.assert @@ -37,11 +40,13 @@ class ImportServiceTest : AbstractSpringTest() { importService.selectExistingLanguage(importFrench, french) assertThat(importFrench.existingLanguage).isEqualTo(french) val translations = importService.findTranslations(importTestData.importFrench.id) - assertThat(translations[0].conflict).isNotNull - assertThat(translations[1].conflict).isNull() + assertThat(translations.getByKey("what a key").conflict).isNotNull + assertThat(translations.getByKey("what a beautiful key").conflict).isNull() } } + fun Collection.getByKey(key: String) = this.find { it.key.name == key } ?: error("Key not found") + @Test fun `deletes import language`() { val testData = executeInNewTransaction { @@ -63,7 +68,30 @@ class ImportServiceTest : AbstractSpringTest() { } @Test - fun `deletes import`() { + fun `hard deletes import`() { + val testData = ImportTestData() + executeInNewTransaction { + testData.addFileIssues() + testData.addKeyMetadata() + testDataService.saveTestData(testData.root) + } + + executeInNewTransaction { + val import = importService.get(testData.import.id) + importService.hardDeleteImport(import) + } + } + + private fun checkImportHardDeleted(id: Long) { + executeInNewTransaction { + entityManager.createQuery("from Import i where i.id = :id", Import::class.java) + .setParameter("id", id) + .resultList.firstOrNull().assert.isNull() + } + } + + @Test + fun `soft deletes import`() { val testData = ImportTestData() executeInNewTransaction { testData.addFileIssues() @@ -79,6 +107,10 @@ class ImportServiceTest : AbstractSpringTest() { executeInNewTransaction { assertThat(importService.find(testData.import.project.id, testData.import.author.id)).isNull() } + + waitForNotThrowing(pollTime = 200, timeout = 10000) { + checkImportHardDeleted(testData.import.id) + } } @Test diff --git a/backend/data/build.gradle b/backend/data/build.gradle index 93828b32a8..69151933cc 100644 --- a/backend/data/build.gradle +++ b/backend/data/build.gradle @@ -110,7 +110,7 @@ dependencies { /** * DB */ - runtimeOnly 'org.postgresql:postgresql' + implementation 'org.postgresql:postgresql' implementation "org.hibernate:hibernate-jpamodelgen:$hibernateVersion" kapt "org.hibernate:hibernate-jpamodelgen:$hibernateVersion" diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt index e5874a760a..5bdc95a252 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt @@ -3,8 +3,10 @@ package io.tolgee.activity import io.tolgee.activity.data.ActivityType import io.tolgee.activity.iterceptor.InterceptedEventsManager import io.tolgee.model.EntityWithId +import io.tolgee.model.activity.ActivityDescribingEntity import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.activity.ActivityRevision +import jakarta.annotation.PreDestroy import org.springframework.context.ApplicationContext import kotlin.reflect.KClass @@ -44,6 +46,25 @@ open class ActivityHolder(val applicationContext: ApplicationContext) { this.applicationContext.getBean(InterceptedEventsManager::class.java).initActivityHolder() field = value } + + var destroyer: (() -> Unit)? = null + + open val describingRelationCache: MutableMap, ActivityDescribingEntity> by lazy { + activityRevision.describingRelations.associateBy { it.entityId to it.entityClass }.toMutableMap() + } + + fun getDescribingRelationFromCache( + entityId: Long, + entityClass: String, + provider: () -> ActivityDescribingEntity + ): ActivityDescribingEntity { + return describingRelationCache.getOrPut(entityId to entityClass, provider) + } + + @PreDestroy + fun destroy() { + destroyer?.invoke() + } } typealias ModifiedEntitiesType = MutableMap, MutableMap> diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt index 1d1b85a040..e43ad59425 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt @@ -1,20 +1,25 @@ package io.tolgee.activity +import com.fasterxml.jackson.databind.ObjectMapper import io.tolgee.activity.data.ActivityType +import io.tolgee.activity.data.RevisionType import io.tolgee.activity.projectActivityView.ProjectActivityViewByPageableProvider import io.tolgee.activity.projectActivityView.ProjectActivityViewByRevisionProvider +import io.tolgee.component.CurrentDateProvider import io.tolgee.dtos.query_results.TranslationHistoryView import io.tolgee.events.OnProjectActivityStoredEvent +import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.activity.ActivityRevision import io.tolgee.model.views.activity.ProjectActivityView import io.tolgee.repository.activity.ActivityModifiedEntityRepository import io.tolgee.util.Logging -import io.tolgee.util.logger -import jakarta.persistence.EntityExistsException +import io.tolgee.util.flushAndClear import jakarta.persistence.EntityManager +import org.postgresql.util.PGobject import org.springframework.context.ApplicationContext import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable +import org.springframework.jdbc.core.JdbcTemplate import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional @@ -22,44 +27,75 @@ import org.springframework.transaction.annotation.Transactional class ActivityService( private val entityManager: EntityManager, private val applicationContext: ApplicationContext, - private val activityModifiedEntityRepository: ActivityModifiedEntityRepository + private val activityModifiedEntityRepository: ActivityModifiedEntityRepository, + private val currentDateProvider: CurrentDateProvider, + private val objectMapper: ObjectMapper, + private val jdbcTemplate: JdbcTemplate ) : Logging { @Transactional fun storeActivityData(activityRevision: ActivityRevision, modifiedEntities: ModifiedEntitiesType) { - val mergedActivityRevision = activityRevision.persist() - mergedActivityRevision.persistedDescribingRelations() + // let's keep the persistent context small + entityManager.flushAndClear() - mergedActivityRevision.modifiedEntities = modifiedEntities.values.flatMap { it.values }.toMutableList() - mergedActivityRevision.persistModifiedEntities() - - entityManager.flush() + val mergedActivityRevision = persistAcitivyRevision(activityRevision) + persistedDescribingRelations(mergedActivityRevision) + mergedActivityRevision.modifiedEntities = persistModifiedEntities(modifiedEntities) applicationContext.publishEvent(OnProjectActivityStoredEvent(this, mergedActivityRevision)) } - private fun ActivityRevision.persistModifiedEntities() { - modifiedEntities.forEach { activityModifiedEntity -> - try { - entityManager.persist(activityModifiedEntity) - } catch (e: EntityExistsException) { - logger.debug("ModifiedEntity entity already exists in persistence context, skipping", e) - } + private fun persistModifiedEntities(modifiedEntities: ModifiedEntitiesType): MutableList { + val list = modifiedEntities.values.flatMap { it.values }.toMutableList() + jdbcTemplate.batchUpdate( + "INSERT INTO activity_modified_entity " + + "(entity_class, entity_id, describing_data, " + + "describing_relations, modifications, revision_type, activity_revision_id) " + + "VALUES (?, ?, ?, ?, ?, ?, ?)", + list, + 100 + ) { ps, entity -> + ps.setString(1, entity.entityClass) + ps.setLong(2, entity.entityId) + ps.setObject(3, getJsonbObject(entity.describingData)) + ps.setObject(4, getJsonbObject(entity.describingRelations)) + ps.setObject(5, getJsonbObject(entity.modifications)) + ps.setInt(6, RevisionType.values().indexOf(entity.revisionType)) + ps.setLong(7, entity.activityRevision.id) } + + return list } - private fun ActivityRevision.persistedDescribingRelations() { - @Suppress("UselessCallOnCollection") - describingRelations.filterNotNull().forEach { - entityManager.persist(it) + private fun persistedDescribingRelations(activityRevision: ActivityRevision) { + jdbcTemplate.batchUpdate( + "INSERT INTO activity_describing_entity " + + "(entity_class, entity_id, data, describing_relations, activity_revision_id) " + + "VALUES (?, ?, ?, ?, ?)", + activityRevision.describingRelations, + 100 + ) { ps, entity -> + ps.setString(1, entity.entityClass) + ps.setLong(2, entity.entityId) + ps.setObject(3, getJsonbObject(entity.data)) + ps.setObject(4, getJsonbObject(entity.describingRelations)) + ps.setLong(5, activityRevision.id) } } - private fun ActivityRevision.persist(): ActivityRevision { - return if (id == 0L) { - entityManager.persist(this) - this + private fun getJsonbObject(data: Any?): PGobject { + val pgObject = PGobject() + pgObject.type = "jsonb" + pgObject.value = objectMapper.writeValueAsString(data) + return pgObject + } + + private fun persistAcitivyRevision(activityRevision: ActivityRevision): ActivityRevision { + return if (activityRevision.id == 0L) { + entityManager.persist(activityRevision) + entityManager.flushAndClear() + activityRevision } else { - entityManager.getReference(ActivityRevision::class.java, id) + entityManager.getReference(ActivityRevision::class.java, activityRevision.id) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityDatabaseInterceptor.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityDatabaseInterceptor.kt index 9921c7cb3e..a6df5ebe64 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityDatabaseInterceptor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityDatabaseInterceptor.kt @@ -77,9 +77,10 @@ class ActivityDatabaseInterceptor : Interceptor, Logging { interceptedEventsManager.onCollectionModification(collection, key) } - val interceptedEventsManager: InterceptedEventsManager - get() = applicationContext.getBean(InterceptedEventsManager::class.java) - - val preCommitEventsPublisher: PreCommitEventPublisher - get() = applicationContext.getBean(PreCommitEventPublisher::class.java) + val interceptedEventsManager: InterceptedEventsManager by lazy { + applicationContext.getBean(InterceptedEventsManager::class.java) + } + val preCommitEventsPublisher: PreCommitEventPublisher by lazy { + applicationContext.getBean(PreCommitEventPublisher::class.java) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt index fb2b17d610..74ec83fc3a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt @@ -1,6 +1,5 @@ package io.tolgee.activity.iterceptor -import io.tolgee.activity.ActivityHolder import io.tolgee.activity.ActivityService import io.tolgee.activity.EntityDescriptionProvider import io.tolgee.activity.annotation.ActivityLoggedEntity @@ -10,6 +9,7 @@ import io.tolgee.activity.data.EntityDescriptionWithRelations import io.tolgee.activity.data.PropertyModification import io.tolgee.activity.data.RevisionType import io.tolgee.activity.propChangesProvider.PropChangesProvider +import io.tolgee.component.ActivityHolderProvider import io.tolgee.dtos.cacheable.UserAccountDto import io.tolgee.events.OnProjectActivityEvent import io.tolgee.model.EntityWithId @@ -68,7 +68,7 @@ class InterceptedEventsManager( } val changes = provider.getChanges(old, collection) ?: return - val activityModifiedEntity = getModifiedEntity(collectionOwner) + val activityModifiedEntity = getModifiedEntity(collectionOwner, RevisionType.MOD) val newChanges = activityModifiedEntity.modifications + mutableMapOf(ownerField.name to changes) activityModifiedEntity.modifications = (newChanges).toMutableMap() @@ -89,7 +89,7 @@ class InterceptedEventsManager( entity as EntityWithId - val activityModifiedEntity = getModifiedEntity(entity) + val activityModifiedEntity = getModifiedEntity(entity, revisionType) val changesMap = getChangesMap(entity, currentState, previousState, propertyNames) @@ -107,7 +107,7 @@ class InterceptedEventsManager( this.describingRelations = describingData.second } - private fun getModifiedEntity(entity: EntityWithId): ActivityModifiedEntity { + private fun getModifiedEntity(entity: EntityWithId, revisionType: RevisionType): ActivityModifiedEntity { val activityModifiedEntity = activityHolder.modifiedEntities .computeIfAbsent(entity::class) { mutableMapOf() } .computeIfAbsent( @@ -117,7 +117,7 @@ class InterceptedEventsManager( activityRevision, entity::class.simpleName!!, entity.id - ) + ).also { it.revisionType = revisionType } } return activityModifiedEntity @@ -141,9 +141,8 @@ class InterceptedEventsManager( value: EntityDescriptionWithRelations, activityRevision: ActivityRevision ): EntityDescriptionRef { - val activityDescribingEntity = activityRevision.describingRelations - .find { it.entityId == value.entityId && it.entityClass == value.entityClass } - ?: let { + val activityDescribingEntity = activityHolder + .getDescribingRelationFromCache(value.entityId, value.entityClass) { val compressedRelations = value.relations.map { relation -> relation.key to compressRelation(relation.value, activityRevision) }.toMap() @@ -235,6 +234,8 @@ class InterceptedEventsManager( logger.debug("Publishing project activity event") try { publishOnActivityEvent(activityRevision) + entityManager.flush() + entityManager.clear() activityService.storeActivityData(activityRevision, activityHolder.modifiedEntities) activityHolder.afterActivityFlushed?.invoke() } catch (e: Throwable) { @@ -274,10 +275,13 @@ class InterceptedEventsManager( applicationContext.getBean(ActivityService::class.java) } - private val activityHolder: ActivityHolder by lazy { - applicationContext.getBean(ActivityHolder::class.java) + private val activityHolderProvider: ActivityHolderProvider by lazy { + applicationContext.getBean(ActivityHolderProvider::class.java) } + private val activityHolder + get() = activityHolderProvider.getActivityHolder() + private val userAccount: UserAccountDto? get() = authenticationFacade.authenticatedUserOrNull } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt b/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt index 792f50ec5f..96b921aee4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt @@ -9,6 +9,10 @@ class TagsPropChangesProvider : PropChangesProvider { override fun getChanges(old: Any?, new: Any?): PropertyModification? { if (old is Collection<*> && new is Collection<*>) { + if (old === new) { + return null + } + val oldTagNames = mapSetToTagNames(old) val newTagNames = mapSetToTagNames(new) if (oldTagNames.containsAll(newTagNames) && newTagNames.containsAll(oldTagNames)) { diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt index 71b4f25af0..ae70495d7e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt @@ -56,10 +56,6 @@ class BatchJobActionService( @EventListener(ApplicationReadyEvent::class) fun run() { println("Application ready") - executeInNewTransaction(transactionManager) { - batchJobChunkExecutionQueue.populateQueue() - } - concurrentExecutionLauncher.run { executionItem, coroutineContext -> var retryExecution: BatchJobChunkExecution? = null try { diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt index 008cae89c6..dbc4607fd3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt @@ -2,6 +2,7 @@ package io.tolgee.batch import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.tolgee.Metrics +import io.tolgee.batch.data.BatchJobChunkExecutionDto import io.tolgee.batch.data.ExecutionQueueItem import io.tolgee.batch.data.QueueEventType import io.tolgee.batch.events.JobQueueItemsEvent @@ -45,35 +46,41 @@ class BatchJobChunkExecutionQueue( } } - @Scheduled(fixedRate = 60000) + @Scheduled(fixedDelay = 60000, initialDelay = 0) fun populateQueue() { logger.debug("Running scheduled populate queue") val data = entityManager.createQuery( """ + select new io.tolgee.batch.data.BatchJobChunkExecutionDto(bjce.id, bk.id, bjce.executeAfter, bk.jobCharacter) from BatchJobChunkExecution bjce - join fetch bjce.batchJob bk + join bjce.batchJob bk where bjce.status = :executionStatus order by bjce.createdAt asc, bjce.executeAfter asc, bjce.id asc """.trimIndent(), - BatchJobChunkExecution::class.java + BatchJobChunkExecutionDto::class.java ).setParameter("executionStatus", BatchJobChunkExecutionStatus.PENDING) .setHint( "jakarta.persistence.lock.timeout", LockOptions.SKIP_LOCKED ).resultList + if (data.size > 0) { - logger.debug("Adding ${data.size} items to queue ${System.identityHashCode(this)}") + logger.debug("Attempt to add ${data.size} items to queue ${System.identityHashCode(this)}") + addExecutionsToLocalQueue(data) } - addExecutionsToLocalQueue(data) } - fun addExecutionsToLocalQueue(data: List) { + fun addExecutionsToLocalQueue(data: List) { val ids = queue.map { it.chunkExecutionId }.toSet() + var count = 0 data.forEach { if (!ids.contains(it.id)) { queue.add(it.toItem()) + count++ } } + + logger.debug("Added $count new items to queue ${System.identityHashCode(this)}") } fun addItemsToLocalQueue(data: List) { @@ -121,6 +128,9 @@ class BatchJobChunkExecutionQueue( ) = ExecutionQueueItem(id, batchJob.id, executeAfter?.time, jobCharacter ?: batchJob.jobCharacter) + private fun BatchJobChunkExecutionDto.toItem(providedJobCharacter: JobCharacter? = null) = + ExecutionQueueItem(id, batchJobId, executeAfter?.time, providedJobCharacter ?: jobCharacter) + val size get() = queue.size fun joinToString(separator: String = ", ", transform: (item: ExecutionQueueItem) -> String) = diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt index 6ba8ab59a9..3f62bcd5f2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt @@ -1,5 +1,6 @@ package io.tolgee.batch +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.tolgee.batch.data.BatchJobDto import io.tolgee.batch.data.BatchJobType @@ -9,7 +10,9 @@ import io.tolgee.constants.Message import io.tolgee.dtos.cacheable.UserAccountDto import io.tolgee.exceptions.NotFoundException import io.tolgee.exceptions.PermissionException +import io.tolgee.model.ALLOCATION_SIZE import io.tolgee.model.Project +import io.tolgee.model.SEQUENCE_NAME import io.tolgee.model.UserAccount import io.tolgee.model.batch.BatchJob import io.tolgee.model.batch.BatchJobChunkExecution @@ -21,18 +24,23 @@ import io.tolgee.repository.BatchJobRepository import io.tolgee.security.authentication.AuthenticationFacade import io.tolgee.service.security.SecurityService import io.tolgee.util.Logging +import io.tolgee.util.SequenceIdProvider import io.tolgee.util.addMinutes +import io.tolgee.util.flushAndClear import io.tolgee.util.logger import jakarta.persistence.EntityManager import org.apache.commons.codec.digest.DigestUtils.sha256Hex import org.hibernate.LockOptions +import org.postgresql.util.PGobject import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Lazy import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable +import org.springframework.jdbc.core.JdbcTemplate import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.event.TransactionalEventListener +import java.sql.Timestamp import java.time.Duration import java.util.* @@ -48,6 +56,8 @@ class BatchJobService( private val currentDateProvider: CurrentDateProvider, private val securityService: SecurityService, private val authenticationFacade: AuthenticationFacade, + private val objectMapper: ObjectMapper, + private val jdbcTemplate: JdbcTemplate ) : Logging { companion object { @@ -95,18 +105,60 @@ class BatchJobService( job.params = processor.getParams(request) + entityManager.flushAndClear() + + val executions = storeExecutions(chunked, job) + + applicationContext.publishEvent(OnBatchJobCreated(job, executions)) + + return job + } + + private fun storeExecutions( + chunked: List>, + job: BatchJob + ): List { val executions = List(chunked.size) { chunkNumber -> BatchJobChunkExecution().apply { batchJob = job this.chunkNumber = chunkNumber - entityManager.persist(this) } } - entityManager.flush() - applicationContext.publishEvent(OnBatchJobCreated(job, executions)) + insertExecutionsViaBatchStatement(executions) - return job + entityManager.clear() + + return executions + } + + private fun insertExecutionsViaBatchStatement(executions: List) { + val sequenceIdProvider = SequenceIdProvider(SEQUENCE_NAME, ALLOCATION_SIZE) + jdbcTemplate.batchUpdate( + """ + insert into tolgee_batch_job_chunk_execution + (id, batch_job_id, chunk_number, status, created_at, updated_at, success_targets) + values (?, ?, ?, ?, ?, ?, ?) + """, + executions, + 100 + ) { ps, execution -> + val id = sequenceIdProvider.next(ps.connection) + execution.id = id + ps.setLong(1, id) + ps.setLong(2, execution.batchJob.id) + ps.setInt(3, execution.chunkNumber) + ps.setString(4, execution.status.name) + ps.setTimestamp(5, Timestamp(currentDateProvider.date.time)) + ps.setTimestamp(6, Timestamp(currentDateProvider.date.time)) + ps.setObject( + 7, + PGobject().apply { + type = "jsonb" + value = objectMapper.writeValueAsString(execution.successTargets) + } + ) + } } private fun tryDebounceJob( diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt b/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt index b31a0c8f1c..2ada024156 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt @@ -16,6 +16,7 @@ import io.tolgee.util.Logging import io.tolgee.util.debug import io.tolgee.util.executeInNewTransaction import io.tolgee.util.logger +import io.tolgee.util.trace import org.springframework.context.ApplicationEventPublisher import org.springframework.stereotype.Component import org.springframework.transaction.PlatformTransactionManager @@ -139,8 +140,8 @@ class ProgressManager( errorMessage: Message? = null, failOnly: Boolean = false ) { - logger.debug("Job ${job.id} completed chunks: $completedChunks of ${job.totalChunks}") - logger.debug("Job ${job.id} progress: $progress of ${job.totalItems}") + logger.debug { "Job ${job.id} completed chunks: $completedChunks of ${job.totalChunks}" } + logger.debug { "Job ${job.id} progress: $progress of ${job.totalItems}" } if (job.totalChunks.toLong() != completedChunks) { return @@ -165,7 +166,7 @@ class ProgressManager( } jobEntity.status = BatchJobStatus.SUCCESS - logger.debug("Publishing success event for job ${job.id}") + logger.debug { "Publishing success event for job ${job.id}" } eventPublisher.publishEvent(OnBatchJobSucceeded(jobEntity.dto)) cachingBatchJobService.saveJob(jobEntity) } @@ -192,10 +193,10 @@ class ProgressManager( fun handleJobRunning(id: Long) { executeInNewTransaction(transactionManager) { - logger.trace("""Fetching job $id""") + logger.trace { """Fetching job $id""" } val job = batchJobService.getJobDto(id) if (job.status == BatchJobStatus.PENDING) { - logger.debug("""Updating job state to running ${job.id}""") + logger.debug { """Updating job state to running ${job.id}""" } cachingBatchJobService.setRunningState(job.id) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt b/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt new file mode 100644 index 0000000000..d65f59760e --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt @@ -0,0 +1,17 @@ +package io.tolgee.batch.data + +import io.tolgee.batch.JobCharacter +import java.util.Date + +/** + * DTO object for the BatchJobChunkExecution. Contains the bare minimum needed for the + * BatchJobChunckExecutionQueue. + * + * @author Geert Zondervan + */ +class BatchJobChunkExecutionDto( + val id: Long, + val batchJobId: Long, + var executeAfter: Date?, + val jobCharacter: JobCharacter, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt new file mode 100644 index 0000000000..0372d35d95 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt @@ -0,0 +1,65 @@ +package io.tolgee.component + +import io.tolgee.activity.ActivityHolder +import jakarta.annotation.PreDestroy +import org.springframework.beans.factory.support.ScopeNotActiveException +import org.springframework.context.ApplicationContext +import org.springframework.stereotype.Component +import org.springframework.web.context.request.RequestContextHolder + +/** + * Class providing Activity Holder, while caching it in ThreadLocal. + * I also registers a transaction synchronization, which clears the ThreadLocal after transaction + * is completed, this enables us to be safe even when running activities in main thred. + */ +@Component +class ActivityHolderProvider(private val applicationContext: ApplicationContext) { + private val threadLocal = ThreadLocal>() + + fun getActivityHolder(): ActivityHolder { + // Get the activity holder from ThreadLocal. + var (scope, activityHolder) = threadLocal.get() ?: (null to null) + + val currentScope = getCurrentScope() + if (scope != currentScope) { + // If the scope has changed, clear the ThreadLocal and fetch a new activity holder. + clearThreadLocal() + activityHolder = null + } + + if (activityHolder == null) { + // If no activity holder exists for this thread, fetch one and store it in ThreadLocal. + activityHolder = fetchActivityHolder() + threadLocal.set(currentScope to activityHolder) + } + return activityHolder + } + + private fun fetchActivityHolder(): ActivityHolder { + return try { + applicationContext.getBean("requestActivityHolder", ActivityHolder::class.java).also { + it.activityRevision + } + } catch (e: ScopeNotActiveException) { + applicationContext.getBean("transactionActivityHolder", ActivityHolder::class.java) + }.also { + it.destroyer = this::clearThreadLocal + } + } + + private fun getCurrentScope(): Scope { + if (RequestContextHolder.getRequestAttributes() == null) { + return Scope.TRANSACTION + } + return Scope.REQUEST + } + + @PreDestroy + fun clearThreadLocal() { + threadLocal.remove() + } + + enum class Scope { + REQUEST, TRANSACTION + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationEventHandler.kt similarity index 62% rename from backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt rename to backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationEventHandler.kt index f4f314721a..b27310d3c9 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationEventHandler.kt @@ -1,4 +1,4 @@ -package io.tolgee.component +package io.tolgee.component.autoTranslation import com.google.cloud.translate.Translation import io.tolgee.activity.data.ActivityType @@ -9,42 +9,30 @@ import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.key.Key import io.tolgee.service.project.ProjectService import io.tolgee.service.translation.AutoTranslationService -import io.tolgee.util.Logging import jakarta.persistence.EntityManager -import org.springframework.context.event.EventListener -import org.springframework.core.annotation.Order -import org.springframework.stereotype.Component - -@Component -class AutoTranslationListener( - private val autoTranslationService: AutoTranslationService, - private val projectService: ProjectService, - private val entityManager: EntityManager, -) : Logging { - - companion object { - /** - * import activities start visible batch job - */ - private val IMPORT_ACTIVITIES = listOf(ActivityType.IMPORT) - - /** - * Low volume activities start hidden batch job - */ - private val LOW_VOLUME_ACTIVITIES = - listOf(ActivityType.SET_TRANSLATIONS, ActivityType.CREATE_KEY, ActivityType.COMPLEX_EDIT) - } - - @Order(2) - @EventListener - fun onApplicationEvent(event: OnProjectActivityStoredEvent) { - val projectId = event.activityRevision.projectId ?: return - - if (!shouldRunTheOperation(event, projectId)) { +import org.springframework.context.ApplicationContext +import kotlin.properties.Delegates + +/** + * Utility class which handles auto translation. + * + * Separated from [AutoTranslationListener] to be able to cache some data in local + * variables without need to pass the params to so many methods. + */ +class AutoTranslationEventHandler( + private val event: OnProjectActivityStoredEvent, + private val applicationContext: ApplicationContext +) { + var projectId by Delegates.notNull() + + fun handle() { + projectId = event.activityRevision.projectId ?: return + + if (!shouldRunTheOperation()) { return } - val keyIds = getKeyIdsToAutoTranslate(projectId, event.activityRevision.modifiedEntities) + val keyIds = getKeyIdsToAutoTranslate() if (keyIds.isEmpty()) { return @@ -55,12 +43,13 @@ class AutoTranslationListener( projectId = projectId, keyIds = keyIds, isBatch = true, + baseLanguageId = baseLanguageId ?: return, isHiddenJob = event.isLowVolumeActivity() ) } } - private fun shouldRunTheOperation(event: OnProjectActivityStoredEvent, projectId: Long): Boolean { + private fun shouldRunTheOperation(): Boolean { val configs = autoTranslationService.getConfigs( entityManager.getReference(Project::class.java, projectId) ) @@ -84,17 +73,17 @@ class AutoTranslationListener( private fun OnProjectActivityStoredEvent.isLowVolumeActivity() = activityRevision.type in LOW_VOLUME_ACTIVITIES - fun getKeyIdsToAutoTranslate(projectId: Long, modifiedEntities: MutableList): List { - return modifiedEntities.mapNotNull { modifiedEntity -> - if (!modifiedEntity.isBaseTranslationTextChanged(projectId)) { + private fun getKeyIdsToAutoTranslate(): List { + return event.activityRevision.modifiedEntities.mapNotNull { modifiedEntity -> + if (!modifiedEntity.isBaseTranslationTextChanged()) { return@mapNotNull null } getKeyId(modifiedEntity) } } - private fun ActivityModifiedEntity.isBaseTranslationTextChanged(projectId: Long): Boolean { - return this.isTranslation() && this.isBaseTranslation(projectId) && this.isTextChanged() + private fun ActivityModifiedEntity.isBaseTranslationTextChanged(): Boolean { + return this.isTranslation() && this.isBaseTranslation() && this.isTextChanged() } private fun getKeyId(modifiedEntity: ActivityModifiedEntity) = @@ -110,11 +99,38 @@ class AutoTranslationListener( private fun ActivityModifiedEntity.isTranslation() = entityClass == Translation::class.simpleName - private fun ActivityModifiedEntity.isBaseTranslation(projectId: Long): Boolean { - val baseLanguageId = projectService.get(projectId).baseLanguage?.id ?: return false - + private fun ActivityModifiedEntity.isBaseTranslation(): Boolean { return describingRelations?.values ?.any { it.entityClass == Language::class.simpleName && it.entityId == baseLanguageId } ?: false } + + private val baseLanguageId: Long? by lazy { + projectService.get(projectId).baseLanguage?.id + } + + private val autoTranslationService: AutoTranslationService by lazy { + applicationContext.getBean(AutoTranslationService::class.java) + } + + private val projectService: ProjectService by lazy { + applicationContext.getBean(ProjectService::class.java) + } + + private val entityManager: EntityManager by lazy { + applicationContext.getBean(EntityManager::class.java) + } + + companion object { + /** + * import activities start visible batch job + */ + private val IMPORT_ACTIVITIES = listOf(ActivityType.IMPORT) + + /** + * Low volume activities start hidden batch job + */ + private val LOW_VOLUME_ACTIVITIES = + listOf(ActivityType.SET_TRANSLATIONS, ActivityType.CREATE_KEY, ActivityType.COMPLEX_EDIT) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt new file mode 100644 index 0000000000..5f36506ca3 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt @@ -0,0 +1,20 @@ +package io.tolgee.component.autoTranslation + +import io.tolgee.events.OnProjectActivityStoredEvent +import io.tolgee.util.Logging +import org.springframework.context.ApplicationContext +import org.springframework.context.event.EventListener +import org.springframework.core.annotation.Order +import org.springframework.stereotype.Component + +@Component +class AutoTranslationListener( + private val applicationContext: ApplicationContext +) : Logging { + + @Order(2) + @EventListener + fun onApplicationEvent(event: OnProjectActivityStoredEvent) { + AutoTranslationEventHandler(event, applicationContext).handle() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt index 645a146a36..0890fb07b9 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt @@ -1,10 +1,10 @@ package io.tolgee.configuration import io.tolgee.activity.ActivityHolder +import io.tolgee.component.ActivityHolderProvider import io.tolgee.configuration.TransactionScopeConfig.Companion.SCOPE_TRANSACTION import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.config.BeanDefinition -import org.springframework.beans.factory.support.ScopeNotActiveException import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Bean @@ -31,16 +31,16 @@ class ActivityHolderConfig { return ActivityHolder(applicationContext) } + /** + * This method for getting the activity holder is slow, since it + * needs to create new bean every time holder is requested. Which is pretty often. + * + * Use the activityHolderProvider when possible. + */ @Bean @Primary @Scope(BeanDefinition.SCOPE_PROTOTYPE, proxyMode = ScopedProxyMode.TARGET_CLASS) - fun activityHolder(applicationContext: ApplicationContext): ActivityHolder { - return try { - applicationContext.getBean("requestActivityHolder", ActivityHolder::class.java).also { - it.activityRevision - } - } catch (e: ScopeNotActiveException) { - return applicationContext.getBean("transactionActivityHolder", ActivityHolder::class.java) - } + fun activityHolder(activityHolderProvider: ActivityHolderProvider): ActivityHolder { + return activityHolderProvider.getActivityHolder() } } diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/HibernateSession.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/HibernateSession.kt deleted file mode 100644 index 16607834e3..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/HibernateSession.kt +++ /dev/null @@ -1,16 +0,0 @@ -package io.tolgee.configuration - -// -// @Configuration -// class HibernateSessionConfiguration( -// private val dataSource: DataSource -// ) { -// @Bean -// fun sessionFactory(emf: EntityManagerFactory?): LocalSessionFactoryBean { -// val sessionFactory = LocalSessionFactoryBean() -// sessionFactory.setDataSource(dataSource) -// sessionFactory.setPackagesToScan("io.tolgee.model") -// sessionFactory.hibernateProperties = hibernateProperties() -// return sessionFactory -// } -// } diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnImportSoftDeleted.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnImportSoftDeleted.kt new file mode 100644 index 0000000000..4453642383 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/events/OnImportSoftDeleted.kt @@ -0,0 +1,5 @@ +package io.tolgee.events + +class OnImportSoftDeleted( + val importId: Long +) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/StandardAuditModel.kt b/backend/data/src/main/kotlin/io/tolgee/model/StandardAuditModel.kt index a82db38c84..4949890de1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/StandardAuditModel.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/StandardAuditModel.kt @@ -7,14 +7,17 @@ import jakarta.persistence.MappedSuperclass import jakarta.persistence.SequenceGenerator import org.springframework.data.util.ProxyUtils +const val SEQUENCE_NAME = "hibernate_sequence" +const val ALLOCATION_SIZE = 1000 + @MappedSuperclass abstract class StandardAuditModel : AuditModel(), EntityWithId { @Id @SequenceGenerator( name = "sequenceGenerator", - sequenceName = "hibernate_sequence", + sequenceName = SEQUENCE_NAME, initialValue = 1000000000, - allocationSize = 1000 + allocationSize = ALLOCATION_SIZE ) @GeneratedValue( strategy = GenerationType.SEQUENCE, diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt index 5dd54909c4..100c1dd835 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt @@ -2,9 +2,7 @@ package io.tolgee.model.activity import io.hypersistence.utils.hibernate.type.json.JsonBinaryType import io.tolgee.activity.data.EntityDescriptionRef -import io.tolgee.activity.data.RevisionType import jakarta.persistence.Entity -import jakarta.persistence.Enumerated import jakarta.persistence.Id import jakarta.persistence.IdClass import jakarta.persistence.ManyToOne @@ -33,7 +31,4 @@ class ActivityDescribingEntity( @Type(JsonBinaryType::class) var describingRelations: Map? = null - - @Enumerated - lateinit var revisionType: RevisionType } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt index 4ba74cf8e4..e91acf15ea 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt @@ -56,5 +56,5 @@ class ActivityModifiedEntity( var describingRelations: Map? = null @Enumerated - lateinit var revisionType: RevisionType + var revisionType: RevisionType = RevisionType.MOD } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/dataImport/Import.kt b/backend/data/src/main/kotlin/io/tolgee/model/dataImport/Import.kt index 0f7e3b4aa3..99328ffad3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/dataImport/Import.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/dataImport/Import.kt @@ -1,6 +1,7 @@ package io.tolgee.model.dataImport import io.tolgee.model.Project +import io.tolgee.model.SoftDeletable import io.tolgee.model.StandardAuditModel import io.tolgee.model.UserAccount import jakarta.persistence.Entity @@ -9,6 +10,7 @@ import jakarta.persistence.OneToMany import jakarta.persistence.Table import jakarta.persistence.UniqueConstraint import jakarta.validation.constraints.NotNull +import java.util.* @Entity @Table(uniqueConstraints = [UniqueConstraint(columnNames = ["author_id", "project_id"])]) @@ -16,7 +18,7 @@ class Import( @field:NotNull @ManyToOne(optional = false) val project: Project -) : StandardAuditModel() { +) : StandardAuditModel(), SoftDeletable { @field:NotNull @ManyToOne(optional = false) @@ -24,4 +26,6 @@ class Import( @OneToMany(mappedBy = "import", orphanRemoval = true) var files = mutableListOf() + + override var deletedAt: Date? = null } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/AutoTranslationConfigRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/AutoTranslationConfigRepository.kt index ed23c8c3ad..0fcfda563a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/AutoTranslationConfigRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/AutoTranslationConfigRepository.kt @@ -9,6 +9,7 @@ import org.springframework.stereotype.Repository @Repository interface AutoTranslationConfigRepository : JpaRepository { fun findOneByProjectAndTargetLanguageId(project: Project, languageId: Long?): AutoTranslationConfig? + fun findByProjectAndTargetLanguageIdIn(project: Project, languageIds: List): List @Query( """ diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt index 9c89a8553e..6d6200583f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt @@ -172,4 +172,11 @@ interface TranslationRepository : JpaRepository { """ ) fun getAllByProjectId(projectId: Long): List + @Query( + """ + from Translation t + where t.key = :key and t.language.tag in :languageTags + """ + ) + fun findForKeyByLanguages(key: Key, languageTags: Collection): List } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportRepository.kt index 2ae5a134d5..86cd65fc3d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportRepository.kt @@ -7,8 +7,19 @@ import org.springframework.stereotype.Repository @Repository interface ImportRepository : JpaRepository { + + @Query( + """ + select i from Import i where i.project.id = :projectId and i.author.id = :authorId and i.deletedAt is null + """ + ) fun findByProjectIdAndAuthorId(projectId: Long, authorId: Long): Import? + @Query( + """ + select i from Import i where i.project.id = :projectId and i.deletedAt is null + """ + ) fun findAllByProjectId(projectId: Long): List @Query( @@ -17,4 +28,11 @@ interface ImportRepository : JpaRepository { """ ) fun getAllNamespaces(importId: Long): Set + + @Query( + """ + from Import i where i.id = :importId and i.deletedAt is not null + """ + ) + fun findDeleted(importId: Long): Import? } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt index 2aca539683..243c05fac6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt @@ -12,7 +12,7 @@ import org.springframework.stereotype.Repository interface TranslationCommentRepository : JpaRepository { fun deleteAllByIdIn(ids: List) - @Query("select tc from TranslationComment tc where tc.translation = :translation") + @Query("select tc from TranslationComment tc left join fetch tc.author where tc.translation = :translation") fun getPagedByTranslation(translation: Translation, pageable: Pageable): Page fun deleteAllByTranslationIdIn(translationIds: Collection) @@ -28,4 +28,13 @@ interface TranslationCommentRepository : JpaRepository """ ) fun getAllByProjectId(projectId: Long): List + + @Query( + """ + from TranslationComment tc + left join fetch tc.author + where tc.id = :id + """ + ) + fun findWithFetchedAuthor(id: Long): TranslationComment? } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/AsyncImportHardDeleter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/AsyncImportHardDeleter.kt new file mode 100644 index 0000000000..0608b41c78 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/AsyncImportHardDeleter.kt @@ -0,0 +1,19 @@ +package io.tolgee.service.dataImport + +import io.tolgee.events.OnImportSoftDeleted +import org.springframework.scheduling.annotation.Async +import org.springframework.stereotype.Component +import org.springframework.transaction.event.TransactionalEventListener + +@Component +class AsyncImportHardDeleter( + private val importService: ImportService +) { + @Async + @TransactionalEventListener + fun hardDelete(event: OnImportSoftDeleted) { + importService.findDeleted(importId = event.importId)?.let { + importService.hardDeleteImport(it) + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDeleteService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDeleteService.kt new file mode 100644 index 0000000000..29fc7f5646 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDeleteService.kt @@ -0,0 +1,123 @@ +package io.tolgee.service.dataImport + +import jakarta.persistence.EntityManager +import org.hibernate.Session +import org.springframework.stereotype.Service + +@Service +class ImportDeleteService( + private val entityManager: EntityManager +) { + fun deleteImport(importId: Long) { + entityManager.unwrap(Session::class.java).doWork { connection -> + deleteImportTranslations(connection, importId) + deleteImportLanguages(connection, importId) + deleteImportKeyMetaTags(connection, importId) + deleteImportKeyMetaComments(connection, importId) + deleteImportKeyMetaCodeReferences(connection, importId) + deleteImportKeyMeta(connection, importId) + deleteImportKeys(connection, importId) + deleteImportFileIssueParams(connection, importId) + deleteImportFileIssues(connection, importId) + deleteImportFiles(connection, importId) + deleteTheImport(connection, importId) + } + } + + fun executeUpdate(connection: java.sql.Connection, query: String, importId: Long) { + @Suppress("SqlSourceToSinkFlow") + connection.prepareStatement(query).use { statement -> + statement.setLong(1, importId) + statement.executeUpdate() + } + } + + fun deleteImportTranslations(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_translation " + + "where id in (" + + "select it.id from import_translation it " + + "join import_language il on il.id = it.language_id " + + "join import_file if on il.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportLanguages(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_language " + + "where id in (select il.id from import_language il " + + "join import_file if on il.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeys(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_key " + + "where id in (select ik.id from import_key ik " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeyMetaTags(connection: java.sql.Connection, importId: Long) { + val query = + "delete from key_meta_tags " + + "where key_metas_id in (select ikm.id from key_meta ikm " + + "join import_key ik on ikm.import_key_id = ik.id " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeyMetaComments(connection: java.sql.Connection, importId: Long) { + val query = + "delete from key_comment " + + "where key_meta_id in (select ikm.id from key_meta ikm " + + "join import_key ik on ikm.import_key_id = ik.id " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeyMetaCodeReferences(connection: java.sql.Connection, importId: Long) { + val query = + "delete from key_code_reference " + + "where key_meta_id in (select ikm.id from key_meta ikm " + + "join import_key ik on ikm.import_key_id = ik.id " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeyMeta(connection: java.sql.Connection, importId: Long) { + val query = + "delete from key_meta " + + "where id in (select ikm.id from key_meta ikm " + + "join import_key ik on ikm.import_key_id = ik.id " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportFileIssueParams(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_file_issue_param " + + "where import_file_issue_param.issue_id in (select ifi.id from import_file_issue ifi " + + "join import_file if on ifi.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportFileIssues(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_file_issue " + + "where id in (select ifi.id from import_file_issue ifi " + + "join import_file if on ifi.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportFiles(connection: java.sql.Connection, importId: Long) { + val query = "delete from import_file " + + "where id in (select if.id from import_file if where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteTheImport(connection: java.sql.Connection, importId: Long) { + val query = "delete from import where id = ?" + executeUpdate(connection, query, importId) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index 2e7ae1a23b..3aa049b966 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -1,10 +1,12 @@ package io.tolgee.service.dataImport +import io.tolgee.component.CurrentDateProvider import io.tolgee.component.reporting.BusinessEventPublisher import io.tolgee.component.reporting.OnBusinessEventToCaptureEvent import io.tolgee.constants.Message import io.tolgee.dtos.dataImport.ImportAddFilesParams import io.tolgee.dtos.dataImport.ImportFileDto +import io.tolgee.events.OnImportSoftDeleted import io.tolgee.exceptions.BadRequestException import io.tolgee.exceptions.ErrorResponseBody import io.tolgee.exceptions.ImportConflictNotResolvedException @@ -29,7 +31,6 @@ import io.tolgee.repository.dataImport.ImportRepository import io.tolgee.repository.dataImport.ImportTranslationRepository import io.tolgee.repository.dataImport.issues.ImportFileIssueParamRepository import io.tolgee.repository.dataImport.issues.ImportFileIssueRepository -import io.tolgee.service.key.KeyMetaService import io.tolgee.util.getSafeNamespace import jakarta.persistence.EntityManager import org.springframework.context.ApplicationContext @@ -50,10 +51,11 @@ class ImportService( private val applicationContext: ApplicationContext, private val importTranslationRepository: ImportTranslationRepository, private val importFileIssueParamRepository: ImportFileIssueParamRepository, - private val keyMetaService: KeyMetaService, private val removeExpiredImportService: RemoveExpiredImportService, private val entityManager: EntityManager, - private val businessEventPublisher: BusinessEventPublisher + private val businessEventPublisher: BusinessEventPublisher, + private val importDeleteService: ImportDeleteService, + private val currentDateProvider: CurrentDateProvider ) { @Transactional fun addFiles( @@ -150,6 +152,10 @@ class ImportService( return findNotExpired(projectId, authorId) ?: throw NotFoundException() } + fun findDeleted(importId: Long): Import? { + return importRepository.findDeleted(importId) + } + private fun findNotExpired(projectId: Long, userAccountId: Long): Import? { val import = this.find(projectId, userAccountId) return removeExpiredImportService.removeIfExpired(import) @@ -251,16 +257,16 @@ class ImportService( ) } + @Transactional fun deleteImport(import: Import) { - this.importTranslationRepository.deleteAllByImport(import) - this.importLanguageRepository.deleteAllByImport(import) - val keyIds = this.importKeyRepository.getAllIdsByImport(import) - this.keyMetaService.deleteAllByImportKeyIdIn(keyIds) - this.importKeyRepository.deleteByIdIn(keyIds) - this.importFileIssueParamRepository.deleteAllByImport(import) - this.importFileIssueRepository.deleteAllByImport(import) - this.importFileRepository.deleteAllByImport(import) - this.importRepository.delete(import) + import.deletedAt = currentDateProvider.date + importRepository.save(import) + applicationContext.publishEvent(OnImportSoftDeleted(import.id)) + } + + @Transactional + fun hardDeleteImport(import: Import) { + importDeleteService.deleteImport(import.id) } @Transactional diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt index b6d1e1dfbf..45aa54c0fc 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt @@ -15,6 +15,8 @@ import io.tolgee.service.key.KeyService import io.tolgee.service.key.NamespaceService import io.tolgee.service.security.SecurityService import io.tolgee.service.translation.TranslationService +import io.tolgee.util.flushAndClear +import jakarta.persistence.EntityManager import org.springframework.context.ApplicationContext class StoredDataImporter( @@ -46,6 +48,8 @@ class StoredDataImporter( */ private val translationService = applicationContext.getBean(TranslationService::class.java) + private val entityManager = applicationContext.getBean(EntityManager::class.java) + private val namespacesToSave = mutableMapOf() /** @@ -89,6 +93,8 @@ class StoredDataImporter( saveMetaData(keyEntitiesToSave) translationService.setOutdatedBatch(outdatedFlagKeys) + + entityManager.flushAndClear() } private fun saveMetaData(keyEntitiesToSave: MutableCollection) { diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyMetaService.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyMetaService.kt index 224fd08dae..44ddd5b564 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyMetaService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyMetaService.kt @@ -121,13 +121,6 @@ class KeyMetaService( fun save(meta: KeyMeta): KeyMeta = this.keyMetaRepository.save(meta) - fun deleteAllByImportKeyIdIn(importKeyIds: List) { - tagService.deleteAllByImportKeyIdIn(importKeyIds) - keyCommentRepository.deleteAllByImportKeyIds(importKeyIds) - keyCodeReferenceRepository.deleteAllByImportKeyIds(importKeyIds) - this.keyMetaRepository.deleteAllByImportKeyIdIn(importKeyIds) - } - fun deleteAllByKeyIdIn(ids: Collection) { tagService.deleteAllByKeyIdIn(ids) keyCommentRepository.deleteAllByKeyIds(ids) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt b/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt index 507b487f71..6147040074 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt @@ -245,7 +245,7 @@ class ProjectService( } importService.getAllByProject(id).forEach { - importService.deleteImport(it) + importService.hardDeleteImport(it) } // otherwise we cannot delete the languages diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt index 31c02911c4..20c53b181c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt @@ -47,15 +47,20 @@ class AutoTranslationService( fun autoTranslateViaBatchJob( projectId: Long, keyIds: List, + baseLanguageId: Long, useTranslationMemory: Boolean? = null, useMachineTranslation: Boolean? = null, isBatch: Boolean, isHiddenJob: Boolean, ) { val project = entityManager.getReference(Project::class.java, projectId) - val languageIds = languageService.findAll(projectId).map { it.id } + val languageIds = languageService.findAll(projectId).map { it.id }.filter { it != baseLanguageId } + val configs = this.getConfigs(project, languageIds) val request = AutoTranslationRequest().apply { target = languageIds.flatMap { languageId -> + if (configs[languageId]?.usingTm == false && configs[languageId]?.usingPrimaryMtService == false) { + return@flatMap listOf() + } keyIds.map { keyId -> BatchTranslationTargetItem( keyId = keyId, @@ -288,6 +293,18 @@ class AutoTranslationService( return list!! } + fun getConfigs(project: Project, targetLanguageIds: List): Map { + val configs = autoTranslationConfigRepository.findByProjectAndTargetLanguageIdIn(project, targetLanguageIds) + val default = autoTranslationConfigRepository.findDefaultForProject(project) + ?: autoTranslationConfigRepository.findDefaultForProject(project) ?: AutoTranslationConfig().also { + it.project = project + } + + return targetLanguageIds.associateWith { languageId -> + (configs.find { it.targetLanguage?.id == languageId } ?: default) + } + } + fun getConfig(project: Project, targetLanguageId: Long) = autoTranslationConfigRepository.findOneByProjectAndTargetLanguageId(project, targetLanguageId) ?: autoTranslationConfigRepository.findDefaultForProject(project) ?: AutoTranslationConfig().also { diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt index 53994590e9..a082c98c84 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt @@ -37,14 +37,26 @@ class TranslationCommentService( } } + @Transactional fun find(id: Long): TranslationComment? { return translationCommentRepository.findById(id).orElse(null) } + @Transactional fun get(id: Long): TranslationComment { return find(id) ?: throw NotFoundException() } + @Transactional + fun findWithAuthorFetched(id: Long): TranslationComment? { + return translationCommentRepository.findWithFetchedAuthor(id) + } + + @Transactional + fun getWithAuthorFetched(id: Long): TranslationComment { + return findWithAuthorFetched(id) ?: throw NotFoundException() + } + @Transactional fun update(dto: TranslationCommentDto, entity: TranslationComment): TranslationComment { entity.text = dto.text diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt index 1a7c7bf74b..5ce23a670a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt @@ -191,6 +191,10 @@ class TranslationService( ).mapKeys { it.key.tag } } + fun findForKeyByLanguages(key: Key, languageTags: Collection): List { + return translationRepository.findForKeyByLanguages(key, languageTags) + } + private fun languageByTagFromLanguages(tag: String, languages: Collection) = languages.find { it.tag == tag } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) diff --git a/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt new file mode 100644 index 0000000000..3f54dc5ee6 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt @@ -0,0 +1,34 @@ +package io.tolgee.util + +import java.sql.Connection + +class SequenceIdProvider( + private val sequenceName: String, + private val allocationSize: Int +) { + + private var currentId: Long? = null + private var currentMaxId: Long? = null + + fun next(connection: Connection): Long { + allocateIfRequired(connection) + val currentId = currentId + this.currentId = currentId!! + 1 + return currentId + } + + private fun allocateIfRequired(connection: Connection) { + if (currentId == null || currentMaxId == null || currentId!! >= currentMaxId!!) { + allocate(connection) + } + } + + private fun allocate(connection: Connection) { + @Suppress("SqlSourceToSinkFlow") + val statement = connection.prepareStatement("select nextval('$sequenceName')") + val resultSet = statement.executeQuery() + resultSet.next() + currentMaxId = resultSet.getLong(1) + currentId = currentMaxId!! - allocationSize + 1 + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt new file mode 100644 index 0000000000..bc55bd8855 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt @@ -0,0 +1,12 @@ +package io.tolgee.util + +import jakarta.persistence.EntityManager +import org.hibernate.Session + +val EntityManager.session: Session + get() = this.unwrap(org.hibernate.Session::class.java)!! + +fun EntityManager.flushAndClear() { + this.flush() + this.clear() +} diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index 593c903964..a38b77e9de 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -2949,4 +2949,13 @@ CREATE INDEX organization_deleted_at_null ON organization((deleted_at IS NULL)); + + + + + + + CREATE INDEX import_deleted_at_null ON import((deleted_at IS NULL)); + + diff --git a/gradle.properties b/gradle.properties index 598eae6393..75119bfea2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -kotlinVersion=1.9.10 +kotlinVersion=1.9.21 springBootVersion=3.1.5 springDocVersion=2.2.0 jjwtVersion=0.11.2