From d143559f69b2db3b496c3d559dd980fe094e6125 Mon Sep 17 00:00:00 2001 From: Geert Zondervan Date: Tue, 12 Dec 2023 12:11:51 +0100 Subject: [PATCH 01/26] Only fetch required fields for job queue. --- .../batch/BatchJobChunkExecutionQueue.kt | 44 ++++++++++++++++--- .../batch/data/BatchJobChunkExecutionDto.kt | 17 +++++++ 2 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt index 008cae89c6..a62542cf9a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt @@ -3,6 +3,7 @@ package io.tolgee.batch import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.tolgee.Metrics import io.tolgee.batch.data.ExecutionQueueItem +import io.tolgee.batch.data.BatchJobChunkExecutionDto import io.tolgee.batch.data.QueueEventType import io.tolgee.batch.events.JobQueueItemsEvent import io.tolgee.component.UsingRedisProvider @@ -48,26 +49,49 @@ class BatchJobChunkExecutionQueue( @Scheduled(fixedRate = 60000) fun populateQueue() { logger.debug("Running scheduled populate queue") + +// val data1 = entityManager.createQuery( +// """ +// select bjce.id, bk.id, bjce.executeAfter, bk.jobCharacter +// from BatchJobChunkExecution bjce +// join bjce.batchJob bk +// where bjce.status = :executionStatus +// order by bjce.createdAt asc, bjce.executeAfter asc, bjce.id asc +// """.trimIndent(), +// Array::class.java +// ).setParameter("executionStatus", BatchJobChunkExecutionStatus.PENDING) +// .setHint( +// "javax.persistence.lock.timeout", +// LockOptions.SKIP_LOCKED +// ).resultList + +// for (result in data) { +// println( +// "bjce.i: " + result[0] + ", bk.id: " + result[1] + ", executeAfter " + result[2] + ", jobCharacter " + result[3]) +// } + val data = entityManager.createQuery( """ + select new io.tolgee.batch.data.BatchJobChunkExecutionDto(bjce.id, bk.id, bjce.executeAfter, bk.jobCharacter) from BatchJobChunkExecution bjce - join fetch bjce.batchJob bk + join bjce.batchJob bk where bjce.status = :executionStatus order by bjce.createdAt asc, bjce.executeAfter asc, bjce.id asc """.trimIndent(), - BatchJobChunkExecution::class.java + BatchJobChunkExecutionDto::class.java ).setParameter("executionStatus", BatchJobChunkExecutionStatus.PENDING) .setHint( "jakarta.persistence.lock.timeout", LockOptions.SKIP_LOCKED ).resultList + if (data.size > 0) { - logger.debug("Adding ${data.size} items to queue ${System.identityHashCode(this)}") + logger.info("Adding ${data.size} items to queue ${System.identityHashCode(this)}") + addExecutionsToLocalQueue(data) } - addExecutionsToLocalQueue(data) } - - fun addExecutionsToLocalQueue(data: List) { + + fun addExecutionsToLocalQueue(data: List) { val ids = queue.map { it.chunkExecutionId }.toSet() data.forEach { if (!ids.contains(it.id)) { @@ -121,6 +145,14 @@ class BatchJobChunkExecutionQueue( ) = ExecutionQueueItem(id, batchJob.id, executeAfter?.time, jobCharacter ?: batchJob.jobCharacter) + private fun BatchJobChunkExecutionDto.toItem( + // Yes. jobCharacter is part of the batchJob entity. + // However, we don't want to fetch it here, because it would be a waste of resources. + // So we can provide the jobCharacter here. + providedJobCharacter: JobCharacter? = null + ) = + ExecutionQueueItem(id, batchJobId, executeAfter?.time, providedJobCharacter ?: jobCharacter) + val size get() = queue.size fun joinToString(separator: String = ", ", transform: (item: ExecutionQueueItem) -> String) = diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt b/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt new file mode 100644 index 0000000000..975e860e71 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt @@ -0,0 +1,17 @@ +package io.tolgee.batch.data + +import io.tolgee.batch.JobCharacter +import java.util.Date + +/** + * DTO object for the BatchJobChunkExecution. Contains the bare minimum needed for the + * BatchJobChunckExecutionQueue. + * + * @author Geert Zondervan + */ +class BatchJobChunkExecutionDto ( + val id: Long, + val batchJobId: Long, + var executeAfter: Date?, + val jobCharacter: JobCharacter, +) From 0c0fc5195338e9d03ad6eff476162f39fa0ec3b0 Mon Sep 17 00:00:00 2001 From: Geert Zondervan Date: Wed, 13 Dec 2023 13:06:34 +0100 Subject: [PATCH 02/26] Set initialDelay in Scheduled annotation. --- .../io/tolgee/batch/BatchJobActionService.kt | 4 --- .../batch/BatchJobChunkExecutionQueue.kt | 31 +++++-------------- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt index 71b4f25af0..ae70495d7e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt @@ -56,10 +56,6 @@ class BatchJobActionService( @EventListener(ApplicationReadyEvent::class) fun run() { println("Application ready") - executeInNewTransaction(transactionManager) { - batchJobChunkExecutionQueue.populateQueue() - } - concurrentExecutionLauncher.run { executionItem, coroutineContext -> var retryExecution: BatchJobChunkExecution? = null try { diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt index a62542cf9a..5d87c3aa1a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt @@ -46,30 +46,9 @@ class BatchJobChunkExecutionQueue( } } - @Scheduled(fixedRate = 60000) + @Scheduled(fixedDelay = 60000, initialDelay = 0) fun populateQueue() { - logger.debug("Running scheduled populate queue") - -// val data1 = entityManager.createQuery( -// """ -// select bjce.id, bk.id, bjce.executeAfter, bk.jobCharacter -// from BatchJobChunkExecution bjce -// join bjce.batchJob bk -// where bjce.status = :executionStatus -// order by bjce.createdAt asc, bjce.executeAfter asc, bjce.id asc -// """.trimIndent(), -// Array::class.java -// ).setParameter("executionStatus", BatchJobChunkExecutionStatus.PENDING) -// .setHint( -// "javax.persistence.lock.timeout", -// LockOptions.SKIP_LOCKED -// ).resultList - -// for (result in data) { -// println( -// "bjce.i: " + result[0] + ", bk.id: " + result[1] + ", executeAfter " + result[2] + ", jobCharacter " + result[3]) -// } - + logger.info("Running scheduled populate queue") val data = entityManager.createQuery( """ select new io.tolgee.batch.data.BatchJobChunkExecutionDto(bjce.id, bk.id, bjce.executeAfter, bk.jobCharacter) @@ -86,18 +65,22 @@ class BatchJobChunkExecutionQueue( ).resultList if (data.size > 0) { - logger.info("Adding ${data.size} items to queue ${System.identityHashCode(this)}") + logger.debug("Attempt to add ${data.size} items to queue ${System.identityHashCode(this)}") addExecutionsToLocalQueue(data) } } fun addExecutionsToLocalQueue(data: List) { val ids = queue.map { it.chunkExecutionId }.toSet() + var count = 0 data.forEach { if (!ids.contains(it.id)) { queue.add(it.toItem()) + count++ } } + + logger.debug("Added ${count} new items to queue ${System.identityHashCode(this)}") } fun addItemsToLocalQueue(data: List) { From a84383d3c2c1518d766c392cae8a4cd28a315ee8 Mon Sep 17 00:00:00 2001 From: Geert Zondervan Date: Wed, 13 Dec 2023 13:09:26 +0100 Subject: [PATCH 03/26] Fix formatting. --- .../kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt | 8 ++++---- .../io/tolgee/batch/data/BatchJobChunkExecutionDto.kt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt index 5d87c3aa1a..6e37be34ba 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt @@ -2,8 +2,8 @@ package io.tolgee.batch import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.tolgee.Metrics -import io.tolgee.batch.data.ExecutionQueueItem import io.tolgee.batch.data.BatchJobChunkExecutionDto +import io.tolgee.batch.data.ExecutionQueueItem import io.tolgee.batch.data.QueueEventType import io.tolgee.batch.events.JobQueueItemsEvent import io.tolgee.component.UsingRedisProvider @@ -48,7 +48,7 @@ class BatchJobChunkExecutionQueue( @Scheduled(fixedDelay = 60000, initialDelay = 0) fun populateQueue() { - logger.info("Running scheduled populate queue") + logger.debug("Running scheduled populate queue") val data = entityManager.createQuery( """ select new io.tolgee.batch.data.BatchJobChunkExecutionDto(bjce.id, bk.id, bjce.executeAfter, bk.jobCharacter) @@ -69,7 +69,7 @@ class BatchJobChunkExecutionQueue( addExecutionsToLocalQueue(data) } } - + fun addExecutionsToLocalQueue(data: List) { val ids = queue.map { it.chunkExecutionId }.toSet() var count = 0 @@ -80,7 +80,7 @@ class BatchJobChunkExecutionQueue( } } - logger.debug("Added ${count} new items to queue ${System.identityHashCode(this)}") + logger.debug("Added $count new items to queue ${System.identityHashCode(this)}") } fun addItemsToLocalQueue(data: List) { diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt b/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt index 975e860e71..d65f59760e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/data/BatchJobChunkExecutionDto.kt @@ -9,7 +9,7 @@ import java.util.Date * * @author Geert Zondervan */ -class BatchJobChunkExecutionDto ( +class BatchJobChunkExecutionDto( val id: Long, val batchJobId: Long, var executeAfter: Date?, From 10ecfd310fec57da0ab84f65cbac95555d8d85c8 Mon Sep 17 00:00:00 2001 From: Geert Zondervan Date: Wed, 13 Dec 2023 13:45:21 +0100 Subject: [PATCH 04/26] Cleanup. --- .../kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt index 6e37be34ba..dbc4607fd3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobChunkExecutionQueue.kt @@ -128,12 +128,7 @@ class BatchJobChunkExecutionQueue( ) = ExecutionQueueItem(id, batchJob.id, executeAfter?.time, jobCharacter ?: batchJob.jobCharacter) - private fun BatchJobChunkExecutionDto.toItem( - // Yes. jobCharacter is part of the batchJob entity. - // However, we don't want to fetch it here, because it would be a waste of resources. - // So we can provide the jobCharacter here. - providedJobCharacter: JobCharacter? = null - ) = + private fun BatchJobChunkExecutionDto.toItem(providedJobCharacter: JobCharacter? = null) = ExecutionQueueItem(id, batchJobId, executeAfter?.time, providedJobCharacter ?: jobCharacter) val size get() = queue.size From 1ea93f0c6743c1f96a0cd453843726d8f4afa202 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Fri, 15 Dec 2023 13:12:22 +0100 Subject: [PATCH 05/26] fix: Try to remove enhancement config --- backend/app/src/main/resources/application.yaml | 3 --- backend/app/src/test/resources/application.yaml | 3 --- backend/data/build.gradle | 7 ------- 3 files changed, 13 deletions(-) diff --git a/backend/app/src/main/resources/application.yaml b/backend/app/src/main/resources/application.yaml index 280c38e0ba..1c636f1817 100644 --- a/backend/app/src/main/resources/application.yaml +++ b/backend/app/src/main/resources/application.yaml @@ -22,9 +22,6 @@ spring: types: print: banner: false - enhancer: - enableLazyInitialization: true - enableDirtyTracking: true # open-in-view: false batch: job: diff --git a/backend/app/src/test/resources/application.yaml b/backend/app/src/test/resources/application.yaml index 4404906561..fe6235cb02 100644 --- a/backend/app/src/test/resources/application.yaml +++ b/backend/app/src/test/resources/application.yaml @@ -14,9 +14,6 @@ spring: order_inserts: true order_updates: true dialect: io.tolgee.dialects.postgres.CustomPostgreSQLDialect - enhancer: - enableLazyInitialization: true - enableDirtyTracking: true mvc: pathmatch: matching-strategy: ant_path_matcher diff --git a/backend/data/build.gradle b/backend/data/build.gradle index 93828b32a8..abf05bf927 100644 --- a/backend/data/build.gradle +++ b/backend/data/build.gradle @@ -82,13 +82,6 @@ kotlin { jvmToolchain(17) } -hibernate { - enhancement { - lazyInitialization = true - dirtyTracking = true - } -} - dependencies { /** * SPRING From 2187a615d22989ed31dfa08c296fcd23763c4d7f Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Fri, 15 Dec 2023 13:14:16 +0100 Subject: [PATCH 06/26] fix: Import optimization --- .../AutoTranslationEventHandler.kt} | 100 ++++++++++-------- .../AutoTranslationListener.kt | 29 +++++ .../AutoTranslationConfigRepository.kt | 1 + .../translation/AutoTranslationService.kt | 24 ++++- 4 files changed, 109 insertions(+), 45 deletions(-) rename backend/data/src/main/kotlin/io/tolgee/component/{AutoTranslationListener.kt => autoTranslation/AutoTranslationEventHandler.kt} (62%) create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationEventHandler.kt similarity index 62% rename from backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt rename to backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationEventHandler.kt index f4f314721a..b27310d3c9 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationEventHandler.kt @@ -1,4 +1,4 @@ -package io.tolgee.component +package io.tolgee.component.autoTranslation import com.google.cloud.translate.Translation import io.tolgee.activity.data.ActivityType @@ -9,42 +9,30 @@ import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.key.Key import io.tolgee.service.project.ProjectService import io.tolgee.service.translation.AutoTranslationService -import io.tolgee.util.Logging import jakarta.persistence.EntityManager -import org.springframework.context.event.EventListener -import org.springframework.core.annotation.Order -import org.springframework.stereotype.Component - -@Component -class AutoTranslationListener( - private val autoTranslationService: AutoTranslationService, - private val projectService: ProjectService, - private val entityManager: EntityManager, -) : Logging { - - companion object { - /** - * import activities start visible batch job - */ - private val IMPORT_ACTIVITIES = listOf(ActivityType.IMPORT) - - /** - * Low volume activities start hidden batch job - */ - private val LOW_VOLUME_ACTIVITIES = - listOf(ActivityType.SET_TRANSLATIONS, ActivityType.CREATE_KEY, ActivityType.COMPLEX_EDIT) - } - - @Order(2) - @EventListener - fun onApplicationEvent(event: OnProjectActivityStoredEvent) { - val projectId = event.activityRevision.projectId ?: return - - if (!shouldRunTheOperation(event, projectId)) { +import org.springframework.context.ApplicationContext +import kotlin.properties.Delegates + +/** + * Utility class which handles auto translation. + * + * Separated from [AutoTranslationListener] to be able to cache some data in local + * variables without need to pass the params to so many methods. + */ +class AutoTranslationEventHandler( + private val event: OnProjectActivityStoredEvent, + private val applicationContext: ApplicationContext +) { + var projectId by Delegates.notNull() + + fun handle() { + projectId = event.activityRevision.projectId ?: return + + if (!shouldRunTheOperation()) { return } - val keyIds = getKeyIdsToAutoTranslate(projectId, event.activityRevision.modifiedEntities) + val keyIds = getKeyIdsToAutoTranslate() if (keyIds.isEmpty()) { return @@ -55,12 +43,13 @@ class AutoTranslationListener( projectId = projectId, keyIds = keyIds, isBatch = true, + baseLanguageId = baseLanguageId ?: return, isHiddenJob = event.isLowVolumeActivity() ) } } - private fun shouldRunTheOperation(event: OnProjectActivityStoredEvent, projectId: Long): Boolean { + private fun shouldRunTheOperation(): Boolean { val configs = autoTranslationService.getConfigs( entityManager.getReference(Project::class.java, projectId) ) @@ -84,17 +73,17 @@ class AutoTranslationListener( private fun OnProjectActivityStoredEvent.isLowVolumeActivity() = activityRevision.type in LOW_VOLUME_ACTIVITIES - fun getKeyIdsToAutoTranslate(projectId: Long, modifiedEntities: MutableList): List { - return modifiedEntities.mapNotNull { modifiedEntity -> - if (!modifiedEntity.isBaseTranslationTextChanged(projectId)) { + private fun getKeyIdsToAutoTranslate(): List { + return event.activityRevision.modifiedEntities.mapNotNull { modifiedEntity -> + if (!modifiedEntity.isBaseTranslationTextChanged()) { return@mapNotNull null } getKeyId(modifiedEntity) } } - private fun ActivityModifiedEntity.isBaseTranslationTextChanged(projectId: Long): Boolean { - return this.isTranslation() && this.isBaseTranslation(projectId) && this.isTextChanged() + private fun ActivityModifiedEntity.isBaseTranslationTextChanged(): Boolean { + return this.isTranslation() && this.isBaseTranslation() && this.isTextChanged() } private fun getKeyId(modifiedEntity: ActivityModifiedEntity) = @@ -110,11 +99,38 @@ class AutoTranslationListener( private fun ActivityModifiedEntity.isTranslation() = entityClass == Translation::class.simpleName - private fun ActivityModifiedEntity.isBaseTranslation(projectId: Long): Boolean { - val baseLanguageId = projectService.get(projectId).baseLanguage?.id ?: return false - + private fun ActivityModifiedEntity.isBaseTranslation(): Boolean { return describingRelations?.values ?.any { it.entityClass == Language::class.simpleName && it.entityId == baseLanguageId } ?: false } + + private val baseLanguageId: Long? by lazy { + projectService.get(projectId).baseLanguage?.id + } + + private val autoTranslationService: AutoTranslationService by lazy { + applicationContext.getBean(AutoTranslationService::class.java) + } + + private val projectService: ProjectService by lazy { + applicationContext.getBean(ProjectService::class.java) + } + + private val entityManager: EntityManager by lazy { + applicationContext.getBean(EntityManager::class.java) + } + + companion object { + /** + * import activities start visible batch job + */ + private val IMPORT_ACTIVITIES = listOf(ActivityType.IMPORT) + + /** + * Low volume activities start hidden batch job + */ + private val LOW_VOLUME_ACTIVITIES = + listOf(ActivityType.SET_TRANSLATIONS, ActivityType.CREATE_KEY, ActivityType.COMPLEX_EDIT) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt new file mode 100644 index 0000000000..1691dadf16 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt @@ -0,0 +1,29 @@ +package io.tolgee.component.autoTranslation + +import com.google.cloud.translate.Translation +import io.tolgee.activity.data.ActivityType +import io.tolgee.events.OnProjectActivityStoredEvent +import io.tolgee.model.Language +import io.tolgee.model.Project +import io.tolgee.model.activity.ActivityModifiedEntity +import io.tolgee.model.key.Key +import io.tolgee.service.project.ProjectService +import io.tolgee.service.translation.AutoTranslationService +import io.tolgee.util.Logging +import jakarta.persistence.EntityManager +import org.springframework.context.ApplicationContext +import org.springframework.context.event.EventListener +import org.springframework.core.annotation.Order +import org.springframework.stereotype.Component + +@Component +class AutoTranslationListener( + private val applicationContext: ApplicationContext +) : Logging { + + @Order(2) + @EventListener + fun onApplicationEvent(event: OnProjectActivityStoredEvent) { + AutoTranslationEventHandler(event, applicationContext).handle() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/AutoTranslationConfigRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/AutoTranslationConfigRepository.kt index ed23c8c3ad..0fcfda563a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/AutoTranslationConfigRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/AutoTranslationConfigRepository.kt @@ -9,6 +9,7 @@ import org.springframework.stereotype.Repository @Repository interface AutoTranslationConfigRepository : JpaRepository { fun findOneByProjectAndTargetLanguageId(project: Project, languageId: Long?): AutoTranslationConfig? + fun findByProjectAndTargetLanguageIdIn(project: Project, languageIds: List): List @Query( """ diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt index 31c02911c4..4fa2ce6bf5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt @@ -47,15 +47,20 @@ class AutoTranslationService( fun autoTranslateViaBatchJob( projectId: Long, keyIds: List, + baseLanguageId: Long, useTranslationMemory: Boolean? = null, useMachineTranslation: Boolean? = null, isBatch: Boolean, isHiddenJob: Boolean, ) { val project = entityManager.getReference(Project::class.java, projectId) - val languageIds = languageService.findAll(projectId).map { it.id } + val languageIds = languageService.findAll(projectId).map { it.id }.filter { it != baseLanguageId } + val configs = this.getConfigs(project, languageIds) val request = AutoTranslationRequest().apply { target = languageIds.flatMap { languageId -> + if(configs[languageId]?.usingTm == false && configs[languageId]?.usingPrimaryMtService == false) { + return@flatMap listOf() + } keyIds.map { keyId -> BatchTranslationTargetItem( keyId = keyId, @@ -288,11 +293,24 @@ class AutoTranslationService( return list!! } + + fun getConfigs(project: Project, targetLanguageIds: List): Map { + val configs = autoTranslationConfigRepository.findByProjectAndTargetLanguageIdIn(project, targetLanguageIds) + val default = autoTranslationConfigRepository.findDefaultForProject(project) + ?: autoTranslationConfigRepository.findDefaultForProject(project) ?: AutoTranslationConfig().also { + it.project = project + } + + return targetLanguageIds.associateWith { languageId -> + (configs.find { it.targetLanguage?.id == languageId } ?: default) + } + } + fun getConfig(project: Project, targetLanguageId: Long) = autoTranslationConfigRepository.findOneByProjectAndTargetLanguageId(project, targetLanguageId) ?: autoTranslationConfigRepository.findDefaultForProject(project) ?: AutoTranslationConfig().also { - it.project = project - } + it.project = project + } fun getDefaultConfig(project: Project) = autoTranslationConfigRepository.findOneByProjectAndTargetLanguageId(project, null) ?: AutoTranslationConfig() From d54e93de451f583a50dde51c964e646f936dfa9f Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Fri, 15 Dec 2023 21:56:15 +0100 Subject: [PATCH 07/26] fix: Import optimization --- .../iterceptor/InterceptedEventsManager.kt | 8 +- .../TagsPropChangesProvider.kt | 4 + .../component/ActivityHolderProvider.kt | 58 +++++++++ .../configuration/ActivityHolderConfig.kt | 17 +-- .../service/dataImport/ImportDeleteService.kt | 123 ++++++++++++++++++ .../service/dataImport/ImportService.kt | 14 +- .../io/tolgee/service/key/KeyMetaService.kt | 7 - 7 files changed, 204 insertions(+), 27 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDeleteService.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt index fb2b17d610..6062fd7c1f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt @@ -10,6 +10,7 @@ import io.tolgee.activity.data.EntityDescriptionWithRelations import io.tolgee.activity.data.PropertyModification import io.tolgee.activity.data.RevisionType import io.tolgee.activity.propChangesProvider.PropChangesProvider +import io.tolgee.component.ActivityHolderProvider import io.tolgee.dtos.cacheable.UserAccountDto import io.tolgee.events.OnProjectActivityEvent import io.tolgee.model.EntityWithId @@ -274,10 +275,13 @@ class InterceptedEventsManager( applicationContext.getBean(ActivityService::class.java) } - private val activityHolder: ActivityHolder by lazy { - applicationContext.getBean(ActivityHolder::class.java) + private val activityHolderProvider: ActivityHolderProvider by lazy { + applicationContext.getBean(ActivityHolderProvider::class.java) } + private val activityHolder + get() = activityHolderProvider.getActivityHolder() + private val userAccount: UserAccountDto? get() = authenticationFacade.authenticatedUserOrNull } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt b/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt index 792f50ec5f..601b399ca1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt @@ -9,6 +9,10 @@ class TagsPropChangesProvider : PropChangesProvider { override fun getChanges(old: Any?, new: Any?): PropertyModification? { if (old is Collection<*> && new is Collection<*>) { + if(old === new){ + return null + } + val oldTagNames = mapSetToTagNames(old) val newTagNames = mapSetToTagNames(new) if (oldTagNames.containsAll(newTagNames) && newTagNames.containsAll(oldTagNames)) { diff --git a/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt new file mode 100644 index 0000000000..05743cc520 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt @@ -0,0 +1,58 @@ +package io.tolgee.component + +import io.tolgee.activity.ActivityHolder +import jakarta.annotation.PreDestroy +import org.springframework.beans.factory.support.ScopeNotActiveException +import org.springframework.context.ApplicationContext +import org.springframework.stereotype.Component +import org.springframework.transaction.support.TransactionSynchronization +import org.springframework.transaction.support.TransactionSynchronizationManager + +/** + * Class providing Activity Holder, while caching it in ThreadLocal. + * I also registers a transaction synchronization, which clears the ThreadLocal after transaction + * is completed, this enables us to be safe even when running activities in main thred. + */ +@Component +class ActivityHolderProvider(private val applicationContext: ApplicationContext) { + private val threadLocal = ThreadLocal() + + fun getActivityHolder(): ActivityHolder { + // Get the activity holder from ThreadLocal. + var activityHolder = threadLocal.get() + if (activityHolder == null) { + // If no activity holder exists for this thread, fetch one and store it in ThreadLocal. + activityHolder = fetchActivityHolder() + threadLocal.set(activityHolder) + } + return activityHolder + } + + private fun fetchActivityHolder(): ActivityHolder { + return try { + applicationContext.getBean("requestActivityHolder", ActivityHolder::class.java).also { + it.activityRevision + } + } catch (e: ScopeNotActiveException) { + val transactionActivityHolder = + applicationContext.getBean("transactionActivityHolder", ActivityHolder::class.java) + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization( + object : TransactionSynchronization { + override fun afterCompletion(status: Int) { + clearThreadLocal() + } + } + ) + return transactionActivityHolder + } + + throw IllegalStateException("Transaction synchronization is not active.") + } + } + + @PreDestroy + fun clearThreadLocal() { + threadLocal.remove() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt index 645a146a36..c2e64910d2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt @@ -1,6 +1,7 @@ package io.tolgee.configuration import io.tolgee.activity.ActivityHolder +import io.tolgee.component.ActivityHolderProvider import io.tolgee.configuration.TransactionScopeConfig.Companion.SCOPE_TRANSACTION import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.config.BeanDefinition @@ -31,16 +32,16 @@ class ActivityHolderConfig { return ActivityHolder(applicationContext) } + /** + * This method for getting the activity holder is slow, since it + * needs to create new bean every time holder is requested. Which is pretty often. + * + * Use the activityHolderProvider when possible. + */ @Bean @Primary @Scope(BeanDefinition.SCOPE_PROTOTYPE, proxyMode = ScopedProxyMode.TARGET_CLASS) - fun activityHolder(applicationContext: ApplicationContext): ActivityHolder { - return try { - applicationContext.getBean("requestActivityHolder", ActivityHolder::class.java).also { - it.activityRevision - } - } catch (e: ScopeNotActiveException) { - return applicationContext.getBean("transactionActivityHolder", ActivityHolder::class.java) - } + fun activityHolder(activityHolderProvider: ActivityHolderProvider): ActivityHolder { + return activityHolderProvider.getActivityHolder() } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDeleteService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDeleteService.kt new file mode 100644 index 0000000000..29fc7f5646 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDeleteService.kt @@ -0,0 +1,123 @@ +package io.tolgee.service.dataImport + +import jakarta.persistence.EntityManager +import org.hibernate.Session +import org.springframework.stereotype.Service + +@Service +class ImportDeleteService( + private val entityManager: EntityManager +) { + fun deleteImport(importId: Long) { + entityManager.unwrap(Session::class.java).doWork { connection -> + deleteImportTranslations(connection, importId) + deleteImportLanguages(connection, importId) + deleteImportKeyMetaTags(connection, importId) + deleteImportKeyMetaComments(connection, importId) + deleteImportKeyMetaCodeReferences(connection, importId) + deleteImportKeyMeta(connection, importId) + deleteImportKeys(connection, importId) + deleteImportFileIssueParams(connection, importId) + deleteImportFileIssues(connection, importId) + deleteImportFiles(connection, importId) + deleteTheImport(connection, importId) + } + } + + fun executeUpdate(connection: java.sql.Connection, query: String, importId: Long) { + @Suppress("SqlSourceToSinkFlow") + connection.prepareStatement(query).use { statement -> + statement.setLong(1, importId) + statement.executeUpdate() + } + } + + fun deleteImportTranslations(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_translation " + + "where id in (" + + "select it.id from import_translation it " + + "join import_language il on il.id = it.language_id " + + "join import_file if on il.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportLanguages(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_language " + + "where id in (select il.id from import_language il " + + "join import_file if on il.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeys(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_key " + + "where id in (select ik.id from import_key ik " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeyMetaTags(connection: java.sql.Connection, importId: Long) { + val query = + "delete from key_meta_tags " + + "where key_metas_id in (select ikm.id from key_meta ikm " + + "join import_key ik on ikm.import_key_id = ik.id " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeyMetaComments(connection: java.sql.Connection, importId: Long) { + val query = + "delete from key_comment " + + "where key_meta_id in (select ikm.id from key_meta ikm " + + "join import_key ik on ikm.import_key_id = ik.id " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeyMetaCodeReferences(connection: java.sql.Connection, importId: Long) { + val query = + "delete from key_code_reference " + + "where key_meta_id in (select ikm.id from key_meta ikm " + + "join import_key ik on ikm.import_key_id = ik.id " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportKeyMeta(connection: java.sql.Connection, importId: Long) { + val query = + "delete from key_meta " + + "where id in (select ikm.id from key_meta ikm " + + "join import_key ik on ikm.import_key_id = ik.id " + + "join import_file if on ik.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportFileIssueParams(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_file_issue_param " + + "where import_file_issue_param.issue_id in (select ifi.id from import_file_issue ifi " + + "join import_file if on ifi.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportFileIssues(connection: java.sql.Connection, importId: Long) { + val query = + "delete from import_file_issue " + + "where id in (select ifi.id from import_file_issue ifi " + + "join import_file if on ifi.file_id = if.id where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteImportFiles(connection: java.sql.Connection, importId: Long) { + val query = "delete from import_file " + + "where id in (select if.id from import_file if where if.import_id = ?)" + executeUpdate(connection, query, importId) + } + + fun deleteTheImport(connection: java.sql.Connection, importId: Long) { + val query = "delete from import where id = ?" + executeUpdate(connection, query, importId) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index 2e7ae1a23b..80703eb59e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -53,7 +53,8 @@ class ImportService( private val keyMetaService: KeyMetaService, private val removeExpiredImportService: RemoveExpiredImportService, private val entityManager: EntityManager, - private val businessEventPublisher: BusinessEventPublisher + private val businessEventPublisher: BusinessEventPublisher, + private val importDeleteService: ImportDeleteService ) { @Transactional fun addFiles( @@ -251,16 +252,9 @@ class ImportService( ) } + @Transactional fun deleteImport(import: Import) { - this.importTranslationRepository.deleteAllByImport(import) - this.importLanguageRepository.deleteAllByImport(import) - val keyIds = this.importKeyRepository.getAllIdsByImport(import) - this.keyMetaService.deleteAllByImportKeyIdIn(keyIds) - this.importKeyRepository.deleteByIdIn(keyIds) - this.importFileIssueParamRepository.deleteAllByImport(import) - this.importFileIssueRepository.deleteAllByImport(import) - this.importFileRepository.deleteAllByImport(import) - this.importRepository.delete(import) + importDeleteService.deleteImport(import.id) } @Transactional diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyMetaService.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyMetaService.kt index 224fd08dae..44ddd5b564 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyMetaService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyMetaService.kt @@ -121,13 +121,6 @@ class KeyMetaService( fun save(meta: KeyMeta): KeyMeta = this.keyMetaRepository.save(meta) - fun deleteAllByImportKeyIdIn(importKeyIds: List) { - tagService.deleteAllByImportKeyIdIn(importKeyIds) - keyCommentRepository.deleteAllByImportKeyIds(importKeyIds) - keyCodeReferenceRepository.deleteAllByImportKeyIds(importKeyIds) - this.keyMetaRepository.deleteAllByImportKeyIdIn(importKeyIds) - } - fun deleteAllByKeyIdIn(ids: Collection) { tagService.deleteAllByKeyIdIn(ids) keyCommentRepository.deleteAllByKeyIds(ids) From 620dabf478536725df642d9bd981efd354367b6a Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Fri, 15 Dec 2023 22:47:55 +0100 Subject: [PATCH 08/26] fix: ActivityHolder, log batch --- .../batch/StartBatchJobControllerTest.kt | 37 ++++++++++++++----- .../io/tolgee/activity/ActivityHolder.kt | 8 ++++ .../component/ActivityHolderProvider.kt | 17 ++------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt index b69d0301d1..59dc2e37e2 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt @@ -18,6 +18,7 @@ import io.tolgee.model.translation.Translation import io.tolgee.testing.ContextRecreatingTest import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert +import io.tolgee.util.BatchDumper import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -38,6 +39,9 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Autowired lateinit var batchJobService: BatchJobService + @Autowired + lateinit var batchDumper: BatchDumper + @BeforeEach fun setup() { batchJobOperationQueue.clear() @@ -377,17 +381,30 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { val allKeyIds = keys.map { it.id }.toList() val keyIds = allKeyIds.take(700) - performProjectAuthPost( + var batchJobId = 0L + + val res = performProjectAuthPost( "start-batch-job/set-keys-namespace", mapOf( "keyIds" to keyIds, "namespace" to "other-namespace" ) - ).andIsOk.waitForJobCompleted() + ).andIsOk.andAssertThatJson { + node("id").isNumber.satisfies { id -> + batchJobId = id.toLong() + } + + } - val all = keyService.find(keyIds) - all.count { it.namespace?.name == "other-namespace" }.assert.isEqualTo(keyIds.size) - namespaceService.find(testData.projectBuilder.self.id, "namespace1").assert.isNull() + try { + res.waitForJobCompleted() + val all = keyService.find(keyIds) + all.count { it.namespace?.name == "other-namespace" }.assert.isEqualTo(keyIds.size) + namespaceService.find(testData.projectBuilder.self.id, "namespace1").assert.isNull() + } catch (e: Throwable) { + batchDumper.dump(batchJobId) + throw e + } } @Test @@ -410,10 +427,12 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { fun ResultActions.waitForJobCompleted() = andAssertThatJson { node("id").isNumber.satisfies( - Consumer { - waitFor(pollTime = 2000) { - val job = batchJobService.findJobDto(it.toLong()) - job?.status?.completed == true + Consumer { id -> + waitFor(pollTime = 200, timeout = 120000) { + executeInNewTransaction { + val job = batchJobService.findJobDto(id.toLong()) + job?.status?.completed == true + } } } ) diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt index e5874a760a..b0d6ebd819 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt @@ -5,6 +5,7 @@ import io.tolgee.activity.iterceptor.InterceptedEventsManager import io.tolgee.model.EntityWithId import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.activity.ActivityRevision +import jakarta.annotation.PreDestroy import org.springframework.context.ApplicationContext import kotlin.reflect.KClass @@ -44,6 +45,13 @@ open class ActivityHolder(val applicationContext: ApplicationContext) { this.applicationContext.getBean(InterceptedEventsManager::class.java).initActivityHolder() field = value } + + var destroyer: (() -> Unit)? = null + + @PreDestroy + fun destroy() { + destroyer?.invoke() + } } typealias ModifiedEntitiesType = MutableMap, MutableMap> diff --git a/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt index 05743cc520..904037cd40 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt @@ -34,20 +34,9 @@ class ActivityHolderProvider(private val applicationContext: ApplicationContext) it.activityRevision } } catch (e: ScopeNotActiveException) { - val transactionActivityHolder = - applicationContext.getBean("transactionActivityHolder", ActivityHolder::class.java) - if (TransactionSynchronizationManager.isSynchronizationActive()) { - TransactionSynchronizationManager.registerSynchronization( - object : TransactionSynchronization { - override fun afterCompletion(status: Int) { - clearThreadLocal() - } - } - ) - return transactionActivityHolder - } - - throw IllegalStateException("Transaction synchronization is not active.") + applicationContext.getBean("transactionActivityHolder", ActivityHolder::class.java) + }.also { + it.destroyer = this::clearThreadLocal } } From 2d067e4afc4c826e85c1a83a9641107660265561 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sat, 16 Dec 2023 09:14:36 +0100 Subject: [PATCH 09/26] Revert "fix: Try to remove enhancement config" This reverts commit afacd4f5449ec48ae57749a3ccbbd505c21b4205. --- backend/app/src/main/resources/application.yaml | 3 +++ backend/app/src/test/resources/application.yaml | 3 +++ backend/data/build.gradle | 7 +++++++ ee/backend/app/build.gradle | 7 +++++++ 4 files changed, 20 insertions(+) diff --git a/backend/app/src/main/resources/application.yaml b/backend/app/src/main/resources/application.yaml index 1c636f1817..280c38e0ba 100644 --- a/backend/app/src/main/resources/application.yaml +++ b/backend/app/src/main/resources/application.yaml @@ -22,6 +22,9 @@ spring: types: print: banner: false + enhancer: + enableLazyInitialization: true + enableDirtyTracking: true # open-in-view: false batch: job: diff --git a/backend/app/src/test/resources/application.yaml b/backend/app/src/test/resources/application.yaml index fe6235cb02..4404906561 100644 --- a/backend/app/src/test/resources/application.yaml +++ b/backend/app/src/test/resources/application.yaml @@ -14,6 +14,9 @@ spring: order_inserts: true order_updates: true dialect: io.tolgee.dialects.postgres.CustomPostgreSQLDialect + enhancer: + enableLazyInitialization: true + enableDirtyTracking: true mvc: pathmatch: matching-strategy: ant_path_matcher diff --git a/backend/data/build.gradle b/backend/data/build.gradle index abf05bf927..93828b32a8 100644 --- a/backend/data/build.gradle +++ b/backend/data/build.gradle @@ -82,6 +82,13 @@ kotlin { jvmToolchain(17) } +hibernate { + enhancement { + lazyInitialization = true + dirtyTracking = true + } +} + dependencies { /** * SPRING diff --git a/ee/backend/app/build.gradle b/ee/backend/app/build.gradle index 1f9c6881a7..9745b2be2b 100644 --- a/ee/backend/app/build.gradle +++ b/ee/backend/app/build.gradle @@ -89,6 +89,13 @@ kotlin { jvmToolchain(17) } +hibernate { + enhancement { + lazyInitialization = true + dirtyTracking = true + } +} + dependencyManagement { imports { mavenBom org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES From df332368dc187506187469ce069282216bdba0cf Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sat, 16 Dec 2023 10:00:43 +0100 Subject: [PATCH 10/26] chore: Remove build-time enhancement config from ee module, add readable test reports --- .github/workflows/test.yml | 9 +++++ .../batch/StartBatchJobControllerTest.kt | 37 +++++-------------- ee/backend/app/build.gradle | 7 ---- 3 files changed, 18 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 072a931f3b..416755a936 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -139,6 +139,15 @@ jobs: path: | ./**/build/reports/**/* + - name: Test Report + uses: dorny/test-reporter@v1 + if: always() + with: + name: Backend Tests + path: "**/build/test-results/TEST-*.xml" + reporter: java-junit + + e2e: needs: [frontend-build, backend-build, e2e-install-deps] runs-on: ubuntu-latest diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt index 59dc2e37e2..b69d0301d1 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt @@ -18,7 +18,6 @@ import io.tolgee.model.translation.Translation import io.tolgee.testing.ContextRecreatingTest import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import io.tolgee.util.BatchDumper import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -39,9 +38,6 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Autowired lateinit var batchJobService: BatchJobService - @Autowired - lateinit var batchDumper: BatchDumper - @BeforeEach fun setup() { batchJobOperationQueue.clear() @@ -381,30 +377,17 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { val allKeyIds = keys.map { it.id }.toList() val keyIds = allKeyIds.take(700) - var batchJobId = 0L - - val res = performProjectAuthPost( + performProjectAuthPost( "start-batch-job/set-keys-namespace", mapOf( "keyIds" to keyIds, "namespace" to "other-namespace" ) - ).andIsOk.andAssertThatJson { - node("id").isNumber.satisfies { id -> - batchJobId = id.toLong() - } - - } + ).andIsOk.waitForJobCompleted() - try { - res.waitForJobCompleted() - val all = keyService.find(keyIds) - all.count { it.namespace?.name == "other-namespace" }.assert.isEqualTo(keyIds.size) - namespaceService.find(testData.projectBuilder.self.id, "namespace1").assert.isNull() - } catch (e: Throwable) { - batchDumper.dump(batchJobId) - throw e - } + val all = keyService.find(keyIds) + all.count { it.namespace?.name == "other-namespace" }.assert.isEqualTo(keyIds.size) + namespaceService.find(testData.projectBuilder.self.id, "namespace1").assert.isNull() } @Test @@ -427,12 +410,10 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { fun ResultActions.waitForJobCompleted() = andAssertThatJson { node("id").isNumber.satisfies( - Consumer { id -> - waitFor(pollTime = 200, timeout = 120000) { - executeInNewTransaction { - val job = batchJobService.findJobDto(id.toLong()) - job?.status?.completed == true - } + Consumer { + waitFor(pollTime = 2000) { + val job = batchJobService.findJobDto(it.toLong()) + job?.status?.completed == true } } ) diff --git a/ee/backend/app/build.gradle b/ee/backend/app/build.gradle index 9745b2be2b..1f9c6881a7 100644 --- a/ee/backend/app/build.gradle +++ b/ee/backend/app/build.gradle @@ -89,13 +89,6 @@ kotlin { jvmToolchain(17) } -hibernate { - enhancement { - lazyInitialization = true - dirtyTracking = true - } -} - dependencyManagement { imports { mavenBom org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES From e33c68a96385e5898b00a8bae548f8f12558a224 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sat, 16 Dec 2023 11:11:18 +0100 Subject: [PATCH 11/26] fix: Import async soft delete, test reports --- .github/workflows/test.yml | 2 +- .../service/dataImport/ImportServiceTest.kt | 38 +++++++++++++++++-- .../iterceptor/InterceptedEventsManager.kt | 1 - .../TagsPropChangesProvider.kt | 2 +- .../component/ActivityHolderProvider.kt | 2 - .../AutoTranslationListener.kt | 9 ----- .../configuration/ActivityHolderConfig.kt | 1 - .../io/tolgee/events/OnImportSoftDeleted.kt | 5 +++ .../io/tolgee/model/dataImport/Import.kt | 6 ++- .../repository/dataImport/ImportRepository.kt | 18 +++++++++ .../dataImport/AsyncImportHardDeleter.kt | 19 ++++++++++ .../service/dataImport/ImportService.kt | 18 ++++++++- .../tolgee/service/project/ProjectService.kt | 2 +- .../translation/AutoTranslationService.kt | 11 +++--- .../main/resources/db/changelog/schema.xml | 9 +++++ 15 files changed, 115 insertions(+), 28 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/events/OnImportSoftDeleted.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/dataImport/AsyncImportHardDeleter.kt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 416755a936..bd10db573f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,7 +144,7 @@ jobs: if: always() with: name: Backend Tests - path: "**/build/test-results/TEST-*.xml" + path: "**/build/test-results/**/TEST-*.xml" reporter: java-junit diff --git a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/ImportServiceTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/ImportServiceTest.kt index 67a86a9233..8fd91800cc 100644 --- a/backend/app/src/test/kotlin/io/tolgee/service/dataImport/ImportServiceTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/service/dataImport/ImportServiceTest.kt @@ -4,6 +4,9 @@ import io.tolgee.AbstractSpringTest import io.tolgee.development.testDataBuilder.data.dataImport.ImportNamespacesTestData import io.tolgee.development.testDataBuilder.data.dataImport.ImportTestData import io.tolgee.dtos.cacheable.UserAccountDto +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.dataImport.Import +import io.tolgee.model.dataImport.ImportTranslation import io.tolgee.security.authentication.TolgeeAuthentication import io.tolgee.security.authentication.TolgeeAuthenticationDetails import io.tolgee.testing.assert @@ -37,11 +40,13 @@ class ImportServiceTest : AbstractSpringTest() { importService.selectExistingLanguage(importFrench, french) assertThat(importFrench.existingLanguage).isEqualTo(french) val translations = importService.findTranslations(importTestData.importFrench.id) - assertThat(translations[0].conflict).isNotNull - assertThat(translations[1].conflict).isNull() + assertThat(translations.getByKey("what a key").conflict).isNotNull + assertThat(translations.getByKey("what a beautiful key").conflict).isNull() } } + fun Collection.getByKey(key: String) = this.find { it.key.name == key } ?: error("Key not found") + @Test fun `deletes import language`() { val testData = executeInNewTransaction { @@ -63,7 +68,30 @@ class ImportServiceTest : AbstractSpringTest() { } @Test - fun `deletes import`() { + fun `hard deletes import`() { + val testData = ImportTestData() + executeInNewTransaction { + testData.addFileIssues() + testData.addKeyMetadata() + testDataService.saveTestData(testData.root) + } + + executeInNewTransaction { + val import = importService.get(testData.import.id) + importService.hardDeleteImport(import) + } + } + + private fun checkImportHardDeleted(id: Long) { + executeInNewTransaction { + entityManager.createQuery("from Import i where i.id = :id", Import::class.java) + .setParameter("id", id) + .resultList.firstOrNull().assert.isNull() + } + } + + @Test + fun `soft deletes import`() { val testData = ImportTestData() executeInNewTransaction { testData.addFileIssues() @@ -79,6 +107,10 @@ class ImportServiceTest : AbstractSpringTest() { executeInNewTransaction { assertThat(importService.find(testData.import.project.id, testData.import.author.id)).isNull() } + + waitForNotThrowing(pollTime = 200, timeout = 10000) { + checkImportHardDeleted(testData.import.id) + } } @Test diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt index 6062fd7c1f..2ac5ba67e4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt @@ -1,6 +1,5 @@ package io.tolgee.activity.iterceptor -import io.tolgee.activity.ActivityHolder import io.tolgee.activity.ActivityService import io.tolgee.activity.EntityDescriptionProvider import io.tolgee.activity.annotation.ActivityLoggedEntity diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt b/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt index 601b399ca1..96b921aee4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/propChangesProvider/TagsPropChangesProvider.kt @@ -9,7 +9,7 @@ class TagsPropChangesProvider : PropChangesProvider { override fun getChanges(old: Any?, new: Any?): PropertyModification? { if (old is Collection<*> && new is Collection<*>) { - if(old === new){ + if (old === new) { return null } diff --git a/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt index 904037cd40..4f3f35782d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt @@ -5,8 +5,6 @@ import jakarta.annotation.PreDestroy import org.springframework.beans.factory.support.ScopeNotActiveException import org.springframework.context.ApplicationContext import org.springframework.stereotype.Component -import org.springframework.transaction.support.TransactionSynchronization -import org.springframework.transaction.support.TransactionSynchronizationManager /** * Class providing Activity Holder, while caching it in ThreadLocal. diff --git a/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt index 1691dadf16..5f36506ca3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/autoTranslation/AutoTranslationListener.kt @@ -1,16 +1,7 @@ package io.tolgee.component.autoTranslation -import com.google.cloud.translate.Translation -import io.tolgee.activity.data.ActivityType import io.tolgee.events.OnProjectActivityStoredEvent -import io.tolgee.model.Language -import io.tolgee.model.Project -import io.tolgee.model.activity.ActivityModifiedEntity -import io.tolgee.model.key.Key -import io.tolgee.service.project.ProjectService -import io.tolgee.service.translation.AutoTranslationService import io.tolgee.util.Logging -import jakarta.persistence.EntityManager import org.springframework.context.ApplicationContext import org.springframework.context.event.EventListener import org.springframework.core.annotation.Order diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt index c2e64910d2..0890fb07b9 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt @@ -5,7 +5,6 @@ import io.tolgee.component.ActivityHolderProvider import io.tolgee.configuration.TransactionScopeConfig.Companion.SCOPE_TRANSACTION import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.config.BeanDefinition -import org.springframework.beans.factory.support.ScopeNotActiveException import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Bean diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnImportSoftDeleted.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnImportSoftDeleted.kt new file mode 100644 index 0000000000..4453642383 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/events/OnImportSoftDeleted.kt @@ -0,0 +1,5 @@ +package io.tolgee.events + +class OnImportSoftDeleted( + val importId: Long +) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/dataImport/Import.kt b/backend/data/src/main/kotlin/io/tolgee/model/dataImport/Import.kt index 0f7e3b4aa3..99328ffad3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/dataImport/Import.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/dataImport/Import.kt @@ -1,6 +1,7 @@ package io.tolgee.model.dataImport import io.tolgee.model.Project +import io.tolgee.model.SoftDeletable import io.tolgee.model.StandardAuditModel import io.tolgee.model.UserAccount import jakarta.persistence.Entity @@ -9,6 +10,7 @@ import jakarta.persistence.OneToMany import jakarta.persistence.Table import jakarta.persistence.UniqueConstraint import jakarta.validation.constraints.NotNull +import java.util.* @Entity @Table(uniqueConstraints = [UniqueConstraint(columnNames = ["author_id", "project_id"])]) @@ -16,7 +18,7 @@ class Import( @field:NotNull @ManyToOne(optional = false) val project: Project -) : StandardAuditModel() { +) : StandardAuditModel(), SoftDeletable { @field:NotNull @ManyToOne(optional = false) @@ -24,4 +26,6 @@ class Import( @OneToMany(mappedBy = "import", orphanRemoval = true) var files = mutableListOf() + + override var deletedAt: Date? = null } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportRepository.kt index 2ae5a134d5..86cd65fc3d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportRepository.kt @@ -7,8 +7,19 @@ import org.springframework.stereotype.Repository @Repository interface ImportRepository : JpaRepository { + + @Query( + """ + select i from Import i where i.project.id = :projectId and i.author.id = :authorId and i.deletedAt is null + """ + ) fun findByProjectIdAndAuthorId(projectId: Long, authorId: Long): Import? + @Query( + """ + select i from Import i where i.project.id = :projectId and i.deletedAt is null + """ + ) fun findAllByProjectId(projectId: Long): List @Query( @@ -17,4 +28,11 @@ interface ImportRepository : JpaRepository { """ ) fun getAllNamespaces(importId: Long): Set + + @Query( + """ + from Import i where i.id = :importId and i.deletedAt is not null + """ + ) + fun findDeleted(importId: Long): Import? } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/AsyncImportHardDeleter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/AsyncImportHardDeleter.kt new file mode 100644 index 0000000000..0608b41c78 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/AsyncImportHardDeleter.kt @@ -0,0 +1,19 @@ +package io.tolgee.service.dataImport + +import io.tolgee.events.OnImportSoftDeleted +import org.springframework.scheduling.annotation.Async +import org.springframework.stereotype.Component +import org.springframework.transaction.event.TransactionalEventListener + +@Component +class AsyncImportHardDeleter( + private val importService: ImportService +) { + @Async + @TransactionalEventListener + fun hardDelete(event: OnImportSoftDeleted) { + importService.findDeleted(importId = event.importId)?.let { + importService.hardDeleteImport(it) + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index 80703eb59e..e1d253cd30 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -1,10 +1,12 @@ package io.tolgee.service.dataImport +import io.tolgee.component.CurrentDateProvider import io.tolgee.component.reporting.BusinessEventPublisher import io.tolgee.component.reporting.OnBusinessEventToCaptureEvent import io.tolgee.constants.Message import io.tolgee.dtos.dataImport.ImportAddFilesParams import io.tolgee.dtos.dataImport.ImportFileDto +import io.tolgee.events.OnImportSoftDeleted import io.tolgee.exceptions.BadRequestException import io.tolgee.exceptions.ErrorResponseBody import io.tolgee.exceptions.ImportConflictNotResolvedException @@ -54,7 +56,8 @@ class ImportService( private val removeExpiredImportService: RemoveExpiredImportService, private val entityManager: EntityManager, private val businessEventPublisher: BusinessEventPublisher, - private val importDeleteService: ImportDeleteService + private val importDeleteService: ImportDeleteService, + private val currentDateProvider: CurrentDateProvider ) { @Transactional fun addFiles( @@ -151,6 +154,10 @@ class ImportService( return findNotExpired(projectId, authorId) ?: throw NotFoundException() } + fun findDeleted(importId: Long): Import? { + return importRepository.findDeleted(importId) + } + private fun findNotExpired(projectId: Long, userAccountId: Long): Import? { val import = this.find(projectId, userAccountId) return removeExpiredImportService.removeIfExpired(import) @@ -254,6 +261,13 @@ class ImportService( @Transactional fun deleteImport(import: Import) { + import.deletedAt = currentDateProvider.date + importRepository.save(import) + applicationContext.publishEvent(OnImportSoftDeleted(import.id)) + } + + @Transactional + fun hardDeleteImport(import: Import) { importDeleteService.deleteImport(import.id) } @@ -267,7 +281,7 @@ class ImportService( this.importTranslationRepository.deleteAllByLanguage(language) this.importLanguageRepository.delete(language) if (this.findLanguages(import = language.file.import).isEmpty()) { - deleteImport(import) + hardDeleteImport(import) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt b/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt index 507b487f71..6147040074 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt @@ -245,7 +245,7 @@ class ProjectService( } importService.getAllByProject(id).forEach { - importService.deleteImport(it) + importService.hardDeleteImport(it) } // otherwise we cannot delete the languages diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt index 4fa2ce6bf5..20c53b181c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt @@ -58,7 +58,7 @@ class AutoTranslationService( val configs = this.getConfigs(project, languageIds) val request = AutoTranslationRequest().apply { target = languageIds.flatMap { languageId -> - if(configs[languageId]?.usingTm == false && configs[languageId]?.usingPrimaryMtService == false) { + if (configs[languageId]?.usingTm == false && configs[languageId]?.usingPrimaryMtService == false) { return@flatMap listOf() } keyIds.map { keyId -> @@ -293,13 +293,12 @@ class AutoTranslationService( return list!! } - fun getConfigs(project: Project, targetLanguageIds: List): Map { val configs = autoTranslationConfigRepository.findByProjectAndTargetLanguageIdIn(project, targetLanguageIds) val default = autoTranslationConfigRepository.findDefaultForProject(project) ?: autoTranslationConfigRepository.findDefaultForProject(project) ?: AutoTranslationConfig().also { - it.project = project - } + it.project = project + } return targetLanguageIds.associateWith { languageId -> (configs.find { it.targetLanguage?.id == languageId } ?: default) @@ -309,8 +308,8 @@ class AutoTranslationService( fun getConfig(project: Project, targetLanguageId: Long) = autoTranslationConfigRepository.findOneByProjectAndTargetLanguageId(project, targetLanguageId) ?: autoTranslationConfigRepository.findDefaultForProject(project) ?: AutoTranslationConfig().also { - it.project = project - } + it.project = project + } fun getDefaultConfig(project: Project) = autoTranslationConfigRepository.findOneByProjectAndTargetLanguageId(project, null) ?: AutoTranslationConfig() diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index 593c903964..a38b77e9de 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -2949,4 +2949,13 @@ CREATE INDEX organization_deleted_at_null ON organization((deleted_at IS NULL)); + + + + + + + CREATE INDEX import_deleted_at_null ON import((deleted_at IS NULL)); + + From 9a1017d5d8d751ae229d203aac8ce6aafd731022 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sat, 16 Dec 2023 12:08:54 +0100 Subject: [PATCH 12/26] fix: Return new activity holder when scope changes --- .../component/ActivityHolderProvider.kt | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt index 4f3f35782d..0372d35d95 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/ActivityHolderProvider.kt @@ -5,6 +5,7 @@ import jakarta.annotation.PreDestroy import org.springframework.beans.factory.support.ScopeNotActiveException import org.springframework.context.ApplicationContext import org.springframework.stereotype.Component +import org.springframework.web.context.request.RequestContextHolder /** * Class providing Activity Holder, while caching it in ThreadLocal. @@ -13,15 +14,23 @@ import org.springframework.stereotype.Component */ @Component class ActivityHolderProvider(private val applicationContext: ApplicationContext) { - private val threadLocal = ThreadLocal() + private val threadLocal = ThreadLocal>() fun getActivityHolder(): ActivityHolder { // Get the activity holder from ThreadLocal. - var activityHolder = threadLocal.get() + var (scope, activityHolder) = threadLocal.get() ?: (null to null) + + val currentScope = getCurrentScope() + if (scope != currentScope) { + // If the scope has changed, clear the ThreadLocal and fetch a new activity holder. + clearThreadLocal() + activityHolder = null + } + if (activityHolder == null) { // If no activity holder exists for this thread, fetch one and store it in ThreadLocal. activityHolder = fetchActivityHolder() - threadLocal.set(activityHolder) + threadLocal.set(currentScope to activityHolder) } return activityHolder } @@ -38,8 +47,19 @@ class ActivityHolderProvider(private val applicationContext: ApplicationContext) } } + private fun getCurrentScope(): Scope { + if (RequestContextHolder.getRequestAttributes() == null) { + return Scope.TRANSACTION + } + return Scope.REQUEST + } + @PreDestroy fun clearThreadLocal() { threadLocal.remove() } + + enum class Scope { + REQUEST, TRANSACTION + } } From 3fd03670040534d8e9aa651c5dcb4aa896acfe0a Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sat, 16 Dec 2023 16:35:07 +0100 Subject: [PATCH 13/26] fix: Clear and flush more often to keep the persistent context small --- .../kotlin/io/tolgee/activity/ActivityService.kt | 10 +++++++++- .../iterceptor/ActivityDatabaseInterceptor.kt | 11 ++++++----- .../activity/iterceptor/InterceptedEventsManager.kt | 2 ++ .../main/kotlin/io/tolgee/batch/BatchJobService.kt | 10 ++++++---- .../io/tolgee/model/activity/ActivityRevision.kt | 4 ++-- .../io/tolgee/service/dataImport/ImportService.kt | 1 - .../main/kotlin/io/tolgee/util/entityManagerExt.kt | 12 ++++++++++++ 7 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt index 1d1b85a040..7bb2263bd0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt @@ -9,6 +9,7 @@ import io.tolgee.model.activity.ActivityRevision import io.tolgee.model.views.activity.ProjectActivityView import io.tolgee.repository.activity.ActivityModifiedEntityRepository import io.tolgee.util.Logging +import io.tolgee.util.flushAndClear import io.tolgee.util.logger import jakarta.persistence.EntityExistsException import jakarta.persistence.EntityManager @@ -26,15 +27,22 @@ class ActivityService( ) : Logging { @Transactional fun storeActivityData(activityRevision: ActivityRevision, modifiedEntities: ModifiedEntitiesType) { + // let's keep the persistent context small + entityManager.flushAndClear() + val mergedActivityRevision = activityRevision.persist() mergedActivityRevision.persistedDescribingRelations() + entityManager.flushAndClear() + mergedActivityRevision.modifiedEntities = modifiedEntities.values.flatMap { it.values }.toMutableList() mergedActivityRevision.persistModifiedEntities() - entityManager.flush() + entityManager.flushAndClear() applicationContext.publishEvent(OnProjectActivityStoredEvent(this, mergedActivityRevision)) + + entityManager.flushAndClear() } private fun ActivityRevision.persistModifiedEntities() { diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityDatabaseInterceptor.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityDatabaseInterceptor.kt index 9921c7cb3e..a6df5ebe64 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityDatabaseInterceptor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityDatabaseInterceptor.kt @@ -77,9 +77,10 @@ class ActivityDatabaseInterceptor : Interceptor, Logging { interceptedEventsManager.onCollectionModification(collection, key) } - val interceptedEventsManager: InterceptedEventsManager - get() = applicationContext.getBean(InterceptedEventsManager::class.java) - - val preCommitEventsPublisher: PreCommitEventPublisher - get() = applicationContext.getBean(PreCommitEventPublisher::class.java) + val interceptedEventsManager: InterceptedEventsManager by lazy { + applicationContext.getBean(InterceptedEventsManager::class.java) + } + val preCommitEventsPublisher: PreCommitEventPublisher by lazy { + applicationContext.getBean(PreCommitEventPublisher::class.java) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt index 2ac5ba67e4..37b0c90867 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt @@ -235,6 +235,8 @@ class InterceptedEventsManager( logger.debug("Publishing project activity event") try { publishOnActivityEvent(activityRevision) + entityManager.flush() + entityManager.clear() activityService.storeActivityData(activityRevision, activityHolder.modifiedEntities) activityHolder.afterActivityFlushed?.invoke() } catch (e: Throwable) { diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt index 6ba8ab59a9..0bc0e07120 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt @@ -22,6 +22,7 @@ import io.tolgee.security.authentication.AuthenticationFacade import io.tolgee.service.security.SecurityService import io.tolgee.util.Logging import io.tolgee.util.addMinutes +import io.tolgee.util.flushAndClear import io.tolgee.util.logger import jakarta.persistence.EntityManager import org.apache.commons.codec.digest.DigestUtils.sha256Hex @@ -103,7 +104,8 @@ class BatchJobService( } } - entityManager.flush() + entityManager.flushAndClear() + applicationContext.publishEvent(OnBatchJobCreated(job, executions)) return job @@ -139,9 +141,9 @@ class BatchJobService( executions.let { batchJobChunkExecutionQueue.addToQueue(it) } logger.debug( "Starting job ${job.id}, aadded ${executions.size} executions to queue ${ - System.identityHashCode( - batchJobChunkExecutionQueue - ) + System.identityHashCode( + batchJobChunkExecutionQueue + ) }" ) } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt index f518c29e7d..ceeb92ef79 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt @@ -72,8 +72,8 @@ class ActivityRevision : java.io.Serializable { */ var projectId: Long? = null - @OneToMany(mappedBy = "activityRevision") - var describingRelations: MutableList = mutableListOf() + @OneToMany(mappedBy = "activityRevision", targetEntity = ActivityDescribingEntity::class) + var describingRelations: DescribingRelationsList = DescribingRelationsList() @OneToMany(mappedBy = "activityRevision") var modifiedEntities: MutableList = mutableListOf() diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index e1d253cd30..1850d4062e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -52,7 +52,6 @@ class ImportService( private val applicationContext: ApplicationContext, private val importTranslationRepository: ImportTranslationRepository, private val importFileIssueParamRepository: ImportFileIssueParamRepository, - private val keyMetaService: KeyMetaService, private val removeExpiredImportService: RemoveExpiredImportService, private val entityManager: EntityManager, private val businessEventPublisher: BusinessEventPublisher, diff --git a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt new file mode 100644 index 0000000000..bc55bd8855 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt @@ -0,0 +1,12 @@ +package io.tolgee.util + +import jakarta.persistence.EntityManager +import org.hibernate.Session + +val EntityManager.session: Session + get() = this.unwrap(org.hibernate.Session::class.java)!! + +fun EntityManager.flushAndClear() { + this.flush() + this.clear() +} From a557d905d52e531c0005cbdba51628bc251cb1ac Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sun, 17 Dec 2023 10:22:39 +0100 Subject: [PATCH 14/26] fix: Describing relation cache --- .../kotlin/io/tolgee/activity/ActivityHolder.kt | 13 +++++++++++++ .../kotlin/io/tolgee/activity/ActivityService.kt | 12 ++++++------ .../activity/iterceptor/InterceptedEventsManager.kt | 5 ++--- .../io/tolgee/model/activity/ActivityRevision.kt | 4 ++-- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt index b0d6ebd819..5bdc95a252 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt @@ -3,6 +3,7 @@ package io.tolgee.activity import io.tolgee.activity.data.ActivityType import io.tolgee.activity.iterceptor.InterceptedEventsManager import io.tolgee.model.EntityWithId +import io.tolgee.model.activity.ActivityDescribingEntity import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.activity.ActivityRevision import jakarta.annotation.PreDestroy @@ -48,6 +49,18 @@ open class ActivityHolder(val applicationContext: ApplicationContext) { var destroyer: (() -> Unit)? = null + open val describingRelationCache: MutableMap, ActivityDescribingEntity> by lazy { + activityRevision.describingRelations.associateBy { it.entityId to it.entityClass }.toMutableMap() + } + + fun getDescribingRelationFromCache( + entityId: Long, + entityClass: String, + provider: () -> ActivityDescribingEntity + ): ActivityDescribingEntity { + return describingRelationCache.getOrPut(entityId to entityClass, provider) + } + @PreDestroy fun destroy() { destroyer?.invoke() diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt index 7bb2263bd0..d792505fa0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt @@ -5,6 +5,7 @@ import io.tolgee.activity.projectActivityView.ProjectActivityViewByPageableProvi import io.tolgee.activity.projectActivityView.ProjectActivityViewByRevisionProvider import io.tolgee.dtos.query_results.TranslationHistoryView import io.tolgee.events.OnProjectActivityStoredEvent +import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.activity.ActivityRevision import io.tolgee.model.views.activity.ProjectActivityView import io.tolgee.repository.activity.ActivityModifiedEntityRepository @@ -33,10 +34,7 @@ class ActivityService( val mergedActivityRevision = activityRevision.persist() mergedActivityRevision.persistedDescribingRelations() - entityManager.flushAndClear() - - mergedActivityRevision.modifiedEntities = modifiedEntities.values.flatMap { it.values }.toMutableList() - mergedActivityRevision.persistModifiedEntities() + mergedActivityRevision.modifiedEntities = modifiedEntities.persist() entityManager.flushAndClear() @@ -45,14 +43,16 @@ class ActivityService( entityManager.flushAndClear() } - private fun ActivityRevision.persistModifiedEntities() { - modifiedEntities.forEach { activityModifiedEntity -> + private fun ModifiedEntitiesType.persist(): MutableList { + val list = this.values.flatMap { it.values }.toMutableList() + list.forEach { activityModifiedEntity -> try { entityManager.persist(activityModifiedEntity) } catch (e: EntityExistsException) { logger.debug("ModifiedEntity entity already exists in persistence context, skipping", e) } } + return list } private fun ActivityRevision.persistedDescribingRelations() { diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt index 37b0c90867..642807d91f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt @@ -141,9 +141,8 @@ class InterceptedEventsManager( value: EntityDescriptionWithRelations, activityRevision: ActivityRevision ): EntityDescriptionRef { - val activityDescribingEntity = activityRevision.describingRelations - .find { it.entityId == value.entityId && it.entityClass == value.entityClass } - ?: let { + val activityDescribingEntity = activityHolder + .getDescribingRelationFromCache(value.entityId, value.entityClass) { val compressedRelations = value.relations.map { relation -> relation.key to compressRelation(relation.value, activityRevision) }.toMap() diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt index ceeb92ef79..f518c29e7d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt @@ -72,8 +72,8 @@ class ActivityRevision : java.io.Serializable { */ var projectId: Long? = null - @OneToMany(mappedBy = "activityRevision", targetEntity = ActivityDescribingEntity::class) - var describingRelations: DescribingRelationsList = DescribingRelationsList() + @OneToMany(mappedBy = "activityRevision") + var describingRelations: MutableList = mutableListOf() @OneToMany(mappedBy = "activityRevision") var modifiedEntities: MutableList = mutableListOf() From ed48930443e5ec1a6a2fc0517306d9ff973c8310 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sun, 17 Dec 2023 12:17:11 +0100 Subject: [PATCH 15/26] fix: Store activity using stateless session --- .../io/tolgee/activity/ActivityService.kt | 50 +++++++++---------- .../tolgee/configuration/HibernateSession.kt | 16 ------ .../kotlin/io/tolgee/util/entityManagerExt.kt | 13 +++++ 3 files changed, 36 insertions(+), 43 deletions(-) delete mode 100644 backend/data/src/main/kotlin/io/tolgee/configuration/HibernateSession.kt diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt index d792505fa0..3da119fba0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt @@ -3,6 +3,7 @@ package io.tolgee.activity import io.tolgee.activity.data.ActivityType import io.tolgee.activity.projectActivityView.ProjectActivityViewByPageableProvider import io.tolgee.activity.projectActivityView.ProjectActivityViewByRevisionProvider +import io.tolgee.component.CurrentDateProvider import io.tolgee.dtos.query_results.TranslationHistoryView import io.tolgee.events.OnProjectActivityStoredEvent import io.tolgee.model.activity.ActivityModifiedEntity @@ -10,10 +11,10 @@ import io.tolgee.model.activity.ActivityRevision import io.tolgee.model.views.activity.ProjectActivityView import io.tolgee.repository.activity.ActivityModifiedEntityRepository import io.tolgee.util.Logging +import io.tolgee.util.doInStatelessSession import io.tolgee.util.flushAndClear -import io.tolgee.util.logger -import jakarta.persistence.EntityExistsException import jakarta.persistence.EntityManager +import org.hibernate.StatelessSession import org.springframework.context.ApplicationContext import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable @@ -24,50 +25,45 @@ import org.springframework.transaction.annotation.Transactional class ActivityService( private val entityManager: EntityManager, private val applicationContext: ApplicationContext, - private val activityModifiedEntityRepository: ActivityModifiedEntityRepository + private val activityModifiedEntityRepository: ActivityModifiedEntityRepository, + private val currentDateProvider: CurrentDateProvider ) : Logging { @Transactional fun storeActivityData(activityRevision: ActivityRevision, modifiedEntities: ModifiedEntitiesType) { // let's keep the persistent context small entityManager.flushAndClear() - val mergedActivityRevision = activityRevision.persist() - mergedActivityRevision.persistedDescribingRelations() - - mergedActivityRevision.modifiedEntities = modifiedEntities.persist() - - entityManager.flushAndClear() - + val mergedActivityRevision = entityManager.doInStatelessSession { statelessSession -> + val mergedActivityRevision = statelessSession.persist(activityRevision) + statelessSession.persistedDescribingRelations(mergedActivityRevision) + mergedActivityRevision.modifiedEntities = statelessSession.persist(modifiedEntities) + mergedActivityRevision + } applicationContext.publishEvent(OnProjectActivityStoredEvent(this, mergedActivityRevision)) - - entityManager.flushAndClear() } - private fun ModifiedEntitiesType.persist(): MutableList { - val list = this.values.flatMap { it.values }.toMutableList() + private fun StatelessSession.persist(modifiedEntities: ModifiedEntitiesType): MutableList { + val list = modifiedEntities.values.flatMap { it.values }.toMutableList() list.forEach { activityModifiedEntity -> - try { - entityManager.persist(activityModifiedEntity) - } catch (e: EntityExistsException) { - logger.debug("ModifiedEntity entity already exists in persistence context, skipping", e) - } + this.insert(activityModifiedEntity) } return list } - private fun ActivityRevision.persistedDescribingRelations() { + private fun StatelessSession.persistedDescribingRelations(activityRevision: ActivityRevision) { @Suppress("UselessCallOnCollection") - describingRelations.filterNotNull().forEach { - entityManager.persist(it) + activityRevision.describingRelations.filterNotNull().forEach { + this.insert(it) } } - private fun ActivityRevision.persist(): ActivityRevision { - return if (id == 0L) { - entityManager.persist(this) - this + private fun StatelessSession.persist(activityRevision: ActivityRevision): ActivityRevision { + return if (activityRevision.id == 0L) { + activityRevision.timestamp = currentDateProvider.date + this.insert(activityRevision) + activityRevision } else { - entityManager.getReference(ActivityRevision::class.java, id) + entityManager.getReference(ActivityRevision::class.java, activityRevision.id) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/HibernateSession.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/HibernateSession.kt deleted file mode 100644 index 16607834e3..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/HibernateSession.kt +++ /dev/null @@ -1,16 +0,0 @@ -package io.tolgee.configuration - -// -// @Configuration -// class HibernateSessionConfiguration( -// private val dataSource: DataSource -// ) { -// @Bean -// fun sessionFactory(emf: EntityManagerFactory?): LocalSessionFactoryBean { -// val sessionFactory = LocalSessionFactoryBean() -// sessionFactory.setDataSource(dataSource) -// sessionFactory.setPackagesToScan("io.tolgee.model") -// sessionFactory.hibernateProperties = hibernateProperties() -// return sessionFactory -// } -// } diff --git a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt index bc55bd8855..859a4a0cbc 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt @@ -2,6 +2,7 @@ package io.tolgee.util import jakarta.persistence.EntityManager import org.hibernate.Session +import org.hibernate.StatelessSession val EntityManager.session: Session get() = this.unwrap(org.hibernate.Session::class.java)!! @@ -10,3 +11,15 @@ fun EntityManager.flushAndClear() { this.flush() this.clear() } + + +inline fun EntityManager.doInStatelessSession( + crossinline block: (StatelessSession) -> T +): T { + return unwrap(Session::class.java).doReturningWork { connection -> + val statelessSession = unwrap(Session::class.java).sessionFactory.openStatelessSession(connection) + statelessSession.use { ss -> + block(ss) + } + } +} From c4befb060a136ece684f26fa29760c0856772a3f Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sun, 17 Dec 2023 12:30:17 +0100 Subject: [PATCH 16/26] fix: Back to the standard hibernate session --- .../io/tolgee/activity/ActivityService.kt | 50 ++++++++++--------- .../kotlin/io/tolgee/util/entityManagerExt.kt | 13 ----- 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt index 3da119fba0..d792505fa0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt @@ -3,7 +3,6 @@ package io.tolgee.activity import io.tolgee.activity.data.ActivityType import io.tolgee.activity.projectActivityView.ProjectActivityViewByPageableProvider import io.tolgee.activity.projectActivityView.ProjectActivityViewByRevisionProvider -import io.tolgee.component.CurrentDateProvider import io.tolgee.dtos.query_results.TranslationHistoryView import io.tolgee.events.OnProjectActivityStoredEvent import io.tolgee.model.activity.ActivityModifiedEntity @@ -11,10 +10,10 @@ import io.tolgee.model.activity.ActivityRevision import io.tolgee.model.views.activity.ProjectActivityView import io.tolgee.repository.activity.ActivityModifiedEntityRepository import io.tolgee.util.Logging -import io.tolgee.util.doInStatelessSession import io.tolgee.util.flushAndClear +import io.tolgee.util.logger +import jakarta.persistence.EntityExistsException import jakarta.persistence.EntityManager -import org.hibernate.StatelessSession import org.springframework.context.ApplicationContext import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable @@ -25,45 +24,50 @@ import org.springframework.transaction.annotation.Transactional class ActivityService( private val entityManager: EntityManager, private val applicationContext: ApplicationContext, - private val activityModifiedEntityRepository: ActivityModifiedEntityRepository, - private val currentDateProvider: CurrentDateProvider + private val activityModifiedEntityRepository: ActivityModifiedEntityRepository ) : Logging { @Transactional fun storeActivityData(activityRevision: ActivityRevision, modifiedEntities: ModifiedEntitiesType) { // let's keep the persistent context small entityManager.flushAndClear() - val mergedActivityRevision = entityManager.doInStatelessSession { statelessSession -> - val mergedActivityRevision = statelessSession.persist(activityRevision) - statelessSession.persistedDescribingRelations(mergedActivityRevision) - mergedActivityRevision.modifiedEntities = statelessSession.persist(modifiedEntities) - mergedActivityRevision - } + val mergedActivityRevision = activityRevision.persist() + mergedActivityRevision.persistedDescribingRelations() + + mergedActivityRevision.modifiedEntities = modifiedEntities.persist() + + entityManager.flushAndClear() + applicationContext.publishEvent(OnProjectActivityStoredEvent(this, mergedActivityRevision)) + + entityManager.flushAndClear() } - private fun StatelessSession.persist(modifiedEntities: ModifiedEntitiesType): MutableList { - val list = modifiedEntities.values.flatMap { it.values }.toMutableList() + private fun ModifiedEntitiesType.persist(): MutableList { + val list = this.values.flatMap { it.values }.toMutableList() list.forEach { activityModifiedEntity -> - this.insert(activityModifiedEntity) + try { + entityManager.persist(activityModifiedEntity) + } catch (e: EntityExistsException) { + logger.debug("ModifiedEntity entity already exists in persistence context, skipping", e) + } } return list } - private fun StatelessSession.persistedDescribingRelations(activityRevision: ActivityRevision) { + private fun ActivityRevision.persistedDescribingRelations() { @Suppress("UselessCallOnCollection") - activityRevision.describingRelations.filterNotNull().forEach { - this.insert(it) + describingRelations.filterNotNull().forEach { + entityManager.persist(it) } } - private fun StatelessSession.persist(activityRevision: ActivityRevision): ActivityRevision { - return if (activityRevision.id == 0L) { - activityRevision.timestamp = currentDateProvider.date - this.insert(activityRevision) - activityRevision + private fun ActivityRevision.persist(): ActivityRevision { + return if (id == 0L) { + entityManager.persist(this) + this } else { - entityManager.getReference(ActivityRevision::class.java, activityRevision.id) + entityManager.getReference(ActivityRevision::class.java, id) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt index 859a4a0cbc..bc55bd8855 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt @@ -2,7 +2,6 @@ package io.tolgee.util import jakarta.persistence.EntityManager import org.hibernate.Session -import org.hibernate.StatelessSession val EntityManager.session: Session get() = this.unwrap(org.hibernate.Session::class.java)!! @@ -11,15 +10,3 @@ fun EntityManager.flushAndClear() { this.flush() this.clear() } - - -inline fun EntityManager.doInStatelessSession( - crossinline block: (StatelessSession) -> T -): T { - return unwrap(Session::class.java).doReturningWork { connection -> - val statelessSession = unwrap(Session::class.java).sessionFactory.openStatelessSession(connection) - statelessSession.use { ss -> - block(ss) - } - } -} From a0ab25249d5684a42063915b9680d1eab46d4bce Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sun, 17 Dec 2023 20:10:47 +0100 Subject: [PATCH 17/26] fix: Store activity using batch sql updates --- .../app/src/main/resources/application.yaml | 2 - backend/data/build.gradle | 2 +- .../io/tolgee/activity/ActivityService.kt | 91 +++++++++++++------ .../activity/ActivityDescribingEntity.kt | 3 - .../kotlin/io/tolgee/util/entityManagerExt.kt | 13 +++ 5 files changed, 77 insertions(+), 34 deletions(-) diff --git a/backend/app/src/main/resources/application.yaml b/backend/app/src/main/resources/application.yaml index 280c38e0ba..6e2b4d1c74 100644 --- a/backend/app/src/main/resources/application.yaml +++ b/backend/app/src/main/resources/application.yaml @@ -9,7 +9,6 @@ spring: pathmatch: matching-strategy: ant_path_matcher jpa: -# open-in-view: false properties: hibernate: order_by: @@ -25,7 +24,6 @@ spring: enhancer: enableLazyInitialization: true enableDirtyTracking: true - # open-in-view: false batch: job: enabled: false diff --git a/backend/data/build.gradle b/backend/data/build.gradle index 93828b32a8..69151933cc 100644 --- a/backend/data/build.gradle +++ b/backend/data/build.gradle @@ -110,7 +110,7 @@ dependencies { /** * DB */ - runtimeOnly 'org.postgresql:postgresql' + implementation 'org.postgresql:postgresql' implementation "org.hibernate:hibernate-jpamodelgen:$hibernateVersion" kapt "org.hibernate:hibernate-jpamodelgen:$hibernateVersion" diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt index d792505fa0..2daaeb2f6d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt @@ -1,8 +1,11 @@ package io.tolgee.activity +import com.fasterxml.jackson.databind.ObjectMapper import io.tolgee.activity.data.ActivityType +import io.tolgee.activity.data.RevisionType import io.tolgee.activity.projectActivityView.ProjectActivityViewByPageableProvider import io.tolgee.activity.projectActivityView.ProjectActivityViewByRevisionProvider +import io.tolgee.component.CurrentDateProvider import io.tolgee.dtos.query_results.TranslationHistoryView import io.tolgee.events.OnProjectActivityStoredEvent import io.tolgee.model.activity.ActivityModifiedEntity @@ -10,10 +13,11 @@ import io.tolgee.model.activity.ActivityRevision import io.tolgee.model.views.activity.ProjectActivityView import io.tolgee.repository.activity.ActivityModifiedEntityRepository import io.tolgee.util.Logging +import io.tolgee.util.doInStatelessSession import io.tolgee.util.flushAndClear -import io.tolgee.util.logger -import jakarta.persistence.EntityExistsException import jakarta.persistence.EntityManager +import org.hibernate.StatelessSession +import org.postgresql.util.PGobject import org.springframework.context.ApplicationContext import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable @@ -24,50 +28,81 @@ import org.springframework.transaction.annotation.Transactional class ActivityService( private val entityManager: EntityManager, private val applicationContext: ApplicationContext, - private val activityModifiedEntityRepository: ActivityModifiedEntityRepository + private val activityModifiedEntityRepository: ActivityModifiedEntityRepository, + private val currentDateProvider: CurrentDateProvider, + private val objectMapper: ObjectMapper ) : Logging { @Transactional fun storeActivityData(activityRevision: ActivityRevision, modifiedEntities: ModifiedEntitiesType) { // let's keep the persistent context small entityManager.flushAndClear() - val mergedActivityRevision = activityRevision.persist() - mergedActivityRevision.persistedDescribingRelations() - - mergedActivityRevision.modifiedEntities = modifiedEntities.persist() - - entityManager.flushAndClear() - + val mergedActivityRevision = entityManager.doInStatelessSession { statelessSession -> + val mergedActivityRevision = statelessSession.persistAcitivyRevision(activityRevision) + statelessSession.persistedDescribingRelations(mergedActivityRevision) + mergedActivityRevision.modifiedEntities = statelessSession.persistModifiedEntities(modifiedEntities) + mergedActivityRevision + } applicationContext.publishEvent(OnProjectActivityStoredEvent(this, mergedActivityRevision)) - - entityManager.flushAndClear() } - private fun ModifiedEntitiesType.persist(): MutableList { - val list = this.values.flatMap { it.values }.toMutableList() - list.forEach { activityModifiedEntity -> - try { - entityManager.persist(activityModifiedEntity) - } catch (e: EntityExistsException) { - logger.debug("ModifiedEntity entity already exists in persistence context, skipping", e) + private fun StatelessSession.persistModifiedEntities(modifiedEntities: ModifiedEntitiesType): MutableList { + val list = modifiedEntities.values.flatMap { it.values }.toMutableList() + + this.doWork { connection -> + val describingRelationQuery = "INSERT INTO activity_modified_entity " + + "(entity_class, entity_id, describing_data, " + + "describing_relations, modifications, revision_type, activity_revision_id) " + + "VALUES (?, ?, ?, ?, ?, ?, ?)" + val preparedStatement = connection.prepareStatement(describingRelationQuery) + list.forEach { entity -> + preparedStatement.setString(1, entity.entityClass) + preparedStatement.setLong(2, entity.entityId) + preparedStatement.setObject(3, getJsonbObject(entity.describingData)) + preparedStatement.setObject(4, getJsonbObject(entity.describingRelations)) + preparedStatement.setObject(5, getJsonbObject(entity.modifications)) + preparedStatement.setInt(6, RevisionType.values().indexOf(entity.revisionType)) + preparedStatement.setLong(7, entity.activityRevision.id) + preparedStatement.addBatch() } + preparedStatement.executeBatch() } return list } - private fun ActivityRevision.persistedDescribingRelations() { - @Suppress("UselessCallOnCollection") - describingRelations.filterNotNull().forEach { - entityManager.persist(it) + + private fun StatelessSession.persistedDescribingRelations(activityRevision: ActivityRevision) { + this.doWork { connection -> + val describingRelationQuery = "INSERT INTO activity_describing_entity " + + "(entity_class, entity_id, data, describing_relations, activity_revision_id) " + + "VALUES (?, ?, ?, ?, ?)" + val preparedStatement = connection.prepareStatement(describingRelationQuery) + activityRevision.describingRelations.forEach { entity -> + preparedStatement.setString(1, entity.entityClass) + preparedStatement.setLong(2, entity.entityId) + preparedStatement.setObject(3, getJsonbObject(entity.data)) + preparedStatement.setObject(4, getJsonbObject(entity.describingRelations)) + preparedStatement.setLong(5, activityRevision.id) + preparedStatement.addBatch() + } + preparedStatement.executeBatch() } } - private fun ActivityRevision.persist(): ActivityRevision { - return if (id == 0L) { - entityManager.persist(this) - this + private fun getJsonbObject(data: Any?): PGobject { + val pgObject = PGobject() + pgObject.type = "jsonb" + pgObject.value = objectMapper.writeValueAsString(data) + return pgObject + } + + private fun StatelessSession.persistAcitivyRevision(activityRevision: ActivityRevision): ActivityRevision { + return if (activityRevision.id == 0L) { + activityRevision.timestamp = currentDateProvider.date + this.insert(activityRevision) + activityRevision } else { - entityManager.getReference(ActivityRevision::class.java, id) + entityManager.getReference(ActivityRevision::class.java, activityRevision.id) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt index 5dd54909c4..59afd37523 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt @@ -33,7 +33,4 @@ class ActivityDescribingEntity( @Type(JsonBinaryType::class) var describingRelations: Map? = null - - @Enumerated - lateinit var revisionType: RevisionType } diff --git a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt index bc55bd8855..859a4a0cbc 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt @@ -2,6 +2,7 @@ package io.tolgee.util import jakarta.persistence.EntityManager import org.hibernate.Session +import org.hibernate.StatelessSession val EntityManager.session: Session get() = this.unwrap(org.hibernate.Session::class.java)!! @@ -10,3 +11,15 @@ fun EntityManager.flushAndClear() { this.flush() this.clear() } + + +inline fun EntityManager.doInStatelessSession( + crossinline block: (StatelessSession) -> T +): T { + return unwrap(Session::class.java).doReturningWork { connection -> + val statelessSession = unwrap(Session::class.java).sessionFactory.openStatelessSession(connection) + statelessSession.use { ss -> + block(ss) + } + } +} From 1776733a61c98096f1ad0386c5ffb6ff6ab15f71 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 11:57:43 +0100 Subject: [PATCH 18/26] fix: Missing revision type --- .../activity/iterceptor/InterceptedEventsManager.kt | 8 ++++---- .../io/tolgee/model/activity/ActivityModifiedEntity.kt | 2 +- gradle.properties | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt index 642807d91f..74ec83fc3a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt @@ -68,7 +68,7 @@ class InterceptedEventsManager( } val changes = provider.getChanges(old, collection) ?: return - val activityModifiedEntity = getModifiedEntity(collectionOwner) + val activityModifiedEntity = getModifiedEntity(collectionOwner, RevisionType.MOD) val newChanges = activityModifiedEntity.modifications + mutableMapOf(ownerField.name to changes) activityModifiedEntity.modifications = (newChanges).toMutableMap() @@ -89,7 +89,7 @@ class InterceptedEventsManager( entity as EntityWithId - val activityModifiedEntity = getModifiedEntity(entity) + val activityModifiedEntity = getModifiedEntity(entity, revisionType) val changesMap = getChangesMap(entity, currentState, previousState, propertyNames) @@ -107,7 +107,7 @@ class InterceptedEventsManager( this.describingRelations = describingData.second } - private fun getModifiedEntity(entity: EntityWithId): ActivityModifiedEntity { + private fun getModifiedEntity(entity: EntityWithId, revisionType: RevisionType): ActivityModifiedEntity { val activityModifiedEntity = activityHolder.modifiedEntities .computeIfAbsent(entity::class) { mutableMapOf() } .computeIfAbsent( @@ -117,7 +117,7 @@ class InterceptedEventsManager( activityRevision, entity::class.simpleName!!, entity.id - ) + ).also { it.revisionType = revisionType } } return activityModifiedEntity diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt index 4ba74cf8e4..e91acf15ea 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt @@ -56,5 +56,5 @@ class ActivityModifiedEntity( var describingRelations: Map? = null @Enumerated - lateinit var revisionType: RevisionType + var revisionType: RevisionType = RevisionType.MOD } diff --git a/gradle.properties b/gradle.properties index 598eae6393..75119bfea2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -kotlinVersion=1.9.10 +kotlinVersion=1.9.21 springBootVersion=3.1.5 springDocVersion=2.2.0 jjwtVersion=0.11.2 From 166bbc005a3ea222c27132433d4eb5530001ed05 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 13:44:34 +0100 Subject: [PATCH 19/26] fix: No Session issues --- .../translation/TranslationCommentController.kt | 6 +++--- .../translation/TranslationsController.kt | 17 ++++++++--------- .../comments/TranslationCommentModel.kt | 3 ++- .../TranslationCommentModelAssembler.kt | 5 +++-- .../tolgee/repository/TranslationRepository.kt | 5 +++++ .../translation/TranslationCommentRepository.kt | 12 +++++++++++- .../translation/TranslationCommentService.kt | 12 ++++++++++++ .../service/translation/TranslationService.kt | 4 ++++ 8 files changed, 48 insertions(+), 16 deletions(-) diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt index 95ee2aff3d..86857432d4 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt @@ -95,7 +95,7 @@ class TranslationCommentController( @UseDefaultPermissions @AllowApiAccess fun get(@PathVariable translationId: Long, @PathVariable commentId: Long): TranslationCommentModel { - val comment = translationCommentService.get(commentId) + val comment = translationCommentService.getWithAuthorFetched(commentId) comment.checkFromProject() return translationCommentModelAssembler.toModel(comment) } @@ -106,7 +106,7 @@ class TranslationCommentController( @UseDefaultPermissions // Security: Permission check done inside; users should be able to edit their comments @AllowApiAccess fun update(@PathVariable commentId: Long, @RequestBody @Valid dto: TranslationCommentDto): TranslationCommentModel { - val comment = translationCommentService.get(commentId) + val comment = translationCommentService.getWithAuthorFetched(commentId) if (comment.author.id != authenticationFacade.authenticatedUser.id) { throw BadRequestException(io.tolgee.constants.Message.CAN_EDIT_ONLY_OWN_COMMENT) } @@ -123,7 +123,7 @@ class TranslationCommentController( @PathVariable commentId: Long, @PathVariable state: TranslationCommentState ): TranslationCommentModel { - val comment = translationCommentService.get(commentId) + val comment = translationCommentService.getWithAuthorFetched(commentId) comment.checkFromProject() translationCommentService.setState(comment, state) return translationCommentModelAssembler.toModel(comment) diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt index 783a3c74cf..50b99a9997 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt @@ -172,7 +172,7 @@ When null, resulting file will be a flat key-value object. @PutMapping("") @Operation(summary = "Sets translations for existing key") @RequestActivity(ActivityType.SET_TRANSLATIONS) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun setTranslations(@RequestBody @Valid dto: SetTranslationsWithKeyDto): SetTranslationsResponseModel { val key = keyService.get(projectHolder.project.id, dto.key, dto.namespace) @@ -182,8 +182,7 @@ When null, resulting file will be a flat key-value object. val translations = dto.languagesToReturn ?.let { languagesToReturn -> - key.translations - .filter { languagesToReturn.contains(it.language.tag) } + translationService.findForKeyByLanguages(key, languagesToReturn) .associateBy { it.language.tag } } ?: modifiedTranslations @@ -194,7 +193,7 @@ When null, resulting file will be a flat key-value object. @PostMapping("") @Operation(summary = "Sets translations for existing or not existing key.") @RequestActivity(ActivityType.SET_TRANSLATIONS) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun createOrUpdateTranslations(@RequestBody @Valid dto: SetTranslationsWithKeyDto): SetTranslationsResponseModel { val key = keyService.find(projectHolder.projectEntity.id, dto.key, dto.namespace)?.also { @@ -211,7 +210,7 @@ When null, resulting file will be a flat key-value object. @PutMapping("/{translationId}/set-state/{state}") @Operation(summary = "Sets translation state") @RequestActivity(ActivityType.SET_TRANSLATION_STATE) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_STATE_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_STATE_EDIT]) @AllowApiAccess fun setTranslationState( @PathVariable translationId: Long, @@ -266,7 +265,7 @@ When null, resulting file will be a flat key-value object. @GetMapping(value = ["select-all"]) @Operation(summary = "Get select all keys") - @RequiresProjectPermissions([ Scope.KEYS_VIEW ]) + @RequiresProjectPermissions([Scope.KEYS_VIEW]) @AllowApiAccess fun getSelectAllKeyIds( @ParameterObject @ModelAttribute("translationFilters") params: TranslationFilters, @@ -289,7 +288,7 @@ When null, resulting file will be a flat key-value object. @PutMapping(value = ["/{translationId:[0-9]+}/dismiss-auto-translated-state"]) @Operation(summary = """Removes "auto translated" indication""") @RequestActivity(ActivityType.DISMISS_AUTO_TRANSLATED_STATE) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_STATE_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_STATE_EDIT]) @AllowApiAccess fun dismissAutoTranslatedState( @PathVariable translationId: Long @@ -304,7 +303,7 @@ When null, resulting file will be a flat key-value object. @PutMapping(value = ["/{translationId:[0-9]+}/set-outdated-flag/{state}"]) @Operation(summary = """Set's "outdated" indication""") @RequestActivity(ActivityType.SET_OUTDATED_FLAG) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_STATE_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_STATE_EDIT]) @AllowApiAccess fun setOutdated( @PathVariable translationId: Long, @@ -322,7 +321,7 @@ When null, resulting file will be a flat key-value object. Sorting is not supported for supported. It is automatically sorted from newest to oldest.""" ) - @RequiresProjectPermissions([ Scope.TRANSLATIONS_VIEW ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) @AllowApiAccess fun getTranslationHistory( @PathVariable translationId: Long, diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt index 48c57bc389..822fca7d76 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt @@ -1,6 +1,7 @@ package io.tolgee.hateoas.translations.comments import io.swagger.v3.oas.annotations.media.Schema +import io.tolgee.hateoas.user_account.SimpleUserAccountModel import io.tolgee.hateoas.user_account.UserAccountModel import io.tolgee.model.enums.TranslationCommentState import org.springframework.hateoas.RepresentationModel @@ -20,7 +21,7 @@ open class TranslationCommentModel( val state: TranslationCommentState, @Schema(description = "User who created the comment") - val author: UserAccountModel, + val author: SimpleUserAccountModel, @Schema(description = "Date when it was created") val createdAt: Date, diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt index 94d63b9136..e813b6747f 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt @@ -1,6 +1,7 @@ package io.tolgee.hateoas.translations.comments import io.tolgee.api.v2.controllers.translation.TranslationCommentController +import io.tolgee.hateoas.user_account.SimpleUserAccountModelAssembler import io.tolgee.hateoas.user_account.UserAccountModelAssembler import io.tolgee.model.translation.TranslationComment import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport @@ -9,7 +10,7 @@ import java.util.* @Component class TranslationCommentModelAssembler( - private val userAccountModelAssembler: UserAccountModelAssembler + private val simpleUserAccountModelAssembler: SimpleUserAccountModelAssembler ) : RepresentationModelAssemblerSupport( TranslationCommentController::class.java, TranslationCommentModel::class.java ) { @@ -18,7 +19,7 @@ class TranslationCommentModelAssembler( id = entity.id, text = entity.text, state = entity.state, - author = entity.author.let { userAccountModelAssembler.toModel(it) }, + author = entity.author.let { simpleUserAccountModelAssembler.toModel(it) }, createdAt = entity.createdAt ?: Date(), updatedAt = entity.updatedAt ?: Date() ) diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt index 9c89a8553e..4930717013 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt @@ -172,4 +172,9 @@ interface TranslationRepository : JpaRepository { """ ) fun getAllByProjectId(projectId: Long): List + @Query(""" + from Translation t + where t.key = :key and t.language.tag in :languageTags + """) + fun findForKeyByLanguages(key: Key, languageTags: Collection): List } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt index 2aca539683..3bbac78caa 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt @@ -12,7 +12,7 @@ import org.springframework.stereotype.Repository interface TranslationCommentRepository : JpaRepository { fun deleteAllByIdIn(ids: List) - @Query("select tc from TranslationComment tc where tc.translation = :translation") + @Query("select tc from TranslationComment tc left join fetch tc.author where tc.translation = :translation") fun getPagedByTranslation(translation: Translation, pageable: Pageable): Page fun deleteAllByTranslationIdIn(translationIds: Collection) @@ -28,4 +28,14 @@ interface TranslationCommentRepository : JpaRepository """ ) fun getAllByProjectId(projectId: Long): List + + + @Query( + """ + from TranslationComment tc + left join fetch tc.author + where tc.id = :id + """ + ) + fun findWithFetchedAuthor(id: Long): TranslationComment? } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt index 53994590e9..a082c98c84 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt @@ -37,14 +37,26 @@ class TranslationCommentService( } } + @Transactional fun find(id: Long): TranslationComment? { return translationCommentRepository.findById(id).orElse(null) } + @Transactional fun get(id: Long): TranslationComment { return find(id) ?: throw NotFoundException() } + @Transactional + fun findWithAuthorFetched(id: Long): TranslationComment? { + return translationCommentRepository.findWithFetchedAuthor(id) + } + + @Transactional + fun getWithAuthorFetched(id: Long): TranslationComment { + return findWithAuthorFetched(id) ?: throw NotFoundException() + } + @Transactional fun update(dto: TranslationCommentDto, entity: TranslationComment): TranslationComment { entity.text = dto.text diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt index 1a7c7bf74b..5ce23a670a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt @@ -191,6 +191,10 @@ class TranslationService( ).mapKeys { it.key.tag } } + fun findForKeyByLanguages(key: Key, languageTags: Collection): List { + return translationRepository.findForKeyByLanguages(key, languageTags) + } + private fun languageByTagFromLanguages(tag: String, languages: Collection) = languages.find { it.tag == tag } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) From a1011de92a0bc08ef91442939667b844f98752e1 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 13:52:48 +0100 Subject: [PATCH 20/26] chore: fix bad language controller tests --- .../controllers/V2LanguageControllerTest.kt | 29 ++++++++++--------- .../main/kotlin/io/tolgee/model/Project.kt | 4 +-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2LanguageControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2LanguageControllerTest.kt index 1ef17f93ed..033ef25669 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2LanguageControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2LanguageControllerTest.kt @@ -69,11 +69,11 @@ class V2LanguageControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Test fun deleteLanguage() { + val base = dbPopulator.createBase(generateUniqueString()) + val project = base.project + val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } + performDelete(project.id, deutsch.id).andExpect(MockMvcResultMatchers.status().isOk) executeInNewTransaction { - val base = dbPopulator.createBase(generateUniqueString()) - val project = base.project - val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } - performDelete(project.id, deutsch.id).andExpect(MockMvcResultMatchers.status().isOk) Assertions.assertThat(languageService.findById(deutsch.id)).isEmpty } } @@ -81,13 +81,14 @@ class V2LanguageControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Test @ProjectApiKeyAuthTestMethod(scopes = [Scope.LANGUAGES_EDIT]) fun `deletes language with API key`() { + val base = dbPopulator.createBase(generateUniqueString()) + this.userAccount = base.userAccount + this.projectSupplier = { base.project } + val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } + + performProjectAuthDelete("languages/${deutsch.id}", null) + .andExpect(MockMvcResultMatchers.status().isOk) executeInNewTransaction { - val base = dbPopulator.createBase(generateUniqueString()) - this.userAccount = base.userAccount - this.projectSupplier = { base.project } - val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } - performProjectAuthDelete("languages/${deutsch.id}", null) - .andExpect(MockMvcResultMatchers.status().isOk) Assertions.assertThat(languageService.findById(deutsch.id)).isEmpty } } @@ -95,11 +96,11 @@ class V2LanguageControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Test @ProjectApiKeyAuthTestMethod(scopes = [Scope.TRANSLATIONS_VIEW]) fun `does not delete language with API key (permissions)`() { + val base = dbPopulator.createBase(generateUniqueString()) + this.userAccount = base.userAccount + this.projectSupplier = { base.project } + val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } executeInNewTransaction { - val base = dbPopulator.createBase(generateUniqueString()) - this.userAccount = base.userAccount - this.projectSupplier = { base.project } - val deutsch = project.findLanguageOptional("de").orElseThrow { NotFoundException() } performProjectAuthDelete("languages/${deutsch.id}", null).andIsForbidden } } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/Project.kt b/backend/data/src/main/kotlin/io/tolgee/model/Project.kt index 01a1fa9d6d..0b1597a0fa 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/Project.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/Project.kt @@ -109,8 +109,8 @@ class Project( constructor(name: String, description: String? = null, slug: String?, organizationOwner: Organization) : this(id = 0L, name, description, slug) { - this.organizationOwner = organizationOwner - } + this.organizationOwner = organizationOwner + } fun findLanguageOptional(tag: String): Optional { return languages.stream().filter { l: Language -> (l.tag == tag) }.findFirst() From fa477dc7b65da693fee057b4cc159447ad837197 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 15:04:53 +0100 Subject: [PATCH 21/26] chore: Add batch job executions via batch statement --- .../SecuredKeyScreenshotControllerTest.kt | 6 +- .../kotlin/io/tolgee/batch/BatchJobService.kt | 57 +++++++++++++++++-- .../kotlin/io/tolgee/batch/ProgressManager.kt | 11 ++-- .../io/tolgee/model/StandardAuditModel.kt | 7 ++- .../io/tolgee/util/SequenceIdProvider.kt | 35 ++++++++++++ 5 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt index 25fa0bf65e..17551e5c92 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ScreenshotController/SecuredKeyScreenshotControllerTest.kt @@ -95,10 +95,10 @@ class SecuredKeyScreenshotControllerTest : AbstractV2ScreenshotControllerTest() @Test @ProjectJWTAuthTestMethod fun uploadScreenshot() { - executeInNewTransaction { - val key = keyService.create(project, CreateKeyDto("test")) + val key = keyService.create(project, CreateKeyDto("test")) - performStoreScreenshot(project, key).andIsCreated.andAssertThatJson { + performStoreScreenshot(project, key).andIsCreated.andAssertThatJson { + executeInNewTransaction { val screenshots = screenshotService.findAll(key = key) assertThat(screenshots).hasSize(1) val file = File(tolgeeProperties.fileStorage.fsDataPath + "/screenshots/" + screenshots[0].filename) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt index 0bc0e07120..3a3cf866dd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt @@ -1,5 +1,6 @@ package io.tolgee.batch +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.tolgee.batch.data.BatchJobDto import io.tolgee.batch.data.BatchJobType @@ -9,7 +10,9 @@ import io.tolgee.constants.Message import io.tolgee.dtos.cacheable.UserAccountDto import io.tolgee.exceptions.NotFoundException import io.tolgee.exceptions.PermissionException +import io.tolgee.model.ALLOCATION_SIZE import io.tolgee.model.Project +import io.tolgee.model.SEQUENCE_NAME import io.tolgee.model.UserAccount import io.tolgee.model.batch.BatchJob import io.tolgee.model.batch.BatchJobChunkExecution @@ -21,12 +24,15 @@ import io.tolgee.repository.BatchJobRepository import io.tolgee.security.authentication.AuthenticationFacade import io.tolgee.service.security.SecurityService import io.tolgee.util.Logging +import io.tolgee.util.SequenceIdProvider import io.tolgee.util.addMinutes import io.tolgee.util.flushAndClear import io.tolgee.util.logger import jakarta.persistence.EntityManager import org.apache.commons.codec.digest.DigestUtils.sha256Hex import org.hibernate.LockOptions +import org.hibernate.Session +import org.postgresql.util.PGobject import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Lazy import org.springframework.data.domain.Page @@ -34,6 +40,7 @@ import org.springframework.data.domain.Pageable import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.event.TransactionalEventListener +import java.sql.Timestamp import java.time.Duration import java.util.* @@ -49,6 +56,7 @@ class BatchJobService( private val currentDateProvider: CurrentDateProvider, private val securityService: SecurityService, private val authenticationFacade: AuthenticationFacade, + private val objectMapper: ObjectMapper ) : Logging { companion object { @@ -96,19 +104,60 @@ class BatchJobService( job.params = processor.getParams(request) + entityManager.flushAndClear() + + val executions = storeExecutions(chunked, job) + + applicationContext.publishEvent(OnBatchJobCreated(job, executions)) + + return job + } + + private fun storeExecutions( + chunked: List>, + job: BatchJob + ): List { val executions = List(chunked.size) { chunkNumber -> BatchJobChunkExecution().apply { batchJob = job this.chunkNumber = chunkNumber - entityManager.persist(this) } } - entityManager.flushAndClear() + insertExecutionsViaBatchStatement(executions) - applicationContext.publishEvent(OnBatchJobCreated(job, executions)) + entityManager.clear() - return job + return executions + } + + private fun insertExecutionsViaBatchStatement(executions: List) { + entityManager.unwrap(Session::class.java).doWork { connection -> + val query = """ + insert into tolgee_batch_job_chunk_execution + (id, batch_job_id, chunk_number, status, created_at, updated_at, success_targets) + values (?, ?, ?, ?, ?, ?, ?) + """ + val statement = connection.prepareStatement(query) + val sequenceIdProvider = SequenceIdProvider(connection, SEQUENCE_NAME, ALLOCATION_SIZE) + val timestamp = Timestamp(currentDateProvider.date.time) + executions.forEach { + val id = sequenceIdProvider.next() + it.id = id + statement.setLong(1, id) + statement.setLong(2, it.batchJob.id) + statement.setInt(3, it.chunkNumber) + statement.setString(4, it.status.name) + statement.setTimestamp(5, timestamp) + statement.setTimestamp(6, timestamp) + statement.setObject(7, PGobject().apply { + type = "jsonb" + value = objectMapper.writeValueAsString(it.successTargets) + }) + statement.addBatch() + } + statement.executeBatch() + } } private fun tryDebounceJob( diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt b/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt index b31a0c8f1c..2ada024156 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt @@ -16,6 +16,7 @@ import io.tolgee.util.Logging import io.tolgee.util.debug import io.tolgee.util.executeInNewTransaction import io.tolgee.util.logger +import io.tolgee.util.trace import org.springframework.context.ApplicationEventPublisher import org.springframework.stereotype.Component import org.springframework.transaction.PlatformTransactionManager @@ -139,8 +140,8 @@ class ProgressManager( errorMessage: Message? = null, failOnly: Boolean = false ) { - logger.debug("Job ${job.id} completed chunks: $completedChunks of ${job.totalChunks}") - logger.debug("Job ${job.id} progress: $progress of ${job.totalItems}") + logger.debug { "Job ${job.id} completed chunks: $completedChunks of ${job.totalChunks}" } + logger.debug { "Job ${job.id} progress: $progress of ${job.totalItems}" } if (job.totalChunks.toLong() != completedChunks) { return @@ -165,7 +166,7 @@ class ProgressManager( } jobEntity.status = BatchJobStatus.SUCCESS - logger.debug("Publishing success event for job ${job.id}") + logger.debug { "Publishing success event for job ${job.id}" } eventPublisher.publishEvent(OnBatchJobSucceeded(jobEntity.dto)) cachingBatchJobService.saveJob(jobEntity) } @@ -192,10 +193,10 @@ class ProgressManager( fun handleJobRunning(id: Long) { executeInNewTransaction(transactionManager) { - logger.trace("""Fetching job $id""") + logger.trace { """Fetching job $id""" } val job = batchJobService.getJobDto(id) if (job.status == BatchJobStatus.PENDING) { - logger.debug("""Updating job state to running ${job.id}""") + logger.debug { """Updating job state to running ${job.id}""" } cachingBatchJobService.setRunningState(job.id) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/StandardAuditModel.kt b/backend/data/src/main/kotlin/io/tolgee/model/StandardAuditModel.kt index a82db38c84..4949890de1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/StandardAuditModel.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/StandardAuditModel.kt @@ -7,14 +7,17 @@ import jakarta.persistence.MappedSuperclass import jakarta.persistence.SequenceGenerator import org.springframework.data.util.ProxyUtils +const val SEQUENCE_NAME = "hibernate_sequence" +const val ALLOCATION_SIZE = 1000 + @MappedSuperclass abstract class StandardAuditModel : AuditModel(), EntityWithId { @Id @SequenceGenerator( name = "sequenceGenerator", - sequenceName = "hibernate_sequence", + sequenceName = SEQUENCE_NAME, initialValue = 1000000000, - allocationSize = 1000 + allocationSize = ALLOCATION_SIZE ) @GeneratedValue( strategy = GenerationType.SEQUENCE, diff --git a/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt new file mode 100644 index 0000000000..7c3452b067 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt @@ -0,0 +1,35 @@ +package io.tolgee.util + +import java.sql.Connection + +class SequenceIdProvider( + private val connection: Connection, + private val sequenceName: String, + private val allocationSize: Int +) { + + private var currentId: Long? = null + private var currentMaxId: Long? = null + + fun next(): Long { + allocateIfRequired() + val currentId = currentId + this.currentId = currentId!! + 1 + return currentId + } + + private fun allocateIfRequired() { + if (currentId == null || currentMaxId == null || currentId!! >= currentMaxId!!) { + allocate() + } + } + + private fun allocate() { + @Suppress("SqlSourceToSinkFlow") + val statement = connection.prepareStatement("select nextval('$sequenceName')") + val resultSet = statement.executeQuery() + resultSet.next() + currentId = resultSet.getLong(1) + currentMaxId = currentId!! + allocationSize - 1 + } +} From 7e9d1f7890620c95569149cfa4dbb64d404c2f2d Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 16:28:22 +0100 Subject: [PATCH 22/26] fix: Ditch stateless session and use jdbcTemplate --- .../io/tolgee/activity/ActivityService.kt | 81 +++++++++---------- .../kotlin/io/tolgee/batch/BatchJobService.kt | 45 +++++------ .../service/dataImport/StoredDataImporter.kt | 6 ++ .../io/tolgee/util/SequenceIdProvider.kt | 11 ++- .../kotlin/io/tolgee/util/entityManagerExt.kt | 12 --- 5 files changed, 70 insertions(+), 85 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt index 2daaeb2f6d..e43ad59425 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt @@ -13,14 +13,13 @@ import io.tolgee.model.activity.ActivityRevision import io.tolgee.model.views.activity.ProjectActivityView import io.tolgee.repository.activity.ActivityModifiedEntityRepository import io.tolgee.util.Logging -import io.tolgee.util.doInStatelessSession import io.tolgee.util.flushAndClear import jakarta.persistence.EntityManager -import org.hibernate.StatelessSession import org.postgresql.util.PGobject import org.springframework.context.ApplicationContext import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable +import org.springframework.jdbc.core.JdbcTemplate import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional @@ -30,62 +29,56 @@ class ActivityService( private val applicationContext: ApplicationContext, private val activityModifiedEntityRepository: ActivityModifiedEntityRepository, private val currentDateProvider: CurrentDateProvider, - private val objectMapper: ObjectMapper + private val objectMapper: ObjectMapper, + private val jdbcTemplate: JdbcTemplate ) : Logging { @Transactional fun storeActivityData(activityRevision: ActivityRevision, modifiedEntities: ModifiedEntitiesType) { // let's keep the persistent context small entityManager.flushAndClear() - val mergedActivityRevision = entityManager.doInStatelessSession { statelessSession -> - val mergedActivityRevision = statelessSession.persistAcitivyRevision(activityRevision) - statelessSession.persistedDescribingRelations(mergedActivityRevision) - mergedActivityRevision.modifiedEntities = statelessSession.persistModifiedEntities(modifiedEntities) - mergedActivityRevision - } + val mergedActivityRevision = persistAcitivyRevision(activityRevision) + + persistedDescribingRelations(mergedActivityRevision) + mergedActivityRevision.modifiedEntities = persistModifiedEntities(modifiedEntities) applicationContext.publishEvent(OnProjectActivityStoredEvent(this, mergedActivityRevision)) } - private fun StatelessSession.persistModifiedEntities(modifiedEntities: ModifiedEntitiesType): MutableList { + private fun persistModifiedEntities(modifiedEntities: ModifiedEntitiesType): MutableList { val list = modifiedEntities.values.flatMap { it.values }.toMutableList() - - this.doWork { connection -> - val describingRelationQuery = "INSERT INTO activity_modified_entity " + + jdbcTemplate.batchUpdate( + "INSERT INTO activity_modified_entity " + "(entity_class, entity_id, describing_data, " + "describing_relations, modifications, revision_type, activity_revision_id) " + - "VALUES (?, ?, ?, ?, ?, ?, ?)" - val preparedStatement = connection.prepareStatement(describingRelationQuery) - list.forEach { entity -> - preparedStatement.setString(1, entity.entityClass) - preparedStatement.setLong(2, entity.entityId) - preparedStatement.setObject(3, getJsonbObject(entity.describingData)) - preparedStatement.setObject(4, getJsonbObject(entity.describingRelations)) - preparedStatement.setObject(5, getJsonbObject(entity.modifications)) - preparedStatement.setInt(6, RevisionType.values().indexOf(entity.revisionType)) - preparedStatement.setLong(7, entity.activityRevision.id) - preparedStatement.addBatch() - } - preparedStatement.executeBatch() + "VALUES (?, ?, ?, ?, ?, ?, ?)", + list, + 100 + ) { ps, entity -> + ps.setString(1, entity.entityClass) + ps.setLong(2, entity.entityId) + ps.setObject(3, getJsonbObject(entity.describingData)) + ps.setObject(4, getJsonbObject(entity.describingRelations)) + ps.setObject(5, getJsonbObject(entity.modifications)) + ps.setInt(6, RevisionType.values().indexOf(entity.revisionType)) + ps.setLong(7, entity.activityRevision.id) } + return list } - - private fun StatelessSession.persistedDescribingRelations(activityRevision: ActivityRevision) { - this.doWork { connection -> - val describingRelationQuery = "INSERT INTO activity_describing_entity " + + private fun persistedDescribingRelations(activityRevision: ActivityRevision) { + jdbcTemplate.batchUpdate( + "INSERT INTO activity_describing_entity " + "(entity_class, entity_id, data, describing_relations, activity_revision_id) " + - "VALUES (?, ?, ?, ?, ?)" - val preparedStatement = connection.prepareStatement(describingRelationQuery) - activityRevision.describingRelations.forEach { entity -> - preparedStatement.setString(1, entity.entityClass) - preparedStatement.setLong(2, entity.entityId) - preparedStatement.setObject(3, getJsonbObject(entity.data)) - preparedStatement.setObject(4, getJsonbObject(entity.describingRelations)) - preparedStatement.setLong(5, activityRevision.id) - preparedStatement.addBatch() - } - preparedStatement.executeBatch() + "VALUES (?, ?, ?, ?, ?)", + activityRevision.describingRelations, + 100 + ) { ps, entity -> + ps.setString(1, entity.entityClass) + ps.setLong(2, entity.entityId) + ps.setObject(3, getJsonbObject(entity.data)) + ps.setObject(4, getJsonbObject(entity.describingRelations)) + ps.setLong(5, activityRevision.id) } } @@ -96,10 +89,10 @@ class ActivityService( return pgObject } - private fun StatelessSession.persistAcitivyRevision(activityRevision: ActivityRevision): ActivityRevision { + private fun persistAcitivyRevision(activityRevision: ActivityRevision): ActivityRevision { return if (activityRevision.id == 0L) { - activityRevision.timestamp = currentDateProvider.date - this.insert(activityRevision) + entityManager.persist(activityRevision) + entityManager.flushAndClear() activityRevision } else { entityManager.getReference(ActivityRevision::class.java, activityRevision.id) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt index 3a3cf866dd..0de17ccc18 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt @@ -37,6 +37,7 @@ import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Lazy import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable +import org.springframework.jdbc.core.JdbcTemplate import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.event.TransactionalEventListener @@ -56,7 +57,8 @@ class BatchJobService( private val currentDateProvider: CurrentDateProvider, private val securityService: SecurityService, private val authenticationFacade: AuthenticationFacade, - private val objectMapper: ObjectMapper + private val objectMapper: ObjectMapper, + private val jdbcTemplate: JdbcTemplate ) : Logging { companion object { @@ -132,31 +134,28 @@ class BatchJobService( } private fun insertExecutionsViaBatchStatement(executions: List) { - entityManager.unwrap(Session::class.java).doWork { connection -> - val query = """ + val sequenceIdProvider = SequenceIdProvider(SEQUENCE_NAME, ALLOCATION_SIZE) + jdbcTemplate.batchUpdate( + """ insert into tolgee_batch_job_chunk_execution (id, batch_job_id, chunk_number, status, created_at, updated_at, success_targets) values (?, ?, ?, ?, ?, ?, ?) - """ - val statement = connection.prepareStatement(query) - val sequenceIdProvider = SequenceIdProvider(connection, SEQUENCE_NAME, ALLOCATION_SIZE) - val timestamp = Timestamp(currentDateProvider.date.time) - executions.forEach { - val id = sequenceIdProvider.next() - it.id = id - statement.setLong(1, id) - statement.setLong(2, it.batchJob.id) - statement.setInt(3, it.chunkNumber) - statement.setString(4, it.status.name) - statement.setTimestamp(5, timestamp) - statement.setTimestamp(6, timestamp) - statement.setObject(7, PGobject().apply { - type = "jsonb" - value = objectMapper.writeValueAsString(it.successTargets) - }) - statement.addBatch() - } - statement.executeBatch() + """, + executions, + 100 + ) { ps, execution -> + val id = sequenceIdProvider.next(ps.connection) + execution.id = id + ps.setLong(1, id) + ps.setLong(2, execution.batchJob.id) + ps.setInt(3, execution.chunkNumber) + ps.setString(4, execution.status.name) + ps.setTimestamp(5, Timestamp(currentDateProvider.date.time)) + ps.setTimestamp(6, Timestamp(currentDateProvider.date.time)) + ps.setObject(7, PGobject().apply { + type = "jsonb" + value = objectMapper.writeValueAsString(execution.successTargets) + }) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt index b6d1e1dfbf..45aa54c0fc 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt @@ -15,6 +15,8 @@ import io.tolgee.service.key.KeyService import io.tolgee.service.key.NamespaceService import io.tolgee.service.security.SecurityService import io.tolgee.service.translation.TranslationService +import io.tolgee.util.flushAndClear +import jakarta.persistence.EntityManager import org.springframework.context.ApplicationContext class StoredDataImporter( @@ -46,6 +48,8 @@ class StoredDataImporter( */ private val translationService = applicationContext.getBean(TranslationService::class.java) + private val entityManager = applicationContext.getBean(EntityManager::class.java) + private val namespacesToSave = mutableMapOf() /** @@ -89,6 +93,8 @@ class StoredDataImporter( saveMetaData(keyEntitiesToSave) translationService.setOutdatedBatch(outdatedFlagKeys) + + entityManager.flushAndClear() } private fun saveMetaData(keyEntitiesToSave: MutableCollection) { diff --git a/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt index 7c3452b067..9939f24b40 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt @@ -3,7 +3,6 @@ package io.tolgee.util import java.sql.Connection class SequenceIdProvider( - private val connection: Connection, private val sequenceName: String, private val allocationSize: Int ) { @@ -11,20 +10,20 @@ class SequenceIdProvider( private var currentId: Long? = null private var currentMaxId: Long? = null - fun next(): Long { - allocateIfRequired() + fun next(connection: Connection): Long { + allocateIfRequired(connection) val currentId = currentId this.currentId = currentId!! + 1 return currentId } - private fun allocateIfRequired() { + private fun allocateIfRequired(connection: Connection) { if (currentId == null || currentMaxId == null || currentId!! >= currentMaxId!!) { - allocate() + allocate(connection) } } - private fun allocate() { + private fun allocate(connection: Connection) { @Suppress("SqlSourceToSinkFlow") val statement = connection.prepareStatement("select nextval('$sequenceName')") val resultSet = statement.executeQuery() diff --git a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt index 859a4a0cbc..c910945b53 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt @@ -11,15 +11,3 @@ fun EntityManager.flushAndClear() { this.flush() this.clear() } - - -inline fun EntityManager.doInStatelessSession( - crossinline block: (StatelessSession) -> T -): T { - return unwrap(Session::class.java).doReturningWork { connection -> - val statelessSession = unwrap(Session::class.java).sessionFactory.openStatelessSession(connection) - statelessSession.use { ss -> - block(ss) - } - } -} From 4de243839b1293e3c048d5780febaedc55e81bd4 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 16:30:28 +0100 Subject: [PATCH 23/26] chore: Formatting --- .../comments/TranslationCommentModel.kt | 1 - .../TranslationCommentModelAssembler.kt | 1 - .../kotlin/io/tolgee/batch/BatchJobService.kt | 18 ++++++++++-------- .../src/main/kotlin/io/tolgee/model/Project.kt | 4 ++-- .../model/activity/ActivityDescribingEntity.kt | 2 -- .../tolgee/repository/TranslationRepository.kt | 6 ++++-- .../TranslationCommentRepository.kt | 1 - .../tolgee/service/dataImport/ImportService.kt | 1 - .../kotlin/io/tolgee/util/entityManagerExt.kt | 1 - 9 files changed, 16 insertions(+), 19 deletions(-) diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt index 822fca7d76..dfe6d0e11d 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModel.kt @@ -2,7 +2,6 @@ package io.tolgee.hateoas.translations.comments import io.swagger.v3.oas.annotations.media.Schema import io.tolgee.hateoas.user_account.SimpleUserAccountModel -import io.tolgee.hateoas.user_account.UserAccountModel import io.tolgee.model.enums.TranslationCommentState import org.springframework.hateoas.RepresentationModel import org.springframework.hateoas.server.core.Relation diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt index e813b6747f..ae623a6dcb 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/translations/comments/TranslationCommentModelAssembler.kt @@ -2,7 +2,6 @@ package io.tolgee.hateoas.translations.comments import io.tolgee.api.v2.controllers.translation.TranslationCommentController import io.tolgee.hateoas.user_account.SimpleUserAccountModelAssembler -import io.tolgee.hateoas.user_account.UserAccountModelAssembler import io.tolgee.model.translation.TranslationComment import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport import org.springframework.stereotype.Component diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt index 0de17ccc18..3f62bcd5f2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt @@ -31,7 +31,6 @@ import io.tolgee.util.logger import jakarta.persistence.EntityManager import org.apache.commons.codec.digest.DigestUtils.sha256Hex import org.hibernate.LockOptions -import org.hibernate.Session import org.postgresql.util.PGobject import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Lazy @@ -152,10 +151,13 @@ class BatchJobService( ps.setString(4, execution.status.name) ps.setTimestamp(5, Timestamp(currentDateProvider.date.time)) ps.setTimestamp(6, Timestamp(currentDateProvider.date.time)) - ps.setObject(7, PGobject().apply { - type = "jsonb" - value = objectMapper.writeValueAsString(execution.successTargets) - }) + ps.setObject( + 7, + PGobject().apply { + type = "jsonb" + value = objectMapper.writeValueAsString(execution.successTargets) + } + ) } } @@ -189,9 +191,9 @@ class BatchJobService( executions.let { batchJobChunkExecutionQueue.addToQueue(it) } logger.debug( "Starting job ${job.id}, aadded ${executions.size} executions to queue ${ - System.identityHashCode( - batchJobChunkExecutionQueue - ) + System.identityHashCode( + batchJobChunkExecutionQueue + ) }" ) } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/Project.kt b/backend/data/src/main/kotlin/io/tolgee/model/Project.kt index 0b1597a0fa..01a1fa9d6d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/Project.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/Project.kt @@ -109,8 +109,8 @@ class Project( constructor(name: String, description: String? = null, slug: String?, organizationOwner: Organization) : this(id = 0L, name, description, slug) { - this.organizationOwner = organizationOwner - } + this.organizationOwner = organizationOwner + } fun findLanguageOptional(tag: String): Optional { return languages.stream().filter { l: Language -> (l.tag == tag) }.findFirst() diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt index 59afd37523..100c1dd835 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt @@ -2,9 +2,7 @@ package io.tolgee.model.activity import io.hypersistence.utils.hibernate.type.json.JsonBinaryType import io.tolgee.activity.data.EntityDescriptionRef -import io.tolgee.activity.data.RevisionType import jakarta.persistence.Entity -import jakarta.persistence.Enumerated import jakarta.persistence.Id import jakarta.persistence.IdClass import jakarta.persistence.ManyToOne diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt index 4930717013..6d6200583f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt @@ -172,9 +172,11 @@ interface TranslationRepository : JpaRepository { """ ) fun getAllByProjectId(projectId: Long): List - @Query(""" + @Query( + """ from Translation t where t.key = :key and t.language.tag in :languageTags - """) + """ + ) fun findForKeyByLanguages(key: Key, languageTags: Collection): List } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt index 3bbac78caa..243c05fac6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt @@ -29,7 +29,6 @@ interface TranslationCommentRepository : JpaRepository ) fun getAllByProjectId(projectId: Long): List - @Query( """ from TranslationComment tc diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index 1850d4062e..f8c64ae189 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -31,7 +31,6 @@ import io.tolgee.repository.dataImport.ImportRepository import io.tolgee.repository.dataImport.ImportTranslationRepository import io.tolgee.repository.dataImport.issues.ImportFileIssueParamRepository import io.tolgee.repository.dataImport.issues.ImportFileIssueRepository -import io.tolgee.service.key.KeyMetaService import io.tolgee.util.getSafeNamespace import jakarta.persistence.EntityManager import org.springframework.context.ApplicationContext diff --git a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt index c910945b53..bc55bd8855 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/entityManagerExt.kt @@ -2,7 +2,6 @@ package io.tolgee.util import jakarta.persistence.EntityManager import org.hibernate.Session -import org.hibernate.StatelessSession val EntityManager.session: Session get() = this.unwrap(org.hibernate.Session::class.java)!! From 6d71673b3497179ecca4e21785388c909db7dbdf Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 16:47:12 +0100 Subject: [PATCH 24/26] fix: SequenceIdProvider --- .../src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt index 9939f24b40..dd87ce6082 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt @@ -1,5 +1,6 @@ package io.tolgee.util +import io.tolgee.model.ALLOCATION_SIZE import java.sql.Connection class SequenceIdProvider( @@ -28,7 +29,7 @@ class SequenceIdProvider( val statement = connection.prepareStatement("select nextval('$sequenceName')") val resultSet = statement.executeQuery() resultSet.next() - currentId = resultSet.getLong(1) - currentMaxId = currentId!! + allocationSize - 1 + currentMaxId = resultSet.getLong(1) + currentId = currentMaxId!! - allocationSize + 1 } } From 4944ba3a5bb75a6a045b1d801cea413b1b8e99e6 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 16:53:26 +0100 Subject: [PATCH 25/26] chore: KtLint --- .../data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt index dd87ce6082..3f54dc5ee6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/SequenceIdProvider.kt @@ -1,6 +1,5 @@ package io.tolgee.util -import io.tolgee.model.ALLOCATION_SIZE import java.sql.Connection class SequenceIdProvider( From 90ea484f59c09d198ceabf41dfa7f3d4d3e5ac1e Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 18 Dec 2023 19:18:32 +0100 Subject: [PATCH 26/26] fix: Soft delete on last language delete --- .../main/kotlin/io/tolgee/service/dataImport/ImportService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt index f8c64ae189..3aa049b966 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportService.kt @@ -279,7 +279,7 @@ class ImportService( this.importTranslationRepository.deleteAllByLanguage(language) this.importLanguageRepository.delete(language) if (this.findLanguages(import = language.file.import).isEmpty()) { - hardDeleteImport(import) + deleteImport(import) } }