diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt index 77c6533e1b..8871f02c03 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt @@ -54,7 +54,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin ) ).andIsOk - bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(1) + bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(1) } @OptIn(ExperimentalTime::class) @@ -84,7 +84,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin storeLogOfBigMeta(keys, 800, 50) }.inWholeSeconds.assert.isLessThan(10) - bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(104790) + bigMetaService.findExistingKeysDistancesDtosByIds(keys.map { it.id }).assert.hasSize(104790) } private fun storeLogOfBigMeta(keys: List, drop: Int, take: Int) { diff --git a/backend/app/src/test/kotlin/io/tolgee/service/BigMetaServiceTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/BigMetaServiceTest.kt index 779721dd6c..f10ac63daf 100644 --- a/backend/app/src/test/kotlin/io/tolgee/service/BigMetaServiceTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/service/BigMetaServiceTest.kt @@ -48,14 +48,14 @@ class BigMetaServiceTest : ProjectAuthControllerTest("/v2/projects/") { ) ).andIsOk - bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(1) + bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(1) executeInNewTransaction { keyService.delete(testData.yepKey.id) } waitForNotThrowing(pollTime = 1000) { - bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(0) + bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(0) } } } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt b/backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt index 2d8e22a88a..9ed1c72147 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt @@ -30,7 +30,7 @@ class KeysDistance( @ManyToOne(fetch = FetchType.LAZY) lateinit var project: Project - var score: Long = 10000 + var score: Long = MAX_SCORE var hits: Long = 1 override fun equals(other: Any?): Boolean { @@ -63,4 +63,8 @@ class KeysDistance( @Transient var new = false + + companion object { + const val MAX_SCORE = 10000L + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt index f527099e72..537e26863b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt @@ -10,17 +10,6 @@ import org.springframework.stereotype.Repository @Repository interface KeysDistanceRepository : JpaRepository { - @Query( - """ - from KeysDistance kd - where kd.key1Id in ( - select kd2.key1Id from KeysDistance kd2 where kd2.key1Id in :data or kd2.key2Id in :data - ) or kd.key2Id in ( - select kd3.key2Id from KeysDistance kd3 where kd3.key1Id in :data or kd3.key2Id in :data - ) - """ - ) - fun findForKeyIdsWithRelations(data: Collection): List @Query( """ diff --git a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt index bb87c160ff..8efbb92944 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt @@ -1,5 +1,6 @@ package io.tolgee.service.bigMeta +import io.tolgee.component.CurrentDateProvider import io.tolgee.dtos.BigMetaDto import io.tolgee.dtos.RelatedKeyDto import io.tolgee.dtos.query_results.KeyIdFindResult @@ -21,16 +22,20 @@ import jakarta.persistence.criteria.CriteriaBuilder import jakarta.persistence.criteria.CriteriaQuery import jakarta.persistence.criteria.JoinType import org.springframework.context.event.EventListener +import org.springframework.jdbc.core.JdbcTemplate import org.springframework.scheduling.annotation.Async import org.springframework.stereotype.Service import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.annotation.Transactional +import java.sql.Timestamp @Service class BigMetaService( private val keysDistanceRepository: KeysDistanceRepository, private val entityManager: EntityManager, - private val transactionManager: PlatformTransactionManager + private val transactionManager: PlatformTransactionManager, + private val jdbcTemplate: JdbcTemplate, + private val currentDateProvider: CurrentDateProvider ) : Logging { companion object { const val MAX_DISTANCE_SCORE = 10000L @@ -56,8 +61,31 @@ class BigMetaService( return } - val distances = KeysDistanceUtil(relatedKeysInOrder, project, this).newDistances - keysDistanceRepository.saveAll(distances) + val util = KeysDistanceUtil(relatedKeysInOrder, project, this) + + insertNewDistances(util.newDistances) + } + + private fun insertNewDistances(toInsert: List) { + val timestamp = Timestamp(currentDateProvider.date.time) + jdbcTemplate.batchUpdate( + """ + insert into keys_distance (key1id, key2id, score, hits, created_at, updated_at, project_id) + values (?, ?, ?, ?, ?, ?, ?) + on conflict (key1id, key2id) do update set score = excluded.score, hits = excluded.hits, updated_at = ? + """, + toInsert, + 100 + ) { ps, dto -> + ps.setLong(1, dto.key1Id) + ps.setLong(2, dto.key2Id) + ps.setLong(3, dto.score) + ps.setLong(4, dto.hits) + ps.setTimestamp(5, timestamp) + ps.setTimestamp(6, timestamp) + ps.setLong(7, dto.projectId) + ps.setTimestamp(8, timestamp) + } } fun getKeyIdsForItems( @@ -73,14 +101,26 @@ class BigMetaService( } @Transactional - fun findExistingKeyDistances(keys: List, project: Project): List { + fun findExistingKeyDistances(keys: List, project: Project): Set { val keyIds = keys.map { it.id } - return findExistingKeysDistancesByIds(keyIds) + return findExistingKeysDistancesDtosByIds(keyIds) } @Transactional - fun findExistingKeysDistancesByIds(keyIds: List): List { - return keysDistanceRepository.findForKeyIdsWithRelations(keyIds) + fun findExistingKeysDistancesDtosByIds(keyIds: List): Set { + return entityManager.createQuery( + """ + select new io.tolgee.service.bigMeta.KeysDistanceDto(kd.key1Id, kd.key2Id, kd.score, kd.project.id, kd.hits) from KeysDistance kd + where kd.key1Id in ( + select kd2.key1Id from KeysDistance kd2 where kd2.key1Id in :data or kd2.key2Id in :data + ) or kd.key2Id in ( + select kd3.key2Id from KeysDistance kd3 where kd3.key1Id in :data or kd3.key2Id in :data + ) + """, + KeysDistanceDto::class.java + ) + .setParameter("data", keyIds) + .resultList.toSet() } fun get(id: Long): KeysDistance { diff --git a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceDto.kt b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceDto.kt new file mode 100644 index 0000000000..265d7402cb --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceDto.kt @@ -0,0 +1,11 @@ +package io.tolgee.service.bigMeta + +import io.tolgee.model.keyBigMeta.KeysDistance.Companion.MAX_SCORE + +data class KeysDistanceDto( + var key1Id: Long = 0, + var key2Id: Long = 0, + var score: Long = MAX_SCORE, + var projectId: Long, + var hits: Long = 1 +) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt index da4e15685f..7599f86f30 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt @@ -1,9 +1,7 @@ package io.tolgee.service.bigMeta -import com.google.common.primitives.Longs import io.tolgee.dtos.RelatedKeyDto import io.tolgee.model.Project -import io.tolgee.model.keyBigMeta.KeysDistance import io.tolgee.util.Logging import kotlin.math.abs import kotlin.math.max @@ -17,7 +15,7 @@ class KeysDistanceUtil( val newDistances by lazy { increaseRelevant() decreaseOthers() - distances.values + distances.values.toList() } private fun increaseRelevant() { @@ -51,7 +49,7 @@ class KeysDistanceUtil( bigMetaService.getKeyIdsForItems(relatedKeysInOrder, project.id) } - private val relevant = mutableMapOf, KeysDistance>() + private val relevant = mutableMapOf, KeysDistanceDto>() private val keyIdMap by lazy { keys.associate { (it.namespace to it.name) to it.id } @@ -69,15 +67,14 @@ class KeysDistanceUtil( private val relatedKeysSize = relatedKeysInOrder.size - private fun createDistance(key1Id: Long, key2Id: Long): KeysDistance { - return KeysDistance() - .apply { - this.key1Id = Longs.min(key1Id, key2Id) - this.key2Id = max(key1Id, key2Id) - this.project = this@KeysDistanceUtil.project - this.new = true - distances[this.key1Id to this.key2Id] = this - } + private fun createDistance(key1Id: Long, key2Id: Long): KeysDistanceDto { + return KeysDistanceDto( + key1Id = min(a = key1Id, b = key2Id), + key2Id = max(key1Id, key2Id), + projectId = project.id, + ).apply { + distances[this.key1Id to this.key2Id] = this + } } private fun computeDistanceScore( diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/KeysDistanceUtilTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/KeysDistanceUtilTest.kt index f9e4f6647d..797dfc326b 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/KeysDistanceUtilTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/KeysDistanceUtilTest.kt @@ -3,8 +3,8 @@ package io.tolgee.unit import io.tolgee.dtos.RelatedKeyDto import io.tolgee.dtos.query_results.KeyIdFindResult import io.tolgee.model.Project -import io.tolgee.model.keyBigMeta.KeysDistance import io.tolgee.service.bigMeta.BigMetaService +import io.tolgee.service.bigMeta.KeysDistanceDto import io.tolgee.service.bigMeta.KeysDistanceUtil import io.tolgee.testing.assert import org.junit.jupiter.api.Test @@ -28,12 +28,12 @@ class KeysDistanceUtilTest { whenever(bigMetaService.findExistingKeyDistances(any(), any())) .thenReturn( - listOf( - KeysDistance(1, 3).also { keysDistance -> + setOf( + KeysDistanceDto(1, 3, projectId = 0).also { keysDistance -> keysDistance.score = 10000 keysDistance.hits = 10 }, - KeysDistance(3, 4).also { keysDistance -> + KeysDistanceDto(3, 4, projectId = 0).also { keysDistance -> keysDistance.score = 10000 keysDistance.hits = 1 }