fix: Faster context storing (#2048)

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt

@@ -54,7 +54,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
       )
     ).andIsOk
 
-    bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
+    bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
   }
 
   @OptIn(ExperimentalTime::class)
@@ -84,7 +84,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
       storeLogOfBigMeta(keys, 800, 50)
     }.inWholeSeconds.assert.isLessThan(10)
 
-    bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(104790)
+    bigMetaService.findExistingKeysDistancesDtosByIds(keys.map { it.id }).assert.hasSize(104790)
   }
 
   private fun storeLogOfBigMeta(keys: List<Key>, drop: Int, take: Int) {

backend/app/src/test/kotlin/io/tolgee/service/BigMetaServiceTest.kt

@@ -48,14 +48,14 @@ class BigMetaServiceTest : ProjectAuthControllerTest("/v2/projects/") {
       )
     ).andIsOk
 
-    bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
+    bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
 
     executeInNewTransaction {
       keyService.delete(testData.yepKey.id)
     }
 
     waitForNotThrowing(pollTime = 1000) {
-      bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(0)
+      bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(0)
     }
   }
 }

backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt

@@ -30,7 +30,7 @@ class KeysDistance(
   @ManyToOne(fetch = FetchType.LAZY)
   lateinit var project: Project
 
-  var score: Long = 10000
+  var score: Long = MAX_SCORE
 
   var hits: Long = 1
   override fun equals(other: Any?): Boolean {
@@ -63,4 +63,8 @@ class KeysDistance(
 
   @Transient
   var new = false
+
+  companion object {
+    const val MAX_SCORE = 10000L
+  }
 }

backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt

@@ -10,17 +10,6 @@ import org.springframework.stereotype.Repository
 
 @Repository
 interface KeysDistanceRepository : JpaRepository<KeysDistance, Long> {
-  @Query(
-    """
-        from KeysDistance kd
-        where kd.key1Id in (
-            select kd2.key1Id from KeysDistance kd2 where kd2.key1Id in :data or kd2.key2Id in :data
-        ) or kd.key2Id in (
-            select kd3.key2Id from KeysDistance kd3 where kd3.key1Id in :data or kd3.key2Id in :data
-        )
- """
-  )
-  fun findForKeyIdsWithRelations(data: Collection<Long>): List<KeysDistance>
 
   @Query(
     """

backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt

@@ -1,5 +1,6 @@
 package io.tolgee.service.bigMeta
 
+import io.tolgee.component.CurrentDateProvider
 import io.tolgee.dtos.BigMetaDto
 import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.query_results.KeyIdFindResult
@@ -21,16 +22,20 @@ import jakarta.persistence.criteria.CriteriaBuilder
 import jakarta.persistence.criteria.CriteriaQuery
 import jakarta.persistence.criteria.JoinType
 import org.springframework.context.event.EventListener
+import org.springframework.jdbc.core.JdbcTemplate
 import org.springframework.scheduling.annotation.Async
 import org.springframework.stereotype.Service
 import org.springframework.transaction.PlatformTransactionManager
 import org.springframework.transaction.annotation.Transactional
+import java.sql.Timestamp
 
 @Service
 class BigMetaService(
   private val keysDistanceRepository: KeysDistanceRepository,
   private val entityManager: EntityManager,
-  private val transactionManager: PlatformTransactionManager
+  private val transactionManager: PlatformTransactionManager,
+  private val jdbcTemplate: JdbcTemplate,
+  private val currentDateProvider: CurrentDateProvider
 ) : Logging {
   companion object {
     const val MAX_DISTANCE_SCORE = 10000L
@@ -56,8 +61,31 @@ class BigMetaService(
       return
     }
 
-    val distances = KeysDistanceUtil(relatedKeysInOrder, project, this).newDistances
-    keysDistanceRepository.saveAll(distances)
+    val util = KeysDistanceUtil(relatedKeysInOrder, project, this)
+
+    insertNewDistances(util.newDistances)
+  }
+
+  private fun insertNewDistances(toInsert: List<KeysDistanceDto>) {
+    val timestamp = Timestamp(currentDateProvider.date.time)
+    jdbcTemplate.batchUpdate(
+      """
+      insert into keys_distance (key1id, key2id, score, hits, created_at, updated_at, project_id) 
+      values (?, ?, ?, ?, ?, ?, ?)
+      on conflict (key1id, key2id) do update set score = excluded.score, hits = excluded.hits, updated_at = ?
+      """,
+      toInsert,
+      100
+    ) { ps, dto ->
+      ps.setLong(1, dto.key1Id)
+      ps.setLong(2, dto.key2Id)
+      ps.setLong(3, dto.score)
+      ps.setLong(4, dto.hits)
+      ps.setTimestamp(5, timestamp)
+      ps.setTimestamp(6, timestamp)
+      ps.setLong(7, dto.projectId)
+      ps.setTimestamp(8, timestamp)
+    }
   }
 
   fun getKeyIdsForItems(
@@ -73,14 +101,26 @@ class BigMetaService(
   }
 
   @Transactional
-  fun findExistingKeyDistances(keys: List<KeyIdFindResult>, project: Project): List<KeysDistance> {
+  fun findExistingKeyDistances(keys: List<KeyIdFindResult>, project: Project): Set<KeysDistanceDto> {
     val keyIds = keys.map { it.id }
-    return findExistingKeysDistancesByIds(keyIds)
+    return findExistingKeysDistancesDtosByIds(keyIds)
   }
 
   @Transactional
-  fun findExistingKeysDistancesByIds(keyIds: List<Long>): List<KeysDistance> {
-    return keysDistanceRepository.findForKeyIdsWithRelations(keyIds)
+  fun findExistingKeysDistancesDtosByIds(keyIds: List<Long>): Set<KeysDistanceDto> {
+    return entityManager.createQuery(
+      """
+       select new io.tolgee.service.bigMeta.KeysDistanceDto(kd.key1Id, kd.key2Id, kd.score, kd.project.id, kd.hits) from KeysDistance kd
+        where kd.key1Id in (
+            select kd2.key1Id from KeysDistance kd2 where kd2.key1Id in :data or kd2.key2Id in :data
+        ) or kd.key2Id in (
+            select kd3.key2Id from KeysDistance kd3 where kd3.key1Id in :data or kd3.key2Id in :data
+        )
+    """,
+      KeysDistanceDto::class.java
+    )
+      .setParameter("data", keyIds)
+      .resultList.toSet()
   }
 
   fun get(id: Long): KeysDistance {

backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceDto.kt

@@ -0,0 +1,11 @@
+package io.tolgee.service.bigMeta
+
+import io.tolgee.model.keyBigMeta.KeysDistance.Companion.MAX_SCORE
+
+data class KeysDistanceDto(
+  var key1Id: Long = 0,
+  var key2Id: Long = 0,
+  var score: Long = MAX_SCORE,
+  var projectId: Long,
+  var hits: Long = 1
+)

backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt

@@ -1,9 +1,7 @@
 package io.tolgee.service.bigMeta
 
-import com.google.common.primitives.Longs
 import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.model.Project
-import io.tolgee.model.keyBigMeta.KeysDistance
 import io.tolgee.util.Logging
 import kotlin.math.abs
 import kotlin.math.max
@@ -17,7 +15,7 @@ class KeysDistanceUtil(
   val newDistances by lazy {
     increaseRelevant()
     decreaseOthers()
-    distances.values
+    distances.values.toList()
   }
 
   private fun increaseRelevant() {
@@ -51,7 +49,7 @@ class KeysDistanceUtil(
     bigMetaService.getKeyIdsForItems(relatedKeysInOrder, project.id)
   }
 
-  private val relevant = mutableMapOf<Pair<Long, Long>, KeysDistance>()
+  private val relevant = mutableMapOf<Pair<Long, Long>, KeysDistanceDto>()
 
   private val keyIdMap by lazy {
     keys.associate { (it.namespace to it.name) to it.id }
@@ -69,15 +67,14 @@ class KeysDistanceUtil(
 
   private val relatedKeysSize = relatedKeysInOrder.size
 
-  private fun createDistance(key1Id: Long, key2Id: Long): KeysDistance {
-    return KeysDistance()
-      .apply {
-        this.key1Id = Longs.min(key1Id, key2Id)
-        this.key2Id = max(key1Id, key2Id)
-        this.project = this@KeysDistanceUtil.project
-        this.new = true
-        distances[this.key1Id to this.key2Id] = this
-      }
+  private fun createDistance(key1Id: Long, key2Id: Long): KeysDistanceDto {
+    return KeysDistanceDto(
+      key1Id = min(a = key1Id, b = key2Id),
+      key2Id = max(key1Id, key2Id),
+      projectId = project.id,
+    ).apply {
+      distances[this.key1Id to this.key2Id] = this
+    }
   }
 
   private fun computeDistanceScore(

backend/data/src/test/kotlin/io/tolgee/unit/KeysDistanceUtilTest.kt

@@ -3,8 +3,8 @@ package io.tolgee.unit
 import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.query_results.KeyIdFindResult
 import io.tolgee.model.Project
-import io.tolgee.model.keyBigMeta.KeysDistance
 import io.tolgee.service.bigMeta.BigMetaService
+import io.tolgee.service.bigMeta.KeysDistanceDto
 import io.tolgee.service.bigMeta.KeysDistanceUtil
 import io.tolgee.testing.assert
 import org.junit.jupiter.api.Test
@@ -28,12 +28,12 @@ class KeysDistanceUtilTest {
 
     whenever(bigMetaService.findExistingKeyDistances(any(), any()))
       .thenReturn(
-        listOf(
-          KeysDistance(1, 3).also { keysDistance ->
+        setOf(
+          KeysDistanceDto(1, 3, projectId = 0).also { keysDistance ->
             keysDistance.score = 10000
             keysDistance.hits = 10
           },
-          KeysDistance(3, 4).also { keysDistance ->
+          KeysDistanceDto(3, 4, projectId = 0).also { keysDistance ->
             keysDistance.score = 10000
             keysDistance.hits = 1
           }