fix: Faster context storing

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt

@@ -54,7 +54,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
  )
  ).andIsOk
 
- bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
+ bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
  }
 
  @OptIn(ExperimentalTime::class)
@@ -84,7 +84,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
  storeLogOfBigMeta(keys, 800, 50)
  }.inWholeSeconds.assert.isLessThan(10)
 
- bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(104790)
+ bigMetaService.findExistingKeysDistancesDtosByIds(keys.map { it.id }).assert.hasSize(104790)
  }
 
  private fun storeLogOfBigMeta(keys: List<Key>, drop: Int, take: Int) {

backend/app/src/test/kotlin/io/tolgee/service/BigMetaServiceTest.kt

@@ -48,14 +48,14 @@ class BigMetaServiceTest : ProjectAuthControllerTest("/v2/projects/") {
  )
  ).andIsOk
 
- bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
+ bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
 
  executeInNewTransaction {
  keyService.delete(testData.yepKey.id)
  }
 
  waitForNotThrowing(pollTime = 1000) {
- bigMetaService.findExistingKeysDistancesByIds(listOf(testData.yepKey.id)).assert.hasSize(0)
+ bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(0)
  }
  }
 }

backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt

@@ -30,7 +30,7 @@ class KeysDistance(
  @ManyToOne(fetch = FetchType.LAZY)
  lateinit var project: Project
 
- var score: Long = 10000
+ var score: Long = MAX_SCORE
 
  var hits: Long = 1
  override fun equals(other: Any?): Boolean {
@@ -63,4 +63,8 @@ class KeysDistance(
 
  @Transient
  var new = false
+
+ companion object {
+ const val MAX_SCORE = 10000L
+ }
 }

backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt

@@ -10,17 +10,6 @@ import org.springframework.stereotype.Repository
 
 @Repository
 interface KeysDistanceRepository : JpaRepository<KeysDistance, Long> {
- @Query(
- """
- from KeysDistance kd
- where kd.key1Id in (
- select kd2.key1Id from KeysDistance kd2 where kd2.key1Id in :data or kd2.key2Id in :data
- ) or kd.key2Id in (
- select kd3.key2Id from KeysDistance kd3 where kd3.key1Id in :data or kd3.key2Id in :data
- )
- """
- )
- fun findForKeyIdsWithRelations(data: Collection<Long>): List<KeysDistance>
 
  @Query(
  """

backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt

@@ -1,5 +1,6 @@
 package io.tolgee.service.bigMeta
 
+import io.tolgee.component.CurrentDateProvider
 import io.tolgee.dtos.BigMetaDto
 import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.query_results.KeyIdFindResult
@@ -21,16 +22,20 @@ import jakarta.persistence.criteria.CriteriaBuilder
 import jakarta.persistence.criteria.CriteriaQuery
 import jakarta.persistence.criteria.JoinType
 import org.springframework.context.event.EventListener
+import org.springframework.jdbc.core.JdbcTemplate
 import org.springframework.scheduling.annotation.Async
 import org.springframework.stereotype.Service
 import org.springframework.transaction.PlatformTransactionManager
 import org.springframework.transaction.annotation.Transactional
+import java.sql.Timestamp
 
 @Service
 class BigMetaService(
  private val keysDistanceRepository: KeysDistanceRepository,
  private val entityManager: EntityManager,
- private val transactionManager: PlatformTransactionManager
+ private val transactionManager: PlatformTransactionManager,
+ private val jdbcTemplate: JdbcTemplate,
+ private val currentDateProvider: CurrentDateProvider
 ) : Logging {
  companion object {
  const val MAX_DISTANCE_SCORE = 10000L
@@ -56,8 +61,31 @@ class BigMetaService(
  return
  }
 
- val distances = KeysDistanceUtil(relatedKeysInOrder, project, this).newDistances
- keysDistanceRepository.saveAll(distances)
+ val util = KeysDistanceUtil(relatedKeysInOrder, project, this)
+
+ insertNewDistances(util.newDistances)
+ }
+
+ private fun insertNewDistances(toInsert: List<KeysDistanceDto>) {
+ val timestamp = Timestamp(currentDateProvider.date.time)
+ jdbcTemplate.batchUpdate(
+ """
+ insert into keys_distance (key1id, key2id, score, hits, created_at, updated_at, project_id) 
+ values (?, ?, ?, ?, ?, ?, ?)
+ on conflict (key1id, key2id) do update set score = excluded.score, hits = excluded.hits, updated_at = ?
+ """,
+ toInsert,
+ 100
+ ) { ps, dto ->
+ ps.setLong(1, dto.key1Id)
+ ps.setLong(2, dto.key2Id)
+ ps.setLong(3, dto.score)
+ ps.setLong(4, dto.hits)
+ ps.setTimestamp(5, timestamp)
+ ps.setTimestamp(6, timestamp)
+ ps.setLong(7, dto.projectId)
+ ps.setTimestamp(8, timestamp)
+ }
  }
 
  fun getKeyIdsForItems(
@@ -73,14 +101,26 @@ class BigMetaService(
  }
 
  @Transactional
- fun findExistingKeyDistances(keys: List<KeyIdFindResult>, project: Project): List<KeysDistance> {
+ fun findExistingKeyDistances(keys: List<KeyIdFindResult>, project: Project): Set<KeysDistanceDto> {
  val keyIds = keys.map { it.id }
- return findExistingKeysDistancesByIds(keyIds)
+ return findExistingKeysDistancesDtosByIds(keyIds)
  }
 
  @Transactional
- fun findExistingKeysDistancesByIds(keyIds: List<Long>): List<KeysDistance> {
- return keysDistanceRepository.findForKeyIdsWithRelations(keyIds)
+ fun findExistingKeysDistancesDtosByIds(keyIds: List<Long>): Set<KeysDistanceDto> {
+ return entityManager.createQuery(
+ """
+ select new io.tolgee.service.bigMeta.KeysDistanceDto(kd.key1Id, kd.key2Id, kd.score, kd.project.id, kd.hits) from KeysDistance kd
+ where kd.key1Id in (
+ select kd2.key1Id from KeysDistance kd2 where kd2.key1Id in :data or kd2.key2Id in :data
+ ) or kd.key2Id in (
+ select kd3.key2Id from KeysDistance kd3 where kd3.key1Id in :data or kd3.key2Id in :data
+ )
+ """,
+ KeysDistanceDto::class.java
+ )
+ .setParameter("data", keyIds)
+ .resultList.toSet()
  }
 
  fun get(id: Long): KeysDistance {

backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceDto.kt

@@ -0,0 +1,11 @@
+package io.tolgee.service.bigMeta
+
+import io.tolgee.model.keyBigMeta.KeysDistance.Companion.MAX_SCORE
+
+data class KeysDistanceDto(
+ var key1Id: Long = 0,
+ var key2Id: Long = 0,
+ var score: Long = MAX_SCORE,
+ var projectId: Long,
+ var hits: Long = 1
+)

backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt

@@ -1,9 +1,7 @@
 package io.tolgee.service.bigMeta
 
-import com.google.common.primitives.Longs
 import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.model.Project
-import io.tolgee.model.keyBigMeta.KeysDistance
 import io.tolgee.util.Logging
 import kotlin.math.abs
 import kotlin.math.max
@@ -17,7 +15,7 @@ class KeysDistanceUtil(
  val newDistances by lazy {
  increaseRelevant()
  decreaseOthers()
- distances.values
+ distances.values.toList()
  }
 
  private fun increaseRelevant() {
@@ -51,7 +49,7 @@ class KeysDistanceUtil(
  bigMetaService.getKeyIdsForItems(relatedKeysInOrder, project.id)
  }
 
- private val relevant = mutableMapOf<Pair<Long, Long>, KeysDistance>()
+ private val relevant = mutableMapOf<Pair<Long, Long>, KeysDistanceDto>()
 
  private val keyIdMap by lazy {
  keys.associate { (it.namespace to it.name) to it.id }
@@ -69,15 +67,14 @@ class KeysDistanceUtil(
 
  private val relatedKeysSize = relatedKeysInOrder.size
 
- private fun createDistance(key1Id: Long, key2Id: Long): KeysDistance {
- return KeysDistance()
- .apply {
- this.key1Id = Longs.min(key1Id, key2Id)
- this.key2Id = max(key1Id, key2Id)
- this.project = this@KeysDistanceUtil.project
- this.new = true
- distances[this.key1Id to this.key2Id] = this
- }
+ private fun createDistance(key1Id: Long, key2Id: Long): KeysDistanceDto {
+ return KeysDistanceDto(
+ key1Id = min(a = key1Id, b = key2Id),
+ key2Id = max(key1Id, key2Id),
+ projectId = project.id,
+ ).apply {
+ distances[this.key1Id to this.key2Id] = this
+ }
  }
 
  private fun computeDistanceScore(

backend/data/src/test/kotlin/io/tolgee/unit/KeysDistanceUtilTest.kt

@@ -3,8 +3,8 @@ package io.tolgee.unit
 import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.query_results.KeyIdFindResult
 import io.tolgee.model.Project
-import io.tolgee.model.keyBigMeta.KeysDistance
 import io.tolgee.service.bigMeta.BigMetaService
+import io.tolgee.service.bigMeta.KeysDistanceDto
 import io.tolgee.service.bigMeta.KeysDistanceUtil
 import io.tolgee.testing.assert
 import org.junit.jupiter.api.Test
@@ -28,12 +28,12 @@ class KeysDistanceUtilTest {
 
  whenever(bigMetaService.findExistingKeyDistances(any(), any()))
  .thenReturn(
- listOf(
- KeysDistance(1, 3).also { keysDistance ->
+ setOf(
+ KeysDistanceDto(1, 3, projectId = 0).also { keysDistance ->
  keysDistance.score = 10000
  keysDistance.hits = 10
  },
- KeysDistance(3, 4).also { keysDistance ->
+ KeysDistanceDto(3, 4, projectId = 0).also { keysDistance ->
  keysDistance.score = 10000
  keysDistance.hits = 1
  }