fix: Add metrics for big meta storing (#2476)

tolgee · Sep 20, 2024 · bbb2feb · bbb2feb
1 parent b2533ca
commit bbb2feb
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 8 deletions.
diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt
@@ -14,7 +14,6 @@ import io.tolgee.util.logger
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.springframework.beans.factory.annotation.Autowired
-import kotlin.time.ExperimentalTime
 import kotlin.time.measureTime
 
 class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Logging {
@@ -57,11 +56,11 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
  bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
  }
 
- @OptIn(ExperimentalTime::class)
  @Test
  @ProjectJWTAuthTestMethod
  fun `it performs well`() {
  val keys = testData.addLotOfData()
+ testData.addLotOfReferences(keys)
  saveTestDataAndPrepare()
 
  logger.infoMeasureTime("it performs well time 1") {
@@ -87,6 +86,16 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
  bigMetaService.findExistingKeysDistancesDtosByIds(keys.map { it.id }).assert.hasSize(104790)
  }
 
+ @Test
+ @ProjectJWTAuthTestMethod
+ fun `it performs well (large)`() {
+ val keys = testData.addLotOfData()
+ testData.addLotOfReferences(keys)
+ saveTestDataAndPrepare()
+
+ storeLogOfBigMeta(keys, 0, 200)
+ }
+
  private fun storeLogOfBigMeta(
  keys: List<Key>,
  drop: Int,
@@ -96,7 +105,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
  "big-meta",
  mapOf(
  "relatedKeysInOrder" to
- keys.drop(drop).take(take).map {
+ keys.drop(drop).take(take).reversed().map {
  mapOf(
  "namespace" to it.namespace,
  "keyName" to it.name,
@@ -110,6 +119,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
  @ProjectJWTAuthTestMethod
  fun `it returns correct data`() {
  val keys = testData.addLotOfData()
+ testData.addLotOfReferences(keys)
  saveTestDataAndPrepare()
 
  performProjectAuthGet("keys/${keys.first().id}/big-meta").andIsOk.andAssertThatJson {

diff --git a/backend/data/src/main/kotlin/io/tolgee/Metrics.kt b/backend/data/src/main/kotlin/io/tolgee/Metrics.kt
@@ -3,6 +3,7 @@ package io.tolgee
 import io.micrometer.core.instrument.Counter
 import io.micrometer.core.instrument.Gauge
 import io.micrometer.core.instrument.MeterRegistry
+import io.micrometer.core.instrument.Timer
 import org.springframework.stereotype.Component
 import java.util.concurrent.ConcurrentLinkedQueue
 
@@ -27,4 +28,16 @@ class Metrics(
  .description("Total number of failures when trying to store data about batch job execution (execution failed)")
  .register(meterRegistry)
  }
+
+ val bigMetaStoringTimer: Timer by lazy {
+ Timer.builder("tolgee.big_meta.storing.timer")
+ .description("Time spent storing big meta data")
+ .register(meterRegistry)
+ }
+
+ val bigMetaNewDistancesComputeTimer: Timer by lazy {
+ Timer.builder("tolgee.big_meta.new_distances.compute.timer")
+ .description("Time spent computing new distances for big meta data")
+ .register(meterRegistry)
+ }
 }
diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt
@@ -36,7 +36,10 @@ class BigMetaTestData {
  (0..5000).map {
  projectBuilder.addKey(null, "key$it").self
  }
+ return keys
+ }
 
+ fun addLotOfReferences(keys: List<Key>) {
  keys.forEachIndexed forEach1@{ idx1, key1 ->
  keys.forEachIndexed forEach2@{ idx2, key2 ->
  if (idx1 >= idx2 || abs(idx1 - idx2) > (BigMetaService.MAX_ORDER_DISTANCE + 1)) return@forEach2
@@ -46,7 +49,5 @@ class BigMetaTestData {
  }
  }
  }
-
- return keys
  }
 }
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt
@@ -1,5 +1,6 @@
 package io.tolgee.service.bigMeta
 
+import io.tolgee.Metrics
 import io.tolgee.component.CurrentDateProvider
 import io.tolgee.dtos.BigMetaDto
 import io.tolgee.dtos.RelatedKeyDto
@@ -36,6 +37,7 @@ class BigMetaService(
  private val transactionManager: PlatformTransactionManager,
  private val jdbcTemplate: JdbcTemplate,
  private val currentDateProvider: CurrentDateProvider,
+ private val metrics: Metrics,
 ) : Logging {
  companion object {
  const val MAX_DISTANCE_SCORE = 10000L
@@ -64,9 +66,14 @@ class BigMetaService(
  return
  }
 
- val util = KeysDistanceUtil(relatedKeysInOrder, project, this)
+ val util =
+ metrics.bigMetaNewDistancesComputeTimer.recordCallable {
+ KeysDistanceUtil(relatedKeysInOrder, project, this)
+ }!!
 
- insertNewDistances(util.newDistances)
+ metrics.bigMetaStoringTimer.recordCallable {
+ insertNewDistances(util.newDistances)
+ }
  }
 
  private fun insertNewDistances(toInsert: List<KeysDistanceDto>) {
@@ -78,7 +85,7 @@ class BigMetaService(
  on conflict (key1id, key2id) do update set score = excluded.score, hits = excluded.hits, updated_at = ?
  """,
  toInsert,
- 100,
+ 10000,
  ) { ps, dto ->
  ps.setLong(1, dto.key1Id)
  ps.setLong(2, dto.key2Id)