feat: Enable big-meta (context storing) in key create / edit endpoint…

…s & optimize (#2005)

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt

@@ -10,6 +10,7 @@ import io.tolgee.security.ProjectHolder
 import io.tolgee.security.authentication.AllowApiAccess
 import io.tolgee.security.authorization.RequiresProjectPermissions
 import io.tolgee.service.bigMeta.BigMetaService
+import io.tolgee.util.Logging
 import jakarta.validation.Valid
 import org.springframework.hateoas.CollectionModel
 import org.springframework.web.bind.annotation.GetMapping
@@ -29,18 +30,18 @@ class BigMetaController(
   private val bigMetaService: BigMetaService,
   private val projectHolder: ProjectHolder,
   private val keyWithBaseTranslationModelAssembler: KeyWithBaseTranslationModelAssembler,
-) {
+) : Logging {
   @PostMapping("/big-meta")
   @Operation(summary = "Stores a bigMeta for a project")
-  @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ])
+  @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT])
   @AllowApiAccess
   fun store(@RequestBody @Valid data: BigMetaDto) {
     bigMetaService.store(data, projectHolder.projectEntity)
   }
 
   @GetMapping("/keys/{id}/big-meta")
   @Operation(summary = "Returns a bigMeta for given key")
-  @RequiresProjectPermissions([ Scope.TRANSLATIONS_VIEW ])
+  @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW])
   @AllowApiAccess
   fun getBigMeta(
     @PathVariable("id") id: Long

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/KeyController.kt

@@ -95,24 +95,41 @@ class KeyController(
   @RequiresProjectPermissions([Scope.KEYS_CREATE])
   @AllowApiAccess
   fun create(@RequestBody @Valid dto: CreateKeyDto): ResponseEntity<KeyWithDataModel> {
-    if (dto.screenshotUploadedImageIds != null || !dto.screenshots.isNullOrEmpty()) {
-      projectHolder.projectEntity.checkScreenshotsUploadPermission()
+    checkScrenshotUploadPermissions(dto)
+    checkTranslatePermission(dto)
+    checkCanStoreBigMeta(dto)
+    checkStateChangePermission(dto)
+
+    val key = keyService.create(projectHolder.projectEntity, dto)
+    return ResponseEntity(keyWithDataModelAssembler.toModel(key), HttpStatus.CREATED)
+  }
+
+  private fun checkCanStoreBigMeta(dto: CreateKeyDto) {
+    if (!dto.relatedKeysInOrder.isNullOrEmpty()) {
+      securityService.checkBigMetaUploadPermission(projectHolder.project.id)
     }
+  }
 
+  private fun checkTranslatePermission(dto: CreateKeyDto) {
     dto.translations?.filterValues { !it.isNullOrEmpty() }?.keys?.let { languageTags ->
       if (languageTags.isNotEmpty()) {
         securityService.checkLanguageTranslatePermissionByTag(projectHolder.project.id, languageTags)
       }
     }
+  }
 
+  private fun checkStateChangePermission(dto: CreateKeyDto) {
     dto.states?.filterValues { it != AssignableTranslationState.TRANSLATED }?.keys?.let { languageTags ->
       if (languageTags.isNotEmpty()) {
         securityService.checkLanguageStateChangePermissionsByTag(projectHolder.project.id, languageTags)
       }
     }
+  }
 
-    val key = keyService.create(projectHolder.projectEntity, dto)
-    return ResponseEntity(keyWithDataModelAssembler.toModel(key), HttpStatus.CREATED)
+  private fun checkScrenshotUploadPermissions(dto: CreateKeyDto) {
+    if (dto.screenshotUploadedImageIds != null || !dto.screenshots.isNullOrEmpty()) {
+      projectHolder.projectEntity.checkScreenshotsUploadPermission()
+    }
   }
 
   @PutMapping(value = ["/{id}/complex-update"])

backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt

@@ -14,6 +14,7 @@ import io.tolgee.model.key.Key
 import io.tolgee.model.translation.Translation
 import io.tolgee.security.ProjectHolder
 import io.tolgee.service.LanguageService
+import io.tolgee.service.bigMeta.BigMetaService
 import io.tolgee.service.key.KeyService
 import io.tolgee.service.key.ScreenshotService
 import io.tolgee.service.key.TagService
@@ -42,6 +43,7 @@ class KeyComplexEditHelper(
   private val activityHolder: ActivityHolder = applicationContext.getBean(ActivityHolder::class.java)
   private val transactionManager: PlatformTransactionManager =
     applicationContext.getBean(PlatformTransactionManager::class.java)
+  private val bigMetaService = applicationContext.getBean(BigMetaService::class.java)
 
   private lateinit var key: Key
   private var modifiedTranslations: Map<Long, String?>? = null
@@ -54,6 +56,7 @@ class KeyComplexEditHelper(
   private var isKeyModified by Delegates.notNull<Boolean>()
   private var isScreenshotDeleted by Delegates.notNull<Boolean>()
   private var isScreenshotAdded by Delegates.notNull<Boolean>()
+  private var isBigMetaProvided by Delegates.notNull<Boolean>()
 
   private val languages by lazy {
     val translationLanguages = dto.translations?.keys ?: setOf()
@@ -84,7 +87,16 @@ class KeyComplexEditHelper(
       doStateUpdate()
       doUpdateTags()
       doUpdateScreenshots()
-      doUpdateKey()
+      val result = doUpdateKey()
+      storeBigMeta()
+      result
+    }
+  }
+
+  private fun storeBigMeta() {
+    if (isBigMetaProvided) {
+      securityService.checkBigMetaUploadPermission(projectHolder.project.id)
+      bigMetaService.store(dto.relatedKeysInOrder!!, projectHolder.projectEntity)
     }
   }
 
@@ -225,6 +237,7 @@ class KeyComplexEditHelper(
     isKeyModified = key.name != dto.name || getSafeNamespace(key.namespace?.name) != getSafeNamespace(dto.namespace)
     isScreenshotDeleted = !dto.screenshotIdsToDelete.isNullOrEmpty()
     isScreenshotAdded = !dto.screenshotUploadedImageIds.isNullOrEmpty() || !dto.screenshotsToAdd.isNullOrEmpty()
+    isBigMetaProvided = !dto.relatedKeysInOrder.isNullOrEmpty()
   }
 
   private fun areTagsModified(

backend/app/src/test/kotlin/io/tolgee/StreamingBodyDatabasePoolHealthTest.kt

@@ -20,6 +20,7 @@ import com.zaxxer.hikari.HikariDataSource
 import io.tolgee.development.testDataBuilder.data.TranslationsTestData
 import io.tolgee.fixtures.andIsOk
 import io.tolgee.fixtures.retry
+import io.tolgee.fixtures.waitForNotThrowing
 import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod
 import io.tolgee.testing.assert
 import org.junit.jupiter.api.BeforeEach
@@ -73,7 +74,9 @@ class StreamingBodyDatabasePoolHealthTest : ProjectAuthControllerTest("/v2/proje
           performProjectAuthGet("export").andIsOk
           Thread.sleep(sleepBetweenMs)
         }
-        pool.idleConnections.assert.isGreaterThan(90)
+        waitForNotThrowing(pollTime = 50, timeout = 5000) {
+          pool.idleConnections.assert.isGreaterThan(90)
+        }
       } finally {
         sleepBetweenMs += 10
       }

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt

@@ -4,16 +4,20 @@ import io.tolgee.ProjectAuthControllerTest
 import io.tolgee.development.testDataBuilder.data.BigMetaTestData
 import io.tolgee.fixtures.andAssertThatJson
 import io.tolgee.fixtures.andIsOk
+import io.tolgee.model.key.Key
 import io.tolgee.service.bigMeta.BigMetaService
 import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod
 import io.tolgee.testing.assert
+import io.tolgee.util.Logging
+import io.tolgee.util.infoMeasureTime
+import io.tolgee.util.logger
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.springframework.beans.factory.annotation.Autowired
 import kotlin.time.ExperimentalTime
 import kotlin.time.measureTime
 
-class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/") {
+class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Logging {
 
   lateinit var testData: BigMetaTestData
 
@@ -60,22 +64,41 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/") {
     val keys = testData.addLotOfData()
     saveTestDataAndPrepare()
 
-    val time = measureTime {
-      performProjectAuthPost(
-        "big-meta",
-        mapOf(
-          "relatedKeysInOrder" to keys.take(100).map {
-            mapOf(
-              "namespace" to it.namespace,
-              "keyName" to it.name
-            )
-          }
-        )
-      ).andIsOk
+    logger.infoMeasureTime("it performs well time 1") {
+      storeLogOfBigMeta(keys, 500, 100)
+    }
+
+    logger.infoMeasureTime("it performs well time 2") {
+      storeLogOfBigMeta(keys, 500, 100)
+    }
+
+    logger.infoMeasureTime("it performs well time 3") {
+      storeLogOfBigMeta(keys, 10, 200)
+    }
+
+    logger.infoMeasureTime("it performs well time 4") {
+      storeLogOfBigMeta(keys, 800, 50)
     }
 
-    time.inWholeSeconds.assert.isLessThan(10)
-    bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(20790)
+    measureTime {
+      storeLogOfBigMeta(keys, 800, 50)
+    }.inWholeSeconds.assert.isLessThan(10)
+
+    bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(104790)
+  }
+
+  private fun storeLogOfBigMeta(keys: List<Key>, drop: Int, take: Int) {
+    performProjectAuthPost(
+      "big-meta",
+      mapOf(
+        "relatedKeysInOrder" to keys.drop(drop).take(take).map {
+          mapOf(
+            "namespace" to it.namespace,
+            "keyName" to it.name
+          )
+        }
+      )
+    ).andIsOk
   }
 
   @Test

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexUpdateTest.kt

@@ -2,6 +2,7 @@ package io.tolgee.api.v2.controllers.v2KeyController
 
 import io.tolgee.ProjectAuthControllerTest
 import io.tolgee.development.testDataBuilder.data.KeysTestData
+import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.request.KeyInScreenshotPositionDto
 import io.tolgee.dtos.request.key.ComplexEditKeyDto
 import io.tolgee.dtos.request.key.KeyScreenshotDto
@@ -16,12 +17,15 @@ import io.tolgee.model.enums.AssignableTranslationState
 import io.tolgee.model.enums.Scope
 import io.tolgee.model.enums.TranslationState
 import io.tolgee.service.ImageUploadService
+import io.tolgee.service.bigMeta.BigMetaService
 import io.tolgee.testing.annotations.ProjectApiKeyAuthTestMethod
+import io.tolgee.testing.assert
 import io.tolgee.testing.assertions.Assertions.assertThat
 import io.tolgee.util.generateImage
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc
 import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.core.io.InputStreamSource
@@ -33,6 +37,9 @@ class KeyControllerComplexUpdateTest : ProjectAuthControllerTest("/v2/projects/"
 
   lateinit var testData: KeysTestData
 
+  @Autowired
+  lateinit var bigMetaService: BigMetaService
+
   val screenshotFile: InputStreamSource by lazy {
     generateImage(2000, 3000)
   }
@@ -278,6 +285,28 @@ class KeyControllerComplexUpdateTest : ProjectAuthControllerTest("/v2/projects/"
     ).andIsOk
   }
 
+  @ProjectApiKeyAuthTestMethod(
+    scopes = [
+      Scope.TRANSLATIONS_EDIT,
+    ]
+  )
+  @Test
+  fun `stores big meta`() {
+    this.userAccount = testData.enOnlyUserAccount
+    performProjectAuthPut(
+      "keys/${testData.firstKey.id}/complex-update",
+      ComplexEditKeyDto(
+        name = testData.firstKey.name,
+        relatedKeysInOrder = mutableListOf(
+          RelatedKeyDto(null, "first_key"),
+          RelatedKeyDto(null, testData.firstKey.name)
+        )
+      )
+    ).andIsOk
+
+    bigMetaService.getCloseKeyIds(testData.firstKey.id).assert.hasSize(1)
+  }
+
   @Test
   @ProjectApiKeyAuthTestMethod(
     scopes = [

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerCreationTest.kt

@@ -3,6 +3,7 @@ package io.tolgee.api.v2.controllers.v2KeyController
 import io.tolgee.ProjectAuthControllerTest
 import io.tolgee.development.testDataBuilder.data.KeysTestData
 import io.tolgee.development.testDataBuilder.data.PermissionsTestData
+import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.request.KeyInScreenshotPositionDto
 import io.tolgee.dtos.request.key.CreateKeyDto
 import io.tolgee.dtos.request.key.KeyScreenshotDto
@@ -17,6 +18,7 @@ import io.tolgee.model.enums.AssignableTranslationState
 import io.tolgee.model.enums.Scope
 import io.tolgee.model.enums.TranslationState
 import io.tolgee.service.ImageUploadService
+import io.tolgee.service.bigMeta.BigMetaService
 import io.tolgee.testing.annotations.ProjectApiKeyAuthTestMethod
 import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod
 import io.tolgee.testing.assert
@@ -25,6 +27,7 @@ import io.tolgee.util.generateImage
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc
 import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.core.io.InputStreamSource
@@ -36,6 +39,9 @@ class KeyControllerCreationTest : ProjectAuthControllerTest("/v2/projects/") {
 
   lateinit var testData: KeysTestData
 
+  @Autowired
+  lateinit var bigMetaService: BigMetaService
+
   val screenshotFile: InputStreamSource by lazy {
     generateImage(2000, 3000)
   }
@@ -234,6 +240,28 @@ class KeyControllerCreationTest : ProjectAuthControllerTest("/v2/projects/") {
     }
   }
 
+  @ProjectJWTAuthTestMethod
+  @Test
+  fun `creates key with big meta`() {
+    val keyName = "super_key"
+
+    performProjectAuthPost(
+      "keys",
+      CreateKeyDto(
+        name = keyName,
+        translations = mapOf("en" to "EN", "de" to "DE"),
+        relatedKeysInOrder = mutableListOf(
+          RelatedKeyDto(null, "first_key"),
+          RelatedKeyDto(null, "super_key")
+        )
+      )
+    ).andIsCreated.andAssertThatJson {
+      node("id").isNumber.satisfies { it ->
+        bigMetaService.getCloseKeyIds(it.toLong()).assert.hasSize(1)
+      }
+    }
+  }
+
   @ProjectJWTAuthTestMethod
   @Test
   fun `creates key with translation state`() {

backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt

@@ -2,7 +2,6 @@ package io.tolgee.component.demoProject
 
 import io.tolgee.activity.ActivityHolder
 import io.tolgee.activity.data.ActivityType
-import io.tolgee.dtos.BigMetaDto
 import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.request.KeyInScreenshotPositionDto
 import io.tolgee.dtos.request.ScreenshotInfoDto
@@ -62,16 +61,15 @@ class DemoProjectCreator(
   }
 
   private fun addBigMeta() {
-    val bigMetaDto = BigMetaDto().apply {
-      keys.forEach { (keyName, _) ->
-        relatedKeysInOrder.add(
-          RelatedKeyDto().apply {
-            this.keyName = keyName
-          }
-        )
-      }
+    val relatedKeysInOrder = mutableListOf<RelatedKeyDto>()
+    keys.forEach { (keyName, _) ->
+      relatedKeysInOrder.add(
+        RelatedKeyDto().apply {
+          this.keyName = keyName
+        }
+      )
     }
-    bigMetaService.store(bigMetaDto, project)
+    bigMetaService.store(relatedKeysInOrder, project)
   }
 
   private fun setStates() {

backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt

@@ -30,7 +30,7 @@ class BigMetaTestData {
   }
 
   fun addLotOfData(): List<Key> {
-    val keys = (0..1000).map {
+    val keys = (0..5000).map {
       projectBuilder.addKey(null, "key$it").self
     }