feat: Enable big-meta (context storing) in key create / edit endpoint…

…s & optimize (#2005)

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt

@@ -10,6 +10,7 @@ import io.tolgee.security.ProjectHolder
 import io.tolgee.security.authentication.AllowApiAccess
 import io.tolgee.security.authorization.RequiresProjectPermissions
 import io.tolgee.service.bigMeta.BigMetaService
+import io.tolgee.util.Logging
 import jakarta.validation.Valid
 import org.springframework.hateoas.CollectionModel
 import org.springframework.web.bind.annotation.GetMapping
@@ -29,18 +30,18 @@ class BigMetaController(
  private val bigMetaService: BigMetaService,
  private val projectHolder: ProjectHolder,
  private val keyWithBaseTranslationModelAssembler: KeyWithBaseTranslationModelAssembler,
-) {
+) : Logging {
  @PostMapping("/big-meta")
  @Operation(summary = "Stores a bigMeta for a project")
- @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ])
+ @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT])
  @AllowApiAccess
  fun store(@RequestBody @Valid data: BigMetaDto) {
  bigMetaService.store(data, projectHolder.projectEntity)
  }
 
  @GetMapping("/keys/{id}/big-meta")
  @Operation(summary = "Returns a bigMeta for given key")
- @RequiresProjectPermissions([ Scope.TRANSLATIONS_VIEW ])
+ @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW])
  @AllowApiAccess
  fun getBigMeta(
  @PathVariable("id") id: Long

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/KeyController.kt

@@ -95,24 +95,41 @@ class KeyController(
  @RequiresProjectPermissions([Scope.KEYS_CREATE])
  @AllowApiAccess
  fun create(@RequestBody @Valid dto: CreateKeyDto): ResponseEntity<KeyWithDataModel> {
- if (dto.screenshotUploadedImageIds != null || !dto.screenshots.isNullOrEmpty()) {
- projectHolder.projectEntity.checkScreenshotsUploadPermission()
+ checkScrenshotUploadPermissions(dto)
+ checkTranslatePermission(dto)
+ checkCanStoreBigMeta(dto)
+ checkStateChangePermission(dto)
+
+ val key = keyService.create(projectHolder.projectEntity, dto)
+ return ResponseEntity(keyWithDataModelAssembler.toModel(key), HttpStatus.CREATED)
+ }
+
+ private fun checkCanStoreBigMeta(dto: CreateKeyDto) {
+ if (!dto.relatedKeysInOrder.isNullOrEmpty()) {
+ securityService.checkBigMetaUploadPermission(projectHolder.project.id)
  }
+ }
 
+ private fun checkTranslatePermission(dto: CreateKeyDto) {
  dto.translations?.filterValues { !it.isNullOrEmpty() }?.keys?.let { languageTags ->
  if (languageTags.isNotEmpty()) {
  securityService.checkLanguageTranslatePermissionByTag(projectHolder.project.id, languageTags)
  }
  }
+ }
 
+ private fun checkStateChangePermission(dto: CreateKeyDto) {
  dto.states?.filterValues { it != AssignableTranslationState.TRANSLATED }?.keys?.let { languageTags ->
  if (languageTags.isNotEmpty()) {
  securityService.checkLanguageStateChangePermissionsByTag(projectHolder.project.id, languageTags)
  }
  }
+ }
 
- val key = keyService.create(projectHolder.projectEntity, dto)
- return ResponseEntity(keyWithDataModelAssembler.toModel(key), HttpStatus.CREATED)
+ private fun checkScrenshotUploadPermissions(dto: CreateKeyDto) {
+ if (dto.screenshotUploadedImageIds != null || !dto.screenshots.isNullOrEmpty()) {
+ projectHolder.projectEntity.checkScreenshotsUploadPermission()
+ }
  }
 
  @PutMapping(value = ["/{id}/complex-update"])

backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt

@@ -14,6 +14,7 @@ import io.tolgee.model.key.Key
 import io.tolgee.model.translation.Translation
 import io.tolgee.security.ProjectHolder
 import io.tolgee.service.LanguageService
+import io.tolgee.service.bigMeta.BigMetaService
 import io.tolgee.service.key.KeyService
 import io.tolgee.service.key.ScreenshotService
 import io.tolgee.service.key.TagService
@@ -42,6 +43,7 @@ class KeyComplexEditHelper(
  private val activityHolder: ActivityHolder = applicationContext.getBean(ActivityHolder::class.java)
  private val transactionManager: PlatformTransactionManager =
  applicationContext.getBean(PlatformTransactionManager::class.java)
+ private val bigMetaService = applicationContext.getBean(BigMetaService::class.java)
 
  private lateinit var key: Key
  private var modifiedTranslations: Map<Long, String?>? = null
@@ -54,6 +56,7 @@ class KeyComplexEditHelper(
  private var isKeyModified by Delegates.notNull<Boolean>()
  private var isScreenshotDeleted by Delegates.notNull<Boolean>()
  private var isScreenshotAdded by Delegates.notNull<Boolean>()
+ private var isBigMetaProvided by Delegates.notNull<Boolean>()
 
  private val languages by lazy {
  val translationLanguages = dto.translations?.keys ?: setOf()
@@ -84,7 +87,16 @@ class KeyComplexEditHelper(
  doStateUpdate()
  doUpdateTags()
  doUpdateScreenshots()
- doUpdateKey()
+ val result = doUpdateKey()
+ storeBigMeta()
+ result
+ }
+ }
+
+ private fun storeBigMeta() {
+ if (isBigMetaProvided) {
+ securityService.checkBigMetaUploadPermission(projectHolder.project.id)
+ bigMetaService.store(dto.relatedKeysInOrder!!, projectHolder.projectEntity)
  }
  }
 
@@ -225,6 +237,7 @@ class KeyComplexEditHelper(
  isKeyModified = key.name != dto.name || getSafeNamespace(key.namespace?.name) != getSafeNamespace(dto.namespace)
  isScreenshotDeleted = !dto.screenshotIdsToDelete.isNullOrEmpty()
  isScreenshotAdded = !dto.screenshotUploadedImageIds.isNullOrEmpty() || !dto.screenshotsToAdd.isNullOrEmpty()
+ isBigMetaProvided = !dto.relatedKeysInOrder.isNullOrEmpty()
  }
 
  private fun areTagsModified(

backend/app/src/test/kotlin/io/tolgee/StreamingBodyDatabasePoolHealthTest.kt

@@ -20,6 +20,7 @@ import com.zaxxer.hikari.HikariDataSource
 import io.tolgee.development.testDataBuilder.data.TranslationsTestData
 import io.tolgee.fixtures.andIsOk
 import io.tolgee.fixtures.retry
+import io.tolgee.fixtures.waitForNotThrowing
 import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod
 import io.tolgee.testing.assert
 import org.junit.jupiter.api.BeforeEach
@@ -73,7 +74,9 @@ class StreamingBodyDatabasePoolHealthTest : ProjectAuthControllerTest("/v2/proje
  performProjectAuthGet("export").andIsOk
  Thread.sleep(sleepBetweenMs)
  }
- pool.idleConnections.assert.isGreaterThan(90)
+ waitForNotThrowing(pollTime = 50, timeout = 5000) {
+ pool.idleConnections.assert.isGreaterThan(90)
+ }
  } finally {
  sleepBetweenMs += 10
  }

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt

@@ -4,16 +4,20 @@ import io.tolgee.ProjectAuthControllerTest
 import io.tolgee.development.testDataBuilder.data.BigMetaTestData
 import io.tolgee.fixtures.andAssertThatJson
 import io.tolgee.fixtures.andIsOk
+import io.tolgee.model.key.Key
 import io.tolgee.service.bigMeta.BigMetaService
 import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod
 import io.tolgee.testing.assert
+import io.tolgee.util.Logging
+import io.tolgee.util.infoMeasureTime
+import io.tolgee.util.logger
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.springframework.beans.factory.annotation.Autowired
 import kotlin.time.ExperimentalTime
 import kotlin.time.measureTime
 
-class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/") {
+class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Logging {
 
  lateinit var testData: BigMetaTestData
 
@@ -60,22 +64,41 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/") {
  val keys = testData.addLotOfData()
  saveTestDataAndPrepare()
 
- val time = measureTime {
- performProjectAuthPost(
- "big-meta",
- mapOf(
- "relatedKeysInOrder" to keys.take(100).map {
- mapOf(
- "namespace" to it.namespace,
- "keyName" to it.name
- )
- }
- )
- ).andIsOk
+ logger.infoMeasureTime("it performs well time 1") {
+ storeLogOfBigMeta(keys, 500, 100)
+ }
+
+ logger.infoMeasureTime("it performs well time 2") {
+ storeLogOfBigMeta(keys, 500, 100)
+ }
+
+ logger.infoMeasureTime("it performs well time 3") {
+ storeLogOfBigMeta(keys, 10, 200)
+ }
+
+ logger.infoMeasureTime("it performs well time 4") {
+ storeLogOfBigMeta(keys, 800, 50)
  }
 
- time.inWholeSeconds.assert.isLessThan(10)
- bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(20790)
+ measureTime {
+ storeLogOfBigMeta(keys, 800, 50)
+ }.inWholeSeconds.assert.isLessThan(10)
+
+ bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(104790)
+ }
+
+ private fun storeLogOfBigMeta(keys: List<Key>, drop: Int, take: Int) {
+ performProjectAuthPost(
+ "big-meta",
+ mapOf(
+ "relatedKeysInOrder" to keys.drop(drop).take(take).map {
+ mapOf(
+ "namespace" to it.namespace,
+ "keyName" to it.name
+ )
+ }
+ )
+ ).andIsOk
  }
 
  @Test

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexUpdateTest.kt

@@ -2,6 +2,7 @@ package io.tolgee.api.v2.controllers.v2KeyController
 
 import io.tolgee.ProjectAuthControllerTest
 import io.tolgee.development.testDataBuilder.data.KeysTestData
+import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.request.KeyInScreenshotPositionDto
 import io.tolgee.dtos.request.key.ComplexEditKeyDto
 import io.tolgee.dtos.request.key.KeyScreenshotDto
@@ -16,12 +17,15 @@ import io.tolgee.model.enums.AssignableTranslationState
 import io.tolgee.model.enums.Scope
 import io.tolgee.model.enums.TranslationState
 import io.tolgee.service.ImageUploadService
+import io.tolgee.service.bigMeta.BigMetaService
 import io.tolgee.testing.annotations.ProjectApiKeyAuthTestMethod
+import io.tolgee.testing.assert
 import io.tolgee.testing.assertions.Assertions.assertThat
 import io.tolgee.util.generateImage
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc
 import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.core.io.InputStreamSource
@@ -33,6 +37,9 @@ class KeyControllerComplexUpdateTest : ProjectAuthControllerTest("/v2/projects/"
 
  lateinit var testData: KeysTestData
 
+ @Autowired
+ lateinit var bigMetaService: BigMetaService
+
  val screenshotFile: InputStreamSource by lazy {
  generateImage(2000, 3000)
  }
@@ -278,6 +285,28 @@ class KeyControllerComplexUpdateTest : ProjectAuthControllerTest("/v2/projects/"
  ).andIsOk
  }
 
+ @ProjectApiKeyAuthTestMethod(
+ scopes = [
+ Scope.TRANSLATIONS_EDIT,
+ ]
+ )
+ @Test
+ fun `stores big meta`() {
+ this.userAccount = testData.enOnlyUserAccount
+ performProjectAuthPut(
+ "keys/${testData.firstKey.id}/complex-update",
+ ComplexEditKeyDto(
+ name = testData.firstKey.name,
+ relatedKeysInOrder = mutableListOf(
+ RelatedKeyDto(null, "first_key"),
+ RelatedKeyDto(null, testData.firstKey.name)
+ )
+ )
+ ).andIsOk
+
+ bigMetaService.getCloseKeyIds(testData.firstKey.id).assert.hasSize(1)
+ }
+
  @Test
  @ProjectApiKeyAuthTestMethod(
  scopes = [

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerCreationTest.kt

@@ -3,6 +3,7 @@ package io.tolgee.api.v2.controllers.v2KeyController
 import io.tolgee.ProjectAuthControllerTest
 import io.tolgee.development.testDataBuilder.data.KeysTestData
 import io.tolgee.development.testDataBuilder.data.PermissionsTestData
+import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.request.KeyInScreenshotPositionDto
 import io.tolgee.dtos.request.key.CreateKeyDto
 import io.tolgee.dtos.request.key.KeyScreenshotDto
@@ -17,6 +18,7 @@ import io.tolgee.model.enums.AssignableTranslationState
 import io.tolgee.model.enums.Scope
 import io.tolgee.model.enums.TranslationState
 import io.tolgee.service.ImageUploadService
+import io.tolgee.service.bigMeta.BigMetaService
 import io.tolgee.testing.annotations.ProjectApiKeyAuthTestMethod
 import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod
 import io.tolgee.testing.assert
@@ -25,6 +27,7 @@ import io.tolgee.util.generateImage
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc
 import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.core.io.InputStreamSource
@@ -36,6 +39,9 @@ class KeyControllerCreationTest : ProjectAuthControllerTest("/v2/projects/") {
 
  lateinit var testData: KeysTestData
 
+ @Autowired
+ lateinit var bigMetaService: BigMetaService
+
  val screenshotFile: InputStreamSource by lazy {
  generateImage(2000, 3000)
  }
@@ -234,6 +240,28 @@ class KeyControllerCreationTest : ProjectAuthControllerTest("/v2/projects/") {
  }
  }
 
+ @ProjectJWTAuthTestMethod
+ @Test
+ fun `creates key with big meta`() {
+ val keyName = "super_key"
+
+ performProjectAuthPost(
+ "keys",
+ CreateKeyDto(
+ name = keyName,
+ translations = mapOf("en" to "EN", "de" to "DE"),
+ relatedKeysInOrder = mutableListOf(
+ RelatedKeyDto(null, "first_key"),
+ RelatedKeyDto(null, "super_key")
+ )
+ )
+ ).andIsCreated.andAssertThatJson {
+ node("id").isNumber.satisfies { it ->
+ bigMetaService.getCloseKeyIds(it.toLong()).assert.hasSize(1)
+ }
+ }
+ }
+
  @ProjectJWTAuthTestMethod
  @Test
  fun `creates key with translation state`() {

backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt

@@ -2,7 +2,6 @@ package io.tolgee.component.demoProject
 
 import io.tolgee.activity.ActivityHolder
 import io.tolgee.activity.data.ActivityType
-import io.tolgee.dtos.BigMetaDto
 import io.tolgee.dtos.RelatedKeyDto
 import io.tolgee.dtos.request.KeyInScreenshotPositionDto
 import io.tolgee.dtos.request.ScreenshotInfoDto
@@ -62,16 +61,15 @@ class DemoProjectCreator(
  }
 
  private fun addBigMeta() {
- val bigMetaDto = BigMetaDto().apply {
- keys.forEach { (keyName, _) ->
- relatedKeysInOrder.add(
- RelatedKeyDto().apply {
- this.keyName = keyName
- }
- )
- }
+ val relatedKeysInOrder = mutableListOf<RelatedKeyDto>()
+ keys.forEach { (keyName, _) ->
+ relatedKeysInOrder.add(
+ RelatedKeyDto().apply {
+ this.keyName = keyName
+ }
+ )
  }
- bigMetaService.store(bigMetaDto, project)
+ bigMetaService.store(relatedKeysInOrder, project)
  }
 
  private fun setStates() {

backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt

@@ -30,7 +30,7 @@ class BigMetaTestData {
  }
 
  fun addLotOfData(): List<Key> {
- val keys = (0..1000).map {
+ val keys = (0..5000).map {
  projectBuilder.addKey(null, "key$it").self
  }