From 166d2765b988b6e1d17b47643bcb1e9271c6e731 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Fri, 22 Dec 2023 11:14:28 +0100 Subject: [PATCH] feat: Enable big-meta (context storing) in key create / edit endpoints & optimize (#2005) --- .../api/v2/controllers/BigMetaController.kt | 7 +-- .../api/v2/controllers/KeyController.kt | 25 +++++++-- .../tolgee/component/KeyComplexEditHelper.kt | 15 +++++- .../StreamingBodyDatabasePoolHealthTest.kt | 5 +- .../v2/controllers/BigMetaControllerTest.kt | 53 +++++++++++++------ .../KeyControllerComplexUpdateTest.kt | 29 ++++++++++ .../KeyControllerCreationTest.kt | 28 ++++++++++ .../demoProject/DemoProjectCreator.kt | 18 +++---- .../testDataBuilder/data/BigMetaTestData.kt | 2 +- .../main/kotlin/io/tolgee/dtos/BigMetaDto.kt | 10 +--- .../io/tolgee/dtos/WithRelatedKeysInOrder.kt | 12 +++++ .../dtos/request/key/ComplexEditKeyDto.kt | 8 ++- .../tolgee/dtos/request/key/CreateKeyDto.kt | 8 ++- .../tolgee/model/keyBigMeta/KeysDistance.kt | 2 + .../repository/KeysDistanceRepository.kt | 13 ++++- .../tolgee/service/bigMeta/BigMetaService.kt | 23 ++++---- .../service/bigMeta/KeysDistanceUtil.kt | 5 +- .../io/tolgee/service/key/KeyService.kt | 6 ++- .../service/security/SecurityService.kt | 4 ++ .../kotlin/io/tolgee/util/loggerExtension.kt | 49 +++++++++++++++++ .../main/resources/db/changelog/schema.xml | 50 ++++++++++------- 21 files changed, 288 insertions(+), 84 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/dtos/WithRelatedKeysInOrder.kt diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt index 7ca945f437..c929ed6fcf 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt @@ -10,6 +10,7 @@ import io.tolgee.security.ProjectHolder import io.tolgee.security.authentication.AllowApiAccess import io.tolgee.security.authorization.RequiresProjectPermissions import io.tolgee.service.bigMeta.BigMetaService +import io.tolgee.util.Logging import jakarta.validation.Valid import org.springframework.hateoas.CollectionModel import org.springframework.web.bind.annotation.GetMapping @@ -29,10 +30,10 @@ class BigMetaController( private val bigMetaService: BigMetaService, private val projectHolder: ProjectHolder, private val keyWithBaseTranslationModelAssembler: KeyWithBaseTranslationModelAssembler, -) { +) : Logging { @PostMapping("/big-meta") @Operation(summary = "Stores a bigMeta for a project") - @RequiresProjectPermissions([ Scope.TRANSLATIONS_EDIT ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_EDIT]) @AllowApiAccess fun store(@RequestBody @Valid data: BigMetaDto) { bigMetaService.store(data, projectHolder.projectEntity) @@ -40,7 +41,7 @@ class BigMetaController( @GetMapping("/keys/{id}/big-meta") @Operation(summary = "Returns a bigMeta for given key") - @RequiresProjectPermissions([ Scope.TRANSLATIONS_VIEW ]) + @RequiresProjectPermissions([Scope.TRANSLATIONS_VIEW]) @AllowApiAccess fun getBigMeta( @PathVariable("id") id: Long diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/KeyController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/KeyController.kt index c334595a5f..e7fe55ae82 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/KeyController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/KeyController.kt @@ -95,24 +95,41 @@ class KeyController( @RequiresProjectPermissions([Scope.KEYS_CREATE]) @AllowApiAccess fun create(@RequestBody @Valid dto: CreateKeyDto): ResponseEntity { - if (dto.screenshotUploadedImageIds != null || !dto.screenshots.isNullOrEmpty()) { - projectHolder.projectEntity.checkScreenshotsUploadPermission() + checkScrenshotUploadPermissions(dto) + checkTranslatePermission(dto) + checkCanStoreBigMeta(dto) + checkStateChangePermission(dto) + + val key = keyService.create(projectHolder.projectEntity, dto) + return ResponseEntity(keyWithDataModelAssembler.toModel(key), HttpStatus.CREATED) + } + + private fun checkCanStoreBigMeta(dto: CreateKeyDto) { + if (!dto.relatedKeysInOrder.isNullOrEmpty()) { + securityService.checkBigMetaUploadPermission(projectHolder.project.id) } + } + private fun checkTranslatePermission(dto: CreateKeyDto) { dto.translations?.filterValues { !it.isNullOrEmpty() }?.keys?.let { languageTags -> if (languageTags.isNotEmpty()) { securityService.checkLanguageTranslatePermissionByTag(projectHolder.project.id, languageTags) } } + } + private fun checkStateChangePermission(dto: CreateKeyDto) { dto.states?.filterValues { it != AssignableTranslationState.TRANSLATED }?.keys?.let { languageTags -> if (languageTags.isNotEmpty()) { securityService.checkLanguageStateChangePermissionsByTag(projectHolder.project.id, languageTags) } } + } - val key = keyService.create(projectHolder.projectEntity, dto) - return ResponseEntity(keyWithDataModelAssembler.toModel(key), HttpStatus.CREATED) + private fun checkScrenshotUploadPermissions(dto: CreateKeyDto) { + if (dto.screenshotUploadedImageIds != null || !dto.screenshots.isNullOrEmpty()) { + projectHolder.projectEntity.checkScreenshotsUploadPermission() + } } @PutMapping(value = ["/{id}/complex-update"]) diff --git a/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt b/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt index bd678de92f..75d08eecbd 100644 --- a/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt +++ b/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt @@ -14,6 +14,7 @@ import io.tolgee.model.key.Key import io.tolgee.model.translation.Translation import io.tolgee.security.ProjectHolder import io.tolgee.service.LanguageService +import io.tolgee.service.bigMeta.BigMetaService import io.tolgee.service.key.KeyService import io.tolgee.service.key.ScreenshotService import io.tolgee.service.key.TagService @@ -42,6 +43,7 @@ class KeyComplexEditHelper( private val activityHolder: ActivityHolder = applicationContext.getBean(ActivityHolder::class.java) private val transactionManager: PlatformTransactionManager = applicationContext.getBean(PlatformTransactionManager::class.java) + private val bigMetaService = applicationContext.getBean(BigMetaService::class.java) private lateinit var key: Key private var modifiedTranslations: Map? = null @@ -54,6 +56,7 @@ class KeyComplexEditHelper( private var isKeyModified by Delegates.notNull() private var isScreenshotDeleted by Delegates.notNull() private var isScreenshotAdded by Delegates.notNull() + private var isBigMetaProvided by Delegates.notNull() private val languages by lazy { val translationLanguages = dto.translations?.keys ?: setOf() @@ -84,7 +87,16 @@ class KeyComplexEditHelper( doStateUpdate() doUpdateTags() doUpdateScreenshots() - doUpdateKey() + val result = doUpdateKey() + storeBigMeta() + result + } + } + + private fun storeBigMeta() { + if (isBigMetaProvided) { + securityService.checkBigMetaUploadPermission(projectHolder.project.id) + bigMetaService.store(dto.relatedKeysInOrder!!, projectHolder.projectEntity) } } @@ -225,6 +237,7 @@ class KeyComplexEditHelper( isKeyModified = key.name != dto.name || getSafeNamespace(key.namespace?.name) != getSafeNamespace(dto.namespace) isScreenshotDeleted = !dto.screenshotIdsToDelete.isNullOrEmpty() isScreenshotAdded = !dto.screenshotUploadedImageIds.isNullOrEmpty() || !dto.screenshotsToAdd.isNullOrEmpty() + isBigMetaProvided = !dto.relatedKeysInOrder.isNullOrEmpty() } private fun areTagsModified( diff --git a/backend/app/src/test/kotlin/io/tolgee/StreamingBodyDatabasePoolHealthTest.kt b/backend/app/src/test/kotlin/io/tolgee/StreamingBodyDatabasePoolHealthTest.kt index 9cce136896..1ad1c70c75 100644 --- a/backend/app/src/test/kotlin/io/tolgee/StreamingBodyDatabasePoolHealthTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/StreamingBodyDatabasePoolHealthTest.kt @@ -20,6 +20,7 @@ import com.zaxxer.hikari.HikariDataSource import io.tolgee.development.testDataBuilder.data.TranslationsTestData import io.tolgee.fixtures.andIsOk import io.tolgee.fixtures.retry +import io.tolgee.fixtures.waitForNotThrowing import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert import org.junit.jupiter.api.BeforeEach @@ -73,7 +74,9 @@ class StreamingBodyDatabasePoolHealthTest : ProjectAuthControllerTest("/v2/proje performProjectAuthGet("export").andIsOk Thread.sleep(sleepBetweenMs) } - pool.idleConnections.assert.isGreaterThan(90) + waitForNotThrowing(pollTime = 50, timeout = 5000) { + pool.idleConnections.assert.isGreaterThan(90) + } } finally { sleepBetweenMs += 10 } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt index db05e25aec..77c6533e1b 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt @@ -4,16 +4,20 @@ import io.tolgee.ProjectAuthControllerTest import io.tolgee.development.testDataBuilder.data.BigMetaTestData import io.tolgee.fixtures.andAssertThatJson import io.tolgee.fixtures.andIsOk +import io.tolgee.model.key.Key import io.tolgee.service.bigMeta.BigMetaService import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert +import io.tolgee.util.Logging +import io.tolgee.util.infoMeasureTime +import io.tolgee.util.logger import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import kotlin.time.ExperimentalTime import kotlin.time.measureTime -class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/") { +class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Logging { lateinit var testData: BigMetaTestData @@ -60,22 +64,41 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/") { val keys = testData.addLotOfData() saveTestDataAndPrepare() - val time = measureTime { - performProjectAuthPost( - "big-meta", - mapOf( - "relatedKeysInOrder" to keys.take(100).map { - mapOf( - "namespace" to it.namespace, - "keyName" to it.name - ) - } - ) - ).andIsOk + logger.infoMeasureTime("it performs well time 1") { + storeLogOfBigMeta(keys, 500, 100) + } + + logger.infoMeasureTime("it performs well time 2") { + storeLogOfBigMeta(keys, 500, 100) + } + + logger.infoMeasureTime("it performs well time 3") { + storeLogOfBigMeta(keys, 10, 200) + } + + logger.infoMeasureTime("it performs well time 4") { + storeLogOfBigMeta(keys, 800, 50) } - time.inWholeSeconds.assert.isLessThan(10) - bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(20790) + measureTime { + storeLogOfBigMeta(keys, 800, 50) + }.inWholeSeconds.assert.isLessThan(10) + + bigMetaService.findExistingKeysDistancesByIds(keys.map { it.id }).assert.hasSize(104790) + } + + private fun storeLogOfBigMeta(keys: List, drop: Int, take: Int) { + performProjectAuthPost( + "big-meta", + mapOf( + "relatedKeysInOrder" to keys.drop(drop).take(take).map { + mapOf( + "namespace" to it.namespace, + "keyName" to it.name + ) + } + ) + ).andIsOk } @Test diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexUpdateTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexUpdateTest.kt index 61d4088992..f9be5959bc 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexUpdateTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexUpdateTest.kt @@ -2,6 +2,7 @@ package io.tolgee.api.v2.controllers.v2KeyController import io.tolgee.ProjectAuthControllerTest import io.tolgee.development.testDataBuilder.data.KeysTestData +import io.tolgee.dtos.RelatedKeyDto import io.tolgee.dtos.request.KeyInScreenshotPositionDto import io.tolgee.dtos.request.key.ComplexEditKeyDto import io.tolgee.dtos.request.key.KeyScreenshotDto @@ -16,12 +17,15 @@ import io.tolgee.model.enums.AssignableTranslationState import io.tolgee.model.enums.Scope import io.tolgee.model.enums.TranslationState import io.tolgee.service.ImageUploadService +import io.tolgee.service.bigMeta.BigMetaService import io.tolgee.testing.annotations.ProjectApiKeyAuthTestMethod +import io.tolgee.testing.assert import io.tolgee.testing.assertions.Assertions.assertThat import io.tolgee.util.generateImage import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc import org.springframework.boot.test.context.SpringBootTest import org.springframework.core.io.InputStreamSource @@ -33,6 +37,9 @@ class KeyControllerComplexUpdateTest : ProjectAuthControllerTest("/v2/projects/" lateinit var testData: KeysTestData + @Autowired + lateinit var bigMetaService: BigMetaService + val screenshotFile: InputStreamSource by lazy { generateImage(2000, 3000) } @@ -278,6 +285,28 @@ class KeyControllerComplexUpdateTest : ProjectAuthControllerTest("/v2/projects/" ).andIsOk } + @ProjectApiKeyAuthTestMethod( + scopes = [ + Scope.TRANSLATIONS_EDIT, + ] + ) + @Test + fun `stores big meta`() { + this.userAccount = testData.enOnlyUserAccount + performProjectAuthPut( + "keys/${testData.firstKey.id}/complex-update", + ComplexEditKeyDto( + name = testData.firstKey.name, + relatedKeysInOrder = mutableListOf( + RelatedKeyDto(null, "first_key"), + RelatedKeyDto(null, testData.firstKey.name) + ) + ) + ).andIsOk + + bigMetaService.getCloseKeyIds(testData.firstKey.id).assert.hasSize(1) + } + @Test @ProjectApiKeyAuthTestMethod( scopes = [ diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerCreationTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerCreationTest.kt index 2f0c3249e3..848a750a15 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerCreationTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerCreationTest.kt @@ -3,6 +3,7 @@ package io.tolgee.api.v2.controllers.v2KeyController import io.tolgee.ProjectAuthControllerTest import io.tolgee.development.testDataBuilder.data.KeysTestData import io.tolgee.development.testDataBuilder.data.PermissionsTestData +import io.tolgee.dtos.RelatedKeyDto import io.tolgee.dtos.request.KeyInScreenshotPositionDto import io.tolgee.dtos.request.key.CreateKeyDto import io.tolgee.dtos.request.key.KeyScreenshotDto @@ -17,6 +18,7 @@ import io.tolgee.model.enums.AssignableTranslationState import io.tolgee.model.enums.Scope import io.tolgee.model.enums.TranslationState import io.tolgee.service.ImageUploadService +import io.tolgee.service.bigMeta.BigMetaService import io.tolgee.testing.annotations.ProjectApiKeyAuthTestMethod import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert @@ -25,6 +27,7 @@ import io.tolgee.util.generateImage import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc import org.springframework.boot.test.context.SpringBootTest import org.springframework.core.io.InputStreamSource @@ -36,6 +39,9 @@ class KeyControllerCreationTest : ProjectAuthControllerTest("/v2/projects/") { lateinit var testData: KeysTestData + @Autowired + lateinit var bigMetaService: BigMetaService + val screenshotFile: InputStreamSource by lazy { generateImage(2000, 3000) } @@ -234,6 +240,28 @@ class KeyControllerCreationTest : ProjectAuthControllerTest("/v2/projects/") { } } + @ProjectJWTAuthTestMethod + @Test + fun `creates key with big meta`() { + val keyName = "super_key" + + performProjectAuthPost( + "keys", + CreateKeyDto( + name = keyName, + translations = mapOf("en" to "EN", "de" to "DE"), + relatedKeysInOrder = mutableListOf( + RelatedKeyDto(null, "first_key"), + RelatedKeyDto(null, "super_key") + ) + ) + ).andIsCreated.andAssertThatJson { + node("id").isNumber.satisfies { it -> + bigMetaService.getCloseKeyIds(it.toLong()).assert.hasSize(1) + } + } + } + @ProjectJWTAuthTestMethod @Test fun `creates key with translation state`() { diff --git a/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt b/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt index ff40e242a8..2dccc95466 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt @@ -2,7 +2,6 @@ package io.tolgee.component.demoProject import io.tolgee.activity.ActivityHolder import io.tolgee.activity.data.ActivityType -import io.tolgee.dtos.BigMetaDto import io.tolgee.dtos.RelatedKeyDto import io.tolgee.dtos.request.KeyInScreenshotPositionDto import io.tolgee.dtos.request.ScreenshotInfoDto @@ -62,16 +61,15 @@ class DemoProjectCreator( } private fun addBigMeta() { - val bigMetaDto = BigMetaDto().apply { - keys.forEach { (keyName, _) -> - relatedKeysInOrder.add( - RelatedKeyDto().apply { - this.keyName = keyName - } - ) - } + val relatedKeysInOrder = mutableListOf() + keys.forEach { (keyName, _) -> + relatedKeysInOrder.add( + RelatedKeyDto().apply { + this.keyName = keyName + } + ) } - bigMetaService.store(bigMetaDto, project) + bigMetaService.store(relatedKeysInOrder, project) } private fun setStates() { diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt index 5eaa7891c3..6bc5ec4a32 100644 --- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt +++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt @@ -30,7 +30,7 @@ class BigMetaTestData { } fun addLotOfData(): List { - val keys = (0..1000).map { + val keys = (0..5000).map { projectBuilder.addKey(null, "key$it").self } diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/BigMetaDto.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/BigMetaDto.kt index cc065b7e78..3b14fc9551 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/BigMetaDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/BigMetaDto.kt @@ -1,11 +1,5 @@ package io.tolgee.dtos -import io.swagger.v3.oas.annotations.media.Schema - -class BigMetaDto { - @field:Schema( - description = "List of keys, visible, in order as they appear in the document. " + - "The order is important! We are using it for graph distance calculation. " - ) - var relatedKeysInOrder: MutableList = mutableListOf() +class BigMetaDto : WithRelatedKeysInOrder { + override var relatedKeysInOrder: MutableList? = null } diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/WithRelatedKeysInOrder.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/WithRelatedKeysInOrder.kt new file mode 100644 index 0000000000..34fbd3d60e --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/WithRelatedKeysInOrder.kt @@ -0,0 +1,12 @@ +package io.tolgee.dtos + +import io.swagger.v3.oas.annotations.media.Schema + +interface WithRelatedKeysInOrder { + @get:Schema( + description = "Keys in the document used as a context for machine translation. " + + "Keys in the same order as they appear in the document. " + + "The order is important! We are using it for graph distance calculation. " + ) + var relatedKeysInOrder: MutableList? +} diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/ComplexEditKeyDto.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/ComplexEditKeyDto.kt index fed7a2326f..caa4e4dd52 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/ComplexEditKeyDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/ComplexEditKeyDto.kt @@ -3,6 +3,8 @@ package io.tolgee.dtos.request.key import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSetter import io.swagger.v3.oas.annotations.media.Schema +import io.tolgee.dtos.RelatedKeyDto +import io.tolgee.dtos.WithRelatedKeysInOrder import io.tolgee.model.enums.AssignableTranslationState import io.tolgee.util.getSafeNamespace import jakarta.validation.constraints.NotBlank @@ -37,8 +39,10 @@ data class ComplexEditKeyDto( @Deprecated("Use screenshotsToAdd instead") val screenshotUploadedImageIds: List? = null, - val screenshotsToAdd: List? = null -) { + val screenshotsToAdd: List? = null, + + override var relatedKeysInOrder: MutableList? = null +) : WithRelatedKeysInOrder { @JsonSetter("namespace") fun setJsonNamespace(namespace: String?) { this.namespace = getSafeNamespace(namespace) diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/CreateKeyDto.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/CreateKeyDto.kt index 4ffd8785a4..99087bd97b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/CreateKeyDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/CreateKeyDto.kt @@ -3,6 +3,8 @@ package io.tolgee.dtos.request.key import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSetter import io.swagger.v3.oas.annotations.media.Schema +import io.tolgee.dtos.RelatedKeyDto +import io.tolgee.dtos.WithRelatedKeysInOrder import io.tolgee.model.enums.AssignableTranslationState import io.tolgee.util.getSafeNamespace import jakarta.validation.constraints.NotBlank @@ -35,8 +37,10 @@ class CreateKeyDto( @Deprecated("Use screenshots instead") val screenshotUploadedImageIds: List? = null, - val screenshots: List? = null -) { + val screenshots: List? = null, + + override var relatedKeysInOrder: MutableList? = null +) : WithRelatedKeysInOrder { @JsonSetter("namespace") fun setJsonNamespace(namespace: String?) { this.namespace = getSafeNamespace(namespace) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt b/backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt index 0ae926aeae..2d8e22a88a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/keyBigMeta/KeysDistance.kt @@ -15,6 +15,8 @@ import org.springframework.data.domain.Persistable @Table( indexes = [ Index(columnList = "key1Id, key2Id", unique = true), + Index(columnList = "key1Id"), + Index(columnList = "key2Id"), ] ) @IdClass(KeysDistanceId::class) diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt index a79c7f345f..f527099e72 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/KeysDistanceRepository.kt @@ -10,8 +10,17 @@ import org.springframework.stereotype.Repository @Repository interface KeysDistanceRepository : JpaRepository { - @Query("""from KeysDistance kd where kd.key1Id in :data or kd.key2Id in :data""") - fun findForKeyIds(data: Collection): List + @Query( + """ + from KeysDistance kd + where kd.key1Id in ( + select kd2.key1Id from KeysDistance kd2 where kd2.key1Id in :data or kd2.key2Id in :data + ) or kd.key2Id in ( + select kd3.key2Id from KeysDistance kd3 where kd3.key1Id in :data or kd3.key2Id in :data + ) + """ + ) + fun findForKeyIdsWithRelations(data: Collection): List @Query( """ diff --git a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt index 5094c08471..bb87c160ff 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt @@ -12,6 +12,7 @@ import io.tolgee.model.key.Key_ import io.tolgee.model.key.Namespace_ import io.tolgee.model.keyBigMeta.KeysDistance import io.tolgee.repository.KeysDistanceRepository +import io.tolgee.util.Logging import io.tolgee.util.equalNullable import io.tolgee.util.executeInNewTransaction import io.tolgee.util.runSentryCatching @@ -30,7 +31,7 @@ class BigMetaService( private val keysDistanceRepository: KeysDistanceRepository, private val entityManager: EntityManager, private val transactionManager: PlatformTransactionManager -) { +) : Logging { companion object { const val MAX_DISTANCE_SCORE = 10000L const val MAX_POINTS = 2000L @@ -43,16 +44,19 @@ class BigMetaService( @Transactional fun store(data: BigMetaDto, project: Project) { - storeRelatedKeysInOrder(data.relatedKeysInOrder, project) + store(data.relatedKeysInOrder, project) } @Transactional - fun storeRelatedKeysInOrder( - relatedKeysInOrder: MutableList, + fun store( + relatedKeysInOrder: MutableList?, project: Project ) { - val distances = KeysDistanceUtil(relatedKeysInOrder, project, this) - .newDistances + if (relatedKeysInOrder.isNullOrEmpty()) { + return + } + + val distances = KeysDistanceUtil(relatedKeysInOrder, project, this).newDistances keysDistanceRepository.saveAll(distances) } @@ -76,12 +80,7 @@ class BigMetaService( @Transactional fun findExistingKeysDistancesByIds(keyIds: List): List { - val directIds = mutableSetOf() - keysDistanceRepository.findForKeyIds(keyIds).forEach { - directIds.add(it.key1Id) - directIds.add(it.key2Id) - } - return keysDistanceRepository.findForKeyIds(directIds) + return keysDistanceRepository.findForKeyIdsWithRelations(keyIds) } fun get(id: Long): KeysDistance { diff --git a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt index 97f3d0584f..da4e15685f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/KeysDistanceUtil.kt @@ -4,6 +4,7 @@ import com.google.common.primitives.Longs import io.tolgee.dtos.RelatedKeyDto import io.tolgee.model.Project import io.tolgee.model.keyBigMeta.KeysDistance +import io.tolgee.util.Logging import kotlin.math.abs import kotlin.math.max import kotlin.math.min @@ -12,8 +13,7 @@ class KeysDistanceUtil( private val relatedKeysInOrder: MutableList, private val project: Project, private val bigMetaService: BigMetaService -) { - +) : Logging { val newDistances by lazy { increaseRelevant() decreaseOthers() @@ -28,6 +28,7 @@ class KeysDistanceUtil( return@forEach2 } val key2Id = keyIdMap[item2.namespace to item2.keyName] ?: return@forEach2 + val distance = distances[min(key1Id, key2Id) to max(key1Id, key2Id)] ?: createDistance(key1Id, key2Id) relevant[distance.key1Id to distance.key2Id] = distance diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt index 46b49cf8dd..a0cf78c451 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt @@ -18,6 +18,7 @@ import io.tolgee.model.enums.TranslationState import io.tolgee.model.key.Key import io.tolgee.repository.KeyRepository import io.tolgee.repository.LanguageRepository +import io.tolgee.service.bigMeta.BigMetaService import io.tolgee.service.key.utils.KeyInfoProvider import io.tolgee.service.key.utils.KeysImporter import io.tolgee.service.translation.TranslationService @@ -44,7 +45,8 @@ class KeyService( private val entityManager: EntityManager, @Lazy private var translationService: TranslationService, - private val languageRepository: LanguageRepository + private val languageRepository: LanguageRepository, + private val bigMetaService: BigMetaService ) : Logging { fun getAll(projectId: Long): Set { return keyRepository.getAllByProjectId(projectId) @@ -108,6 +110,8 @@ class KeyService( tagService.tagKey(key, it) } + bigMetaService.store(dto.relatedKeysInOrder, project) + storeScreenshots(dto, key) return key diff --git a/backend/data/src/main/kotlin/io/tolgee/service/security/SecurityService.kt b/backend/data/src/main/kotlin/io/tolgee/service/security/SecurityService.kt index be91701e5e..f0243529f6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/security/SecurityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/security/SecurityService.kt @@ -215,6 +215,10 @@ class SecurityService( } } + fun checkBigMetaUploadPermission(projectId: Long) { + checkProjectPermission(projectId, Scope.TRANSLATIONS_EDIT) + } + fun fixInvalidApiKeyWhenRequired(apiKey: ApiKey) { val oldSize = apiKey.scopesEnum.size apiKey.scopesEnum.removeIf { diff --git a/backend/data/src/main/kotlin/io/tolgee/util/loggerExtension.kt b/backend/data/src/main/kotlin/io/tolgee/util/loggerExtension.kt index 2300b7bea1..490b74634a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/loggerExtension.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/loggerExtension.kt @@ -2,8 +2,19 @@ package io.tolgee.util import org.slf4j.Logger import org.slf4j.LoggerFactory +import kotlin.reflect.KClass +import kotlin.time.Duration +import kotlin.time.DurationUnit +import kotlin.time.measureTimedValue interface Logging { + companion object { + /** + * Storage for time sums. Used for counting sum times. + */ + val timeSums = mutableMapOf, String>, Duration>() + } + fun traceLogMeasureTime(operationName: String, fn: () -> T): T { if (logger.isTraceEnabled) { val start = System.currentTimeMillis() @@ -30,3 +41,41 @@ fun Logger.debug(message: () -> String) { this.debug(message()) } } + +inline fun Logger.traceMeasureTime(message: String, block: () -> T): T { + if (this.isTraceEnabled) { + return measureTime(message, this::trace, block) + } + return block() +} + +inline fun Logger.infoMeasureTime(message: String, block: () -> T): T { + return measureTime(message, this::info, block) +} + +inline fun Logger.measureTime(message: String, printFn: (String) -> Unit, block: () -> T): T { + val (result, duration) = measureTimedValue(block) + printFn("$message: $duration") + return result +} + +inline fun Logger.storeTraceTimeSum(id: String, block: () -> T): T { + return if (this.isTraceEnabled) { + val (result, duration) = measureTimedValue(block) + Logging.timeSums.compute(this::class to id) { _, v -> + (v ?: Duration.ZERO) + duration + } + result + } else { + block() + } +} + +fun Logger.traceLogTimeSum(id: String, unit: DurationUnit = DurationUnit.MILLISECONDS) { + if (this.isTraceEnabled) { + val duration = Logging.timeSums[this::class to id] + if (duration != null) { + this.trace("SUM $id: ${duration.toString(unit)}") + } + } +} diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index a38b77e9de..e596863c92 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -2652,7 +2652,7 @@ SET is_initial_user = true WHERE id = ( SELECT id FROM user_account WHERE username != '___implicit_user' ORDER BY id LIMIT 1 - ); + ); @@ -2888,25 +2888,25 @@ - - SELECT CASE - WHEN EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'batch_job_execution_params' - AND column_name = 'parameter_value') - THEN 'true' - ELSE 'false' - END; - - - SELECT CASE - WHEN EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name = 'batch_job_execution_params') - THEN 'true' - ELSE 'false' - END; - + + SELECT CASE + WHEN EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'batch_job_execution_params' + AND column_name = 'parameter_value') + THEN 'true' + ELSE 'false' + END; + + + SELECT CASE + WHEN EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'batch_job_execution_params') + THEN 'true' + ELSE 'false' + END; + @@ -2958,4 +2958,14 @@ CREATE INDEX import_deleted_at_null ON import((deleted_at IS NULL)); + + + + + + + + + +