diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt index 3a6437a96d..8fc08b180f 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/V2ExportControllerTest.kt @@ -79,10 +79,7 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Test @ProjectJWTAuthTestMethod fun `it reports business event once in a day`() { - retry( - retries = 10, - exceptionMatcher = { it is ConcurrentModificationException || it is DataIntegrityViolationException } - ) { + retryingOnCommonIssues { initBaseData() try { executeInNewTransaction { @@ -108,10 +105,10 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Transactional @ProjectJWTAuthTestMethod fun `it exports to single json`() { - executeInNewTransaction { - initBaseData() - } - retry { + retryingOnCommonIssues { + executeInNewTransaction { + initBaseData() + } val response = performProjectAuthGet("export?languages=en&zip=false") .andDo { obj: MvcResult -> obj.asyncResult } response.andPrettyPrint.andAssertThatJson { @@ -128,10 +125,10 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Transactional @ProjectJWTAuthTestMethod fun `it exports to single xliff`() { - executeInNewTransaction { - initBaseData() - } - retry { + retryingOnCommonIssues { + executeInNewTransaction { + initBaseData() + } val response = performProjectAuthGet("export?languages=en&zip=false&format=XLIFF") .andDo { obj: MvcResult -> obj.getAsyncResult(30000) } @@ -146,49 +143,54 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Transactional @ProjectJWTAuthTestMethod fun `it filters by keyId in`() { - testData = TranslationsTestData() - testData.generateLotOfData(1000) - testDataService.saveTestData(testData.root) - prepareUserAndProject(testData) - commitTransaction() - - val time = measureTimeMillis { - val selectAllResult = performProjectAuthGet("translations/select-all") - .andIsOk - .andGetContentAsString - val keyIds = jacksonObjectMapper().readValue>>(selectAllResult)["ids"]?.take(500) - val parsed = performExportPost(mapOf("filterKeyId" to keyIds)) - assertThatJson(parsed["en.json"]!!) { - isObject.hasSize(499) + retryingOnCommonIssues { + testData = TranslationsTestData() + testData.generateLotOfData(1000) + testDataService.saveTestData(testData.root) + prepareUserAndProject(testData) + commitTransaction() + + val time = measureTimeMillis { + val selectAllResult = performProjectAuthGet("translations/select-all") + .andIsOk + .andGetContentAsString + val keyIds = jacksonObjectMapper() + .readValue>>(selectAllResult)["ids"]?.take(500) + val parsed = performExportPost(mapOf("filterKeyId" to keyIds)) + assertThatJson(parsed["en.json"]!!) { + isObject.hasSize(499) + } } - } - assertThat(time).isLessThan(2000) + assertThat(time).isLessThan(2000) + } } @Test @Transactional @ProjectJWTAuthTestMethod fun `the structureDelimiter works`() { - testData = TranslationsTestData() - testData.generateScopedData() - testDataService.saveTestData(testData.root) - prepareUserAndProject(testData) - commitTransaction() - - performExport("structureDelimiter=").let { parsed -> - assertThatJson(parsed["en.json"]!!) { - node("hello\\.i\\.am\\.scoped").isEqualTo("yupee!") + retryingOnCommonIssues { + testData = TranslationsTestData() + testData.generateScopedData() + testDataService.saveTestData(testData.root) + prepareUserAndProject(testData) + commitTransaction() + + performExport("structureDelimiter=").let { parsed -> + assertThatJson(parsed["en.json"]!!) { + node("hello\\.i\\.am\\.scoped").isEqualTo("yupee!") + } } - } - performExport("structureDelimiter=+").let { parsed -> - assertThatJson(parsed["en.json"]!!) { - node("hello.i.am.plus.scoped").isEqualTo("yupee!") + performExport("structureDelimiter=+").let { parsed -> + assertThatJson(parsed["en.json"]!!) { + node("hello.i.am.plus.scoped").isEqualTo("yupee!") + } } - } - performExport("").let { parsed -> - assertThatJson(parsed["en.json"]!!) { - node("hello.i.am.scoped").isEqualTo("yupee!") + performExport("").let { parsed -> + assertThatJson(parsed["en.json"]!!) { + node("hello.i.am.scoped").isEqualTo("yupee!") + } } } } @@ -223,18 +225,20 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Transactional @ProjectJWTAuthTestMethod fun `it exports to json with namespaces`() { - val namespacesTestData = NamespacesTestData() - testDataService.saveTestData(namespacesTestData.root) - projectSupplier = { namespacesTestData.projectBuilder.self } - userAccount = namespacesTestData.user + retryingOnCommonIssues { + val namespacesTestData = NamespacesTestData() + testDataService.saveTestData(namespacesTestData.root) + projectSupplier = { namespacesTestData.projectBuilder.self } + userAccount = namespacesTestData.user - val parsed = performExport() + val parsed = performExport() - assertThatJson(parsed["ns-1/en.json"]!!) { - node("key").isEqualTo("hello") - } - assertThatJson(parsed["en.json"]!!) { - node("key").isEqualTo("hello") + assertThatJson(parsed["ns-1/en.json"]!!) { + node("key").isEqualTo("hello") + } + assertThatJson(parsed["en.json"]!!) { + node("key").isEqualTo("hello") + } } } @@ -242,53 +246,57 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Transactional @ProjectJWTAuthTestMethod fun `it exports only allowed languages`() { - val testData = LanguagePermissionsTestData() - testDataService.saveTestData(testData.root) - projectSupplier = { testData.projectBuilder.self } - userAccount = testData.viewEnOnlyUser - - val parsed = performExport() - val files = parsed.keys - files.assert.containsExactly("en.json") + retryingOnCommonIssues { + val testData = LanguagePermissionsTestData() + testDataService.saveTestData(testData.root) + projectSupplier = { testData.projectBuilder.self } + userAccount = testData.viewEnOnlyUser + + val parsed = performExport() + val files = parsed.keys + files.assert.containsExactly("en.json") + } } @Test @Transactional @ProjectJWTAuthTestMethod fun `it exports all languages by default`() { - val testData = TestDataBuilder() - val user = testData.addUserAccount { - username = "user" - } - val projectBuilder = testData.addProject { - name = "Oh my project" - organizationOwner = user.defaultOrganizationBuilder.self - } + retryingOnCommonIssues { + val testData = TestDataBuilder() + val user = testData.addUserAccount { + username = "user" + } + val projectBuilder = testData.addProject { + name = "Oh my project" + organizationOwner = user.defaultOrganizationBuilder.self + } - val langs = arrayOf( - projectBuilder.addEnglish(), - projectBuilder.addCzech(), - projectBuilder.addGerman(), - projectBuilder.addFrench() - ) - - val key = projectBuilder.addKey { name = "key" }.self - langs.forEach { lang -> - projectBuilder.addTranslation { - this.language = lang.self - this.key = key - this.text = "yey" + val langs = arrayOf( + projectBuilder.addEnglish(), + projectBuilder.addCzech(), + projectBuilder.addGerman(), + projectBuilder.addFrench() + ) + + val key = projectBuilder.addKey { name = "key" }.self + langs.forEach { lang -> + projectBuilder.addTranslation { + this.language = lang.self + this.key = key + this.text = "yey" + } } - } - testDataService.saveTestData(testData) + testDataService.saveTestData(testData) - projectSupplier = { projectBuilder.self } - userAccount = user.self + projectSupplier = { projectBuilder.self } + userAccount = user.self - val parsed = performExport() - val files = parsed.keys - files.assert.containsExactlyInAnyOrder(*langs.map { "${it.self.tag}.json" }.toTypedArray()) + val parsed = performExport() + val files = parsed.keys + files.assert.containsExactlyInAnyOrder(*langs.map { "${it.self.tag}.json" }.toTypedArray()) + } } private fun initBaseData() { @@ -301,4 +309,13 @@ class V2ExportControllerTest : ProjectAuthControllerTest("/v2/projects/") { userAccount = testData.user projectSupplier = { testData.project } } + + private fun retryingOnCommonIssues(fn: () -> Unit) { + retry( + retries = 10, + exceptionMatcher = { it is ConcurrentModificationException || it is DataIntegrityViolationException } + ) { + fn() + } + } } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerResultTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerResultTest.kt index 7c957d1f97..a9b674403d 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerResultTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/V2ImportControllerResultTest.kt @@ -194,19 +194,7 @@ class V2ImportControllerResultTest : AuthorizedControllerTest() { @Test fun `onlyUnresolved filter on translations works`() { val testData = ImportTestData() - val resolvedText = "Hello, I am resolved" - - testData { - data.importFiles[0].addImportTranslation { - - conflict = testData.conflict - this.resolve() - key = data.importFiles[0].data.importKeys[0].self - text = resolvedText - language = testData.importEnglish - }.self - } - + testData.translationWithConflict.resolve() testDataService.saveTestData(testData.root) loginAsUser(testData.root.data.userAccounts[0].self.username) @@ -215,13 +203,13 @@ class V2ImportControllerResultTest : AuthorizedControllerTest() { "/import/result/languages/${testData.importEnglish.id}/" + "translations?onlyConflicts=true" ).andIsOk - .andPrettyPrint.andAssertThatJson { node("_embedded.translations").isArray.hasSize(4) } + .andPrettyPrint.andAssertThatJson { node("_embedded.translations").isArray.hasSize(3) } performAuthGet( "/v2/projects/${testData.project.id}" + "/import/result/languages/${testData.importEnglish.id}/translations?onlyUnresolved=true" ).andIsOk - .andPrettyPrint.andAssertThatJson { node("_embedded.translations").isArray.hasSize(3) } + .andPrettyPrint.andAssertThatJson { node("_embedded.translations").isArray.hasSize(2) } } @Test diff --git a/backend/data/build.gradle b/backend/data/build.gradle index 69151933cc..ff145a6a4c 100644 --- a/backend/data/build.gradle +++ b/backend/data/build.gradle @@ -76,7 +76,7 @@ apply from: "$rootDir/gradle/liquibase.gradle" configureLiquibase("public", "hibernate:spring:io.tolgee", 'src/main/resources/db/changelog/schema.xml') diff.dependsOn compileKotlin -diffChangeLog.dependsOn compileKotlin +diffChangelog.dependsOn compileKotlin kotlin { jvmToolchain(17) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt index 100c1dd835..d00e96f3ec 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityDescribingEntity.kt @@ -2,6 +2,7 @@ package io.tolgee.model.activity import io.hypersistence.utils.hibernate.type.json.JsonBinaryType import io.tolgee.activity.data.EntityDescriptionRef +import jakarta.persistence.Column import jakarta.persistence.Entity import jakarta.persistence.Id import jakarta.persistence.IdClass @@ -26,9 +27,11 @@ class ActivityDescribingEntity( val entityId: Long ) : Serializable { + @Column(columnDefinition = "jsonb") @Type(JsonBinaryType::class) var data: Map = mutableMapOf() @Type(JsonBinaryType::class) + @Column(columnDefinition = "jsonb") var describingRelations: Map? = null } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt index e91acf15ea..c35436d43e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityModifiedEntity.kt @@ -4,6 +4,7 @@ import io.hypersistence.utils.hibernate.type.json.JsonBinaryType import io.tolgee.activity.data.EntityDescriptionRef import io.tolgee.activity.data.PropertyModification import io.tolgee.activity.data.RevisionType +import jakarta.persistence.Column import jakarta.persistence.Entity import jakarta.persistence.Enumerated import jakarta.persistence.Id @@ -39,12 +40,14 @@ class ActivityModifiedEntity( /** * Map of field to object containing old and new values */ + @Column(columnDefinition = "jsonb") @Type(JsonBinaryType::class) var modifications: MutableMap = mutableMapOf() /** * Data, which are discribing the entity, but are not modified by the change */ + @Column(columnDefinition = "jsonb") @Type(JsonBinaryType::class) var describingData: Map? = null @@ -52,6 +55,7 @@ class ActivityModifiedEntity( * Relations describing the entity. * e.g. For translation, we would also need key and language data */ + @Column(columnDefinition = "jsonb") @Type(JsonBinaryType::class) var describingRelations: Map? = null diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt index f518c29e7d..320bdad8d0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt @@ -61,6 +61,7 @@ class ActivityRevision : java.io.Serializable { */ var authorId: Long? = null + @Column(columnDefinition = "jsonb") @Type(JsonBinaryType::class) var meta: MutableMap? = null diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt index 0e86cda01a..234e4925a6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt @@ -8,6 +8,7 @@ import io.tolgee.model.Project import io.tolgee.model.StandardAuditModel import io.tolgee.model.UserAccount import io.tolgee.model.activity.ActivityRevision +import jakarta.persistence.Column import jakarta.persistence.Entity import jakarta.persistence.EnumType.STRING import jakarta.persistence.Enumerated @@ -27,6 +28,7 @@ class BatchJob : StandardAuditModel(), IBatchJob { @ManyToOne(fetch = FetchType.LAZY) var author: UserAccount? = null + @Column(columnDefinition = "jsonb") @Type(JsonBinaryType::class) var target: List = listOf() @@ -45,6 +47,7 @@ class BatchJob : StandardAuditModel(), IBatchJob { @OneToOne(mappedBy = "batchJob", fetch = FetchType.LAZY) var activityRevision: ActivityRevision? = null + @Column(columnDefinition = "jsonb") @Type(JsonBinaryType::class) var params: Any? = null diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt index e56075d2b6..226e8d58e3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt @@ -33,6 +33,7 @@ class BatchJobChunkExecution : StandardAuditModel() { var chunkNumber: Int = 0 + @Column(columnDefinition = "jsonb") @Type(JsonBinaryType::class) var successTargets: List = listOf() diff --git a/backend/data/src/main/kotlin/io/tolgee/model/contentDelivery/ContentDeliveryConfig.kt b/backend/data/src/main/kotlin/io/tolgee/model/contentDelivery/ContentDeliveryConfig.kt index d8237cb0da..7a0658cee5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/contentDelivery/ContentDeliveryConfig.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/contentDelivery/ContentDeliveryConfig.kt @@ -7,6 +7,7 @@ import io.tolgee.model.Project import io.tolgee.model.StandardAuditModel import io.tolgee.model.automations.AutomationAction import io.tolgee.model.enums.TranslationState +import jakarta.persistence.Column import jakarta.persistence.Entity import jakarta.persistence.FetchType import jakarta.persistence.ManyToOne @@ -37,25 +38,30 @@ class ContentDeliveryConfig( var lastPublished: Date? = null @Type(JsonBinaryType::class) + @Column(columnDefinition = "jsonb") override var languages: Set? = null override var format: ExportFormat = ExportFormat.JSON override var structureDelimiter: Char? = '.' @Type(JsonBinaryType::class) + @Column(columnDefinition = "jsonb") override var filterKeyId: List? = null @Type(JsonBinaryType::class) + @Column(columnDefinition = "jsonb") override var filterKeyIdNot: List? = null override var filterTag: String? = null override var filterKeyPrefix: String? = null @Type(JsonBinaryType::class) + @Column(columnDefinition = "jsonb") override var filterState: List? = listOf( TranslationState.TRANSLATED, TranslationState.REVIEWED, ) @Type(JsonBinaryType::class) + @Column(columnDefinition = "jsonb") override var filterNamespace: List? = null } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/key/screenshotReference/KeyScreenshotReference.kt b/backend/data/src/main/kotlin/io/tolgee/model/key/screenshotReference/KeyScreenshotReference.kt index ace5bba210..bf0b1db230 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/key/screenshotReference/KeyScreenshotReference.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/key/screenshotReference/KeyScreenshotReference.kt @@ -22,6 +22,7 @@ class KeyScreenshotReference { lateinit var screenshot: Screenshot @Type(JsonBinaryType::class) + @Column(columnDefinition = "jsonb") var positions: MutableList? = mutableListOf() @Column(columnDefinition = "text", length = 5000) diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index e596863c92..2d504019f2 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -2968,4 +2968,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build.gradle b/build.gradle index 39ca5d7b75..624ca19c80 100644 --- a/build.gradle +++ b/build.gradle @@ -103,7 +103,7 @@ project(':server-app').afterEvaluate { task startDbChangelogContainer { doLast { exec { - commandLine "docker", "run", "-e", "POSTGRES_PASSWORD=postgres", "-d", "-p55432:5432", "--name", dbSchemaContainerName, "postgres:13" + commandLine "docker", "run", "-e", "POSTGRES_PASSWORD=postgres", "-d", "-p55438:5432", "--name", dbSchemaContainerName, "postgres:13" } Thread.sleep(5000) } @@ -111,8 +111,8 @@ project(':server-app').afterEvaluate { task stopDbChangelogContainer(type: Exec) { commandLine "docker", "rm", "--force", dbSchemaContainerName - mustRunAfter project(':data').tasks.findByName("diffChangeLog") - mustRunAfter project(':ee-app').tasks.findByName("diffChangeLog") + mustRunAfter project(':data').tasks.findByName("diffChangelog") + mustRunAfter project(':ee-app').tasks.findByName("diffChangelog") } task diffChangeLog { @@ -126,7 +126,7 @@ project(':server-app').afterEvaluate { finalizedBy = [ startDbChangelogContainer, project(':server-app').tasks.findByName("bootRun"), - project(':data').tasks.findByName("diffChangeLog"), + project(':data').tasks.findByName("diffChangelog"), ] def billingDiffChangelog = project(':billing-app').tasks.findByName("diffChangeLog") diff --git a/ee/backend/app/build.gradle b/ee/backend/app/build.gradle index 1f9c6881a7..f4d82094ea 100644 --- a/ee/backend/app/build.gradle +++ b/ee/backend/app/build.gradle @@ -43,7 +43,7 @@ apply from: "$rootDir/gradle/liquibase.gradle" configureLiquibase("ee", "hibernate:spring:io.tolgee.ee.model", 'src/main/resources/db/changelog/ee-schema.xml') diff.dependsOn compileKotlin -diffChangeLog.dependsOn compileKotlin +diffChangelog.dependsOn compileKotlin dependencies { implementation("org.springframework.boot:spring-boot-starter") diff --git a/gradle/liquibase.gradle b/gradle/liquibase.gradle index 1d26c3955a..3f386239c9 100644 --- a/gradle/liquibase.gradle +++ b/gradle/liquibase.gradle @@ -15,7 +15,7 @@ ext { activities { //noinspection GroovyAssignabilityCheck main { - changeLogFile changeLogPah + changeLogFile "${project.projectDir}/${changeLogPah}" url liveDb.url referenceUrl liveDb.referenceUrl username liveDb.username diff --git a/settings.gradle b/settings.gradle index 00ab12899c..da3d374967 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,7 +18,7 @@ pluginManagement { } if (requested.id.id == 'org.liquibase.gradle') { - useModule('org.liquibase.gradle:org.liquibase.gradle.gradle.plugin:2.1.1') + useModule('org.liquibase.gradle:org.liquibase.gradle.gradle.plugin:2.2.1') } if (requested.id.id == 'org.hibernate.orm') { useVersion(hibernateVersion) @@ -58,10 +58,10 @@ dependencyResolutionManagement { library('amazonSTS', "software.amazon.awssdk:sts:$amazonAwsSdkVersion") library('amazonTranslate', "software.amazon.awssdk:translate:$amazonAwsSdkVersion") library('googleCloud', "com.google.cloud:libraries-bom:24.0.0") - library('liquibaseCore', "org.liquibase:liquibase-core:4.25.0") - library('liquibaseHibernate', "org.liquibase.ext:liquibase-hibernate6:4.25.0") + library('liquibaseCore', "org.liquibase:liquibase-core:4.25.1") + library('liquibaseHibernate', "org.liquibase.ext:liquibase-hibernate6:4.25.1") library('liquibasePicoli', "info.picocli:picocli:4.6.3") - library('hibernateTypes', "io.hypersistence:hypersistence-utils-hibernate-62:3.6.0") + library('hibernateTypes', "io.hypersistence:hypersistence-utils-hibernate-63:3.7.0") library('redissonSpringBootStarter', "org.redisson:redisson-spring-boot-starter:3.23.2") library('redissonSpringData', 'org.redisson:redisson-spring-data-27:3.23.2') library('postHog', 'com.posthog.java:posthog:1.1.0')