feat: Ability to only update translations and not add new keys in imp…

…ort (#2413)

[issue](#2411)

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/SingleStepImportControllerTest.kt

@@ -58,6 +58,20 @@ class SingleStepImportControllerTest : ProjectAuthControllerTest("/v2/projects/"
     }
   }
 
+  @Test
+  @ProjectJWTAuthTestMethod
+  fun `does not create new key if option isn't enabled`() {
+    saveAndPrepare()
+    performImport(
+      projectId = testData.project.id,
+      listOf(Pair(jsonFileName, simpleJson)),
+      params = mapOf("createNewKeys" to false),
+    )
+    executeInNewTransaction {
+      keyService.find(testData.project.id, "test", null).assert.isNull()
+    }
+  }
+
   @Test
   @ProjectJWTAuthTestMethod
   fun `correctly maps language in single language file`() {

backend/app/src/test/kotlin/io/tolgee/service/dataImport/StoredDataImporterTest.kt

@@ -25,6 +25,7 @@ class StoredDataImporterTest : AbstractSpringTest() {
     object : IImportSettings {
       override var convertPlaceholdersToIcu: Boolean = true
       override var overrideKeyDescriptions: Boolean = false
+      override var createNewKeys: Boolean = true
     }
 
   @BeforeEach
@@ -163,4 +164,48 @@ class StoredDataImporterTest : AbstractSpringTest() {
     assertThat(overriddenTranslation.text).isEqualTo(importTestData.translationWithConflict.text)
     assertThat(forceKeptTranslation.text).isEqualTo("What a text")
   }
+
+  @Test
+  fun `only updates old keys but does not add new ones when option disabled`() {
+    defaultImportSettings.createNewKeys = false
+    storedDataImporter =
+      StoredDataImporter(
+        applicationContext,
+        importTestData.import,
+        ForceMode.OVERRIDE,
+        importSettings = defaultImportSettings,
+      )
+    importTestData.addImportKeyThatDoesntExistInProject()
+    testDataService.saveTestData(importTestData.root)
+    login()
+
+    storedDataImporter.doImport()
+    val projectId = importTestData.root.data.projects[0].self.id
+    val importedKey = keyService.find(projectId, "I'm new key in project", null)
+    assertThat(importedKey).isNull()
+
+    val forceOverriddenTranslationId = importTestData.root.data.projects[0].data.translations[1].self.id
+    val forceOverriddenTranslation = translationService.find(forceOverriddenTranslationId)!!
+    assertThat(forceOverriddenTranslation.text).isEqualTo("Imported text")
+  }
+
+  @Test
+  fun `add new key when option enabled`() {
+    defaultImportSettings.createNewKeys = true
+    storedDataImporter =
+      StoredDataImporter(
+        applicationContext,
+        importTestData.import,
+        ForceMode.OVERRIDE,
+        importSettings = defaultImportSettings,
+      )
+    importTestData.addImportKeyThatDoesntExistInProject()
+    testDataService.saveTestData(importTestData.root)
+    login()
+
+    storedDataImporter.doImport()
+    val projectId = importTestData.root.data.projects[0].self.id
+    val importedKey = keyService.find(projectId, "I'm new key in project", null)
+    assertThat(importedKey).isNotNull()
+  }
 }

backend/data/src/main/kotlin/io/tolgee/api/IImportSettings.kt

@@ -8,6 +8,11 @@ interface IImportSettings {
   )
   var overrideKeyDescriptions: Boolean
 
+  @get:Schema(
+    description = "If false, only updates keys, skipping the creation of new keys",
+  )
+  var createNewKeys: Boolean
+
   @get:Schema(
     description = "If true, placeholders from other formats will be converted to ICU when possible",
   )
@@ -16,12 +21,14 @@ interface IImportSettings {
   fun assignFrom(other: IImportSettings) {
     this.overrideKeyDescriptions = other.overrideKeyDescriptions
     this.convertPlaceholdersToIcu = other.convertPlaceholdersToIcu
+    this.createNewKeys = other.createNewKeys
   }
 
   fun clone(): IImportSettings {
     return object : IImportSettings {
       override var overrideKeyDescriptions: Boolean = this@IImportSettings.overrideKeyDescriptions
       override var convertPlaceholdersToIcu: Boolean = this@IImportSettings.convertPlaceholdersToIcu
+      override var createNewKeys: Boolean = this@IImportSettings.createNewKeys
     }
   }
 }

backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/dataImport/ImportTestData.kt

@@ -431,6 +431,20 @@ class ImportTestData {
     }
   }
 
+  fun addImportKeyThatDoesntExistInProject() {
+    importBuilder.data.importFiles[0].build {
+      val key =
+        addImportKey {
+          name = "I'm new key in project"
+        }
+      addImportTranslation {
+        text = "Hey!"
+        this.key = key.self
+        language = importEnglish
+      }
+    }
+  }
+
   data class AddFilesWithNamespacesResult(
     val importFrenchInNs: ImportLanguage,
   )

backend/data/src/main/kotlin/io/tolgee/dtos/request/SingleStepImportRequest.kt

@@ -28,6 +28,11 @@ class SingleStepImportRequest : ImportAddFilesParams(), IImportSettings {
   override var overrideKeyDescriptions: Boolean = false
   override var convertPlaceholdersToIcu: Boolean = true
 
+  @get:Schema(
+    description = "If false, only updates keys, skipping the creation of new keys",
+  )
+  override var createNewKeys: Boolean = true
+
   @get:Schema(
     description = "Definition of mapping for each file to import.",
   )

backend/data/src/main/kotlin/io/tolgee/dtos/request/dataImport/ImportSettingsRequest.kt

@@ -8,4 +8,6 @@ class ImportSettingsRequest(
   override var overrideKeyDescriptions: Boolean,
   @NotNull
   override var convertPlaceholdersToIcu: Boolean,
+  @NotNull
+  override var createNewKeys: Boolean,
 ) : IImportSettings

backend/data/src/main/kotlin/io/tolgee/model/dataImport/ImportKey.kt

@@ -2,13 +2,10 @@ package io.tolgee.model.dataImport
 
 import io.tolgee.model.StandardAuditModel
 import io.tolgee.model.key.KeyMeta
-import jakarta.persistence.Column
-import jakarta.persistence.Entity
-import jakarta.persistence.ManyToOne
-import jakarta.persistence.OneToMany
-import jakarta.persistence.OneToOne
+import jakarta.persistence.*
 import jakarta.validation.constraints.NotBlank
 import jakarta.validation.constraints.Size
+import org.hibernate.annotations.ColumnDefault
 
 @Entity
 class ImportKey(
@@ -27,6 +24,9 @@ class ImportKey(
 
   var pluralArgName: String? = null
 
+  @ColumnDefault("true")
+  var shouldBeImported: Boolean = true
+
   override fun equals(other: Any?): Boolean {
     if (this === other) return true
     if (javaClass != other?.javaClass) return false

backend/data/src/main/kotlin/io/tolgee/model/dataImport/ImportSettings.kt

@@ -26,4 +26,7 @@ class ImportSettings(
 
   @ColumnDefault("true")
   override var convertPlaceholdersToIcu: Boolean = true
+
+  @ColumnDefault("true")
+  override var createNewKeys: Boolean = true
 }

backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportLanguageRepository.kt

@@ -17,20 +17,33 @@ import java.util.*
 interface ImportLanguageRepository : JpaRepository<ImportLanguage, Long> {
   companion object {
     private const val VIEW_BASE_QUERY = """
-            select il.id as id, il.name as name, el.id as existingLanguageId, 
-            el.tag as existingLanguageTag, el.name as existingLanguageName,
-            if.name as importFileName, if.id as importFileId,
-            if.namespace as namespace,
-            (select count(*) from if.issues) as importFileIssueCount,
-            count(it) as totalCount, 
-            sum(case when it.conflict is null then 0 else 1 end) as conflictCount,
-            sum(case when (it.conflict is null or it.resolvedHash is null) then 0 else 1 end) as resolvedCount
-            from ImportLanguage il join il.file if left join il.existingLanguage el left 
-            join il.translations it on it.isSelectedToImport = true
+    SELECT 
+    il.id AS id, 
+    il.name AS name, 
+    el.id AS existingLanguageId, 
+    el.tag AS existingLanguageTag, 
+    el.name AS existingLanguageName, 
+    f.name AS importFileName, 
+    f.id AS importFileId, 
+    f.namespace AS namespace, 
+    (SELECT COUNT(i.id) FROM ImportFileIssue i WHERE f.id = i.file.id) AS importFileIssueCount, 
+    (SELECT COUNT(t.id) FROM ImportTranslation t WHERE t.language.id = il.id AND t.isSelectedToImport = TRUE
+and t.key.shouldBeImported
+    ) AS totalCount, 
+    COALESCE ((SELECT SUM(CASE WHEN t.conflict IS NULL THEN 0 ELSE 1 END) FROM ImportTranslation t WHERE t.language.id = il.id AND t.isSelectedToImport = TRUE 
+    and t.key.shouldBeImported
+    ),0) AS conflictCount, 
+    COALESCE ((SELECT SUM(CASE WHEN t.conflict IS NULL OR t.resolvedHash IS NULL THEN 0 ELSE 1 END) FROM ImportTranslation t WHERE t.language.id = il.id AND t.isSelectedToImport = TRUE
+and t.key.shouldBeImported
+    ), 0) AS resolvedCount 
+FROM ImportLanguage il 
+JOIN il.file f 
+LEFT JOIN il.existingLanguage el 
+
         """
 
     private const val VIEW_GROUP_BY = """
-            group by il.id, if.id, el.id
+            group by il.id, f.id, el.id
         """
   }
 
@@ -44,8 +57,8 @@ interface ImportLanguageRepository : JpaRepository<ImportLanguage, Long> {
 
   @Query(
     """
-            $VIEW_BASE_QUERY
-            where if.import.id = :importId
+   $VIEW_BASE_QUERY
+        WHERE f.import.id = :importId
             $VIEW_GROUP_BY
             order by il.id
             """,
@@ -65,8 +78,8 @@ interface ImportLanguageRepository : JpaRepository<ImportLanguage, Long> {
 
   @Query(
     """
-            $VIEW_BASE_QUERY
-            where il.id = :languageId
+      $VIEW_BASE_QUERY
+      where (il.id = :languageId)
             $VIEW_GROUP_BY
             """,
   )

backend/data/src/main/kotlin/io/tolgee/repository/dataImport/ImportTranslationRepository.kt

@@ -52,8 +52,10 @@ interface ImportTranslationRepository : JpaRepository<ImportTranslation, Long> {
         where (itc.id is not null or :onlyConflicts = false)
         and ((itc.id is not null and it.resolvedHash is null) or :onlyUnresolved = false)
         and it.language.id = :languageId
+        and (ik.shouldBeImported)
         and (:search is null or lower(it.text) like lower(concat('%', cast(:search as text), '%'))
         or lower(ik.name) like lower(concat('%', cast(:search as text), '%')))
+
     """,
   )
   fun findImportTranslationsView(

backend/data/src/main/kotlin/io/tolgee/service/dataImport/CoreImportFilesProcessor.kt

@@ -12,14 +12,11 @@ import io.tolgee.exceptions.ErrorResponseBody
 import io.tolgee.exceptions.ImportCannotParseFileException
 import io.tolgee.formats.ImportFileProcessor
 import io.tolgee.formats.ImportFileProcessorFactory
-import io.tolgee.model.dataImport.Import
-import io.tolgee.model.dataImport.ImportFile
-import io.tolgee.model.dataImport.ImportKey
-import io.tolgee.model.dataImport.ImportLanguage
-import io.tolgee.model.dataImport.ImportTranslation
+import io.tolgee.model.dataImport.*
 import io.tolgee.model.dataImport.issues.issueTypes.FileIssueType
 import io.tolgee.model.dataImport.issues.paramTypes.FileIssueParamType
 import io.tolgee.service.dataImport.processors.FileProcessorContext
+import io.tolgee.service.key.KeyService
 import io.tolgee.service.language.LanguageService
 import io.tolgee.util.Logging
 import io.tolgee.util.filterFiles
@@ -36,6 +33,7 @@ class CoreImportFilesProcessor(
   // single step import doesn't save data
   val saveData: Boolean = true,
 ) : Logging {
+  private val keyService: KeyService by lazy { applicationContext.getBean(KeyService::class.java) }
   private val importService: ImportService by lazy { applicationContext.getBean(ImportService::class.java) }
   private val importFileProcessorFactory: ImportFileProcessorFactory by lazy {
     applicationContext.getBean(
@@ -262,6 +260,7 @@ class CoreImportFilesProcessor(
       entry.value.forEach translationForeach@{ newTranslation ->
         processTranslation(newTranslation, keyEntity)
       }
+      keyEntity.shouldBeImported = shouldImportKey(keyEntity.name)
     }
     importDataManager.prepareKeyMetas()
     if (saveData) {
@@ -271,6 +270,14 @@ class CoreImportFilesProcessor(
     }
   }
 
+  private fun FileProcessorContext.shouldImportKey(keyName: String): Boolean {
+    if (importSettings.createNewKeys) {
+      return true
+    }
+    val n = this.getNamespaceToPreselect()
+    return keyService.find(import.project.id, keyName, this.getNamespaceToPreselect()) != null
+  }
+
   private fun FileProcessorContext.processTranslation(
     newTranslation: ImportTranslation,
     keyEntity: ImportKey,

backend/data/src/main/kotlin/io/tolgee/service/dataImport/ImportDataManager.kt

@@ -4,11 +4,7 @@ import io.tolgee.api.IImportSettings
 import io.tolgee.formats.CollisionHandler
 import io.tolgee.formats.isSamePossiblePlural
 import io.tolgee.model.Language
-import io.tolgee.model.dataImport.Import
-import io.tolgee.model.dataImport.ImportFile
-import io.tolgee.model.dataImport.ImportKey
-import io.tolgee.model.dataImport.ImportLanguage
-import io.tolgee.model.dataImport.ImportTranslation
+import io.tolgee.model.dataImport.*
 import io.tolgee.model.dataImport.issues.ImportFileIssue
 import io.tolgee.model.dataImport.issues.issueTypes.FileIssueType
 import io.tolgee.model.dataImport.issues.paramTypes.FileIssueParamType
@@ -371,6 +367,27 @@ class ImportDataManager(
     if (oldSettings.convertPlaceholdersToIcu != newSettings.convertPlaceholdersToIcu) {
       applyConvertPlaceholdersChange(newSettings.convertPlaceholdersToIcu)
     }
+
+    if (oldSettings.createNewKeys != newSettings.createNewKeys) {
+      applyKeyCreateChange(newSettings.createNewKeys)
+    }
+  }
+
+  fun applyKeyCreateChange(createNewKeys: Boolean) {
+    storedKeys.forEach { (_, key) ->
+      if (createNewKeys) {
+        key.shouldBeImported = true
+      } else {
+        key.shouldBeImported = keyService.find(
+          import.project.id,
+          key.name,
+          getSafeNamespace(key.file.namespace),
+        ) != null
+      }
+      if (saveData) {
+        importService.saveKey(key)
+      }
+    }
   }
 
   private fun applyConvertPlaceholdersChange(convertPlaceholdersToIcu: Boolean) {

backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt

@@ -105,7 +105,6 @@ class StoredDataImporter(
     importDataManager.storedLanguages.forEach {
       it.prepareImport()
     }
-
     addKeysAndCheckPermissions()
 
     handleKeyMetas()
@@ -205,6 +204,9 @@ class StoredDataImporter(
 
   private fun handleKeyMetas() {
     this.importDataManager.storedKeys.entries.forEach { (fileNamePair, importKey) ->
+      if (!importKey.shouldBeImported) {
+        return@forEach
+      }
       val importedKeyMeta = storedMetas[fileNamePair.first.namespace to importKey.name]
       // don't touch key meta when imported key has no meta
       if (importedKeyMeta != null) {
@@ -230,18 +232,22 @@ class StoredDataImporter(
 
   private fun addAllKeys() {
     importDataManager.storedKeys.map { (fileNamePair, importKey) ->
+      if (!importKey.shouldBeImported) {
+        return@map
+      }
       addKeyToSave(importKey.file.namespace, importKey.name)
     }
   }
 
   private fun ImportLanguage.prepareImport() {
     importDataManager.populateStoredTranslations(this)
     importDataManager.handleConflicts(true)
+    importDataManager.applyKeyCreateChange(importSettings.createNewKeys)
     importDataManager.getStoredTranslations(this).forEach { it.doImport() }
   }
 
   private fun ImportTranslation.doImport() {
-    if (!this.isSelectedToImport) {
+    if (!this.isSelectedToImport || !this.key.shouldBeImported) {
       return
     }
     this.checkConflictResolved()

backend/data/src/main/kotlin/io/tolgee/service/dataImport/processors/FileProcessorContext.kt

@@ -29,6 +29,7 @@ data class FileProcessorContext(
   val importSettings: IImportSettings =
     object : IImportSettings {
       override var overrideKeyDescriptions: Boolean = false
+      override var createNewKeys: Boolean = true
       override var convertPlaceholdersToIcu: Boolean = true
     },
   val projectIcuPlaceholdersEnabled: Boolean = true,