feat: Automations, CDN, Webhooks (#1905)

Co-authored-by: Štěpán Granát <granat.stepan@gmail.com>

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/contentDelivery/ContentDeliveryConfigController.kt

@@ -0,0 +1,102 @@
+package io.tolgee.api.v2.controllers.contentDelivery
+
+import io.swagger.v3.oas.annotations.Operation
+import io.swagger.v3.oas.annotations.tags.Tag
+import io.tolgee.api.v2.controllers.IController
+import io.tolgee.component.contentDelivery.ContentDeliveryUploader
+import io.tolgee.dtos.request.ContentDeliveryConfigRequest
+import io.tolgee.hateoas.contentDelivery.ContentDeliveryConfigModel
+import io.tolgee.hateoas.contentDelivery.ContentDeliveryConfigModelAssembler
+import io.tolgee.model.contentDelivery.ContentDeliveryConfig
+import io.tolgee.model.enums.Scope
+import io.tolgee.security.ProjectHolder
+import io.tolgee.security.authentication.AllowApiAccess
+import io.tolgee.security.authorization.RequiresProjectPermissions
+import io.tolgee.service.contentDelivery.ContentDeliveryConfigService
+import org.springdoc.api.annotations.ParameterObject
+import org.springframework.data.domain.Pageable
+import org.springframework.data.web.PagedResourcesAssembler
+import org.springframework.hateoas.PagedModel
+import org.springframework.web.bind.annotation.CrossOrigin
+import org.springframework.web.bind.annotation.DeleteMapping
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PathVariable
+import org.springframework.web.bind.annotation.PostMapping
+import org.springframework.web.bind.annotation.PutMapping
+import org.springframework.web.bind.annotation.RequestBody
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+import javax.validation.Valid
+
+@Suppress("MVCPathVariableInspection")
+@RestController
+@CrossOrigin(origins = ["*"])
+@RequestMapping(
+  value = [
+    "/v2/projects/{projectId}/content-delivery-configs",
+  ]
+)
+@Tag(name = "Content Delivery", description = "Endpoints for Content Delivery management")
+class ContentDeliveryConfigController(
+  private val contentDeliveryService: ContentDeliveryConfigService,
+  private val projectHolder: ProjectHolder,
+  private val contentDeliveryConfigModelAssembler: ContentDeliveryConfigModelAssembler,
+  @Suppress("SpringJavaInjectionPointsAutowiringInspection")
+  private val pagedContentDeliveryConfigModelAssemblerExporter: PagedResourcesAssembler<ContentDeliveryConfig>,
+  private val contentDeliveryUploader: ContentDeliveryUploader
+) : IController {
+  @PostMapping("")
+  @Operation(description = "Create Content Delivery Config")
+  @RequiresProjectPermissions([Scope.CONTENT_DELIVERY_MANAGE])
+  @AllowApiAccess
+  fun create(@Valid @RequestBody dto: ContentDeliveryConfigRequest): ContentDeliveryConfigModel {
+    val contentDeliveryConfig = contentDeliveryService.create(projectHolder.project.id, dto)
+    return contentDeliveryConfigModelAssembler.toModel(contentDeliveryConfig)
+  }
+
+  @PutMapping("/{id}")
+  @Operation(description = "Updates Content Delivery Config")
+  @RequiresProjectPermissions([Scope.CONTENT_DELIVERY_MANAGE])
+  @AllowApiAccess
+  fun update(
+    @PathVariable id: Long,
+    @Valid @RequestBody dto: ContentDeliveryConfigRequest
+  ): ContentDeliveryConfigModel {
+    val contentDeliveryConfig = contentDeliveryService.update(projectId = projectHolder.project.id, id, dto)
+    return contentDeliveryConfigModelAssembler.toModel(contentDeliveryConfig)
+  }
+
+  @RequiresProjectPermissions([Scope.CONTENT_DELIVERY_PUBLISH])
+  @GetMapping("")
+  @Operation(description = "List existing Content Delivery Configs")
+  @AllowApiAccess
+  fun list(@ParameterObject pageable: Pageable): PagedModel<ContentDeliveryConfigModel> {
+    val page = contentDeliveryService.getAllInProject(projectHolder.project.id, pageable)
+    return pagedContentDeliveryConfigModelAssemblerExporter.toModel(page, contentDeliveryConfigModelAssembler)
+  }
+
+  @RequiresProjectPermissions([Scope.CONTENT_DELIVERY_MANAGE])
+  @DeleteMapping("/{id}")
+  @Operation(description = "Delete Content Delivery Config")
+  @AllowApiAccess
+  fun delete(@PathVariable id: Long) {
+    contentDeliveryService.delete(projectHolder.project.id, id)
+  }
+
+  @RequiresProjectPermissions([Scope.CONTENT_DELIVERY_PUBLISH])
+  @GetMapping("/{id}")
+  @Operation(description = "Get Content Delivery Config")
+  @AllowApiAccess
+  fun get(@PathVariable id: Long): ContentDeliveryConfigModel {
+    return contentDeliveryConfigModelAssembler.toModel(contentDeliveryService.get(projectHolder.project.id, id))
+  }
+
+  @RequiresProjectPermissions([Scope.CONTENT_DELIVERY_PUBLISH])
+  @PostMapping("/{id}")
+  @Operation(description = "Publish to Content Delivery")
+  @AllowApiAccess
+  fun post(@PathVariable id: Long) {
+    val exporter = contentDeliveryService.get(projectHolder.project.id, id)
+    contentDeliveryUploader.upload(exporter.id)
+  }
+}

backend/api/src/main/kotlin/io/tolgee/configuration/PublicConfigurationDTO.kt

@@ -33,6 +33,7 @@ class PublicConfigurationDTO(
   val ga4Tag = properties.ga4Tag
   val postHogApiKey: String? = properties.postHog.apiKey
   val postHogHost: String? = properties.postHog.host
+  val contentDeliveryConfigured: Boolean = properties.contentDelivery.publicUrlPrefix != null
 
   class AuthMethodsDTO(
     val github: OAuthPublicConfigDTO,

backend/api/src/main/kotlin/io/tolgee/hateoas/activity/ModifiedEntityModel.kt

@@ -2,11 +2,12 @@ package io.tolgee.hateoas.activity
 
 import io.tolgee.activity.data.ExistenceEntityDescription
 import io.tolgee.activity.data.PropertyModification
+import io.tolgee.api.IModifiedEntityModel
 
 data class ModifiedEntityModel(
-  val entityId: Long,
-  val description: Map<String, Any?>? = null,
-  var modifications: Map<String, PropertyModification>? = null,
-  var relations: Map<String, ExistenceEntityDescription>? = null,
-  val exists: Boolean? = null
-)
+  override val entityId: Long,
+  override val description: Map<String, Any?>? = null,
+  override var modifications: Map<String, PropertyModification>? = null,
+  override var relations: Map<String, ExistenceEntityDescription>? = null,
+  override val exists: Boolean? = null
+) : IModifiedEntityModel

backend/api/src/main/kotlin/io/tolgee/hateoas/activity/ProjectActivityAuthorModel.kt

@@ -1,12 +1,13 @@
 package io.tolgee.hateoas.activity
 
+import io.tolgee.api.IProjectActivityAuthorModel
 import io.tolgee.dtos.Avatar
 import org.springframework.hateoas.RepresentationModel
 
 data class ProjectActivityAuthorModel(
-  val id: Long,
-  val username: String?,
-  var name: String?,
-  var avatar: Avatar?,
-  var deleted: Boolean
-) : RepresentationModel<ProjectActivityAuthorModel>()
+  override val id: Long,
+  override val username: String?,
+  override var name: String?,
+  override var avatar: Avatar?,
+  override var deleted: Boolean
+) : RepresentationModel<ProjectActivityAuthorModel>(), IProjectActivityAuthorModel

backend/api/src/main/kotlin/io/tolgee/hateoas/activity/ProjectActivityModel.kt

@@ -1,19 +1,20 @@
 package io.tolgee.hateoas.activity
 
 import io.tolgee.activity.data.ActivityType
+import io.tolgee.api.IProjectActivityModel
 import org.springframework.hateoas.RepresentationModel
 import org.springframework.hateoas.server.core.Relation
 import java.io.Serializable
 
 @Suppress("unused")
 @Relation(collectionRelation = "activities", itemRelation = "activity")
 class ProjectActivityModel(
-  val revisionId: Long,
-  val timestamp: Long,
-  val type: ActivityType,
-  val author: ProjectActivityAuthorModel?,
-  val modifiedEntities: Map<String, List<ModifiedEntityModel>>?,
-  val meta: Map<String, Any?>?,
-  val counts: MutableMap<String, Long>?,
-  val params: Any?,
-) : RepresentationModel<ProjectActivityModel>(), Serializable
+  override val revisionId: Long,
+  override val timestamp: Long,
+  override val type: ActivityType,
+  override val author: ProjectActivityAuthorModel?,
+  override val modifiedEntities: Map<String, List<ModifiedEntityModel>>?,
+  override val meta: Map<String, Any?>?,
+  override val counts: MutableMap<String, Long>?,
+  override val params: Any?,
+) : RepresentationModel<ProjectActivityModel>(), Serializable, IProjectActivityModel

backend/api/src/main/kotlin/io/tolgee/hateoas/activity/ProjectActivityModelAssembler.kt

@@ -1,5 +1,6 @@
 package io.tolgee.hateoas.activity
 
+import io.tolgee.api.IProjectActivityModelAssembler
 import io.tolgee.api.v2.controllers.ApiKeyController
 import io.tolgee.model.views.activity.ProjectActivityView
 import io.tolgee.service.AvatarService
@@ -11,7 +12,8 @@ class ProjectActivityModelAssembler(
   private val avatarService: AvatarService
 ) : RepresentationModelAssemblerSupport<ProjectActivityView, ProjectActivityModel>(
   ApiKeyController::class.java, ProjectActivityModel::class.java
-) {
+),
+  IProjectActivityModelAssembler {
   override fun toModel(view: ProjectActivityView): ProjectActivityModel {
     return ProjectActivityModel(
       revisionId = view.revisionId,

backend/api/src/main/kotlin/io/tolgee/hateoas/automation/AutomationActionModel.kt

@@ -0,0 +1,11 @@
+package io.tolgee.hateoas.automation
+
+import io.tolgee.hateoas.contentDelivery.ContentDeliveryConfigModel
+import io.tolgee.model.automations.AutomationActionType
+
+class AutomationActionModel(
+  var id: Long,
+  var type: AutomationActionType,
+) {
+  var contentDeliveryConfig: ContentDeliveryConfigModel? = null
+}

backend/api/src/main/kotlin/io/tolgee/hateoas/automation/AutomationActionModelFiller.kt

@@ -0,0 +1,10 @@
+package io.tolgee.hateoas.automation
+
+import io.tolgee.model.automations.AutomationAction
+import io.tolgee.model.automations.AutomationActionType
+
+interface AutomationActionModelFiller {
+  fun fill(model: AutomationActionModel, entity: AutomationAction)
+
+  val type: AutomationActionType
+}

backend/api/src/main/kotlin/io/tolgee/hateoas/automation/AutomationModel.kt

@@ -0,0 +1,14 @@
+package io.tolgee.hateoas.automation
+
+import org.springframework.hateoas.RepresentationModel
+import org.springframework.hateoas.server.core.Relation
+import java.io.Serializable
+
+@Suppress("unused")
+@Relation(collectionRelation = "automations", itemRelation = "automation")
+class AutomationModel(
+  val id: Long,
+  val name: String,
+  val triggers: List<AutomationTriggerModel>,
+  val actions: List<AutomationActionModel>,
+) : RepresentationModel<AutomationModel>(), Serializable

backend/api/src/main/kotlin/io/tolgee/hateoas/automation/AutomationTriggerModel.kt

@@ -0,0 +1,11 @@
+package io.tolgee.hateoas.automation
+
+import io.tolgee.activity.data.ActivityType
+import io.tolgee.model.automations.AutomationTriggerType
+
+class AutomationTriggerModel(
+  var id: Long,
+  var type: AutomationTriggerType = AutomationTriggerType.TRANSLATION_DATA_MODIFICATION,
+  var activityType: ActivityType? = null,
+  var debounceDurationInMs: Long? = null
+)

backend/api/src/main/kotlin/io/tolgee/hateoas/contentDelivery/ContentDeliveryConfigModel.kt

@@ -0,0 +1,31 @@
+package io.tolgee.hateoas.contentDelivery
+
+import io.tolgee.dtos.IExportParams
+import io.tolgee.dtos.request.export.ExportFormat
+import io.tolgee.ee.api.v2.hateoas.contentStorage.ContentStorageModel
+import io.tolgee.model.enums.TranslationState
+import org.springframework.hateoas.RepresentationModel
+import org.springframework.hateoas.server.core.Relation
+import java.io.Serializable
+
+@Suppress("unused")
+@Relation(collectionRelation = "contentDeliveryConfigs", itemRelation = "contentDeliveryConfig")
+class ContentDeliveryConfigModel(
+  val id: Long,
+  val name: String,
+  val slug: String,
+  val storage: ContentStorageModel?,
+  val publicUrl: String?,
+  val autoPublish: Boolean,
+  val lastPublished: Long?
+) : RepresentationModel<ContentDeliveryConfigModel>(), Serializable, IExportParams {
+  override var languages: Set<String>? = null
+  override var format: ExportFormat = ExportFormat.JSON
+  override var structureDelimiter: Char? = null
+  override var filterKeyId: List<Long>? = null
+  override var filterKeyIdNot: List<Long>? = null
+  override var filterTag: String? = null
+  override var filterKeyPrefix: String? = null
+  override var filterState: List<TranslationState>? = null
+  override var filterNamespace: List<String?>? = null
+}

backend/api/src/main/kotlin/io/tolgee/hateoas/contentDelivery/ContentDeliveryConfigModelAssembler.kt

@@ -0,0 +1,37 @@
+package io.tolgee.hateoas.contentDelivery
+
+import io.tolgee.api.v2.controllers.contentDelivery.ContentDeliveryConfigController
+import io.tolgee.configuration.tolgee.TolgeeProperties
+import io.tolgee.ee.api.v2.hateoas.contentStorage.ContentStorageModelAssembler
+import io.tolgee.model.contentDelivery.ContentDeliveryConfig
+import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport
+import org.springframework.stereotype.Component
+
+@Component
+class ContentDeliveryConfigModelAssembler(
+  private val contentStorageModelAssembler: ContentStorageModelAssembler,
+  private val tolgeeProperties: TolgeeProperties
+) : RepresentationModelAssemblerSupport<ContentDeliveryConfig, ContentDeliveryConfigModel>(
+  ContentDeliveryConfigController::class.java, ContentDeliveryConfigModel::class.java
+) {
+  override fun toModel(entity: ContentDeliveryConfig): ContentDeliveryConfigModel {
+    return ContentDeliveryConfigModel(
+      id = entity.id,
+      name = entity.name,
+      slug = entity.slug,
+      storage = entity.contentStorage?.let { contentStorageModelAssembler.toModel(it) },
+      publicUrl = getPublicUrl(entity),
+      autoPublish = entity.automationActions.isNotEmpty(),
+      lastPublished = entity.lastPublished?.time
+    ).also {
+      it.copyPropsFrom(entity)
+    }
+  }
+
+  private fun getPublicUrl(entity: ContentDeliveryConfig): String? {
+    if (entity.contentStorage != null) {
+      return entity.contentStorage?.publicUrlPrefix?.let { it.removeSuffix("/") + "/" + entity.slug }
+    }
+    return tolgeeProperties.contentDelivery.publicUrlPrefix?.let { it.removeSuffix("/") + "/" + entity.slug }
+  }
+}

backend/app/build.gradle

@@ -103,6 +103,7 @@ dependencies {
     implementation libs.amazonSTS
     implementation libs.icu4j
     implementation libs.jacksonModuleKotlin
+    implementation libs.jacksonDataFormatXml
 
     testApi dependencies.create(libs.redissonSpringBootStarter.get()) {
         exclude group: 'org.redisson', module: 'redisson-spring-data-31'

backend/app/src/main/kotlin/io/tolgee/configuration/FileStorageConfiguration.kt

@@ -6,24 +6,25 @@ package io.tolgee.configuration
 
 import io.tolgee.component.fileStorage.FileStorage
 import io.tolgee.component.fileStorage.LocalFileStorage
+import io.tolgee.component.fileStorage.S3ClientProvider
 import io.tolgee.component.fileStorage.S3FileStorage
 import io.tolgee.configuration.tolgee.TolgeeProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
-import software.amazon.awssdk.services.s3.S3Client
 
 @Configuration
 class FileStorageConfiguration(
   private val properties: TolgeeProperties,
-  private val amazonS3: S3Client?
 ) {
 
   private val s3config = properties.fileStorage.s3
 
   @Bean
   fun fileStorage(): FileStorage {
-    if (s3config.enabled && amazonS3 != null) {
-      return S3FileStorage(tolgeeProperties = properties, amazonS3)
+    if (s3config.enabled) {
+      val amazonS3 = S3ClientProvider(s3config).provide()
+      val bucketName = properties.fileStorage.s3.bucketName ?: throw RuntimeException("Bucket name is not set")
+      return S3FileStorage(bucketName, amazonS3)
     }
     return LocalFileStorage(tolgeeProperties = properties)
   }