From 8068a03402b4ab99686808c7b787f39c89222766 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Thu, 12 Sep 2024 15:54:03 +0200 Subject: [PATCH] feat: Enable Cloudflare cache purging --- .../automation/AutomationIntegrationTest.kt | 2 +- .../ContentDeliveryUploader.kt | 4 +- .../ContentDeliveryCachePurgingProvider.kt | 18 ++-- .../AzureContentDeliveryCachePurging.kt | 3 +- ...AzureContentDeliveryCachePurgingFactory.kt | 3 +- .../AzureCredentialProvider.kt | 2 +- .../CloudflareContentDeliveryCachePurging.kt | 74 +++++++++++++++ ...flareContentDeliveryCachePurgingFactory.kt | 18 ++++ .../ContentDeliveryCachePurgingProperties.kt | 3 +- .../ContentDeliveryCloudflareProperties.kt | 24 +++++ .../tolgee/ContentDeliveryProperties.kt | 6 +- .../ContentDeliveryCachePurgingType.kt | 4 +- ...ureContentStorageConfigCachePurgingTest.kt | 4 +- ...areContentStorageConfigCachePurgingTest.kt | 90 +++++++++++++++++++ 14 files changed, 233 insertions(+), 22 deletions(-) rename backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/{ => azureFrontDoor}/AzureContentDeliveryCachePurging.kt (94%) rename backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/{ => azureFrontDoor}/AzureContentDeliveryCachePurgingFactory.kt (75%) rename backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/{ => azureFrontDoor}/AzureCredentialProvider.kt (85%) create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/cloudflare/CloudflareContentDeliveryCachePurging.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/cloudflare/CloudflareContentDeliveryCachePurgingFactory.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryCloudflareProperties.kt create mode 100644 backend/data/src/test/kotlin/io/tolgee/unit/cachePurging/CloudflareContentStorageConfigCachePurgingTest.kt diff --git a/backend/app/src/test/kotlin/io/tolgee/automation/AutomationIntegrationTest.kt b/backend/app/src/test/kotlin/io/tolgee/automation/AutomationIntegrationTest.kt index 634e11d09b..782966bb7f 100644 --- a/backend/app/src/test/kotlin/io/tolgee/automation/AutomationIntegrationTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/automation/AutomationIntegrationTest.kt @@ -93,7 +93,7 @@ class AutomationIntegrationTest : ProjectAuthControllerTest("/v2/projects/") { fileStorageMock = mock() doReturn(fileStorageMock).whenever(contentDeliveryFileStorageProvider).getContentStorageWithDefaultClient() purgingMock = mock() - doReturn(purgingMock).whenever(contentDeliveryCachePurgingProvider).defaultPurging + doReturn(purgingMock).whenever(contentDeliveryCachePurgingProvider).purgings // wait for the first invocation happening because of test data saving, then clear invocations Thread.sleep(1000) diff --git a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/ContentDeliveryUploader.kt b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/ContentDeliveryUploader.kt index 3edc760ae5..ce129a99ac 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/ContentDeliveryUploader.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/ContentDeliveryUploader.kt @@ -47,7 +47,9 @@ class ContentDeliveryUploader( ) { val isDefaultStorage = contentDeliveryConfig.contentStorage == null if (isDefaultStorage) { - contentDeliveryCachePurgingProvider.defaultPurging?.purgeForPaths(contentDeliveryConfig, paths) + contentDeliveryCachePurgingProvider.purgings.forEach { + it.purgeForPaths(contentDeliveryConfig, paths) + } } } diff --git a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/ContentDeliveryCachePurgingProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/ContentDeliveryCachePurgingProvider.kt index e2d888dce0..e7b485c67a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/ContentDeliveryCachePurgingProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/ContentDeliveryCachePurgingProvider.kt @@ -9,22 +9,16 @@ class ContentDeliveryCachePurgingProvider( private val applicationContext: AbstractApplicationContext, private val configs: List, ) { - val defaultPurging by lazy { + val purgings by lazy { getDefaultFactory() } - private fun getDefaultFactory(): ContentDeliveryCachePurging? { - val purgings = - configs.mapNotNull { - if (!it.enabled) { - return@mapNotNull null - } - applicationContext.getBean(it.contentDeliveryCachePurgingType.factory.java).create(it) + private fun getDefaultFactory(): List { + return configs.mapNotNull { + if (!it.enabled) { + return@mapNotNull null } - if (purgings.size > 1) { - throw RuntimeException("Exactly one content delivery purging must be set") + applicationContext.getBean(it.contentDeliveryCachePurgingType.factory.java).create(it) } - - return purgings.firstOrNull() } } diff --git a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureContentDeliveryCachePurging.kt b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureContentDeliveryCachePurging.kt similarity index 94% rename from backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureContentDeliveryCachePurging.kt rename to backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureContentDeliveryCachePurging.kt index 4180ab38b1..d05df2f8bb 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureContentDeliveryCachePurging.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureContentDeliveryCachePurging.kt @@ -1,7 +1,8 @@ -package io.tolgee.component.contentDelivery.cachePurging +package io.tolgee.component.contentDelivery.cachePurging.azureFrontDoor import com.azure.core.credential.TokenRequestContext import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.tolgee.component.contentDelivery.cachePurging.ContentDeliveryCachePurging import io.tolgee.model.contentDelivery.AzureFrontDoorConfig import io.tolgee.model.contentDelivery.ContentDeliveryConfig import org.springframework.http.HttpEntity diff --git a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureContentDeliveryCachePurgingFactory.kt b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureContentDeliveryCachePurgingFactory.kt similarity index 75% rename from backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureContentDeliveryCachePurgingFactory.kt rename to backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureContentDeliveryCachePurgingFactory.kt index ef383f4fbc..663023fd24 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureContentDeliveryCachePurgingFactory.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureContentDeliveryCachePurgingFactory.kt @@ -1,5 +1,6 @@ -package io.tolgee.component.contentDelivery.cachePurging +package io.tolgee.component.contentDelivery.cachePurging.azureFrontDoor +import io.tolgee.component.contentDelivery.cachePurging.ContentDeliveryCachePurgingFactory import io.tolgee.model.contentDelivery.AzureFrontDoorConfig import org.springframework.stereotype.Component import org.springframework.web.client.RestTemplate diff --git a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureCredentialProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureCredentialProvider.kt similarity index 85% rename from backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureCredentialProvider.kt rename to backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureCredentialProvider.kt index 481f4260be..5b4db6e99b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/AzureCredentialProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/azureFrontDoor/AzureCredentialProvider.kt @@ -1,4 +1,4 @@ -package io.tolgee.component.contentDelivery.cachePurging +package io.tolgee.component.contentDelivery.cachePurging.azureFrontDoor import com.azure.identity.ClientSecretCredential import com.azure.identity.ClientSecretCredentialBuilder diff --git a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/cloudflare/CloudflareContentDeliveryCachePurging.kt b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/cloudflare/CloudflareContentDeliveryCachePurging.kt new file mode 100644 index 0000000000..085f0df563 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/cloudflare/CloudflareContentDeliveryCachePurging.kt @@ -0,0 +1,74 @@ +package io.tolgee.component.contentDelivery.cachePurging.cloudflare + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.tolgee.component.contentDelivery.cachePurging.ContentDeliveryCachePurging +import io.tolgee.configuration.tolgee.ContentDeliveryCloudflareProperties +import io.tolgee.model.contentDelivery.ContentDeliveryConfig +import org.springframework.http.HttpEntity +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpMethod +import org.springframework.http.MediaType +import org.springframework.web.client.RestTemplate + +class CloudflareContentDeliveryCachePurging( + private val config: ContentDeliveryCloudflareProperties, + private val restTemplate: RestTemplate, +) : ContentDeliveryCachePurging { + override fun purgeForPaths( + contentDeliveryConfig: ContentDeliveryConfig, + paths: Set, + ) { + val bodies = getChunkedBody(paths, contentDeliveryConfig) + executePurgeRequest(bodies) + } + + private fun getChunkedBody( + paths: Set, + contentDeliveryConfig: ContentDeliveryConfig, + ): List>> { + return paths.map { + "$prefix/${contentDeliveryConfig.slug}/$it" + } + .chunked(config.maxFilesPerRequest) + .map { urls -> + mapOf("files" to urls) + } + } + + val prefix by lazy { + config.urlPrefix?.removeSuffix("/") ?: "" + } + + private fun executePurgeRequest(bodies: List>>) { + bodies.forEach { body -> + val entity: HttpEntity = getHttpEntity(body) + + val url = "https://api.cloudflare.com/client/v4/zones/${config.zoneId}/purge_cache" + + val response = + restTemplate.exchange( + url, + HttpMethod.POST, + entity, + String::class.java, + ) + + if (!response.statusCode.is2xxSuccessful) { + throw IllegalStateException("Purging failed with status code ${response.statusCode}") + } + } + } + + private fun getHttpEntity(body: Map>): HttpEntity { + val headers = getHeaders() + val jsonBody = jacksonObjectMapper().writeValueAsString(body) + return HttpEntity(jsonBody, headers) + } + + private fun getHeaders(): HttpHeaders { + val headers = HttpHeaders() + headers.contentType = MediaType.APPLICATION_JSON + headers.set("Authorization", "Bearer ${config.apiKey}") + return headers + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/cloudflare/CloudflareContentDeliveryCachePurgingFactory.kt b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/cloudflare/CloudflareContentDeliveryCachePurgingFactory.kt new file mode 100644 index 0000000000..47d3e0075a --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/contentDelivery/cachePurging/cloudflare/CloudflareContentDeliveryCachePurgingFactory.kt @@ -0,0 +1,18 @@ +package io.tolgee.component.contentDelivery.cachePurging.cloudflare + +import io.tolgee.component.contentDelivery.cachePurging.ContentDeliveryCachePurgingFactory +import io.tolgee.configuration.tolgee.ContentDeliveryCloudflareProperties +import org.springframework.stereotype.Component +import org.springframework.web.client.RestTemplate + +@Component +class CloudflareContentDeliveryCachePurgingFactory( + private val restTemplate: RestTemplate, +) : ContentDeliveryCachePurgingFactory { + override fun create(config: Any): CloudflareContentDeliveryCachePurging { + return CloudflareContentDeliveryCachePurging( + config as ContentDeliveryCloudflareProperties, + restTemplate, + ) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryCachePurgingProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryCachePurgingProperties.kt index bcaed9dba9..b69a4f2a78 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryCachePurgingProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryCachePurgingProperties.kt @@ -4,5 +4,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "tolgee.content-delivery.cache-purging") class ContentDeliveryCachePurgingProperties { - var azureFrontDoor: ContentDeliveryAzureFrontDoorProperties = ContentDeliveryAzureFrontDoorProperties() + var azureFrontDoor = ContentDeliveryAzureFrontDoorProperties() + var cloudflare = ContentDeliveryCloudflareProperties() } diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryCloudflareProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryCloudflareProperties.kt new file mode 100644 index 0000000000..a404821836 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryCloudflareProperties.kt @@ -0,0 +1,24 @@ +package io.tolgee.configuration.tolgee + +import io.tolgee.configuration.annotations.DocProperty +import io.tolgee.model.contentDelivery.ContentDeliveryCachePurgingType +import io.tolgee.model.contentDelivery.ContentDeliveryPurgingConfig +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "tolgee.content-delivery.cache-purging.cloudflare") +class ContentDeliveryCloudflareProperties( + var apiKey: String? = null, + var urlPrefix: String? = null, + var zoneId: String? = null, + @DocProperty( + "Number of paths to purge in one request. " + + "(Cloudflare limit is 30 now, but it might be subject to change)", + ) + var maxFilesPerRequest: Int = 30, +) : ContentDeliveryPurgingConfig { + override val enabled: Boolean + get() = !apiKey.isNullOrEmpty() && !urlPrefix.isNullOrEmpty() && !zoneId.isNullOrEmpty() + + override val contentDeliveryCachePurgingType: ContentDeliveryCachePurgingType + get() = ContentDeliveryCachePurgingType.CLOUDFLARE +} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryProperties.kt index aec34e5a70..7e3d4a59f1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/ContentDeliveryProperties.kt @@ -21,6 +21,10 @@ class ContentDeliveryProperties { @DocProperty(description = "Configuration of the storage. You have to configure exactly one storage.") var storage: ContentStorageProperties = ContentStorageProperties() - @DocProperty(hidden = true) + @DocProperty( + description = + "Several services can be used as cache. Tolgee is able to purge the cache when " + + "new files are published when this configuration is set.", + ) var cachePurging: ContentDeliveryCachePurgingProperties = ContentDeliveryCachePurgingProperties() } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/contentDelivery/ContentDeliveryCachePurgingType.kt b/backend/data/src/main/kotlin/io/tolgee/model/contentDelivery/ContentDeliveryCachePurgingType.kt index 06350456bb..5943653eca 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/contentDelivery/ContentDeliveryCachePurgingType.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/contentDelivery/ContentDeliveryCachePurgingType.kt @@ -1,9 +1,11 @@ package io.tolgee.model.contentDelivery -import io.tolgee.component.contentDelivery.cachePurging.AzureContentDeliveryCachePurgingFactory import io.tolgee.component.contentDelivery.cachePurging.ContentDeliveryCachePurgingFactory +import io.tolgee.component.contentDelivery.cachePurging.azureFrontDoor.AzureContentDeliveryCachePurgingFactory +import io.tolgee.component.contentDelivery.cachePurging.cloudflare.CloudflareContentDeliveryCachePurgingFactory import kotlin.reflect.KClass enum class ContentDeliveryCachePurgingType(val factory: KClass) { AZURE_FRONT_DOOR(AzureContentDeliveryCachePurgingFactory::class), + CLOUDFLARE(CloudflareContentDeliveryCachePurgingFactory::class), } diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/cachePurging/AzureContentStorageConfigCachePurgingTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/cachePurging/AzureContentStorageConfigCachePurgingTest.kt index ab992d94cb..1a27343098 100644 --- a/backend/data/src/test/kotlin/io/tolgee/unit/cachePurging/AzureContentStorageConfigCachePurgingTest.kt +++ b/backend/data/src/test/kotlin/io/tolgee/unit/cachePurging/AzureContentStorageConfigCachePurgingTest.kt @@ -1,8 +1,8 @@ package io.tolgee.unit.cachePurging import com.azure.identity.ClientSecretCredential -import io.tolgee.component.contentDelivery.cachePurging.AzureContentDeliveryCachePurging -import io.tolgee.component.contentDelivery.cachePurging.AzureCredentialProvider +import io.tolgee.component.contentDelivery.cachePurging.azureFrontDoor.AzureContentDeliveryCachePurging +import io.tolgee.component.contentDelivery.cachePurging.azureFrontDoor.AzureCredentialProvider import io.tolgee.model.contentDelivery.AzureFrontDoorConfig import io.tolgee.model.contentDelivery.ContentDeliveryConfig import io.tolgee.testing.assert diff --git a/backend/data/src/test/kotlin/io/tolgee/unit/cachePurging/CloudflareContentStorageConfigCachePurgingTest.kt b/backend/data/src/test/kotlin/io/tolgee/unit/cachePurging/CloudflareContentStorageConfigCachePurgingTest.kt new file mode 100644 index 0000000000..55a83e29f4 --- /dev/null +++ b/backend/data/src/test/kotlin/io/tolgee/unit/cachePurging/CloudflareContentStorageConfigCachePurgingTest.kt @@ -0,0 +1,90 @@ +package io.tolgee.unit.cachePurging + +import io.tolgee.component.contentDelivery.cachePurging.cloudflare.CloudflareContentDeliveryCachePurging +import io.tolgee.configuration.tolgee.ContentDeliveryCloudflareProperties +import io.tolgee.fixtures.node +import io.tolgee.model.contentDelivery.ContentDeliveryConfig +import io.tolgee.testing.assert +import net.javacrumbs.jsonunit.assertj.JsonAssert +import net.javacrumbs.jsonunit.assertj.assertThatJson +import org.junit.jupiter.api.Test +import org.mockito.Mockito +import org.mockito.invocation.Invocation +import org.mockito.kotlin.any +import org.mockito.kotlin.doAnswer +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import org.springframework.http.HttpEntity +import org.springframework.http.HttpMethod +import org.springframework.http.HttpStatusCode +import org.springframework.http.ResponseEntity +import org.springframework.web.client.RestTemplate + +class CloudflareContentStorageConfigCachePurgingTest() { + @Test + fun `correctly purges`() { + val config = + ContentDeliveryCloudflareProperties( + zoneId = "fake-zone-id", + urlPrefix = "fake-url-prefix", + maxFilesPerRequest = 10, + apiKey = "token", + ) + val restTemplateMock: RestTemplate = mock() + val purging = CloudflareContentDeliveryCachePurging(config, restTemplateMock) + val responseMock: ResponseEntity<*> = Mockito.mock(ResponseEntity::class.java) + whenever(restTemplateMock.exchange(any(), any(), any(), eq(String::class.java))).doAnswer { + responseMock as ResponseEntity + } + whenever(responseMock.statusCode).thenReturn(HttpStatusCode.valueOf(200)) + val contentDeliveryConfig = mock() + whenever(contentDeliveryConfig.slug).thenReturn("fake-slug") + + purging.purgeForPaths( + contentDeliveryConfig = contentDeliveryConfig, + paths = (1..15).map { "fake-path-$it" }.toSet(), + ) + + val invocations = Mockito.mockingDetails(restTemplateMock).invocations + val firstInvocation = invocations.first() + assertFiles(firstInvocation) { + isArray + .hasSize(10) + .contains("fake-url-prefix/fake-slug/fake-path-1") + } + assertUrl(firstInvocation) + assertAuthorizationHeader(firstInvocation) + + assertFiles(invocations.toList()[1]) { + isArray.hasSize(5) + } + } + + private fun assertFiles( + invocation: Invocation, + fn: JsonAssert.() -> Unit, + ) { + val httpEntity = getHttpEntity(invocation) + assertThatJson(httpEntity.body) { + node("files") { + fn() + } + } + } + + private fun assertAuthorizationHeader(invocation: Invocation) { + val httpEntity = getHttpEntity(invocation) + val headers = httpEntity.headers + headers["Authorization"].assert.isEqualTo(listOf("Bearer token")) + } + + private fun getHttpEntity(invocation: Invocation) = invocation.arguments[2] as HttpEntity<*> + + private fun assertUrl(invocation: Invocation) { + val url = invocation.arguments[0] + url.assert.isEqualTo( + "https://api.cloudflare.com/client/v4/zones/fake-zone-id/purge_cache", + ) + } +}