diff --git a/libs/utils/algolia/build.gradle.kts b/libs/utils/algolia/build.gradle.kts new file mode 100644 index 00000000..8d134757 --- /dev/null +++ b/libs/utils/algolia/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `java-library` + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + implementation(project(":libs:utils:config")) + implementation(project(":libs:utils:logging")) + + implementation(Ktor.plugins.serialization.kotlinx.json) + + implementation("com.algolia:algoliasearch-client-kotlin:_") +} \ No newline at end of file diff --git a/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/AlgoliaRecommendClient.kt b/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/AlgoliaRecommendClient.kt new file mode 100644 index 00000000..cc425c90 --- /dev/null +++ b/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/AlgoliaRecommendClient.kt @@ -0,0 +1,57 @@ +package com.k33.platform.utils.algolia + +import com.algolia.client.api.RecommendClient +import com.algolia.client.model.recommend.GetRecommendationsParams +import com.algolia.client.model.recommend.RecommendHit +import com.algolia.client.model.recommend.RelatedModel +import com.algolia.client.model.recommend.RelatedQuery + +class AlgoliaRecommendClient( + applicationId: Algolia.ApplicationId, + apiKey: Algolia.ApiKey, + private val index: Algolia.Index, +) { + private val client by lazy { + RecommendClient( + appId = applicationId.value, + apiKey = apiKey.value, + ) + } + + suspend fun getRelated( + objectID: Algolia.ObjectID, + ): List { + return client + .getRecommendations( + getRecommendationsParams = GetRecommendationsParams( + requests = listOf( + RelatedQuery( + indexName = index.name, + maxRecommendations = 3, + threshold = 40.0, + model = RelatedModel.RelatedProducts, + objectID = objectID.value, + ) + ) + ) + ) + .results + .flatMap { recommendationsResult -> + recommendationsResult.hits.map { hit -> + Algolia.ObjectID((hit as RecommendHit).objectID) + } + } + } + + companion object { + fun getInstance( + index: Algolia.Index, + ): AlgoliaRecommendClient { + return AlgoliaRecommendClient( + applicationId = algoliaConfig.applicationId, + apiKey = algoliaConfig.apiKey, + index = index, + ) + } + } +} \ No newline at end of file diff --git a/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/AlgoliaSearchClient.kt b/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/AlgoliaSearchClient.kt new file mode 100644 index 00000000..a27af3b9 --- /dev/null +++ b/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/AlgoliaSearchClient.kt @@ -0,0 +1,76 @@ +package com.k33.platform.utils.algolia + +import com.algolia.client.api.SearchClient +import com.algolia.client.extensions.replaceAllObjects +import com.algolia.client.model.search.BrowseParamsObject +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonPrimitive + +class AlgoliaSearchClient( + applicationId: Algolia.ApplicationId, + apiKey: Algolia.ApiKey, + private val index: Algolia.Index, +) { + + private val client by lazy { + SearchClient( + appId = applicationId.value, + apiKey = apiKey.value, + ) + } + + suspend fun upsert( + objectID: Algolia.ObjectID, + record: JsonObject, + ) { + client.addOrUpdateObject( + indexName = index.name, + objectID = objectID.value, + body = record, + ) + } + + suspend fun batchUpsert( + records: List + ) { + client.replaceAllObjects( + indexName = index.name, + objects = records, + ) + } + + suspend fun delete( + objectID: Algolia.ObjectID, + ) { + client.deleteObject( + indexName = index.name, + objectID = objectID.value, + ) + } + + suspend fun getAllIds(): Map { + return client + .browse( + indexName = index.name, + browseParams = BrowseParamsObject( + attributesToRetrieve = listOf("publishedAt"), + ) + ) + .hits + .associate { + Algolia.ObjectID(it.objectID) to it.additionalProperties!!["publishedAt"]!!.jsonPrimitive.content + } + } + + companion object { + fun getInstance( + index: Algolia.Index, + ): AlgoliaSearchClient { + return AlgoliaSearchClient( + applicationId = algoliaConfig.applicationId, + apiKey = algoliaConfig.apiKey, + index = index, + ) + } + } +} \ No newline at end of file diff --git a/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/Config.kt b/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/Config.kt new file mode 100644 index 00000000..69918172 --- /dev/null +++ b/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/Config.kt @@ -0,0 +1,13 @@ +package com.k33.platform.utils.algolia + +data class AlgoliaConfig( + val applicationId: Algolia.ApplicationId, + val apiKey: Algolia.ApiKey, +) + +val algoliaConfig: AlgoliaConfig by lazy { + AlgoliaConfig( + applicationId = Algolia.ApplicationId(System.getenv("ALGOLIA_APP_ID")), + apiKey = Algolia.ApiKey(System.getenv("ALGOLIA_API_KEY")), + ) +} \ No newline at end of file diff --git a/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/Model.kt b/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/Model.kt new file mode 100644 index 00000000..da519129 --- /dev/null +++ b/libs/utils/algolia/src/main/kotlin/com/k33/platform/utils/algolia/Model.kt @@ -0,0 +1,22 @@ +package com.k33.platform.utils.algolia + +object Algolia { + + object Key { + const val ObjectID = "objectID" + } + + @JvmInline + value class ApplicationId(val value: String) + + @JvmInline + value class ApiKey(val value: String) + + @Suppress("EnumEntryName") + enum class Index { + articles, + } + + @JvmInline + value class ObjectID(val value: String) +} diff --git a/libs/utils/cms/contentful/build.gradle.kts b/libs/utils/cms/contentful/build.gradle.kts index 678a1018..1b811465 100644 --- a/libs/utils/cms/contentful/build.gradle.kts +++ b/libs/utils/cms/contentful/build.gradle.kts @@ -5,6 +5,8 @@ plugins { } dependencies { + implementation(project(":libs:utils:algolia")) + implementation(project(":libs:utils:config")) implementation(project(":libs:utils:logging")) implementation(project(":libs:utils:slack")) @@ -27,8 +29,6 @@ dependencies { implementation("org.apache.logging.log4j:log4j-to-slf4j:_") implementation("org.apache.logging.log4j:log4j-core:_") - implementation("com.algolia:algoliasearch-client-kotlin:_") - // test testImplementation("io.kotest:kotest-runner-junit5-jvm:_") } \ No newline at end of file diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/JsonObjectId.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/JsonObjectId.kt index bc0171e7..74b23cc6 100644 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/JsonObjectId.kt +++ b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/JsonObjectId.kt @@ -1,10 +1,10 @@ package com.k33.platform.cms -import com.algolia.search.helper.toObjectID -import com.k33.platform.cms.sync.Algolia +import com.k33.platform.utils.algolia.Algolia import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonPrimitive val JsonObject.objectIDString get() = get(Algolia.Key.ObjectID)?.jsonPrimitive?.contentOrNull -val JsonObject.objectID get() = objectIDString?.toObjectID() + +val JsonObject.objectID get() = objectIDString?.let(Algolia::ObjectID) diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/config/Config.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/config/Config.kt index a89b1e98..81e66f81 100644 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/config/Config.kt +++ b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/config/Config.kt @@ -2,6 +2,8 @@ package com.k33.platform.cms.config +import com.k33.platform.utils.algolia.Algolia + @Suppress("EnumEntryName") enum class Sync( val config: SyncConfig, @@ -9,21 +11,16 @@ enum class Sync( researchArticles( SyncConfig( ContentfulSpace.research, - AlgoliaIndexName.articles, + Algolia.Index.articles, ) ), } data class SyncConfig( val contentfulSpace: ContentfulSpace, - val algoliaIndexName: AlgoliaIndexName, + val algoliaIndex: Algolia.Index, ) -@Suppress("EnumEntryName") -enum class AlgoliaIndexName { - articles, -} - // // Contentful // @@ -46,19 +43,3 @@ data class ContentfulSpaceConfig( val token: String, val cmaToken: String, ) - -// -// Algolia -// - -data class AlgoliaConfig( - val applicationId: String, - val apiKey: String, -) - -val algoliaCconfig: AlgoliaConfig by lazy { - AlgoliaConfig( - applicationId = System.getenv("ALGOLIA_APP_ID"), - apiKey = System.getenv("ALGOLIA_API_KEY"), - ) -} \ No newline at end of file diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/space/research/article/ResearchArticle.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/space/research/article/ResearchArticle.kt index 1e617894..0d3400b8 100644 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/space/research/article/ResearchArticle.kt +++ b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/space/research/article/ResearchArticle.kt @@ -3,9 +3,9 @@ package com.k33.platform.cms.space.research.article import com.k33.platform.cms.clients.ContentfulGraphql import com.k33.platform.cms.content.Content import com.k33.platform.cms.objectIDString -import com.k33.platform.cms.sync.Algolia import com.k33.platform.cms.utils.optional import com.k33.platform.cms.utils.richToPlainText +import com.k33.platform.utils.algolia.Algolia import com.k33.platform.utils.config.lazyResourceWithoutWhitespace import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.contentOrNull diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/space/research/page/ResearchPage.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/space/research/page/ResearchPage.kt index afd38d93..2120fa9c 100644 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/space/research/page/ResearchPage.kt +++ b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/space/research/page/ResearchPage.kt @@ -3,9 +3,9 @@ package com.k33.platform.cms.space.research.page import com.k33.platform.cms.clients.ContentfulGraphql import com.k33.platform.cms.content.Content import com.k33.platform.cms.objectIDString -import com.k33.platform.cms.sync.Algolia import com.k33.platform.cms.utils.optional import com.k33.platform.cms.utils.richToPlainText +import com.k33.platform.utils.algolia.Algolia import com.k33.platform.utils.config.lazyResourceWithoutWhitespace import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.contentOrNull diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/Algolia.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/Algolia.kt deleted file mode 100644 index ada07251..00000000 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/Algolia.kt +++ /dev/null @@ -1,7 +0,0 @@ -package com.k33.platform.cms.sync - -object Algolia { - object Key { - const val ObjectID = "objectID" - } -} \ No newline at end of file diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaRecommendClient.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaRecommendClient.kt deleted file mode 100644 index c31b70a4..00000000 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaRecommendClient.kt +++ /dev/null @@ -1,58 +0,0 @@ -package com.k33.platform.cms.sync - -import com.algolia.search.client.ClientRecommend -import com.algolia.search.model.APIKey -import com.algolia.search.model.ApplicationID -import com.algolia.search.model.IndexName -import com.algolia.search.model.ObjectID -import com.algolia.search.model.recommend.RelatedProductsQuery -import com.k33.platform.cms.config.Sync -import com.k33.platform.cms.config.algoliaCconfig -import com.k33.platform.cms.objectID - -class AlgoliaRecommendClient( - applicationId: ApplicationID, - apiKey: APIKey, - private val indexName: IndexName, -) { - private val client by lazy { - ClientRecommend( - applicationId, - apiKey, - ) - } - - suspend fun getRelated( - objectID: ObjectID, - ): List { - val request = RelatedProductsQuery( - indexName = indexName, - objectID = objectID, - maxRecommendations = 3, - ) -// println() -// print(objectID) - return client - .getRelatedProducts(listOf(request)) - .flatMap { responseSearch -> - responseSearch.hits.mapNotNull { hit -> -// logger.info("title: {}", hit["title"]?.jsonPrimitive?.contentOrNull) -// logger.info("score: {}", hit.scoreOrNull) -// print(",${hit.json.objectID},${hit.scoreOrNull}") - hit.json.objectID - } - } - } - - companion object { - fun getInstance( - sync: Sync, - ): AlgoliaRecommendClient { - return AlgoliaRecommendClient( - ApplicationID(algoliaCconfig.applicationId), - APIKey(algoliaCconfig.apiKey), - IndexName(sync.config.algoliaIndexName.name), - ) - } - } -} \ No newline at end of file diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaSearchClient.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaSearchClient.kt deleted file mode 100644 index 9a69f440..00000000 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaSearchClient.kt +++ /dev/null @@ -1,78 +0,0 @@ -package com.k33.platform.cms.sync - -import com.algolia.search.client.ClientSearch -import com.algolia.search.dsl.attributesToRetrieve -import com.algolia.search.dsl.query -import com.algolia.search.model.APIKey -import com.algolia.search.model.ApplicationID -import com.algolia.search.model.IndexName -import com.algolia.search.model.ObjectID -import com.k33.platform.cms.config.Sync -import com.k33.platform.cms.config.algoliaCconfig -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.jsonPrimitive - -class AlgoliaSearchClient( - applicationId: ApplicationID, - apiKey: APIKey, - indexName: IndexName, -) { - - private val index by lazy { - val client = ClientSearch( - applicationId, - apiKey, - ) - client.initIndex(indexName) - } - - suspend fun upsert( - objectID: ObjectID, - record: JsonObject, - ) { - index.replaceObject( - objectID, - record, - ) - } - - suspend fun batchUpsert( - records: List> - ) { - index.replaceObjects(records) - } - - suspend fun delete( - objectID: ObjectID, - ) { - index.deleteObject(objectID) - } - - suspend fun getAllIds(): Map { - val query = query("") { - attributesToRetrieve { - +Algolia.Key.ObjectID - +"publishedAt" - } - } - return index.browseObjects(query) - .flatMap { - it.hits.map { hit -> - hit.json[Algolia.Key.ObjectID]!!.jsonPrimitive.content to hit.json["publishedAt"]!!.jsonPrimitive.content - } - } - .toMap() - } - - companion object { - fun getInstance( - sync: Sync - ): AlgoliaSearchClient { - return AlgoliaSearchClient( - ApplicationID(algoliaCconfig.applicationId), - APIKey(algoliaCconfig.apiKey), - IndexName(sync.config.algoliaIndexName.name), - ) - } - } -} \ No newline at end of file diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaToContentful.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaToContentful.kt index 267e74f6..0bd4b393 100644 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaToContentful.kt +++ b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/AlgoliaToContentful.kt @@ -1,7 +1,5 @@ package com.k33.platform.cms.sync -import com.algolia.search.helper.toObjectID -import com.algolia.search.model.ObjectID import com.contentful.java.cma.model.CMALink import com.contentful.java.cma.model.CMAType import com.k33.platform.cms.clients.ContentfulMgmtClient @@ -9,6 +7,8 @@ import com.k33.platform.cms.config.ContentfulSpace import com.k33.platform.cms.config.Sync import com.k33.platform.cms.content.ContentFactory import com.k33.platform.cms.content.ContentField +import com.k33.platform.utils.algolia.Algolia +import com.k33.platform.utils.algolia.AlgoliaRecommendClient import com.k33.platform.utils.logging.getLogger import kotlinx.coroutines.delay import kotlinx.coroutines.flow.asFlow @@ -23,7 +23,9 @@ class AlgoliaToContentful( private val logger by getLogger() private val algoliaClient by lazy { - AlgoliaRecommendClient.getInstance(sync) + AlgoliaRecommendClient.getInstance( + index = sync.config.algoliaIndex, + ) } private val contentfulMgmtClient by lazy { @@ -50,11 +52,13 @@ class AlgoliaToContentful( suspend fun upsert( entryId: String, ): Boolean { - val relatedEntries = algoliaClient.getRelated(entryId.toObjectID()) + val relatedEntries = algoliaClient + .getRelated(Algolia.ObjectID(entryId)) + .map(Algolia.ObjectID::value) return contentfulMgmtClient.updateField( entryId = entryId, key = recommendedArticles.fieldId, - value = recommendedArticles.mapValues(relatedEntries.map(ObjectID::toString)) + value = recommendedArticles.mapValues(relatedEntries) ) } diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/ContentfulToAlgolia.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/ContentfulToAlgolia.kt index b96ad089..deef9fd4 100644 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/ContentfulToAlgolia.kt +++ b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/ContentfulToAlgolia.kt @@ -1,6 +1,5 @@ package com.k33.platform.cms.sync -import com.algolia.search.model.ObjectID import com.k33.platform.cms.config.Sync import com.k33.platform.cms.content.ContentFactory import com.k33.platform.cms.events.Action @@ -9,6 +8,9 @@ import com.k33.platform.cms.events.EventPattern import com.k33.platform.cms.events.EventType import com.k33.platform.cms.events.Resource import com.k33.platform.cms.objectID +import com.k33.platform.cms.objectIDString +import com.k33.platform.utils.algolia.Algolia +import com.k33.platform.utils.algolia.AlgoliaSearchClient import com.k33.platform.utils.logging.getLogger import kotlinx.coroutines.runBlocking @@ -19,14 +21,16 @@ class ContentfulToAlgolia( private val logger by getLogger() private val algoliaClient by lazy { - AlgoliaSearchClient.getInstance(sync) + AlgoliaSearchClient.getInstance( + index = sync.config.algoliaIndex, + ) } private val content by lazy { ContentFactory.getContent(sync) } suspend fun upsert(entryId: String) { val record = content.fetch(entityId = entryId) ?: return - logger.info("Exporting record with objectID: ${record.objectID} to algolia") + logger.info("Exporting record with objectID: ${record.objectIDString} to algolia") record.objectID?.let { objectId -> algoliaClient.upsert( objectID = objectId, @@ -38,19 +42,14 @@ class ContentfulToAlgolia( suspend fun upsertAll() { val records = content .fetchAll() - .mapNotNull { jsonObject -> - jsonObject.objectID?.let { - objectID -> objectID to jsonObject - } - } + .toList() logger.info("Exporting ${records.size} records from contentful to algolia for syncId: ${sync.name}") algoliaClient.batchUpsert(records) } - suspend fun delete(entryId: String) { - val objectID = ObjectID(entryId) - logger.warn("Deleting objectID: $objectID from algolia") - algoliaClient.delete(objectID) + suspend fun delete(objectID: Algolia.ObjectID) { + logger.warn("Deleting objectID: ${objectID.value} from algolia") + algoliaClient.delete(objectID = objectID) } companion object { @@ -77,7 +76,7 @@ class ContentfulToAlgolia( Action.unpublish -> { logger.warn("Removing $entityContentType: $entityId from algolia") - contentfulToAlgolia.delete(entityId) + contentfulToAlgolia.delete(Algolia.ObjectID(entityId)) } } } diff --git a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/Diff.kt b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/Diff.kt index 83d30310..e72b2745 100644 --- a/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/Diff.kt +++ b/libs/utils/cms/contentful/src/main/kotlin/com/k33/platform/cms/sync/Diff.kt @@ -2,6 +2,7 @@ package com.k33.platform.cms.sync import com.k33.platform.cms.config.Sync import com.k33.platform.cms.content.ContentFactory +import com.k33.platform.utils.algolia.AlgoliaSearchClient import com.k33.platform.utils.logging.getLogger import kotlinx.coroutines.runBlocking @@ -15,9 +16,9 @@ object Diff { .fetchIdToModifiedMap() logger.info("Found in contentful: ${entryIdMap.size}") - val algoliaClient = AlgoliaSearchClient.getInstance(sync) + val algoliaClient = AlgoliaSearchClient.getInstance(index = sync.config.algoliaIndex) - val indices = algoliaClient.getAllIds() + val indices = algoliaClient.getAllIds().mapKeys { it.key.value } logger.info("Found in algolia: ${indices.size}") val newInContentful = entryIdMap.keys - indices.keys diff --git a/settings.gradle.kts b/settings.gradle.kts index 8999faaf..452a6910 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -63,6 +63,7 @@ include( "libs:utils:analytics", "libs:utils:analytics:google-analytics", + "libs:utils:algolia", "libs:utils:cms", "libs:utils:cms:contentful", diff --git a/versions.properties b/versions.properties index f451adc6..f74ff291 100644 --- a/versions.properties +++ b/versions.properties @@ -21,8 +21,7 @@ version.ch.qos.logback..logback-classic=1.5.7 version.ch.qos.logback.contrib..logback-json-classic=0.1.5 -version.com.algolia..algoliasearch-client-kotlin=2.1.12 -## # available=3.0.0 +version.com.algolia..algoliasearch-client-kotlin=3.0.0 ## unused version.com.apollographql.apollo3..apollo-runtime=3.8.2