-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feature: Re-write page content extraction for search feature
- Loading branch information
Showing
23 changed files
with
593 additions
and
211 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package dev.triumphteam.backend.meilisearch | ||
|
|
||
| import io.ktor.client.plugins.auth.Auth | ||
| import io.ktor.client.plugins.auth.AuthProvider | ||
| import io.ktor.client.request.HttpRequestBuilder | ||
| import io.ktor.client.request.headers | ||
| import io.ktor.http.HttpHeaders | ||
| import io.ktor.http.auth.AuthScheme | ||
| import io.ktor.http.auth.HttpAuthHeader | ||
|
|
||
| public fun Auth.api( | ||
| token: String, | ||
| sendWithoutRequestCallback: (HttpRequestBuilder) -> Boolean = { true }, | ||
| realm: String? = null, | ||
| ) { | ||
| providers.add(ApiAuthProvider(token, sendWithoutRequestCallback, realm)) | ||
| } | ||
|
|
||
| public class ApiAuthProvider( | ||
| private val token: String, | ||
| private val sendWithoutRequestCallback: (HttpRequestBuilder) -> Boolean = { true }, | ||
| private val realm: String?, | ||
| ) : AuthProvider { | ||
|
|
||
| @Suppress("OverridingDeprecatedMember") | ||
| @Deprecated("Please use sendWithoutRequest function instead", ReplaceWith("sendWithoutRequest")) | ||
| override val sendWithoutRequest: Boolean | ||
| get() = error("Deprecated") | ||
|
|
||
| override fun sendWithoutRequest(request: HttpRequestBuilder): Boolean = sendWithoutRequestCallback(request) | ||
|
|
||
| /** Checks if current provider is applicable to the request. */ | ||
| override fun isApplicable(auth: HttpAuthHeader): Boolean { | ||
| if (auth.authScheme != AuthScheme.Bearer) return false | ||
| if (realm == null) return true | ||
| if (auth !is HttpAuthHeader.Parameterized) return false | ||
|
|
||
| return auth.parameter("realm") == realm | ||
| } | ||
|
|
||
| /** Adds an authentication method headers and credentials. */ | ||
| override suspend fun addRequestHeaders(request: HttpRequestBuilder, authHeader: HttpAuthHeader?) { | ||
| request.headers { | ||
| if (contains(HttpHeaders.Authorization)) remove(HttpHeaders.Authorization) | ||
| append(HttpHeaders.Authorization, "Bearer $token") | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,189 @@ | ||
| package dev.triumphteam.backend.meilisearch | ||
|
|
||
| import dev.triumphteam.backend.meilisearch.annotation.PrimaryKey | ||
| import dev.triumphteam.website.JsonSerializer | ||
| import io.ktor.client.HttpClient | ||
| import io.ktor.client.call.body | ||
| import io.ktor.client.engine.cio.CIO | ||
| import io.ktor.client.plugins.auth.Auth | ||
| import io.ktor.client.plugins.contentnegotiation.ContentNegotiation | ||
| import io.ktor.client.plugins.defaultRequest | ||
| import io.ktor.client.plugins.logging.DEFAULT | ||
| import io.ktor.client.plugins.logging.LogLevel | ||
| import io.ktor.client.plugins.logging.Logger | ||
| import io.ktor.client.plugins.logging.Logging | ||
| import io.ktor.client.plugins.resources.Resources | ||
| import io.ktor.client.plugins.resources.delete | ||
| import io.ktor.client.plugins.resources.post | ||
| import io.ktor.client.request.parameter | ||
| import io.ktor.client.request.setBody | ||
| import io.ktor.client.statement.HttpResponse | ||
| import io.ktor.http.ContentType | ||
| import io.ktor.http.URLProtocol | ||
| import io.ktor.http.contentType | ||
| import io.ktor.http.isSuccess | ||
| import io.ktor.resources.Resource | ||
| import io.ktor.serialization.kotlinx.json.json | ||
| import kotlinx.serialization.Serializable | ||
|
|
||
| /** A client to handle connection between the application and https://www.meilisearch.com/. */ | ||
| public class MeiliClient( | ||
| host: String, | ||
| port: Int, | ||
| private val apiKey: String = "", | ||
| protocol: URLProtocol, | ||
| ) { | ||
|
|
||
| public val client: HttpClient = HttpClient(CIO) { | ||
| install(Resources) | ||
| install(Auth) { api(apiKey) } // Auto setup authentication | ||
| install(ContentNegotiation) { json(JsonSerializer.json) } // Using Kotlin serialization for content negotiation | ||
| install(Logging) { | ||
| logger = Logger.DEFAULT | ||
| level = LogLevel.BODY | ||
| } | ||
|
|
||
| defaultRequest { | ||
| this.host = host | ||
| this.port = port | ||
| url { | ||
| this.protocol = protocol | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** Gets an index, doesn't always mean the index exists within Meili. */ | ||
| public fun index( | ||
| uid: String, | ||
| primaryKey: String? = null, | ||
| ): Index = Index(uid, primaryKey) | ||
|
|
||
| /** Representation of an index. Contains the needed operations with the index. */ | ||
| public inner class Index( | ||
| public val uid: String, | ||
| public val primaryKey: String?, | ||
| ) { | ||
|
|
||
| /** Create the index. Success even if it already exists. */ | ||
| public suspend fun create(): HttpResponse = client.post(Indexes()) { | ||
| contentType(ContentType.Application.Json) | ||
| setBody(Create(uid, primaryKey)) | ||
| }.also { | ||
| println(it) | ||
| } | ||
|
|
||
| /** Deletes the current index. Success even if it doesn't exist. */ | ||
| public suspend fun delete(): HttpResponse = client.delete(Indexes.Uid(uid = uid)) | ||
|
|
||
| /** Search for specific content in the index. */ | ||
| public suspend inline fun <reified T> search(query: String, filter: String?): List<T> { | ||
| return searchFull<T>(query, filter).hits | ||
| } | ||
|
|
||
| /** [search] but returns all the data ([SearchResult]) provided by the search. */ | ||
| public suspend inline fun <reified T> searchFull(query: String, filter: String?): SearchResult<T> { | ||
| // TODO: Handle errors. | ||
| return client.post(Indexes.Uid.Search(Indexes.Uid(uid = uid))) { | ||
| contentType(ContentType.Application.Json) | ||
| setBody(SearchRequest(query, filter)) | ||
| }.body() | ||
| } | ||
|
|
||
| /** Add documents to the index. Creates a new index if none exists. */ | ||
| public suspend inline fun <reified T> addDocuments( | ||
| documents: List<T>, | ||
| primaryKey: String? = null, | ||
| ): HttpResponse { | ||
| val pk = primaryKey ?: this.primaryKey ?: run { | ||
| T::class.java.constructors.firstOrNull()?.parameters?.find { it.isAnnotationPresent(PrimaryKey::class.java) }?.name | ||
| } | ||
|
|
||
| return client.post(Indexes.Uid.Documents(Indexes.Uid(uid = uid))) { | ||
| contentType(ContentType.Application.Json) | ||
| pk?.let { parameter(PRIMARY_KEY_PARAM, it) } | ||
| setBody<List<T>>(documents) | ||
| }.also { | ||
| println(it) | ||
| println(it.body<String>()) | ||
| } | ||
| } | ||
|
|
||
| /** Transfers all the documents from this index into the passed [index], and deletes this. */ | ||
| public suspend fun transferTo(index: Index): HttpResponse { | ||
| val createResponse = index.create() | ||
|
|
||
| // If there was an error creating the new index we return the response | ||
| if (!createResponse.status.isSuccess()) return createResponse | ||
|
|
||
| val swapResponse = client.post(Indexes.Swap()) { | ||
| setBody(listOf(Swap(listOf(uid, this@Index.uid)))) | ||
| } | ||
|
|
||
| // If there was an error swapping the new indexes we return the response | ||
| if (!swapResponse.status.isSuccess()) return swapResponse | ||
|
|
||
| // Then we delete the current index | ||
| return delete() | ||
| } | ||
| } | ||
|
|
||
| /** Resource location for "$url/indexes". */ | ||
| @Serializable | ||
| @Resource("/indexes") | ||
| public class Indexes { | ||
|
|
||
| /** Resource location for "$url/indexes/[uid]". */ | ||
| @Serializable | ||
| @Resource("{uid}") | ||
| public class Uid(public val parent: Indexes = Indexes(), public val uid: String) { | ||
|
|
||
| /** Resource location for "$url/indexes/[uid]/search". */ | ||
| @Serializable | ||
| @Resource("/search") | ||
| public class Search(public val parent: Uid) | ||
|
|
||
| /** Resource location for "$url/indexes/[uid]/documents". */ | ||
| @Serializable | ||
| @Resource("/documents") | ||
| public class Documents(public val parent: Uid) | ||
| } | ||
|
|
||
| /** Resource location for "$url/indexes/swap-indexes". */ | ||
| @Serializable | ||
| @Resource("/swap-indexes") | ||
| public class Swap(public val parent: Indexes = Indexes()) | ||
| } | ||
|
|
||
| /** Serializable class for the create end point. */ | ||
| @Serializable | ||
| public data class Create( | ||
| val uid: String, | ||
| val primaryKey: String?, | ||
| ) | ||
|
|
||
| /** Serializable class for the search result from end point. */ | ||
| @Serializable | ||
| public data class SearchResult<T>( | ||
| val hits: List<T>, | ||
| val query: String, | ||
| val processingTimeMs: Long, | ||
| val limit: Int, | ||
| val offset: Int, | ||
| val estimatedTotalHits: Int, | ||
| ) | ||
|
|
||
| /** Serializable class for the search end point. */ | ||
| @Serializable | ||
| public data class SearchRequest( | ||
| public val q: String, | ||
| public val filter: String?, | ||
| ) | ||
|
|
||
| /** Serializable class for the swap end point. */ | ||
| @Serializable | ||
| public data class Swap(public val indexes: List<String>) | ||
|
|
||
| public companion object { | ||
| public const val PRIMARY_KEY_PARAM: String = "primaryKey" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package dev.triumphteam.backend.meilisearch | ||
|
|
||
| import io.ktor.http.URLProtocol | ||
| import io.ktor.server.application.Application | ||
| import io.ktor.server.application.ApplicationCall | ||
| import io.ktor.server.application.BaseApplicationPlugin | ||
| import io.ktor.server.application.application | ||
| import io.ktor.server.application.plugin | ||
| import io.ktor.util.AttributeKey | ||
| import io.ktor.util.pipeline.PipelineContext | ||
| import kotlinx.serialization.Contextual | ||
| import kotlinx.serialization.Serializable | ||
|
|
||
| public class Meili(config: Configuration) { | ||
|
|
||
| public val client: MeiliClient = config.createClient() | ||
|
|
||
| public class Configuration { | ||
| private var host: String = "0.0.0.0" | ||
| private var port: Int = 7700 | ||
| private var apiKey: String = "masterKey" | ||
| private var protocol: URLProtocol = URLProtocol.HTTP | ||
|
|
||
| internal fun createClient() = MeiliClient(host, port, apiKey, protocol) | ||
| } | ||
|
|
||
| public companion object Plugin : BaseApplicationPlugin<Application, Configuration, Meili> { | ||
|
|
||
| override val key: AttributeKey<Meili> = AttributeKey("Meili") | ||
|
|
||
| override fun install(pipeline: Application, configure: Configuration.() -> Unit): Meili { | ||
| return Meili(Configuration().apply(configure)) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public suspend inline fun <reified T> PipelineContext<*, ApplicationCall>.search( | ||
| index: String, | ||
| query: String, | ||
| filter: String? = null, | ||
| ): List<T> = with(this.application.plugin(Meili).client) { | ||
| return index(index).search(query, filter) | ||
| } | ||
|
|
||
| public suspend inline fun PipelineContext<*, ApplicationCall>.index(index: String): MeiliClient.Index = | ||
| application.plugin(Meili).client.index(index) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package dev.triumphteam.backend.meilisearch.annotation | ||
|
|
||
| @Retention(AnnotationRetention.RUNTIME) | ||
| @Target(AnnotationTarget.VALUE_PARAMETER) | ||
| public annotation class PrimaryKey |
Oops, something went wrong.