feature: Search!

TriumphTeam · Jul 6, 2024 · 07a2b2b · 07a2b2b
1 parent db242e4
commit 07a2b2b
Show file tree

Hide file tree

Showing 11 changed files with 282 additions and 37 deletions.
diff --git a/backend/src/main/kotlin/api/ApiRouting.kt b/backend/src/main/kotlin/api/ApiRouting.kt
@@ -1,12 +1,14 @@
 package dev.triumphteam.backend.api
 
 import dev.triumphteam.backend.DATA_FOLDER
+import dev.triumphteam.backend.meilisearch.Meili
 import dev.triumphteam.website.api.Api
 import io.ktor.http.HttpStatusCode
 import io.ktor.http.content.PartData
 import io.ktor.http.content.forEachPart
 import io.ktor.http.content.streamProvider
 import io.ktor.server.application.call
+import io.ktor.server.application.plugin
 import io.ktor.server.auth.authenticate
 import io.ktor.server.request.receiveMultipart
 import io.ktor.server.resources.post
@@ -19,6 +21,7 @@ import java.io.File
 private val logger: Logger = LoggerFactory.getLogger("api-route")
 
 public fun Routing.apiRoutes() {
+    val meili = plugin(Meili)
 
     authenticate("bearer") {
         post<Api.Setup> {
@@ -35,7 +38,7 @@ public fun Routing.apiRoutes() {
                                 val zip = downloadsFolder.resolve("projects.zip").also {
                                     it.writeBytes(fileBytes)
                                 }
-                                setupRepository(zip)
+                                setupRepository(meili, zip)
                             }
 
                             else -> {}

diff --git a/backend/src/main/kotlin/api/ProjectSetup.kt b/backend/src/main/kotlin/api/ProjectSetup.kt
@@ -5,8 +5,13 @@ import dev.triumphteam.backend.api.database.DocVersionEntity
 import dev.triumphteam.backend.api.database.PageEntity
 import dev.triumphteam.backend.api.database.ProjectEntity
 import dev.triumphteam.backend.banner.BannerMaker
+import dev.triumphteam.backend.meilisearch.Meili
 import dev.triumphteam.website.JsonSerializer
+import dev.triumphteam.website.project.Page
 import dev.triumphteam.website.project.Repository
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import kotlinx.serialization.Serializable
 import net.lingala.zip4j.ZipFile
 import org.jetbrains.exposed.sql.transactions.transaction
 import java.io.File
@@ -16,9 +21,12 @@ import javax.imageio.ImageIO
 
 private val bannerMaker = BannerMaker()
 
-public fun setupRepository(projects: File) {
+public suspend fun setupRepository(meili: Meili, projects: File) {
+
+    val tempFolder = withContext(Dispatchers.IO) {
+        Files.createTempDirectory("zip-temp")
+    }.toFile()
 
-    val tempFolder = Files.createTempDirectory("zip-temp").toFile()
     ZipFile(projects).extractAll(tempFolder.path)
 
     val json = tempFolder.resolve("repository.json")
@@ -96,4 +104,63 @@ public fun setupRepository(projects: File) {
             }
         }
     }
+
+
+    // Search setup
+    repo.projects.forEach { project ->
+        project.versions.forEach { version ->
+
+            val projectId = projectIndex(project.id, version.reference)
+
+            // First delete it all
+            meili.client.index(projectId).delete()
+
+            // Then re-add new stuff
+            meili.client.index(projectId, primaryKey = "id").addDocuments(
+                version.pages.flatMap { page ->
+                    listOf(descriptionDocument(page.id, page.description))
+                        .plus(
+                            page.description.summary.map { summary ->
+                                SearchDocument(
+                                    id = SearchDocument.createId(page.id, summary.href),
+                                    pageId = page.id,
+                                    anchor = summary.href,
+                                    isAnchor = true,
+                                    reference = summary.terms,
+                                )
+                            }
+                        )
+                }
+            )
+        }
+    }
+}
+
+private fun descriptionDocument(id: String, description: Page.Description): SearchDocument {
+    return SearchDocument(
+        id = id,
+        pageId = id,
+        anchor = id,
+        isAnchor = false,
+        reference = listOfNotNull(description.title, description.subTitle),
+    )
 }
+
+@Serializable
+public data class SearchDocument(
+    public val id: String,
+    public val pageId: String,
+    public val anchor: String,
+    public val isAnchor: Boolean,
+    public val reference: List<String>,
+) {
+
+    public companion object {
+
+        public fun createId(page: String, id: String): String {
+            return "$page-$id"
+        }
+    }
+}
+
+public fun projectIndex(project: String, version: String): String = "$project-${version.replace(".", "_")}"
diff --git a/backend/src/main/kotlin/meilisearch/MeiliClient.kt b/backend/src/main/kotlin/meilisearch/MeiliClient.kt
@@ -76,16 +76,24 @@ public class MeiliClient(
         public suspend fun delete(): HttpResponse = client.delete(Indexes.Uid(uid = uid))
 
         /** Search for specific content in the index. */
-        public suspend inline fun <reified T> search(query: String, filter: String?): List<T> {
-            return searchFull<T>(query, filter).hits
+        public suspend inline fun <reified T> search(
+            query: String,
+            limit: Int = 20,
+            filter: String? = null,
+        ): List<T> {
+            return searchFull<T>(query, limit, filter).hits
         }
 
         /** [search] but returns all the data ([SearchResult]) provided by the search. */
-        public suspend inline fun <reified T> searchFull(query: String, filter: String?): SearchResult<T> {
+        public suspend inline fun <reified T> searchFull(
+            query: String,
+            limit: Int,
+            filter: String?,
+        ): SearchResult<T> {
             // TODO: Handle errors.
             return client.post(Indexes.Uid.Search(Indexes.Uid(uid = uid))) {
                 contentType(ContentType.Application.Json)
-                setBody(SearchRequest(query, filter))
+                setBody(SearchRequest(query, limit, filter))
             }.body()
         }
 
@@ -176,6 +184,7 @@ public class MeiliClient(
     @Serializable
     public data class SearchRequest(
         public val q: String,
+        public val limit: Int = 20,
         public val filter: String?,
     )
 

diff --git a/backend/src/main/kotlin/meilisearch/MeiliPlugin.kt b/backend/src/main/kotlin/meilisearch/MeiliPlugin.kt
@@ -8,8 +8,6 @@ import io.ktor.server.application.application
 import io.ktor.server.application.plugin
 import io.ktor.util.AttributeKey
 import io.ktor.util.pipeline.PipelineContext
-import kotlinx.serialization.Contextual
-import kotlinx.serialization.Serializable
 
 public class Meili(config: Configuration) {
 
@@ -37,9 +35,10 @@ public class Meili(config: Configuration) {
 public suspend inline fun <reified T> PipelineContext<*, ApplicationCall>.search(
     index: String,
     query: String,
+    limit: Int = 20,
     filter: String? = null,
 ): List<T> = with(this.application.plugin(Meili).client) {
-    return index(index).search(query, filter)
+    return index(index).search(query, limit, filter)
 }
 
 public suspend inline fun PipelineContext<*, ApplicationCall>.index(index: String): MeiliClient.Index =

diff --git a/backend/src/main/kotlin/website/WebsiteRoutes.kt b/backend/src/main/kotlin/website/WebsiteRoutes.kt
@@ -1,10 +1,14 @@
 package dev.triumphteam.backend.website
 
+import dev.triumphteam.backend.meilisearch.Meili
 import dev.triumphteam.backend.website.pages.docs.docsRoutes
 import dev.triumphteam.backend.website.pages.home.homeRoutes
+import io.ktor.server.application.plugin
 import io.ktor.server.routing.Routing
 
 public fun Routing.websiteRoutes(developmentMode: Boolean) {
+    val meili = plugin(Meili)
+
     homeRoutes(developmentMode)
-    docsRoutes(developmentMode)
+    docsRoutes(meili, developmentMode)
 }
diff --git a/backend/src/main/kotlin/website/pages/docs/DocsPage.kt b/backend/src/main/kotlin/website/pages/docs/DocsPage.kt
@@ -2,23 +2,31 @@ package dev.triumphteam.backend.website.pages.docs
 
 import com.github.benmanes.caffeine.cache.Cache
 import com.github.benmanes.caffeine.cache.Caffeine
+import dev.triumphteam.backend.api.SearchDocument
 import dev.triumphteam.backend.api.database.DocVersionEntity
 import dev.triumphteam.backend.api.database.DocVersions
 import dev.triumphteam.backend.api.database.PageEntity
 import dev.triumphteam.backend.api.database.Pages
 import dev.triumphteam.backend.api.database.ProjectEntity
+import dev.triumphteam.backend.api.projectIndex
+import dev.triumphteam.backend.meilisearch.Meili
 import dev.triumphteam.backend.website.pages.createIconPath
 import dev.triumphteam.backend.website.pages.docs.components.DropdownOption
 import dev.triumphteam.backend.website.pages.docs.components.dropDown
+import dev.triumphteam.website.highlightWord
+import dev.triumphteam.backend.website.pages.docs.components.noResults
 import dev.triumphteam.backend.website.pages.docs.components.search
 import dev.triumphteam.backend.website.pages.docs.components.searchArea
+import dev.triumphteam.backend.website.pages.docs.components.searchResult
 import dev.triumphteam.backend.website.pages.docs.components.toast
+import dev.triumphteam.website.trim
 import dev.triumphteam.backend.website.pages.setupHead
 import dev.triumphteam.backend.website.respondHtmlCached
 import dev.triumphteam.website.project.Navigation
 import dev.triumphteam.website.project.Page
 import io.ktor.http.HttpStatusCode
 import io.ktor.server.application.call
+import io.ktor.server.html.respondHtml
 import io.ktor.server.request.uri
 import io.ktor.server.response.respond
 import io.ktor.server.response.respondRedirect
@@ -27,6 +35,7 @@ import io.ktor.server.routing.get
 import kotlinx.css.Color
 import kotlinx.css.CssBuilder
 import kotlinx.css.backgroundColor
+import kotlinx.css.borderColor
 import kotlinx.css.color
 import kotlinx.css.properties.s
 import kotlinx.css.transitionDuration
@@ -49,6 +58,7 @@ import kotlinx.html.styleLink
 import kotlinx.html.title
 import kotlinx.html.ul
 import kotlinx.html.unsafe
+import kotlinx.serialization.Serializable
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.transactions.transaction
 import java.time.LocalDate
@@ -59,7 +69,7 @@ private val projectCache: Cache<String, ProjectData> = Caffeine.newBuilder()
     .expireAfterWrite(5.minutes.toJavaDuration())
     .build()
 
-public fun Routing.docsRoutes(developmentMode: Boolean) {
+public fun Routing.docsRoutes(meili: Meili, developmentMode: Boolean) {
 
     get("/docs/{param...}") {
 
@@ -89,6 +99,58 @@ public fun Routing.docsRoutes(developmentMode: Boolean) {
             renderFullPage(developmentMode, project, currentVersion, page)
         }
     }
+
+    get("/search") {
+
+        fun HTML.respondEmpty(query: String = "") {
+            body {
+                noResults(query)
+            }
+        }
+
+        val query = call.request.queryParameters["q"] ?: return@get call.respond(HttpStatusCode.BadRequest)
+        val projectParam = call.request.queryParameters["p"] ?: return@get call.respond(HttpStatusCode.BadRequest)
+        val versionParam = call.request.queryParameters["v"] ?: return@get call.respond(HttpStatusCode.BadRequest)
+
+        if (query.isBlank()) {
+            return@get call.respondHtml {
+                respondEmpty()
+            }
+        }
+
+        val project = getProject(projectParam) ?: return@get call.respond(HttpStatusCode.NotFound)
+        val pages = project.versions[versionParam]?.pages ?: return@get call.respond(HttpStatusCode.NotFound)
+
+        val result =
+            meili.client.index(projectIndex(projectParam, versionParam)).search<SearchDocument>(query, limit = 5)
+                .mapNotNull { document ->
+                    val page = pages[document.pageId] ?: return@mapNotNull null
+                    val summary = page.summary.find { it.href == document.anchor } ?: return@mapNotNull null
+
+                    val queryWords = query.split(" ")
+                    val trimmedDescription = summary.terms.joinToString(" ").trim(queryWords.first(), 50)
+
+                    SearchResult(
+                        title = "${page.title} | ${summary.literal}",
+                        description = trimmedDescription.highlightWord(queryWords),
+                        link = "/docs/$versionParam/$projectParam/${page.id}#${document.anchor}",
+                    )
+                }
+
+        if (result.isEmpty()) {
+            return@get call.respondHtml {
+                respondEmpty(query)
+            }
+        }
+
+        return@get call.respondHtml {
+            body {
+                result.forEach { result ->
+                    searchResult(result.title, result.description, result.link)
+                }
+            }
+        }
+    }
 }
 
 private fun HTML.renderFullPage(
@@ -103,6 +165,10 @@ private fun HTML.renderFullPage(
         styleLink("/static/css/docs_content.css")
         styleLink("/static/css/themes/one_dark.css")
 
+        script {
+            src = "https://unpkg.com/htmx.org@2.0.0"
+        }
+
         val title = "TrimphTeam | ${project.name} - ${currentPage.title}"
         // TODO: Replace with final URL, sucks that it can't be relative
         val image = "https://new.triumphteam.dev/assets/${project.id}/${version.reference}/${currentPage.id}/banner.png"
@@ -175,10 +241,18 @@ private fun HTML.renderFullPage(
                     transitionDuration = 0.3.s
                 }
 
+                rule(".project-color-border") {
+                    transitionDuration = 0.3.s
+                }
+
                 rule(".project-color-hover:hover") {
                     color = Color(project.color)
                 }
 
+                rule(".project-color-border:hover") {
+                    borderColor = Color(project.color)
+                }
+
                 rule(".docs-content a") {
                     color = Color(project.color)
                     transitionDuration = 0.3.s
@@ -190,17 +264,16 @@ private fun HTML.renderFullPage(
                 rule(".summary-active *") {
                     color = Color(project.color)
                 }
-
             }.toString()
         }
     }
 
     body {
 
-        classes = setOf("bg-docs-bg", "text-white", "overflow-y-auto overflow-x-hidden", "opened-search")
+        classes = setOf("bg-docs-bg", "text-white", "overflow-y-auto overflow-x-hidden")
 
         // Must be first here
-        searchArea()
+        searchArea(project.id, version.reference)
 
         sideBar(project, version, currentPage)
         content(currentPage)
@@ -494,3 +567,10 @@ public data class ProjectPage(
 private fun cacheId(project: ProjectData, version: Version, page: ProjectPage): String {
     return "${project.id}:${version.reference}:${page.id}"
 }
+
+@Serializable
+private data class SearchResult(
+    val title: String,
+    val description: String,
+    val link: String,
+)