From 07a2b2b5cb4319caf1e17d5a3a9c17d0379babd1 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 6 Jul 2024 21:04:51 +0100 Subject: [PATCH] feature: Search! --- backend/src/main/kotlin/api/ApiRouting.kt | 5 +- backend/src/main/kotlin/api/ProjectSetup.kt | 71 ++++++++++++++- .../main/kotlin/meilisearch/MeiliClient.kt | 17 +++- .../main/kotlin/meilisearch/MeiliPlugin.kt | 5 +- .../src/main/kotlin/website/WebsiteRoutes.kt | 6 +- .../kotlin/website/pages/docs/DocsPage.kt | 88 ++++++++++++++++++- .../website/pages/docs/components/Search.kt | 71 +++++++++++---- .../main/resources/static/css/docs_style.css | 3 +- common/src/main/kotlin/StringExt.kt | 32 +++++++ docs/src/main/kotlin/Application.kt | 3 +- script/src/jsMain/kotlin/Script.kt | 18 +++- 11 files changed, 282 insertions(+), 37 deletions(-) create mode 100644 common/src/main/kotlin/StringExt.kt diff --git a/backend/src/main/kotlin/api/ApiRouting.kt b/backend/src/main/kotlin/api/ApiRouting.kt index f4c8e37..01f033b 100644 --- a/backend/src/main/kotlin/api/ApiRouting.kt +++ b/backend/src/main/kotlin/api/ApiRouting.kt @@ -1,12 +1,14 @@ package dev.triumphteam.backend.api import dev.triumphteam.backend.DATA_FOLDER +import dev.triumphteam.backend.meilisearch.Meili import dev.triumphteam.website.api.Api import io.ktor.http.HttpStatusCode import io.ktor.http.content.PartData import io.ktor.http.content.forEachPart import io.ktor.http.content.streamProvider import io.ktor.server.application.call +import io.ktor.server.application.plugin import io.ktor.server.auth.authenticate import io.ktor.server.request.receiveMultipart import io.ktor.server.resources.post @@ -19,6 +21,7 @@ import java.io.File private val logger: Logger = LoggerFactory.getLogger("api-route") public fun Routing.apiRoutes() { + val meili = plugin(Meili) authenticate("bearer") { post { @@ -35,7 +38,7 @@ public fun Routing.apiRoutes() { val zip = downloadsFolder.resolve("projects.zip").also { it.writeBytes(fileBytes) } - setupRepository(zip) + setupRepository(meili, zip) } else -> {} diff --git a/backend/src/main/kotlin/api/ProjectSetup.kt b/backend/src/main/kotlin/api/ProjectSetup.kt index fc39f18..c884d9d 100644 --- a/backend/src/main/kotlin/api/ProjectSetup.kt +++ b/backend/src/main/kotlin/api/ProjectSetup.kt @@ -5,8 +5,13 @@ import dev.triumphteam.backend.api.database.DocVersionEntity import dev.triumphteam.backend.api.database.PageEntity import dev.triumphteam.backend.api.database.ProjectEntity import dev.triumphteam.backend.banner.BannerMaker +import dev.triumphteam.backend.meilisearch.Meili import dev.triumphteam.website.JsonSerializer +import dev.triumphteam.website.project.Page import dev.triumphteam.website.project.Repository +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.Serializable import net.lingala.zip4j.ZipFile import org.jetbrains.exposed.sql.transactions.transaction import java.io.File @@ -16,9 +21,12 @@ import javax.imageio.ImageIO private val bannerMaker = BannerMaker() -public fun setupRepository(projects: File) { +public suspend fun setupRepository(meili: Meili, projects: File) { + + val tempFolder = withContext(Dispatchers.IO) { + Files.createTempDirectory("zip-temp") + }.toFile() - val tempFolder = Files.createTempDirectory("zip-temp").toFile() ZipFile(projects).extractAll(tempFolder.path) val json = tempFolder.resolve("repository.json") @@ -96,4 +104,63 @@ public fun setupRepository(projects: File) { } } } + + + // Search setup + repo.projects.forEach { project -> + project.versions.forEach { version -> + + val projectId = projectIndex(project.id, version.reference) + + // First delete it all + meili.client.index(projectId).delete() + + // Then re-add new stuff + meili.client.index(projectId, primaryKey = "id").addDocuments( + version.pages.flatMap { page -> + listOf(descriptionDocument(page.id, page.description)) + .plus( + page.description.summary.map { summary -> + SearchDocument( + id = SearchDocument.createId(page.id, summary.href), + pageId = page.id, + anchor = summary.href, + isAnchor = true, + reference = summary.terms, + ) + } + ) + } + ) + } + } +} + +private fun descriptionDocument(id: String, description: Page.Description): SearchDocument { + return SearchDocument( + id = id, + pageId = id, + anchor = id, + isAnchor = false, + reference = listOfNotNull(description.title, description.subTitle), + ) } + +@Serializable +public data class SearchDocument( + public val id: String, + public val pageId: String, + public val anchor: String, + public val isAnchor: Boolean, + public val reference: List, +) { + + public companion object { + + public fun createId(page: String, id: String): String { + return "$page-$id" + } + } +} + +public fun projectIndex(project: String, version: String): String = "$project-${version.replace(".", "_")}" diff --git a/backend/src/main/kotlin/meilisearch/MeiliClient.kt b/backend/src/main/kotlin/meilisearch/MeiliClient.kt index 9f365eb..a346601 100644 --- a/backend/src/main/kotlin/meilisearch/MeiliClient.kt +++ b/backend/src/main/kotlin/meilisearch/MeiliClient.kt @@ -76,16 +76,24 @@ public class MeiliClient( public suspend fun delete(): HttpResponse = client.delete(Indexes.Uid(uid = uid)) /** Search for specific content in the index. */ - public suspend inline fun search(query: String, filter: String?): List { - return searchFull(query, filter).hits + public suspend inline fun search( + query: String, + limit: Int = 20, + filter: String? = null, + ): List { + return searchFull(query, limit, filter).hits } /** [search] but returns all the data ([SearchResult]) provided by the search. */ - public suspend inline fun searchFull(query: String, filter: String?): SearchResult { + public suspend inline fun searchFull( + query: String, + limit: Int, + filter: String?, + ): SearchResult { // TODO: Handle errors. return client.post(Indexes.Uid.Search(Indexes.Uid(uid = uid))) { contentType(ContentType.Application.Json) - setBody(SearchRequest(query, filter)) + setBody(SearchRequest(query, limit, filter)) }.body() } @@ -176,6 +184,7 @@ public class MeiliClient( @Serializable public data class SearchRequest( public val q: String, + public val limit: Int = 20, public val filter: String?, ) diff --git a/backend/src/main/kotlin/meilisearch/MeiliPlugin.kt b/backend/src/main/kotlin/meilisearch/MeiliPlugin.kt index 671fadb..2f87c95 100644 --- a/backend/src/main/kotlin/meilisearch/MeiliPlugin.kt +++ b/backend/src/main/kotlin/meilisearch/MeiliPlugin.kt @@ -8,8 +8,6 @@ import io.ktor.server.application.application import io.ktor.server.application.plugin import io.ktor.util.AttributeKey import io.ktor.util.pipeline.PipelineContext -import kotlinx.serialization.Contextual -import kotlinx.serialization.Serializable public class Meili(config: Configuration) { @@ -37,9 +35,10 @@ public class Meili(config: Configuration) { public suspend inline fun PipelineContext<*, ApplicationCall>.search( index: String, query: String, + limit: Int = 20, filter: String? = null, ): List = with(this.application.plugin(Meili).client) { - return index(index).search(query, filter) + return index(index).search(query, limit, filter) } public suspend inline fun PipelineContext<*, ApplicationCall>.index(index: String): MeiliClient.Index = diff --git a/backend/src/main/kotlin/website/WebsiteRoutes.kt b/backend/src/main/kotlin/website/WebsiteRoutes.kt index cac4b81..6988bed 100644 --- a/backend/src/main/kotlin/website/WebsiteRoutes.kt +++ b/backend/src/main/kotlin/website/WebsiteRoutes.kt @@ -1,10 +1,14 @@ package dev.triumphteam.backend.website +import dev.triumphteam.backend.meilisearch.Meili import dev.triumphteam.backend.website.pages.docs.docsRoutes import dev.triumphteam.backend.website.pages.home.homeRoutes +import io.ktor.server.application.plugin import io.ktor.server.routing.Routing public fun Routing.websiteRoutes(developmentMode: Boolean) { + val meili = plugin(Meili) + homeRoutes(developmentMode) - docsRoutes(developmentMode) + docsRoutes(meili, developmentMode) } diff --git a/backend/src/main/kotlin/website/pages/docs/DocsPage.kt b/backend/src/main/kotlin/website/pages/docs/DocsPage.kt index d5d0e5e..30b5b61 100644 --- a/backend/src/main/kotlin/website/pages/docs/DocsPage.kt +++ b/backend/src/main/kotlin/website/pages/docs/DocsPage.kt @@ -2,23 +2,31 @@ package dev.triumphteam.backend.website.pages.docs import com.github.benmanes.caffeine.cache.Cache import com.github.benmanes.caffeine.cache.Caffeine +import dev.triumphteam.backend.api.SearchDocument import dev.triumphteam.backend.api.database.DocVersionEntity import dev.triumphteam.backend.api.database.DocVersions import dev.triumphteam.backend.api.database.PageEntity import dev.triumphteam.backend.api.database.Pages import dev.triumphteam.backend.api.database.ProjectEntity +import dev.triumphteam.backend.api.projectIndex +import dev.triumphteam.backend.meilisearch.Meili import dev.triumphteam.backend.website.pages.createIconPath import dev.triumphteam.backend.website.pages.docs.components.DropdownOption import dev.triumphteam.backend.website.pages.docs.components.dropDown +import dev.triumphteam.website.highlightWord +import dev.triumphteam.backend.website.pages.docs.components.noResults import dev.triumphteam.backend.website.pages.docs.components.search import dev.triumphteam.backend.website.pages.docs.components.searchArea +import dev.triumphteam.backend.website.pages.docs.components.searchResult import dev.triumphteam.backend.website.pages.docs.components.toast +import dev.triumphteam.website.trim import dev.triumphteam.backend.website.pages.setupHead import dev.triumphteam.backend.website.respondHtmlCached import dev.triumphteam.website.project.Navigation import dev.triumphteam.website.project.Page import io.ktor.http.HttpStatusCode import io.ktor.server.application.call +import io.ktor.server.html.respondHtml import io.ktor.server.request.uri import io.ktor.server.response.respond import io.ktor.server.response.respondRedirect @@ -27,6 +35,7 @@ import io.ktor.server.routing.get import kotlinx.css.Color import kotlinx.css.CssBuilder import kotlinx.css.backgroundColor +import kotlinx.css.borderColor import kotlinx.css.color import kotlinx.css.properties.s import kotlinx.css.transitionDuration @@ -49,6 +58,7 @@ import kotlinx.html.styleLink import kotlinx.html.title import kotlinx.html.ul import kotlinx.html.unsafe +import kotlinx.serialization.Serializable import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.transactions.transaction import java.time.LocalDate @@ -59,7 +69,7 @@ private val projectCache: Cache = Caffeine.newBuilder() .expireAfterWrite(5.minutes.toJavaDuration()) .build() -public fun Routing.docsRoutes(developmentMode: Boolean) { +public fun Routing.docsRoutes(meili: Meili, developmentMode: Boolean) { get("/docs/{param...}") { @@ -89,6 +99,58 @@ public fun Routing.docsRoutes(developmentMode: Boolean) { renderFullPage(developmentMode, project, currentVersion, page) } } + + get("/search") { + + fun HTML.respondEmpty(query: String = "") { + body { + noResults(query) + } + } + + val query = call.request.queryParameters["q"] ?: return@get call.respond(HttpStatusCode.BadRequest) + val projectParam = call.request.queryParameters["p"] ?: return@get call.respond(HttpStatusCode.BadRequest) + val versionParam = call.request.queryParameters["v"] ?: return@get call.respond(HttpStatusCode.BadRequest) + + if (query.isBlank()) { + return@get call.respondHtml { + respondEmpty() + } + } + + val project = getProject(projectParam) ?: return@get call.respond(HttpStatusCode.NotFound) + val pages = project.versions[versionParam]?.pages ?: return@get call.respond(HttpStatusCode.NotFound) + + val result = + meili.client.index(projectIndex(projectParam, versionParam)).search(query, limit = 5) + .mapNotNull { document -> + val page = pages[document.pageId] ?: return@mapNotNull null + val summary = page.summary.find { it.href == document.anchor } ?: return@mapNotNull null + + val queryWords = query.split(" ") + val trimmedDescription = summary.terms.joinToString(" ").trim(queryWords.first(), 50) + + SearchResult( + title = "${page.title} | ${summary.literal}", + description = trimmedDescription.highlightWord(queryWords), + link = "/docs/$versionParam/$projectParam/${page.id}#${document.anchor}", + ) + } + + if (result.isEmpty()) { + return@get call.respondHtml { + respondEmpty(query) + } + } + + return@get call.respondHtml { + body { + result.forEach { result -> + searchResult(result.title, result.description, result.link) + } + } + } + } } private fun HTML.renderFullPage( @@ -103,6 +165,10 @@ private fun HTML.renderFullPage( styleLink("/static/css/docs_content.css") styleLink("/static/css/themes/one_dark.css") + script { + src = "https://unpkg.com/htmx.org@2.0.0" + } + val title = "TrimphTeam | ${project.name} - ${currentPage.title}" // TODO: Replace with final URL, sucks that it can't be relative val image = "https://new.triumphteam.dev/assets/${project.id}/${version.reference}/${currentPage.id}/banner.png" @@ -175,10 +241,18 @@ private fun HTML.renderFullPage( transitionDuration = 0.3.s } + rule(".project-color-border") { + transitionDuration = 0.3.s + } + rule(".project-color-hover:hover") { color = Color(project.color) } + rule(".project-color-border:hover") { + borderColor = Color(project.color) + } + rule(".docs-content a") { color = Color(project.color) transitionDuration = 0.3.s @@ -190,17 +264,16 @@ private fun HTML.renderFullPage( rule(".summary-active *") { color = Color(project.color) } - }.toString() } } body { - classes = setOf("bg-docs-bg", "text-white", "overflow-y-auto overflow-x-hidden", "opened-search") + classes = setOf("bg-docs-bg", "text-white", "overflow-y-auto overflow-x-hidden") // Must be first here - searchArea() + searchArea(project.id, version.reference) sideBar(project, version, currentPage) content(currentPage) @@ -494,3 +567,10 @@ public data class ProjectPage( private fun cacheId(project: ProjectData, version: Version, page: ProjectPage): String { return "${project.id}:${version.reference}:${page.id}" } + +@Serializable +private data class SearchResult( + val title: String, + val description: String, + val link: String, +) diff --git a/backend/src/main/kotlin/website/pages/docs/components/Search.kt b/backend/src/main/kotlin/website/pages/docs/components/Search.kt index b6bca0f..deaa0ce 100644 --- a/backend/src/main/kotlin/website/pages/docs/components/Search.kt +++ b/backend/src/main/kotlin/website/pages/docs/components/Search.kt @@ -2,14 +2,20 @@ package dev.triumphteam.backend.website.pages.docs.components import kotlinx.html.FlowContent import kotlinx.html.InputType +import kotlinx.html.a import kotlinx.html.classes import kotlinx.html.div import kotlinx.html.i import kotlinx.html.id import kotlinx.html.input import kotlinx.html.label +import kotlinx.html.unsafe -public fun FlowContent.search(enabled: Boolean) { +public fun FlowContent.search( + enabled: Boolean, + project: String = "", + version: String = "", +) { div { classes = setOf("flex", "items-center", "w-full", "mx-auto", "bg-search-bg", "rounded-lg", "h-12") @@ -21,9 +27,16 @@ public fun FlowContent.search(enabled: Boolean) { if (enabled) { input { + attributes["hx-get"] = "/search?p=${project}&v=${version}" + attributes["hx-trigger"] = "keyup changed delay:500ms" + attributes["hx-target"] = "#results" + type = InputType.search classes = textClasses placeholder = "Search" + name = "q" + autoComplete = false + autoFocus = true } } else { label { @@ -53,7 +66,7 @@ public fun FlowContent.search(enabled: Boolean) { } } -public fun FlowContent.searchArea() { +public fun FlowContent.searchArea(project: String, version: String) { div { id = "search-area" @@ -79,39 +92,65 @@ public fun FlowContent.searchArea() { "flex flex-col gap-6", ) - search(true) + search(true, project, version) + + div { + id = "results" + + classes = setOf( + "flex flex-col gap-6", + "text-white/80", + ) - noResults() + noResults() + } } } } -private fun FlowContent.noResults() { +public fun FlowContent.noResults(query: String = "") { div { - classes = setOf("w-full text-center", "pt-12") + classes = setOf("w-full text-center", "pt-12", "text-xl", "text-white/50") - +"..." + +if (query.isBlank()) { + "No results" + } else { + "No results for \"$query\"" + } } } -private fun FlowContent.searchResult(title: String, subtitle: String) { - div { +public fun FlowContent.searchResult(title: String, description: String, link: String) { + a { - classes = setOf("w-full", "flex flex-col", "border border-zinc-700 rounded-lg") + href = link div { - classes = setOf("w-full", "p-2", "font-bold text-lg") + classes = setOf( + "w-full", + "flex flex-col", + "border border-zinc-700 rounded-lg", + "project-color-hover", + "project-color-border", + ) + + div { - +title - } + classes = setOf("w-full", "p-2", "font-bold text-lg") - div { + +title + } + + div { - classes = setOf("w-full", "p-2", "text-sm") + classes = setOf("w-full", "p-2", "text-sm") - +subtitle + unsafe { + raw(description) + } + } } } } diff --git a/backend/src/main/resources/static/css/docs_style.css b/backend/src/main/resources/static/css/docs_style.css index ac68226..3c5660f 100644 --- a/backend/src/main/resources/static/css/docs_style.css +++ b/backend/src/main/resources/static/css/docs_style.css @@ -53,6 +53,5 @@ body { .docs-search { z-index: 1; - backdrop-filter: blur(4px); - background: rgba(40, 44, 52, 0.51); + background: rgba(29, 32, 35, 0.9); } diff --git a/common/src/main/kotlin/StringExt.kt b/common/src/main/kotlin/StringExt.kt new file mode 100644 index 0000000..bd65d31 --- /dev/null +++ b/common/src/main/kotlin/StringExt.kt @@ -0,0 +1,32 @@ +package dev.triumphteam.website + +public fun String.trim(word: String? = null, contextLength: Int = 20): String { + val index = word?.let { indexOf(it, ignoreCase = true) } ?: 0 + val wordLength = word?.length ?: 0 + + // Find start index by moving back contextLength characters and then to the previous space + var start = (index - contextLength).coerceAtLeast(0) + while (start > 0 && !this[start - 1].isWhitespace()) { + start-- + } + + // Find end index by moving forward contextLength characters and then to the next space + var end = (index + wordLength + contextLength).coerceAtMost(length) + while (end < length && !this[end].isWhitespace()) { + end++ + } + + val trimmed = substring(start, end).trim() + + val prefix = if (start > 0) "... " else "" + val suffix = if (end < length) " ..." else "" + + return "$prefix$trimmed$suffix" +} + +public fun String.highlightWord(words: List): String { + val regex = Regex(words.joinToString("|", transform = Regex.Companion::escape), RegexOption.IGNORE_CASE) + return regex.replace(this) { + "${it.value}" + } +} diff --git a/docs/src/main/kotlin/Application.kt b/docs/src/main/kotlin/Application.kt index 516281b..0e53f2e 100644 --- a/docs/src/main/kotlin/Application.kt +++ b/docs/src/main/kotlin/Application.kt @@ -18,6 +18,7 @@ import dev.triumphteam.website.project.Navigation import dev.triumphteam.website.project.Page import dev.triumphteam.website.project.Project import dev.triumphteam.website.project.Repository +import dev.triumphteam.website.trim import io.ktor.client.HttpClient import io.ktor.client.engine.cio.CIO import io.ktor.client.plugins.contentnegotiation.ContentNegotiation @@ -228,7 +229,7 @@ private fun parseVersions(versionDirs: List, parentDir: File, repoSettings path = "${repoSettings.editPath.removeSuffix("/")}/${pageFile.relativeTo(parentDir).path}", description = Page.Description( title = title, - subTitle = subTitle, + subTitle = subTitle?.trim(contextLength = 100), group = parsedGroupConfig.header, summary = summaryExtractor.extract(parsedFile), ), diff --git a/script/src/jsMain/kotlin/Script.kt b/script/src/jsMain/kotlin/Script.kt index 4a30132..5b14e73 100644 --- a/script/src/jsMain/kotlin/Script.kt +++ b/script/src/jsMain/kotlin/Script.kt @@ -46,13 +46,17 @@ public fun main() { showListener( buttonId = "searchbar-button", elementId = "search-area", - ) + ) { + document.body?.addClass("opened-search") + } // Hide the search area hideListener( buttonId = "searchbar-button", elementId = "search-area", ignoreElementId = "search-area-container", - ) + ) { + document.body?.removeClass("opened-search") + } copyCodeListener() observer() } @@ -79,7 +83,11 @@ private fun copyToClipboard(toastElement: Element?, target: EventTarget?) { } } -private fun showListener(buttonId: String, elementId: String) { +private fun showListener( + buttonId: String, + elementId: String, + extra: () -> Unit = {}, +) { val buttonElement = document.getElementById(buttonId) @@ -89,6 +97,7 @@ private fun showListener(buttonId: String, elementId: String) { // Only show if hidden if (target?.hasClass(invisibleClass) == true) { target.removeClass(invisibleClass, hiddenClass) + extra() return@addEventListener } @@ -104,6 +113,7 @@ private fun hideListener( buttonId: String, elementId: String, ignoreElementId: String? = null, + extra: () -> Unit = {}, ) { val buttonElement = document.getElementById(buttonId) val ignoreElement = ignoreElementId?.let { document.getElementById(it) } @@ -119,6 +129,8 @@ private fun hideListener( if (element?.hasClass(invisibleClass) == false) { element.addClass(invisibleClass, hiddenClass) } + + extra() }) }