From 18a19d792f91d13fba0e7ab3afb5064d53bfcf37 Mon Sep 17 00:00:00 2001 From: Alexandru Nedelcu Date: Mon, 17 Oct 2022 23:37:35 +0300 Subject: [PATCH] Many fixes --- README.md | 4 + build.gradle.kts | 22 ++- config/application-dummy.conf | 16 ++ gradle/libs.versions.toml | 2 + src/main/docker/Dockerfile.jvm | 2 +- src/main/docker/Dockerfile.native | 5 +- src/main/docker/Dockerfile.native.old | 27 --- src/main/kotlin/org/alexn/hook/AppConfig.kt | 18 +- .../kotlin/org/alexn/hook/EventPayload.kt | 25 +-- src/main/kotlin/org/alexn/hook/Main.kt | 159 +----------------- src/main/kotlin/org/alexn/hook/Server.kt | 41 ++--- .../META-INF/native-image/reflect-config.json | 132 +++++++++++++++ 12 files changed, 212 insertions(+), 241 deletions(-) create mode 100644 config/application-dummy.conf delete mode 100644 src/main/docker/Dockerfile.native.old create mode 100644 src/main/resources/META-INF/native-image/reflect-config.json diff --git a/README.md b/README.md index 6596c86..ba8a912 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,10 @@ Or the native version: make build-native ``` +### Issues with native-image + +- [Kotlinx Serialization with GraalVM Native Images](https://github.com/Kotlin/kotlinx.serialization/issues/1125) + ## License Copyright © 2018-2022 Alexandru Nedelcu, some rights reserved. diff --git a/build.gradle.kts b/build.gradle.kts index 770f51f..fdb1287 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -50,25 +50,22 @@ repositories { } dependencies { - implementation(libs.kotlin.stdlib.jdk8) - implementation(libs.logback.classic) - implementation(libs.kaml) - implementation(libs.commons.codec) implementation(libs.arrow.core) implementation(libs.arrow.fx.coroutines) implementation(libs.arrow.fx.stm) - implementation(libs.ktor.serialization.kotlinx.json) - implementation(libs.ktor.server.content.negotiation) - implementation(libs.ktor.server.core) - // implementation(libs.ktor.server.netty) - implementation(libs.ktor.server.cio) - implementation(libs.ktor.server.tests.jvm) - implementation(libs.ktor.client.cio.jvm) + implementation(libs.commons.codec) implementation(libs.commons.text) + implementation(libs.kaml) + implementation(libs.kotlin.stdlib.jdk8) implementation(libs.kotlin.test.junit) implementation(libs.kotlinx.cli) - implementation(libs.kotlinx.serialization.hocon) implementation(libs.kotlinx.serialization.json) + implementation(libs.ktor.serialization.kotlinx.json) + implementation(libs.ktor.server.cio) + implementation(libs.ktor.server.core) + implementation(libs.ktor.server.html.builder) + implementation(libs.ktor.server.tests.jvm) + implementation(libs.logback.classic) } tasks { @@ -86,3 +83,4 @@ ktor { archiveFileName.set("github-webhook-listener-fat.jar") } } + diff --git a/config/application-dummy.conf b/config/application-dummy.conf new file mode 100644 index 0000000..1a4bcaf --- /dev/null +++ b/config/application-dummy.conf @@ -0,0 +1,16 @@ +http { + host: "0.0.0.0" + port: 8080 + path: "/" +} + +projects { + myproject { + action: "push" + ref: "refs/heads/gh-pages" + directory: "/tmp" + command: "touch ./i-was-here.txt" + timeout: "PT5S" + secret: "xxxxxxxxxxxxxxxxxxxxxxxxxx" + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 20f4560..a57a9e7 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -173,3 +173,5 @@ ktor-server-cio = { group = "io.ktor", name = "ktor-server-cio", version.ref = " kotlin-stdlib-jdk8 = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8" } ktor-client-cio-jvm = { group = "io.ktor", name = "ktor-client-cio-jvm", version.ref = "ktor" } + +ktor-server-html-builder = { group = "io.ktor", name = "ktor-server-html-builder", version.ref = "ktor" } diff --git a/src/main/docker/Dockerfile.jvm b/src/main/docker/Dockerfile.jvm index 1e05ede..abe7465 100644 --- a/src/main/docker/Dockerfile.jvm +++ b/src/main/docker/Dockerfile.jvm @@ -31,4 +31,4 @@ COPY --from=build --chown=appuser:root /home/gradle/src/scripts/java-exec /opt/a EXPOSE 8080 USER appuser -CMD ["/opt/app/java-exec","-jar","/opt/app/github-webhook-listener-fat.jar","start-server","-c","/opt/app/config/config.yaml"] +CMD ["/opt/app/java-exec","-jar","/opt/app/github-webhook-listener-fat.jar","/opt/app/config/config.yaml"] diff --git a/src/main/docker/Dockerfile.native b/src/main/docker/Dockerfile.native index 12d7c37..9cfd63c 100644 --- a/src/main/docker/Dockerfile.native +++ b/src/main/docker/Dockerfile.native @@ -9,8 +9,7 @@ FROM --platform=linux/amd64 ghcr.io/graalvm/native-image:22.2.0 AS build COPY --chown=root:root . /app/source WORKDIR /app/source -RUN ./gradlew run -PnativeAgent --args="run-test-scenario -c ./config/application-dummy.yaml" -RUN ./gradlew nativeCompile +RUN ./gradlew nativeCompile --no-daemon FROM --platform=linux/amd64 alpine:latest RUN mkdir -p /opt/app/config @@ -27,4 +26,4 @@ COPY --from=build --chown=appuser:root /app/source/config/application-dummy.yaml EXPOSE 8080 USER appuser -CMD ["/opt/app/github-webhook-listener","start-server","-c","/opt/app/config/config.yaml"] +CMD ["/opt/app/github-webhook-listener","/opt/app/config/config.yaml"] diff --git a/src/main/docker/Dockerfile.native.old b/src/main/docker/Dockerfile.native.old deleted file mode 100644 index 5d6f9c6..0000000 --- a/src/main/docker/Dockerfile.native.old +++ /dev/null @@ -1,27 +0,0 @@ -#### -# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. -# -# Before building the container image run: -# -# ./gradlew build -Dquarkus.package.type=native -# -# Then, build the image with: -# -# docker build -f src/main/docker/Dockerfile.native -t quarkus/rest-kotlin-quickstart . -# -# Then run the container using: -# -# docker run -i --rm -p 8080:8080 quarkus/rest-kotlin-quickstart -# -### -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5 -WORKDIR /work/ -RUN chown 1001 /work \ - && chmod "g+rwX" /work \ - && chown 1001:root /work -COPY --chown=1001:root build/*-runner /work/application - -EXPOSE 8080 -USER 1001 - -CMD ["./application", "-Dquarkus.http.host=0.0.0.0"] diff --git a/src/main/kotlin/org/alexn/hook/AppConfig.kt b/src/main/kotlin/org/alexn/hook/AppConfig.kt index b578cfb..29affa5 100644 --- a/src/main/kotlin/org/alexn/hook/AppConfig.kt +++ b/src/main/kotlin/org/alexn/hook/AppConfig.kt @@ -2,10 +2,7 @@ package org.alexn.hook import com.charleskorn.kaml.Yaml import com.charleskorn.kaml.YamlConfiguration -import com.typesafe.config.ConfigFactory -import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.Serializable -import kotlinx.serialization.hocon.Hocon import java.io.File import kotlin.time.Duration @@ -39,26 +36,15 @@ data class AppConfig( ) companion object { - @OptIn(ExperimentalSerializationApi::class) - fun parseHocon(string: String): AppConfig = - Hocon.decodeFromConfig( - serializer(), - ConfigFactory.parseString(string).resolve() - ) - fun parseYaml(string: String): AppConfig = yamlParser.decodeFromString( serializer(), string ) - fun loadFromFile(file: File): AppConfig { + fun parseYaml(file: File): AppConfig { val txt = file.readText() - return if (file.extension.matches("(?i)yaml|yml".toRegex())) { - parseYaml(txt) - } else { - parseHocon(txt) - } + return parseYaml(txt) } private val yamlParser = Yaml( diff --git a/src/main/kotlin/org/alexn/hook/EventPayload.kt b/src/main/kotlin/org/alexn/hook/EventPayload.kt index 22cef08..344f5ed 100644 --- a/src/main/kotlin/org/alexn/hook/EventPayload.kt +++ b/src/main/kotlin/org/alexn/hook/EventPayload.kt @@ -7,7 +7,6 @@ import io.ktor.http.ContentType import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.Serializable import kotlinx.serialization.SerializationException -import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json import org.apache.commons.codec.digest.HmacAlgorithms import org.apache.commons.codec.digest.HmacUtils @@ -20,7 +19,7 @@ import java.nio.charset.StandardCharsets.UTF_8 @Serializable data class EventPayload( val action: String?, - val ref: String?, + val ref: String? ) { fun shouldProcess(prj: AppConfig.Project): Boolean = (action ?: "push") == (prj.action ?: "push") && ref == prj.ref @@ -36,40 +35,44 @@ data class EventPayload( fun authenticateRequest( body: String, signatureKey: String, - signatureHeader: String?, + signatureHeader: String? ): Either { - if (signatureHeader == null) + if (signatureHeader == null) { return RequestError.Forbidden("No signature header was provided").left() + } val sha1Prefix = "sha1=" val sha256Prefix = "sha256=" if (signatureHeader.startsWith(sha256Prefix)) { val hmacHex = HmacUtils(HmacAlgorithms.HMAC_SHA_256, signatureKey).hmacHex(body) - if (!signatureHeader.substring(sha256Prefix.length).equals(hmacHex, ignoreCase = true)) + if (!signatureHeader.substring(sha256Prefix.length).equals(hmacHex, ignoreCase = true)) { return RequestError.Forbidden("Invalid checksum (sha256)").left() + } return Unit.right() } if (signatureHeader.startsWith(sha1Prefix)) { val hmacHex = HmacUtils(HmacAlgorithms.HMAC_SHA_1, signatureKey).hmacHex(body) - if (!signatureHeader.substring(sha1Prefix.length).equals(hmacHex, ignoreCase = true)) + if (!signatureHeader.substring(sha1Prefix.length).equals(hmacHex, ignoreCase = true)) { return RequestError.Forbidden("Invalid checksum (sha1)").left() + } return Unit.right() } return RequestError.Forbidden("Unsupported algorithm").left() } fun parse(contentType: ContentType, body: String): Either = - if (contentType.match(ContentType("application", "json"))) + if (contentType.match(ContentType("application", "json"))) { parseJson(body) - else if (contentType.match(ContentType("application", "x-www-form-urlencoded"))) + } else if (contentType.match(ContentType("application", "x-www-form-urlencoded"))) { parseFormData(body) - else + } else { RequestError.UnsupportedMediaType("Cannot process `$contentType` media type").left() + } fun parseJson(json: String): Either { try { - val payload = jsonParser.decodeFromString(json) + val payload = jsonParser.decodeFromString(serializer(), json) return payload.right() } catch (e: SerializationException) { return RequestError.BadInput("Invalid JSON", e).left() @@ -88,7 +91,7 @@ data class EventPayload( } EventPayload( action = map["action"], - ref = map["ref"], + ref = map["ref"] ).right() } catch (e: AssertionError) { RequestError.BadInput("Invalid form-urlencoded data", null).left() diff --git a/src/main/kotlin/org/alexn/hook/Main.kt b/src/main/kotlin/org/alexn/hook/Main.kt index 8084424..309db1f 100644 --- a/src/main/kotlin/org/alexn/hook/Main.kt +++ b/src/main/kotlin/org/alexn/hook/Main.kt @@ -1,164 +1,19 @@ package org.alexn.hook -import io.ktor.client.HttpClient -import io.ktor.client.engine.cio.CIO -import io.ktor.client.request.get -import io.ktor.client.request.headers -import io.ktor.client.request.post -import io.ktor.client.request.setBody -import io.ktor.client.statement.HttpResponse -import io.ktor.client.statement.bodyAsText -import io.ktor.http.HttpHeaders -import io.ktor.http.HttpStatusCode import kotlinx.cli.ArgParser import kotlinx.cli.ArgType -import kotlinx.cli.ExperimentalCli -import kotlinx.cli.Subcommand -import kotlinx.cli.required -import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeout -import org.apache.commons.codec.digest.HmacAlgorithms -import org.apache.commons.codec.digest.HmacUtils -import org.slf4j.LoggerFactory import java.io.File -import kotlin.time.Duration.Companion.seconds -@OptIn(ExperimentalCli::class) fun main(args: Array) { val parser = ArgParser(programName = "github-webhook-listener") - val configPath by parser - .option( - ArgType.String, - fullName = "config-path", - shortName = "c", - description = "Path to the application configuration" - ) - .required() + val configPath by parser.argument( + ArgType.String, + fullName = "config-path", + description = "Path to the application configuration" + ) - val startServer = object : Subcommand( - "start-server", - "Starts the long-running server process." - ) { - override fun execute() = startServer(configPath) - } - - val runScenario = object : Subcommand( - "run-test-scenario", - "Runs a full test scenario." - ) { - override fun execute() = runTestScenario(configPath) - } - - parser.subcommands(startServer, runScenario) parser.parse(args) -} - -fun startServer(configPath: String) = runBlocking { - val config = AppConfig.loadFromFile(File(configPath)) - startServer(config) -} - -fun runTestScenario(configPath: String) { - suspend fun post( - client: HttpClient, - baseURL: String, - projName: String, - projCfg: AppConfig.Project, - algo: HmacAlgorithms, - body: String - ): HttpResponse { - val signKey = projCfg.secret - return client.post("$baseURL/$projName") { - headers { - append(HttpHeaders.ContentType, "application/json") - if (algo == HmacAlgorithms.HMAC_SHA_256) { - append( - "X-Hub-Signature-256", - "sha256=" + HmacUtils(HmacAlgorithms.HMAC_SHA_256, signKey).hmacHex(body) - ) - } else if (algo == HmacAlgorithms.HMAC_SHA_512) { - append( - "X-Hub-Signature-256", - "sha512=" + HmacUtils(HmacAlgorithms.HMAC_SHA_512, signKey).hmacHex(body) - ) - } else { - append( - "X-Hub-Signature", - "sha1=" + HmacUtils(HmacAlgorithms.HMAC_SHA_1, signKey).hmacHex(body) - ) - } - } - setBody(body) - } - } - - runBlocking { - val logger = LoggerFactory.getLogger("Main") - val config = AppConfig.loadFromFile(File(configPath)) - val server = launch { startServer(config) } - val client = HttpClient(CIO) - val baseURL = "http://${config.http.host ?: "0.0.0.0"}:${config.http.port}${config.http.basePath}" - val supportedAlgos = listOf( - HmacAlgorithms.HMAC_SHA_1, - HmacAlgorithms.HMAC_SHA_256, - HmacAlgorithms.HMAC_SHA_512 - ) - try { - // Wait for the server to start, by sending a ping - withTimeout(3.seconds) { - var isUpAndRunning = false - do { - try { - val resp = client.get("$baseURL/") - isUpAndRunning = resp.status == HttpStatusCode.OK - } catch (_: Exception) {} - } while (!isUpAndRunning) - logger.info("Test — server is up and running at $baseURL") - } - // Test each project - val projects = config.projects + mapOf( - "nonExistentProj" to AppConfig.Project( - ref = "refs/heads/gh-pages", - directory = "/tmp", - command = "echo", - secret = "xxxxx" - ) - ) - for (proj in projects) { - for (algo in supportedAlgos) { - val resp1 = post( - client, - baseURL, - proj.key, - proj.value, - algo, - """ - { - "action": "${proj.value.action ?: "push"}", - "ref": "${proj.value.ref}" - } - """.trimIndent() - ) - logger.info("Test (1) — $baseURL/${proj.key} ($algo) — ${resp1.status.description} — ${resp1.bodyAsText()}") - val resp2 = post( - client, - baseURL, - proj.key, - proj.value, - algo, - """ - { - "action": "blah-389iu2", - "ref": "blah/blah/293" - } - """.trimIndent() - ) - logger.info("Test (2) — $baseURL/${proj.key} ($algo) — ${resp2.status.description} — ${resp2.bodyAsText()}") - } - } - } finally { - server.cancel() - } - } + val config = AppConfig.parseYaml(File(configPath)) + runBlocking { startServer(config) } } diff --git a/src/main/kotlin/org/alexn/hook/Server.kt b/src/main/kotlin/org/alexn/hook/Server.kt index 8ff2aa9..5230835 100644 --- a/src/main/kotlin/org/alexn/hook/Server.kt +++ b/src/main/kotlin/org/alexn/hook/Server.kt @@ -4,26 +4,30 @@ import arrow.core.Either import arrow.core.continuations.either import arrow.core.left import io.ktor.http.HttpStatusCode -import io.ktor.serialization.kotlinx.json.json import io.ktor.server.application.Application import io.ktor.server.application.call -import io.ktor.server.application.install import io.ktor.server.cio.CIO import io.ktor.server.engine.embeddedServer -import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.html.respondHtml import io.ktor.server.request.contentType import io.ktor.server.request.header import io.ktor.server.request.receiveText -import io.ktor.server.response.respond import io.ktor.server.response.respondRedirect import io.ktor.server.response.respondText import io.ktor.server.routing.get import io.ktor.server.routing.post import io.ktor.server.routing.routing import kotlinx.coroutines.runInterruptible -import kotlinx.serialization.json.Json +import kotlinx.html.body +import kotlinx.html.head +import kotlinx.html.li +import kotlinx.html.p +import kotlinx.html.title +import kotlinx.html.ul import org.slf4j.Logger import org.slf4j.LoggerFactory +import java.net.URLEncoder +import java.nio.charset.StandardCharsets.UTF_8 suspend fun startServer(appConfig: AppConfig) { val commandTrigger = CommandTrigger(appConfig.projects) @@ -49,15 +53,6 @@ fun Application.configureRouting( val basePath = config.http.basePath routing { - install(ContentNegotiation) { - json( - Json { - prettyPrint = true - isLenient = true - } - ) - } - if (config.http.basePath.isNotEmpty()) { get(config.http.basePath) { call.respondRedirect("$basePath/") @@ -65,11 +60,19 @@ fun Application.configureRouting( } get("$basePath/") { - call.respond( - mapOf( - "configured" to config.projects.map { it.key } - ) - ) + call.respondHtml(HttpStatusCode.OK) { + head { + title { +"GitHub Webhook Listener" } + } + body { + p { +"Configured hooks:" } + ul { + for (p in config.projects) { + li { +URLEncoder.encode(p.key, UTF_8) } + } + } + } + } } post("$basePath/{project}") { diff --git a/src/main/resources/META-INF/native-image/reflect-config.json b/src/main/resources/META-INF/native-image/reflect-config.json new file mode 100644 index 0000000..8aab3e4 --- /dev/null +++ b/src/main/resources/META-INF/native-image/reflect-config.json @@ -0,0 +1,132 @@ +[ +{ + "name":"ch.qos.logback.classic.encoder.PatternLayoutEncoder", + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"ch.qos.logback.classic.pattern.DateConverter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"ch.qos.logback.classic.pattern.LevelConverter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"ch.qos.logback.classic.pattern.LineSeparatorConverter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"ch.qos.logback.classic.pattern.LoggerConverter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"ch.qos.logback.classic.pattern.MessageConverter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"ch.qos.logback.classic.pattern.ThreadConverter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"ch.qos.logback.core.ConsoleAppender", + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"ch.qos.logback.core.OutputStreamAppender", + "methods":[{"name":"setEncoder","parameterTypes":["ch.qos.logback.core.encoder.Encoder"] }] +}, +{ + "name":"ch.qos.logback.core.encoder.LayoutWrappingEncoder", + "methods":[{"name":"setParent","parameterTypes":["ch.qos.logback.core.spi.ContextAware"] }] +}, +{ + "name":"ch.qos.logback.core.pattern.PatternLayoutEncoderBase", + "methods":[{"name":"setPattern","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"com.sun.crypto.provider.HmacCore$HmacSHA256", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.HmacSHA1", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"io.ktor.utils.io.ByteReadChannel" +}, +{ + "name":"java.io.FilePermission" +}, +{ + "name":"java.lang.RuntimePermission" +}, +{ + "name":"java.net.NetPermission" +}, +{ + "name":"java.net.SocketPermission" +}, +{ + "name":"java.net.StandardSocketOptions" +}, +{ + "name":"java.net.URLPermission", + "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String"] }] +}, +{ + "name":"java.security.AllPermission" +}, +{ + "name":"java.security.SecurityPermission" +}, +{ + "name":"java.util.PropertyPermission" +}, +{ + "name":"javax.management.ObjectName" +}, +{ + "name":"kotlin.Metadata", + "queryAllDeclaredMethods":true, + "methods":[ + {"name":"bv","parameterTypes":[] }, + {"name":"d1","parameterTypes":[] }, + {"name":"d2","parameterTypes":[] }, + {"name":"k","parameterTypes":[] }, + {"name":"mv","parameterTypes":[] }, + {"name":"pn","parameterTypes":[] }, + {"name":"xi","parameterTypes":[] }, + {"name":"xs","parameterTypes":[] } + ] +}, +{ + "name":"kotlin.internal.jdk8.JDK8PlatformImplementations", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"kotlin.jvm.internal.DefaultConstructorMarker" +}, +{ + "name":"kotlin.reflect.jvm.internal.ReflectionFactoryImpl", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.alexn.hook.ServerKt$startServer$server$1", + "queryAllPublicMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"sun.security.provider.SHA", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.SHA2$SHA256", + "methods":[{"name":"","parameterTypes":[] }] +} +]