Skip to content

Commit

Permalink
Add Zstd compression to MojangStyleFileAppenderAndRollover
Browse files Browse the repository at this point in the history
  • Loading branch information
@MrPowerGamerBR
MrPowerGamerBR committed Oct 28, 2023
1 parent 7390916 commit 08ca6c5
Show file tree
Hide file tree
Showing 2 changed files with 157 additions and 12 deletions.
4 changes: 3 additions & 1 deletion build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ repositories {

dependencies {
implementation(kotlin("stdlib"))
implementation("ch.qos.logback:logback-classic:1.3.0-alpha14")
implementation("ch.qos.logback:logback-classic:1.4.11")
implementation("io.github.microutils:kotlin-logging:2.1.21")

implementation("com.github.LorittaBot:DeviousJDA:19d95ed662")
Expand Down Expand Up @@ -86,6 +86,8 @@ dependencies {
implementation("io.ktor:ktor-client-cio:2.1.0")

implementation("org.apache.commons:commons-text:1.9")

implementation("com.github.luben:zstd-jni:1.5.5-6")
}

jib {
Expand Down
165 changes: 154 additions & 11 deletions src/main/kotlin/net/perfectdreams/logging/MojangStyleFileAppenderAndRollover.kt
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,25 @@
package net.perfectdreams.logging

import ch.qos.logback.core.FileAppender
import ch.qos.logback.core.rolling.RolloverFailure
import ch.qos.logback.core.rolling.helper.CompressionMode
import ch.qos.logback.core.rolling.helper.Compressor
import ch.qos.logback.core.spi.ContextAwareBase
import ch.qos.logback.core.status.ErrorStatus
import ch.qos.logback.core.status.WarnStatus
import ch.qos.logback.core.util.FileUtil
import com.github.luben.zstd.Zstd
import com.github.luben.zstd.ZstdOutputStream
import java.io.BufferedInputStream
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.time.Instant
import java.time.LocalDateTime
import java.time.ZoneId
import java.util.*
import java.util.concurrent.Future
import java.util.zip.GZIPOutputStream


/**
Expand All @@ -22,17 +33,43 @@ import java.util.concurrent.Future
*/
class MojangStyleFileAppenderAndRollover<E> : FileAppender<E>() {
private lateinit var nextRolloverDate: LocalDateTime
private lateinit var compressor: Compressor
private lateinit var compressor: Any
private var compressionFuture: Future<*>? = null
private var archivedLogsFilePrefix: String? = null
private var archivedLogsFileExtension: String? = null
private var archivedLogsCompression: String? = null

fun setArchivedLogsFilePrefix(archivedLogsFilePrefix: String) {
this.archivedLogsFilePrefix = archivedLogsFilePrefix
}

fun setArchivedLogsFileExtension(archivedLogsFileExtension: String) {
this.archivedLogsFileExtension = archivedLogsFileExtension
}

fun setArchivedLogsCompression(archivedLogsCompression: String) {
this.archivedLogsCompression = archivedLogsCompression
}

override fun start() {
compressor = Compressor(CompressionMode.GZ)
compressor.context = this.context
when (archivedLogsCompression) {
"GZ" -> {
compressor = Compressor(CompressionMode.GZ).apply {
context = this@MojangStyleFileAppenderAndRollover.context
}
}
"ZIP" -> {
compressor = Compressor(CompressionMode.ZIP).apply {
context = this@MojangStyleFileAppenderAndRollover.context
}
}
"ZSTD" -> {
compressor = ZstdCompressor().apply {
context = this@MojangStyleFileAppenderAndRollover.context
}
}
else -> error("Archived Logs Compression type is unsupported! $archivedLogsCompression")
}

// Trigger the rollover when the application first starts, we need to do it here on start instead of on isTriggeringEvent, because Logback WILL write the log line to the log file BEFORE it is triggered!
// This must be ran BEFORE the super.start(), to avoid an empty file being created due to the openFile call
Expand Down Expand Up @@ -80,7 +117,8 @@ class MojangStyleFileAppenderAndRollover<E> : FileAppender<E>() {
while (true) {
val fileName = "${date.year}-${date.monthValue.toString().padStart(2, '0')}-${date.dayOfMonth.toString().padStart(2, '0')}.$index"

val newCompressedFile = File("$archivedLogsFilePrefix$fileName.log.gz")
val newCompressedFile = File("$archivedLogsFilePrefix$fileName$archivedLogsFileExtension")
println(newCompressedFile)
if (!newCompressedFile.exists()) {
compressedFile = newCompressedFile
break
Expand All @@ -99,21 +137,126 @@ class MojangStyleFileAppenderAndRollover<E> : FileAppender<E>() {
// Reopen the latest.log file
openFile(file)

val future = compressor.asyncCompress(
compressedFileRaw.toString(),
compressedFile.toString(),
"xd" // Not used for gz files
)
when (val compressor = compressor) {
is ZstdCompressor -> {
val future = compressor.asyncCompress(
compressedFileRaw.toString(),
compressedFile.toString()
)
this.compressionFuture = future
}

is Compressor -> {
val future = compressor.asyncCompress(
compressedFileRaw.toString(),
compressedFile.toString(),
"xd" // Not used for gz files
)
this.compressionFuture = future
}

else -> error("Invalid compressor $compressor")
}


this.compressionFuture = future
} catch (e: Exception) {
e.printStackTrace()
}
}

// Tomorrow at midnight
private fun getNextRolloverDate(date: LocalDateTime) = date.plusDays(1)
private fun getNextRolloverDate(date: LocalDateTime) = date
.plusDays(1)
.withHour(0)
.withMinute(0)
.withNano(0)

/**
* A make-shift Zstd Compressor class based off Logback's [Compressor] class.
*
* @author MrPowerGamerBR
*/
class ZstdCompressor : ContextAwareBase() {
companion object {
private const val BUFFER_SIZE = 8192
}

/**
* @param nameOfFile2Compress
* @param nameOfCompressedFile
*/
fun compress(nameOfFile2Compress: String, nameOfCompressedFile: String) {
zstdCompress(nameOfFile2Compress, nameOfCompressedFile)
}

private fun zstdCompress(nameOfFile2gz: String, nameOfgzedFile: String) {
val file2gz = File(nameOfFile2gz)
if (!file2gz.exists()) {
addStatus(WarnStatus("The file to compress named [$nameOfFile2gz] does not exist.", this))
return
}
val gzedFile = File(nameOfgzedFile)
if (gzedFile.exists()) {
addWarn(
"The target compressed file named [" + nameOfgzedFile
+ "] exist already. Aborting file compression."
)
return
}
addInfo("ZSTD compressing [$file2gz] as [$gzedFile]")
createMissingTargetDirsIfNecessary(gzedFile)
try {
BufferedInputStream(FileInputStream(nameOfFile2gz)).use { bis ->
ZstdOutputStream(FileOutputStream(nameOfgzedFile)).use { gzos ->
val inbuf = ByteArray(BUFFER_SIZE)
var n: Int
while ((bis.read(inbuf).also { n = it }) != -1) {
gzos.write(inbuf, 0, n)
}
addInfo("Done ZSTD compressing [" + file2gz + "] as [" + gzedFile + "]")
}
}
} catch (e: Exception) {
addStatus(
ErrorStatus(
"Error occurred while compressing [$nameOfFile2gz] into [$nameOfgzedFile].", this,
e
)
)
}
if (!file2gz.delete()) {
addStatus(WarnStatus("Could not delete [$nameOfFile2gz].", this))
}
}

fun createMissingTargetDirsIfNecessary(file: File) {
val result = FileUtil.createMissingParentDirectories(file)
if (!result) {
addError("Failed to create parent directories for [" + file.absolutePath + "]")
}
}

override fun toString(): String {
return this.javaClass.name
}

@Throws(RolloverFailure::class)
fun asyncCompress(
nameOfFile2Compress: String,
nameOfCompressedFile: String
): Future<*> {
val runnable = CompressionRunnable(nameOfFile2Compress, nameOfCompressedFile)
val executorService = context.executorService
return executorService.submit(runnable)
}

internal inner class CompressionRunnable(
val nameOfFile2Compress: String,
val nameOfCompressedFile: String
) : Runnable {
override fun run() {
compress(nameOfFile2Compress, nameOfCompressedFile)
}
}
}
}

0 comments on commit 08ca6c5

Please sign in to comment.