From 8a01b9ad39205d43a3d2bf3c43655e2499ccedd6 Mon Sep 17 00:00:00 2001 From: MrHua269 Date: Thu, 26 Dec 2024 12:53:23 +0800 Subject: [PATCH] Rewrite linear flushing --- ...-region-format-framework-linear-v2-r.patch | 188 ++++++++++-------- 1 file changed, 110 insertions(+), 78 deletions(-) diff --git a/patches/server/0018-Add-configurable-region-format-framework-linear-v2-r.patch b/patches/server/0018-Add-configurable-region-format-framework-linear-v2-r.patch index 9b51d31..4e21654 100644 --- a/patches/server/0018-Add-configurable-region-format-framework-linear-v2-r.patch +++ b/patches/server/0018-Add-configurable-region-format-framework-linear-v2-r.patch @@ -1,7 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: MrHua269 Date: Sun, 15 Dec 2024 12:53:33 +0800 -Subject: [PATCH] Add configurable region format framework & linear v2 region +Subject: [PATCH] Add configurable region format framework & linear v2 region format support @@ -68,15 +68,12 @@ index 0000000000000000000000000000000000000000..d92f1d549c7e01daa6b5bba7d405e462 +} diff --git a/src/main/java/abomination/LinearRegionFile.java b/src/main/java/abomination/LinearRegionFile.java new file mode 100644 -index 0000000000000000000000000000000000000000..4a8a71686dddfc6b0c27882d1f73b92c96f6173e +index 0000000000000000000000000000000000000000..9fdc9aa840c6b2ae027bcf84a2b77856d79a6a38 --- /dev/null +++ b/src/main/java/abomination/LinearRegionFile.java -@@ -0,0 +1,666 @@ +@@ -0,0 +1,608 @@ +package abomination; + -+import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; -+import ca.spottedleaf.concurrentutil.executor.thread.PrioritisedThreadPool; -+import ca.spottedleaf.concurrentutil.util.Priority; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; @@ -85,7 +82,6 @@ index 0000000000000000000000000000000000000000..4a8a71686dddfc6b0c27882d1f73b92c +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; -+import net.minecraft.server.MinecraftServer; +import net.openhft.hashing.LongHashFunction; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.chunk.storage.RegionStorageInfo; @@ -99,12 +95,9 @@ index 0000000000000000000000000000000000000000..4a8a71686dddfc6b0c27882d1f73b92c +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; -+import java.util.concurrent.CompletableFuture; -+import java.util.concurrent.Executor; -+import java.util.concurrent.TimeUnit; ++import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; -+import java.util.stream.IntStream; + +// LinearRegionFile_implementation_version_0_5byXymb +// Just gonna use this string to inform other forks about updates ;-) @@ -136,7 +129,7 @@ index 0000000000000000000000000000000000000000..4a8a71686dddfc6b0c27882d1f73b92c + private int bucketSize = 4; + + private final AtomicInteger modifiedChunkCount = new AtomicInteger(0); -+ private PrioritisedExecutor.PrioritisedTask ioTask = null; ++ private ScheduledFuture ioTask = null; + + public Path getRegionFile() { + return this.regionFile; @@ -354,28 +347,12 @@ index 0000000000000000000000000000000000000000..4a8a71686dddfc6b0c27882d1f73b92c + } + } + -+ private static final AtomicInteger ioThreadId = new AtomicInteger(); -+ private static final PrioritisedThreadPool linearIOThreadPool = new PrioritisedThreadPool(thread -> { -+ thread.setName("Linear RegionFile IO Thread- " + ioThreadId.getAndIncrement()); -+ thread.setContextClassLoader(MinecraftServer.class.getClassLoader()); -+ }); -+ public static final long WORKER_QUEUE_HOLD_TIME = (long)(20.0e6); // 20ms -+ private static PrioritisedExecutor linearIOExecutor; -+ + /* + private static final int SAVE_THREAD_MAX_COUNT = 6; + private static final Object saveLock = new Object(); + private static int activeSaveThreads = 0; + */ + -+ public static void initIOExecutor() { -+ linearIOExecutor = linearIOThreadPool.createExecutorGroup(1, 0).createExecutor(RegionFormatConfig.linearIoThreadCount, WORKER_QUEUE_HOLD_TIME, 0); -+ } -+ -+ public static void shutdownIOExecutor() { -+ linearIOThreadPool.shutdown(true); -+ } -+ + /*public void run() { + try { + while (!close) { @@ -560,51 +537,15 @@ index 0000000000000000000000000000000000000000..4a8a71686dddfc6b0c27882d1f73b92c + markToSave(); + this.modifiedChunkCount.getAndIncrement(); + -+ if (ioTask == null) { -+ this.scheduleSave(); -+ }else { -+ if (this.ioTask.getPriority() == Priority.COMPLETING) { -+ this.scheduleSave(); -+ return; -+ } -+ -+ this.ioTask.raisePriority(this.computeSavePriority()); -+ } ++ this.ioTask = RegionFormatConfig.linearFlusher.claimTask(this.ioTask, this); + } + -+ private Priority computeSavePriority() { -+ final int currentModifiedChunkCount = this.modifiedChunkCount.get(); -+ -+ final int[] ordinals = new int[]{2, 3, 4, 5, 6}; -+ final int[] thresholds = new int[]{20, 40, 60, 80, 100}; -+ final int coverPercent = (currentModifiedChunkCount / 1024) * 100; -+ -+ int actualOrdinal = 6; -+ for (int i = 0; i < thresholds.length; i++) { -+ if (coverPercent >= thresholds[i]) { -+ actualOrdinal = ordinals[i]; -+ } -+ } -+ -+ return Priority.values()[actualOrdinal]; ++ public synchronized boolean isClosed() { ++ return this.close; + } + -+ private void scheduleSave() { -+ final PrioritisedExecutor.PrioritisedTask created = linearIOExecutor.createTask(() -> { -+ try { -+ synchronized (this) { -+ if (!this.close) { -+ this.flush(); -+ } -+ } -+ } catch (IOException e) { -+ throw new RuntimeException(e); -+ } -+ }, this.computeSavePriority()); -+ -+ created.queue(); -+ -+ this.ioTask = created; ++ public double getLoadPercent() { ++ return ((double)this.modifiedChunkCount.get()) / (32 * 32); + } + + public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { @@ -683,6 +624,7 @@ index 0000000000000000000000000000000000000000..4a8a71686dddfc6b0c27882d1f73b92c + openRegionFile(); + close = true; + try { ++ if (this.ioTask != null) this.ioTask.cancel(false); + flush(); + } catch(IOException e) { + throw new IOException("Region flush IOException " + e + " " + this.regionFile); @@ -738,6 +680,67 @@ index 0000000000000000000000000000000000000000..4a8a71686dddfc6b0c27882d1f73b92c + } + } +} +diff --git a/src/main/java/abomination/LinearRegionFileFlusher.java b/src/main/java/abomination/LinearRegionFileFlusher.java +new file mode 100644 +index 0000000000000000000000000000000000000000..b4e4b4a7fb8f37715b6a3099ec2c7f48d65f9390 +--- /dev/null ++++ b/src/main/java/abomination/LinearRegionFileFlusher.java +@@ -0,0 +1,55 @@ ++package abomination; ++ ++import com.google.common.util.concurrent.ThreadFactoryBuilder; ++import org.jetbrains.annotations.NotNull; ++import org.jetbrains.annotations.Nullable; ++ ++import java.util.concurrent.ScheduledFuture; ++import java.util.concurrent.ScheduledThreadPoolExecutor; ++import java.util.concurrent.TimeUnit; ++ ++public class LinearRegionFileFlusher { ++ private final ScheduledThreadPoolExecutor delayedIoSchedule; ++ private final long baseDelay; ++ ++ public LinearRegionFileFlusher(int nThreads, long baseDelay) { ++ this.delayedIoSchedule = new ScheduledThreadPoolExecutor(nThreads, ++ new ThreadFactoryBuilder() ++ .setPriority(3) ++ .setNameFormat("Linear IO Thread - %d") ++ .build() ++ ); ++ this.baseDelay = baseDelay; ++ } ++ ++ public void shutdown() { ++ this.delayedIoSchedule.shutdown(); ++ while (true) { ++ try { ++ if (this.delayedIoSchedule.awaitTermination(1000, TimeUnit.MILLISECONDS)) break; ++ } catch (InterruptedException e) { ++ throw new RuntimeException(e); ++ } ++ } ++ } ++ ++ public ScheduledFuture claimTask(@Nullable ScheduledFuture parent, @NotNull LinearRegionFile file){ ++ if (parent != null) { ++ if (!parent.isCancelled() && !parent.isDone()) { ++ parent.cancel(false); ++ } ++ } ++ ++ final double loadPercent = file.getLoadPercent() * 100; ++ final double delayOffset = Math.max(this.baseDelay, loadPercent * this.baseDelay); ++ ++ final long actualDelay = (long) (this.baseDelay - delayOffset); ++ return this.delayedIoSchedule.schedule(() -> { ++ try { ++ file.flush(); ++ }catch (Exception e) { ++ throw new RuntimeException(e); ++ } ++ }, actualDelay, java.util.concurrent.TimeUnit.MILLISECONDS); ++ } ++} diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java index a814512fcfb85312474ae2c2c21443843bf57831..2e084a5b28cbe4737f48c25e10af589213525362 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java @@ -780,28 +783,35 @@ index 51c126735ace8fdde89ad97b5cab62f244212db0..c7d4d944eb198ac53a3eeae717a25c7d } diff --git a/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java b/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java new file mode 100644 -index 0000000000000000000000000000000000000000..60546260cd1d535cc596485de2ced48b7e045b3a +index 0000000000000000000000000000000000000000..4dc62688fe2731f7b43d7f48c7cb76fef33821c1 --- /dev/null +++ b/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java -@@ -0,0 +1,48 @@ +@@ -0,0 +1,64 @@ +package me.earthme.luminol.config.modules.misc; + +import abomination.LinearRegionFile; ++import abomination.LinearRegionFileFlusher; +import com.electronwill.nightconfig.core.file.CommentedFileConfig; -+import me.earthme.luminol.config.ConfigInfo; -+import me.earthme.luminol.config.DoNotLoad; -+import me.earthme.luminol.config.EnumConfigCategory; -+import me.earthme.luminol.config.IConfigModule; ++import me.earthme.luminol.config.*; +import me.earthme.luminol.utils.EnumRegionFormat; +import net.minecraft.server.MinecraftServer; + +public class RegionFormatConfig implements IConfigModule { ++ @HotReloadUnsupported + @ConfigInfo(baseName = "format") + public static String format = "MCA"; ++ @HotReloadUnsupported + @ConfigInfo(baseName = "linear_compression_level") + public static int linearCompressionLevel = 1; ++ @HotReloadUnsupported + @ConfigInfo(baseName = "linear_io_thread_count") + public static int linearIoThreadCount = 6; ++ @HotReloadUnsupported ++ @ConfigInfo(baseName = "linear_io_flush_delay_ms") ++ public static int linearIoFlushDelayMs = 100; ++ ++ @DoNotLoad ++ public static LinearRegionFileFlusher linearFlusher; + + @DoNotLoad + public static EnumRegionFormat regionFormat; @@ -818,7 +828,6 @@ index 0000000000000000000000000000000000000000..60546260cd1d535cc596485de2ced48b + + @Override + public void onLoaded(CommentedFileConfig configInstance) { -+ LinearRegionFile.initIOExecutor(); + regionFormat = EnumRegionFormat.fromString(format.toUpperCase()); + + if (regionFormat == null) { @@ -830,6 +839,16 @@ index 0000000000000000000000000000000000000000..60546260cd1d535cc596485de2ced48b + MinecraftServer.LOGGER.error("Falling back to compression level 1."); + RegionFormatConfig.linearCompressionLevel = 1; + } ++ ++ if (regionFormat == EnumRegionFormat.LINEAR_V2) { ++ linearFlusher = new LinearRegionFileFlusher(RegionFormatConfig.linearIoThreadCount, RegionFormatConfig.linearIoFlushDelayMs); ++ } ++ } ++ ++ public static void shutdownLinearIOPool() { ++ if (linearFlusher != null) { ++ linearFlusher.shutdown(); ++ } + } +} diff --git a/src/main/java/me/earthme/luminol/utils/EnumRegionFormat.java b/src/main/java/me/earthme/luminol/utils/EnumRegionFormat.java @@ -907,14 +926,27 @@ index 0000000000000000000000000000000000000000..5af068489646ed70330d8c6242ec88f5 + +public record RegionCreatorInfo (RegionStorageInfo info, Path filePath, Path folder, boolean sync) {} diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java -index 8cc0c01a19fc71753d7c3ed4fa7e9992aaf93b5a..88be8a6232bc3311cc0bdb7c697f7a78ce33e38d 100644 +index 8cc0c01a19fc71753d7c3ed4fa7e9992aaf93b5a..ad79dee048a57be6a6997a38195d636ce952c48d 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java +@@ -1036,10 +1036,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop