258 lines
11 KiB
Diff
258 lines
11 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Aikar <aikar@aikar.co>
|
|
Date: Fri, 15 Feb 2019 01:08:19 -0500
|
|
Subject: [PATCH] Allow Saving of Oversized Chunks
|
|
|
|
The Minecraft World Region File format has a hard cap of 1MB per chunk.
|
|
This is due to the fact that the header of the file format only allocates
|
|
a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector.
|
|
|
|
This limit can be reached fairly easily with books, resulting in the chunk being unable
|
|
to save to the world. Worse off, is that nothing printed when this occured, and silently
|
|
performed a chunk rollback on next load.
|
|
|
|
This leads to security risk with duplication and is being actively exploited.
|
|
|
|
This patch catches the too large scenario, falls back and moves any large Entity
|
|
or Tile Entity into a new compound, and this compound is saved into a different file.
|
|
|
|
On Chunk Load, we check for oversized status, and if so, we load the extra file and
|
|
merge the Entities and Tile Entities from the oversized chunk back into the level to
|
|
then be loaded as normal.
|
|
|
|
Once a chunk is returned back to normal size, the oversized flag will clear, and no
|
|
extra data file will exist.
|
|
|
|
This fix maintains compatability with all existing Anvil Region Format tools as it
|
|
does not alter the save format. They will just not know about the extra entities.
|
|
|
|
This fix also maintains compatability if someone switches server jars to one without
|
|
this fix, as the data will remain in the oversized file. Once the server returns
|
|
to a jar with this fix, the data will be restored.
|
|
|
|
diff --git a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
|
|
index db66d4ac7dc1bede8b674cd9ad8f56dd989b6693..2322c0c8c5aacebb6317eab8ce4245554f6d9d55 100644
|
|
--- a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
|
|
+++ b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
|
|
@@ -69,6 +69,7 @@ public class NBTCompressedStreamTools {
|
|
|
|
}
|
|
|
|
+ public static NBTTagCompound readNBT(DataInputStream datainputstream) throws IOException { return a(datainputstream); } // Paper - OBFHELPER
|
|
public static NBTTagCompound a(DataInputStream datainputstream) throws IOException {
|
|
return a((DataInput) datainputstream, NBTReadLimiter.a);
|
|
}
|
|
@@ -89,6 +90,7 @@ public class NBTCompressedStreamTools {
|
|
}
|
|
}
|
|
|
|
+ public static void writeNBT(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException { a(nbttagcompound, dataoutput); } // Paper - OBFHELPER
|
|
public static void a(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException {
|
|
a((NBTBase) nbttagcompound, dataoutput);
|
|
}
|
|
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
|
|
index cee1ea8f43e3135cd72a67c1c8067174feaf956e..6b543f89d4e32fb79bfe9aa7b815e4117dbea602 100644
|
|
--- a/src/main/java/net/minecraft/server/RegionFile.java
|
|
+++ b/src/main/java/net/minecraft/server/RegionFile.java
|
|
@@ -17,6 +17,8 @@ import java.nio.file.Files;
|
|
import java.nio.file.LinkOption;
|
|
import java.nio.file.StandardCopyOption;
|
|
import java.nio.file.StandardOpenOption;
|
|
+import java.util.zip.InflaterInputStream; // Paper
|
|
+
|
|
import javax.annotation.Nullable;
|
|
import org.apache.logging.log4j.LogManager;
|
|
import org.apache.logging.log4j.Logger;
|
|
@@ -32,13 +34,16 @@ public class RegionFile implements AutoCloseable {
|
|
private final IntBuffer g;
|
|
private final IntBuffer h;
|
|
private final RegionFileBitSet freeSectors;
|
|
+ public final File file;
|
|
|
|
public RegionFile(File file, File file1) throws IOException {
|
|
this(file.toPath(), file1.toPath(), RegionFileCompression.b);
|
|
}
|
|
|
|
public RegionFile(java.nio.file.Path java_nio_file_path, java.nio.file.Path java_nio_file_path1, RegionFileCompression regionfilecompression) throws IOException {
|
|
+ this.file = java_nio_file_path.toFile(); // Paper
|
|
this.f = ByteBuffer.allocateDirect(8192);
|
|
+ initOversizedState();
|
|
this.freeSectors = new RegionFileBitSet();
|
|
this.e = regionfilecompression;
|
|
if (!Files.isDirectory(java_nio_file_path1, new LinkOption[0])) {
|
|
@@ -384,6 +389,74 @@ public class RegionFile implements AutoCloseable {
|
|
void run() throws IOException;
|
|
}
|
|
|
|
+ private final byte[] oversized = new byte[1024];
|
|
+ private int oversizedCount = 0;
|
|
+
|
|
+ private synchronized void initOversizedState() throws IOException {
|
|
+ File metaFile = getOversizedMetaFile();
|
|
+ if (metaFile.exists()) {
|
|
+ final byte[] read = java.nio.file.Files.readAllBytes(metaFile.toPath());
|
|
+ System.arraycopy(read, 0, oversized, 0, oversized.length);
|
|
+ for (byte temp : oversized) {
|
|
+ oversizedCount += temp;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ private static int getChunkIndex(int x, int z) {
|
|
+ return (x & 31) + (z & 31) * 32;
|
|
+ }
|
|
+ synchronized boolean isOversized(int x, int z) {
|
|
+ return this.oversized[getChunkIndex(x, z)] == 1;
|
|
+ }
|
|
+ synchronized void setOversized(int x, int z, boolean oversized) throws IOException {
|
|
+ final int offset = getChunkIndex(x, z);
|
|
+ boolean previous = this.oversized[offset] == 1;
|
|
+ this.oversized[offset] = (byte) (oversized ? 1 : 0);
|
|
+ if (!previous && oversized) {
|
|
+ oversizedCount++;
|
|
+ } else if (!oversized && previous) {
|
|
+ oversizedCount--;
|
|
+ }
|
|
+ if (previous && !oversized) {
|
|
+ File oversizedFile = getOversizedFile(x, z);
|
|
+ if (oversizedFile.exists()) {
|
|
+ oversizedFile.delete();
|
|
+ }
|
|
+ }
|
|
+ if (oversizedCount > 0) {
|
|
+ if (previous != oversized) {
|
|
+ writeOversizedMeta();
|
|
+ }
|
|
+ } else if (previous) {
|
|
+ File oversizedMetaFile = getOversizedMetaFile();
|
|
+ if (oversizedMetaFile.exists()) {
|
|
+ oversizedMetaFile.delete();
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ private void writeOversizedMeta() throws IOException {
|
|
+ java.nio.file.Files.write(getOversizedMetaFile().toPath(), oversized);
|
|
+ }
|
|
+
|
|
+ private File getOversizedMetaFile() {
|
|
+ return new File(this.file.getParentFile(), this.file.getName().replaceAll("\\.mca$", "") + ".oversized.nbt");
|
|
+ }
|
|
+
|
|
+ private File getOversizedFile(int x, int z) {
|
|
+ return new File(this.file.getParentFile(), this.file.getName().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt");
|
|
+ }
|
|
+
|
|
+ synchronized NBTTagCompound getOversizedData(int x, int z) throws IOException {
|
|
+ File file = getOversizedFile(x, z);
|
|
+ try (DataInputStream out = new DataInputStream(new BufferedInputStream(new InflaterInputStream(new java.io.FileInputStream(file))))) {
|
|
+ return NBTCompressedStreamTools.readNBT(out);
|
|
+ }
|
|
+
|
|
+ }
|
|
+ // Paper end
|
|
+
|
|
class ChunkBuffer extends ByteArrayOutputStream {
|
|
|
|
private final ChunkCoordIntPair b;
|
|
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
|
|
index 5065ece393a47add3c4e888fd7fb3d9ba47acfed..942b7d323936a872d83c536c1901b4b436aa7e7c 100644
|
|
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
|
|
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
|
|
@@ -42,6 +42,74 @@ public final class RegionFileCache implements AutoCloseable {
|
|
}
|
|
}
|
|
|
|
+ // Paper start
|
|
+ private static void printOversizedLog(String msg, File file, int x, int z) {
|
|
+ org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed.");
|
|
+ }
|
|
+
|
|
+ private static final int DEFAULT_SIZE_THRESHOLD = 1024 * 8;
|
|
+ private static final int OVERZEALOUS_TOTAL_THRESHOLD = 1024 * 64;
|
|
+ private static final int OVERZEALOUS_THRESHOLD = 1024;
|
|
+ private static int SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD;
|
|
+ private static void resetFilterThresholds() {
|
|
+ SIZE_THRESHOLD = Math.max(1024 * 4, Integer.getInteger("Paper.FilterThreshhold", DEFAULT_SIZE_THRESHOLD));
|
|
+ }
|
|
+ static {
|
|
+ resetFilterThresholds();
|
|
+ }
|
|
+
|
|
+ static boolean isOverzealous() {
|
|
+ return SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD;
|
|
+ }
|
|
+
|
|
+
|
|
+ private static NBTTagCompound readOversizedChunk(RegionFile regionfile, ChunkCoordIntPair chunkCoordinate) throws IOException {
|
|
+ synchronized (regionfile) {
|
|
+ try (DataInputStream datainputstream = regionfile.getReadStream(chunkCoordinate)) {
|
|
+ NBTTagCompound oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z);
|
|
+ NBTTagCompound chunk = NBTCompressedStreamTools.readNBT(datainputstream);
|
|
+ if (oversizedData == null) {
|
|
+ return chunk;
|
|
+ }
|
|
+ NBTTagCompound oversizedLevel = oversizedData.getCompound("Level");
|
|
+ NBTTagCompound level = chunk.getCompound("Level");
|
|
+
|
|
+ mergeChunkList(level, oversizedLevel, "Entities");
|
|
+ mergeChunkList(level, oversizedLevel, "TileEntities");
|
|
+
|
|
+ chunk.set("Level", level);
|
|
+
|
|
+ return chunk;
|
|
+ } catch (Throwable throwable) {
|
|
+ throwable.printStackTrace();
|
|
+ throw throwable;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ private static void mergeChunkList(NBTTagCompound level, NBTTagCompound oversizedLevel, String key) {
|
|
+ NBTTagList levelList = level.getList(key, 10);
|
|
+ NBTTagList oversizedList = oversizedLevel.getList(key, 10);
|
|
+
|
|
+ if (!oversizedList.isEmpty()) {
|
|
+ levelList.addAll(oversizedList);
|
|
+ level.set(key, levelList);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ private static int getNBTSize(NBTBase nbtBase) {
|
|
+ DataOutputStream test = new DataOutputStream(new org.apache.commons.io.output.NullOutputStream());
|
|
+ try {
|
|
+ nbtBase.write(test);
|
|
+ return test.size();
|
|
+ } catch (IOException e) {
|
|
+ e.printStackTrace();
|
|
+ return 0;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ // Paper End
|
|
+
|
|
@Nullable
|
|
public NBTTagCompound read(ChunkCoordIntPair chunkcoordintpair) throws IOException {
|
|
// CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing
|
|
@@ -51,6 +119,12 @@ public final class RegionFileCache implements AutoCloseable {
|
|
}
|
|
// CraftBukkit end
|
|
DataInputStream datainputstream = regionfile.a(chunkcoordintpair);
|
|
+ // Paper start
|
|
+ if (regionfile.isOversized(chunkcoordintpair.x, chunkcoordintpair.z)) {
|
|
+ printOversizedLog("Loading Oversized Chunk!", regionfile.file, chunkcoordintpair.x, chunkcoordintpair.z);
|
|
+ return readOversizedChunk(regionfile, chunkcoordintpair);
|
|
+ }
|
|
+ // Paper end
|
|
Throwable throwable = null;
|
|
|
|
NBTTagCompound nbttagcompound;
|
|
@@ -91,6 +165,7 @@ public final class RegionFileCache implements AutoCloseable {
|
|
|
|
try {
|
|
NBTCompressedStreamTools.a(nbttagcompound, (DataOutput) dataoutputstream);
|
|
+ regionfile.setOversized(chunkcoordintpair.x, chunkcoordintpair.z, false); // We don't do this anymore
|
|
} catch (Throwable throwable1) {
|
|
throwable = throwable1;
|
|
throw throwable1;
|