From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Spottedleaf Date: Mon, 31 Aug 2020 11:08:17 -0700 Subject: [PATCH] Actually unload POI data While it's not likely for a poi data leak to be meaningful, sometimes it is. This patch also prevents the saving/unloading of POI data when world saving is disabled. diff --git a/src/main/java/net/minecraft/server/level/ChunkMap.java b/src/main/java/net/minecraft/server/level/ChunkMap.java index 29fdc1b991c67006a13231abbbe50e201744b5c2..79dbffe899dd7e31af648302c557e2993f170623 100644 --- a/src/main/java/net/minecraft/server/level/ChunkMap.java +++ b/src/main/java/net/minecraft/server/level/ChunkMap.java @@ -812,6 +812,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider } // Paper end } + this.getPoiManager().dequeueUnload(holder.pos.longKey); // Paper - unload POI data this.updatingChunks.queueUpdate(pos, holder); // Paper - Don't copy this.modified = true; @@ -957,7 +958,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider gameprofilerfiller.pop(); } - private static final double UNLOAD_QUEUE_RESIZE_FACTOR = 0.90; // Spigot // Paper - unload more + public static final double UNLOAD_QUEUE_RESIZE_FACTOR = 0.90; // Spigot // Paper - unload more private void processUnloads(BooleanSupplier shouldKeepTicking) { LongIterator longiterator = this.toDrop.iterator(); @@ -1021,6 +1022,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider this.regionManagers.get(index).removeChunk(holder.pos.x, holder.pos.z); } // Paper end + this.getPoiManager().queueUnload(holder.pos.longKey, MinecraftServer.currentTickLong + 1); // Paper - unload POI data if (ichunkaccess instanceof LevelChunk) { ((LevelChunk) ichunkaccess).setLoaded(false); } @@ -1049,6 +1051,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider for (int index = 0, len = this.regionManagers.size(); index < len; ++index) { this.regionManagers.get(index).removeChunk(holder.pos.x, holder.pos.z); } + this.getPoiManager().queueUnload(holder.pos.longKey, MinecraftServer.currentTickLong + 1); // Paper - unload POI data } // Paper end } finally { this.unloadingPlayerChunk = unloadingBefore; } // Paper - do not allow ticket level changes while unloading chunks @@ -1125,6 +1128,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider } this.poiManager.loadInData(pos, chunkHolder.poiData); chunkHolder.tasks.forEach(Runnable::run); + this.getPoiManager().dequeueUnload(pos.longKey); // Paper // Paper end if (chunkHolder.protoChunk != null) {try (Timing ignored2 = this.level.timings.chunkLoadLevelTimer.startTimingIfSync()) { // Paper start - timings // Paper - chunk is created async diff --git a/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java b/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java index 8a569e3300543cb171c3befae59969628adc424c..bbd9fdaa4b12543307b144da72b0604eae638cbb 100644 --- a/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java +++ b/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java @@ -1,5 +1,6 @@ package net.minecraft.world.entity.ai.village.poi; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; // Paper import com.mojang.datafixers.DataFixer; import com.mojang.datafixers.util.Pair; import it.unimi.dsi.fastutil.longs.Long2ByteMap; @@ -35,16 +36,145 @@ import net.minecraft.world.level.chunk.storage.SectionStorage; public class PoiManager extends SectionStorage { public static final int MAX_VILLAGE_DISTANCE = 6; public static final int VILLAGE_SECTION_SIZE = 1; - private final PoiManager.DistanceTracker distanceTracker; + // Paper start - unload poi data + // the vanilla tracker needs to be replaced because it does not support level removes + private final io.papermc.paper.util.misc.Delayed26WayDistancePropagator3D villageDistanceTracker = new io.papermc.paper.util.misc.Delayed26WayDistancePropagator3D(); + static final int POI_DATA_SOURCE = 7; + public static int convertBetweenLevels(final int level) { + return POI_DATA_SOURCE - level; + } + + protected void updateDistanceTracking(long section) { + if (this.isVillageCenter(section)) { + this.villageDistanceTracker.setSource(section, POI_DATA_SOURCE); + } else { + this.villageDistanceTracker.removeSource(section); + } + } + // Paper end - unload poi data private final LongSet loadedChunks = new LongOpenHashSet(); public final net.minecraft.server.level.ServerLevel world; // Paper // Paper public public PoiManager(Path path, DataFixer dataFixer, boolean dsync, LevelHeightAccessor world) { super(path, PoiSection::codec, PoiSection::new, dataFixer, DataFixTypes.POI_CHUNK, dsync, world); + if (world == null) { throw new IllegalStateException("world must be non-null"); } // Paper - require non-null this.world = (net.minecraft.server.level.ServerLevel)world; // Paper - this.distanceTracker = new PoiManager.DistanceTracker(); } + // Paper start - actually unload POI data + private final java.util.TreeSet queuedUnloads = new java.util.TreeSet<>(); + private final Long2ObjectOpenHashMap queuedUnloadsByCoordinate = new Long2ObjectOpenHashMap<>(); + + static final class QueuedUnload implements Comparable { + + private final long unloadTick; + private final long coordinate; + + public QueuedUnload(long unloadTick, long coordinate) { + this.unloadTick = unloadTick; + this.coordinate = coordinate; + } + + @Override + public int compareTo(QueuedUnload other) { + if (other.unloadTick == this.unloadTick) { + return Long.compare(this.coordinate, other.coordinate); + } else { + return Long.compare(this.unloadTick, other.unloadTick); + } + } + + @Override + public int hashCode() { + int hash = 1; + hash = hash * 31 + Long.hashCode(this.unloadTick); + hash = hash * 31 + Long.hashCode(this.coordinate); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != QueuedUnload.class) { + return false; + } + QueuedUnload other = (QueuedUnload)obj; + return other.unloadTick == this.unloadTick && other.coordinate == this.coordinate; + } + } + + long determineDelay(long coordinate) { + if (this.isEmpty(coordinate)) { + return 5 * 60 * 20; + } else { + return 60 * 20; + } + } + + public void queueUnload(long coordinate, long minTarget) { + io.papermc.paper.util.TickThread.softEnsureTickThread("async poi unload queue"); + QueuedUnload unload = new QueuedUnload(minTarget + this.determineDelay(coordinate), coordinate); + QueuedUnload existing = this.queuedUnloadsByCoordinate.put(coordinate, unload); + if (existing != null) { + this.queuedUnloads.remove(existing); + } + this.queuedUnloads.add(unload); + } + + public void dequeueUnload(long coordinate) { + io.papermc.paper.util.TickThread.softEnsureTickThread("async poi unload dequeue"); + QueuedUnload unload = this.queuedUnloadsByCoordinate.remove(coordinate); + if (unload != null) { + this.queuedUnloads.remove(unload); + } + } + + public void pollUnloads(BooleanSupplier canSleepForTick) { + io.papermc.paper.util.TickThread.softEnsureTickThread("async poi unload"); + long currentTick = net.minecraft.server.MinecraftServer.currentTickLong; + net.minecraft.server.level.ServerChunkCache chunkProvider = this.world.getChunkSource(); + net.minecraft.server.level.ChunkMap playerChunkMap = chunkProvider.chunkMap; + // copied target determination from PlayerChunkMap + int target = Math.min(this.queuedUnloads.size() - 100, (int) (this.queuedUnloads.size() * net.minecraft.server.level.ChunkMap.UNLOAD_QUEUE_RESIZE_FACTOR)); // Paper - Make more aggressive + for (java.util.Iterator iterator = this.queuedUnloads.iterator(); + iterator.hasNext() && (this.queuedUnloads.size() > target || canSleepForTick.getAsBoolean());) { + QueuedUnload unload = iterator.next(); + if (unload.unloadTick > currentTick) { + break; + } + + long coordinate = unload.coordinate; + + iterator.remove(); + this.queuedUnloadsByCoordinate.remove(coordinate); + + if (playerChunkMap.getUnloadingChunkHolder(net.minecraft.server.MCUtil.getCoordinateX(coordinate), net.minecraft.server.MCUtil.getCoordinateZ(coordinate)) != null + || playerChunkMap.getUpdatingChunkIfPresent(coordinate) != null) { + continue; + } + + this.unloadData(coordinate); + } + } + + @Override + public void unloadData(long coordinate) { + io.papermc.paper.util.TickThread.softEnsureTickThread("async unloading poi data"); + super.unloadData(coordinate); + } + + @Override + protected void onUnload(long coordinate) { + io.papermc.paper.util.TickThread.softEnsureTickThread("async poi unload callback"); + this.loadedChunks.remove(coordinate); + int chunkX = net.minecraft.server.MCUtil.getCoordinateX(coordinate); + int chunkZ = net.minecraft.server.MCUtil.getCoordinateZ(coordinate); + for (int section = this.levelHeightAccessor.getMinSection(); section < this.levelHeightAccessor.getMaxSection(); ++section) { + long sectionPos = SectionPos.asLong(chunkX, section, chunkZ); + this.updateDistanceTracking(sectionPos); + } + } + // Paper end - actually unload POI data + public void add(BlockPos pos, PoiType type) { this.getOrCreate(SectionPos.asLong(pos)).add(pos, type); } @@ -181,8 +311,8 @@ public class PoiManager extends SectionStorage { } public int sectionsToVillage(SectionPos pos) { - this.distanceTracker.runAllUpdates(); - return this.distanceTracker.getLevel(pos.asLong()); + this.villageDistanceTracker.propagateUpdates(); // Paper - replace distance tracking util + return convertBetweenLevels(this.villageDistanceTracker.getLevel(io.papermc.paper.util.CoordinateUtils.getChunkSectionKey(pos))); // Paper - replace distance tracking util } boolean isVillageCenter(long pos) { @@ -195,7 +325,7 @@ public class PoiManager extends SectionStorage { @Override public void tick(BooleanSupplier shouldKeepTicking) { // Paper start - async chunk io - while (!this.dirty.isEmpty() && shouldKeepTicking.getAsBoolean()) { + while (!this.dirty.isEmpty() && shouldKeepTicking.getAsBoolean() && !this.world.noSave()) { // Paper - unload POI data - don't write to disk if saving is disabled ChunkPos chunkcoordintpair = SectionPos.of(this.dirty.firstLong()).chunk(); net.minecraft.nbt.CompoundTag data; @@ -205,19 +335,24 @@ public class PoiManager extends SectionStorage { com.destroystokyo.paper.io.PaperFileIOThread.Holder.INSTANCE.scheduleSave(this.world, chunkcoordintpair.x, chunkcoordintpair.z, data, null, com.destroystokyo.paper.io.PrioritizedTaskQueue.NORMAL_PRIORITY); } + // Paper start - unload POI data + if (!this.world.noSave()) { // don't write to disk if saving is disabled + this.pollUnloads(shouldKeepTicking); + } + // Paper end - unload POI data // Paper end - this.distanceTracker.runAllUpdates(); + this.villageDistanceTracker.propagateUpdates(); // Paper - replace distance tracking until } @Override protected void setDirty(long pos) { super.setDirty(pos); - this.distanceTracker.update(pos, this.distanceTracker.getLevelFromSource(pos), false); + this.updateDistanceTracking(pos); // Paper - move to new distance tracking util } @Override protected void onSectionLoad(long pos) { - this.distanceTracker.update(pos, this.distanceTracker.getLevelFromSource(pos), false); + this.updateDistanceTracking(pos); // Paper - move to new distance tracking util } public void checkConsistencyWithBlocks(ChunkPos chunkPos, LevelChunkSection chunkSection) { @@ -275,7 +410,7 @@ public class PoiManager extends SectionStorage { @Override protected int getLevelFromSource(long id) { - return PoiManager.this.isVillageCenter(id) ? 0 : 7; + return PoiManager.this.isVillageCenter(id) ? 0 : 7; // Paper - unload poi data - diff on change, this specifies the source level to use for distance tracking } @Override diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java index ec7aa86514f89042c885c0515f0744318c9bdf99..ed688841b1a2a48bacf7f69f177afe136468422c 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java @@ -52,6 +52,40 @@ public class SectionStorage extends RegionFileStorage implements AutoCloseabl // Paper - remove mojang I/O thread } + // Paper start - actually unload POI data + public void unloadData(long coordinate) { + ChunkPos chunkPos = new ChunkPos(coordinate); + this.flush(chunkPos); + + Long2ObjectMap> data = this.storage; + int before = data.size(); + + for (int section = this.levelHeightAccessor.getMinSection(); section < this.levelHeightAccessor.getMaxSection(); ++section) { + data.remove(SectionPos.asLong(chunkPos.x, section, chunkPos.z)); + } + + if (before != data.size()) { + this.onUnload(coordinate); + } + } + + protected void onUnload(long coordinate) {} + + public boolean isEmpty(long coordinate) { + Long2ObjectMap> data = this.storage; + int x = net.minecraft.server.MCUtil.getCoordinateX(coordinate); + int z = net.minecraft.server.MCUtil.getCoordinateZ(coordinate); + for (int section = this.levelHeightAccessor.getMinSection(); section < this.levelHeightAccessor.getMaxSection(); ++section) { + Optional optional = data.get(SectionPos.asLong(x, section, z)); + if (optional != null && optional.orElse(null) != null) { + return false; + } + } + + return true; + } + // Paper end - actually unload POI data + protected void tick(BooleanSupplier shouldKeepTicking) { while(!this.dirty.isEmpty() && shouldKeepTicking.getAsBoolean()) { ChunkPos chunkPos = SectionPos.of(this.dirty.firstLong()).chunk(); @@ -162,6 +196,7 @@ public class SectionStorage extends RegionFileStorage implements AutoCloseabl }); } } + if (this instanceof net.minecraft.world.entity.ai.village.poi.PoiManager) { ((net.minecraft.world.entity.ai.village.poi.PoiManager)this).queueUnload(pos.longKey, net.minecraft.server.MinecraftServer.currentTickLong + 1); } // Paper - unload POI data }