Fixed issues where urgent and prioritized chunks didn't actually always get their priority boosted correctly.... Properly deprioritize non ticking chunks. Limit recursion on watchdog prints to stop flooding as much Remove neighbor priorities from watchdog to reduce information reduce synchronization duration so that watch dog won't block main should main actually wake up probably fixed a deadlock risk in watchdog printing also that was leading to crashes fixed chunk holder enqueues not being processed correctly added async catchers in some locations that should not be ran async Fixed upstream bug where VITAL callbacks that must run on main actually could sometimes run on the server thread pool causing alot of these nasty bugs we've seen lately! This build will provide massive improvements to stability as well as even faster sync chunk load/gens now that priority is correctly set. Fixes #3435
303 lines
16 KiB
Diff
303 lines
16 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Aikar <aikar@aikar.co>
|
|
Date: Wed, 8 Apr 2020 03:06:30 -0400
|
|
Subject: [PATCH] Optimize PlayerChunkMap memory use for visibleChunks
|
|
|
|
No longer clones visible chunks which is causing massive memory
|
|
allocation issues, likely the source of Humongous Objects on large servers.
|
|
|
|
Instead we just synchronize, clear and rebuild, reusing the same object buffers
|
|
as before with only 2 small objects created (FastIterator/MapEntry)
|
|
|
|
This should result in siginificant memory use reduction and improved GC behavior.
|
|
|
|
diff --git a/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
|
|
new file mode 100644
|
|
index 0000000000000000000000000000000000000000..f6ff4d8132a95895680f5bc81f8f873e78f0bbdb
|
|
--- /dev/null
|
|
+++ b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
|
|
@@ -0,0 +1,39 @@
|
|
+package com.destroystokyo.paper.util.map;
|
|
+
|
|
+import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
|
|
+
|
|
+public class Long2ObjectLinkedOpenHashMapFastCopy<V> extends Long2ObjectLinkedOpenHashMap<V> {
|
|
+
|
|
+ public void copyFrom(Long2ObjectLinkedOpenHashMapFastCopy<V> map) {
|
|
+ if (key.length != map.key.length) {
|
|
+ key = null;
|
|
+ key = new long[map.key.length];
|
|
+ }
|
|
+ if (value.length != map.value.length) {
|
|
+ value = null;
|
|
+ //noinspection unchecked
|
|
+ value = (V[]) new Object[map.value.length];
|
|
+ }
|
|
+ if (link.length != map.link.length) {
|
|
+ link = null;
|
|
+ link = new long[map.link.length];
|
|
+ }
|
|
+ System.arraycopy(map.key, 0, this.key, 0, map.key.length);
|
|
+ System.arraycopy(map.value, 0, this.value, 0, map.value.length);
|
|
+ System.arraycopy(map.link, 0, this.link, 0, map.link.length);
|
|
+ this.size = map.size;
|
|
+ this.mask = map.mask;
|
|
+ this.first = map.first;
|
|
+ this.last = map.last;
|
|
+ this.n = map.n;
|
|
+ this.maxFill = map.maxFill;
|
|
+ this.containsNullKey = map.containsNullKey;
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public Long2ObjectLinkedOpenHashMapFastCopy<V> clone() {
|
|
+ Long2ObjectLinkedOpenHashMapFastCopy<V> clone = (Long2ObjectLinkedOpenHashMapFastCopy<V>) super.clone();
|
|
+ clone.copyFrom(this);
|
|
+ return clone;
|
|
+ }
|
|
+}
|
|
diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java
|
|
index d53b34ba552771bf271131ce0a56ebb992ccc84c..a1b5e6b90fc93f83186cf3ebf3e158767008c69a 100644
|
|
--- a/src/main/java/net/minecraft/server/ChunkProviderServer.java
|
|
+++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java
|
|
@@ -756,7 +756,7 @@ public class ChunkProviderServer extends IChunkProvider {
|
|
entityPlayer.playerNaturallySpawnedEvent.callEvent();
|
|
};
|
|
// Paper end
|
|
- this.playerChunkMap.f().forEach((playerchunk) -> {
|
|
+ this.playerChunkMap.forEachVisibleChunk((playerchunk) -> { // Paper - safe iterator incase chunk loads, also no wrapping
|
|
Optional<Chunk> optional = ((Either) playerchunk.b().getNow(PlayerChunk.UNLOADED_CHUNK)).left();
|
|
|
|
if (optional.isPresent()) {
|
|
diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/minecraft/server/MCUtil.java
|
|
index 9f8c0e10e42d233a8b74ee5a71fb8fb6ea8e7480..0d1065688b19ceca9440bc8bf2bf65910f03fa46 100644
|
|
--- a/src/main/java/net/minecraft/server/MCUtil.java
|
|
+++ b/src/main/java/net/minecraft/server/MCUtil.java
|
|
@@ -628,7 +628,7 @@ public final class MCUtil {
|
|
|
|
WorldServer world = ((org.bukkit.craftbukkit.CraftWorld)bukkitWorld).getHandle();
|
|
PlayerChunkMap chunkMap = world.getChunkProvider().playerChunkMap;
|
|
- Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks = chunkMap.visibleChunks;
|
|
+ Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks = chunkMap.getVisibleChunks();
|
|
ChunkMapDistance chunkMapDistance = chunkMap.getChunkMapDistanceManager();
|
|
List<PlayerChunk> allChunks = new ArrayList<>(visibleChunks.values());
|
|
List<EntityPlayer> players = world.players;
|
|
diff --git a/src/main/java/net/minecraft/server/PlayerChunkMap.java b/src/main/java/net/minecraft/server/PlayerChunkMap.java
|
|
index 4beae504c875767ff00e26461fe7240498750e27..75906f794205f5b7fe894163e1b13bfd85c2b419 100644
|
|
--- a/src/main/java/net/minecraft/server/PlayerChunkMap.java
|
|
+++ b/src/main/java/net/minecraft/server/PlayerChunkMap.java
|
|
@@ -55,8 +55,33 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
|
|
|
|
private static final Logger LOGGER = LogManager.getLogger();
|
|
public static final int GOLDEN_TICKET = 33 + ChunkStatus.b();
|
|
- public final Long2ObjectLinkedOpenHashMap<PlayerChunk> updatingChunks = new Long2ObjectLinkedOpenHashMap();
|
|
- public volatile Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks;
|
|
+ // Paper start - faster copying
|
|
+ public final Long2ObjectLinkedOpenHashMap<PlayerChunk> updatingChunks = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<>(); // Paper - faster copying
|
|
+ public final Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks = new ProtectedVisibleChunksMap(); // Paper - faster copying
|
|
+
|
|
+ private class ProtectedVisibleChunksMap extends com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk> {
|
|
+ @Override
|
|
+ public PlayerChunk put(long k, PlayerChunk playerChunk) {
|
|
+ throw new UnsupportedOperationException("Updating visible Chunks");
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public PlayerChunk remove(long k) {
|
|
+ throw new UnsupportedOperationException("Removing visible Chunks");
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public PlayerChunk get(long k) {
|
|
+ return PlayerChunkMap.this.getVisibleChunk(k);
|
|
+ }
|
|
+
|
|
+ public PlayerChunk safeGet(long k) {
|
|
+ return super.get(k);
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+ public final com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk> pendingVisibleChunks = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk>(); // Paper - this is used if the visible chunks is updated while iterating only
|
|
+ public transient com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk> visibleChunksClone; // Paper - used for async access of visible chunks, clone and cache only when needed
|
|
private final Long2ObjectLinkedOpenHashMap<PlayerChunk> pendingUnload;
|
|
final LongSet loadedChunks; // Paper - private -> package
|
|
public final WorldServer world;
|
|
@@ -130,7 +155,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
|
|
|
|
public PlayerChunkMap(WorldServer worldserver, File file, DataFixer datafixer, DefinedStructureManager definedstructuremanager, Executor executor, IAsyncTaskHandler<Runnable> iasynctaskhandler, ILightAccess ilightaccess, ChunkGenerator<?> chunkgenerator, WorldLoadListener worldloadlistener, Supplier<WorldPersistentData> supplier, int i) {
|
|
super(new File(worldserver.getWorldProvider().getDimensionManager().a(file), "region"), datafixer);
|
|
- this.visibleChunks = this.updatingChunks.clone();
|
|
+ //this.visibleChunks = this.updatingChunks.clone(); // Paper - no more cloning
|
|
this.pendingUnload = new Long2ObjectLinkedOpenHashMap();
|
|
this.loadedChunks = new LongOpenHashSet();
|
|
this.unloadQueue = new LongOpenHashSet();
|
|
@@ -221,9 +246,52 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
|
|
return (PlayerChunk) this.updatingChunks.get(i);
|
|
}
|
|
|
|
+ // Paper start - remove cloning of visible chunks unless accessed as a collection async
|
|
+ private static final boolean DEBUG_ASYNC_VISIBLE_CHUNKS = Boolean.getBoolean("paper.debug-async-visible-chunks");
|
|
+ private boolean isIterating = false;
|
|
+ private boolean hasPendingVisibleUpdate = false;
|
|
+ public void forEachVisibleChunk(java.util.function.Consumer<PlayerChunk> consumer) {
|
|
+ org.spigotmc.AsyncCatcher.catchOp("forEachVisibleChunk");
|
|
+ boolean prev = isIterating;
|
|
+ isIterating = true;
|
|
+ try {
|
|
+ for (PlayerChunk value : this.visibleChunks.values()) {
|
|
+ consumer.accept(value);
|
|
+ }
|
|
+ } finally {
|
|
+ this.isIterating = prev;
|
|
+ if (!this.isIterating && this.hasPendingVisibleUpdate) {
|
|
+ ((ProtectedVisibleChunksMap)this.visibleChunks).copyFrom(this.pendingVisibleChunks);
|
|
+ this.pendingVisibleChunks.clear();
|
|
+ this.hasPendingVisibleUpdate = false;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ public Long2ObjectLinkedOpenHashMap<PlayerChunk> getVisibleChunks() {
|
|
+ if (Thread.currentThread() == this.world.serverThread) {
|
|
+ return this.visibleChunks;
|
|
+ } else {
|
|
+ synchronized (this.visibleChunks) {
|
|
+ if (DEBUG_ASYNC_VISIBLE_CHUNKS) new Throwable("Async getVisibleChunks").printStackTrace();
|
|
+ if (this.visibleChunksClone == null) {
|
|
+ this.visibleChunksClone = this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.clone() : ((ProtectedVisibleChunksMap)this.visibleChunks).clone();
|
|
+ }
|
|
+ return this.visibleChunksClone;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+
|
|
@Nullable
|
|
public PlayerChunk getVisibleChunk(long i) { // Paper - protected -> public
|
|
- return (PlayerChunk) this.visibleChunks.get(i);
|
|
+ // Paper start - mt safe get
|
|
+ if (Thread.currentThread() != this.world.serverThread) {
|
|
+ synchronized (this.visibleChunks) {
|
|
+ return (PlayerChunk) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(i) : ((ProtectedVisibleChunksMap)this.visibleChunks).safeGet(i));
|
|
+ }
|
|
+ }
|
|
+ return (PlayerChunk) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(i) : ((ProtectedVisibleChunksMap)this.visibleChunks).safeGet(i));
|
|
+ // Paper end
|
|
}
|
|
|
|
protected IntSupplier c(long i) {
|
|
@@ -411,8 +479,9 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
|
|
// Paper end
|
|
|
|
protected void save(boolean flag) {
|
|
+ Long2ObjectLinkedOpenHashMap<PlayerChunk> visibleChunks = this.getVisibleChunks(); // Paper remove clone of visible Chunks unless saving off main thread (watchdog kill)
|
|
if (flag) {
|
|
- List<PlayerChunk> list = (List) this.visibleChunks.values().stream().filter(PlayerChunk::hasBeenLoaded).peek(PlayerChunk::m).collect(Collectors.toList());
|
|
+ List<PlayerChunk> list = (List) visibleChunks.values().stream().filter(PlayerChunk::hasBeenLoaded).peek(PlayerChunk::m).collect(Collectors.toList()); // Paper - remove cloning of visible chunks
|
|
MutableBoolean mutableboolean = new MutableBoolean();
|
|
|
|
do {
|
|
@@ -440,7 +509,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
|
|
// this.i(); // Paper - nuke IOWorker
|
|
PlayerChunkMap.LOGGER.info("ThreadedAnvilChunkStorage ({}): All chunks are saved", this.w.getName());
|
|
} else {
|
|
- this.visibleChunks.values().stream().filter(PlayerChunk::hasBeenLoaded).forEach((playerchunk) -> {
|
|
+ visibleChunks.values().stream().filter(PlayerChunk::hasBeenLoaded).forEach((playerchunk) -> {
|
|
IChunkAccess ichunkaccess = (IChunkAccess) playerchunk.getChunkSave().getNow(null); // CraftBukkit - decompile error
|
|
|
|
if (ichunkaccess instanceof ProtoChunkExtension || ichunkaccess instanceof Chunk) {
|
|
@@ -610,7 +679,20 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
|
|
if (!this.updatingChunksModified) {
|
|
return false;
|
|
} else {
|
|
- this.visibleChunks = this.updatingChunks.clone();
|
|
+ // Paper start - stop cloning visibleChunks
|
|
+ synchronized (this.visibleChunks) {
|
|
+ if (isIterating) {
|
|
+ hasPendingVisibleUpdate = true;
|
|
+ this.pendingVisibleChunks.copyFrom((com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk>)this.updatingChunks);
|
|
+ } else {
|
|
+ hasPendingVisibleUpdate = false;
|
|
+ this.pendingVisibleChunks.clear();
|
|
+ ((ProtectedVisibleChunksMap)this.visibleChunks).copyFrom((com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<PlayerChunk>)this.updatingChunks);
|
|
+ this.visibleChunksClone = null;
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+
|
|
this.updatingChunksModified = false;
|
|
return true;
|
|
}
|
|
@@ -1076,12 +1158,12 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
|
|
}
|
|
|
|
protected Iterable<PlayerChunk> f() {
|
|
- return Iterables.unmodifiableIterable(this.visibleChunks.values());
|
|
+ return Iterables.unmodifiableIterable(this.getVisibleChunks().values()); // Paper
|
|
}
|
|
|
|
void a(Writer writer) throws IOException {
|
|
CSVWriter csvwriter = CSVWriter.a().a("x").a("z").a("level").a("in_memory").a("status").a("full_status").a("accessible_ready").a("ticking_ready").a("entity_ticking_ready").a("ticket").a("spawning").a("entity_count").a("block_entity_count").a(writer);
|
|
- ObjectBidirectionalIterator objectbidirectionaliterator = this.visibleChunks.long2ObjectEntrySet().iterator();
|
|
+ ObjectBidirectionalIterator objectbidirectionaliterator = this.getVisibleChunks().long2ObjectEntrySet().iterator(); // Paper
|
|
|
|
while (objectbidirectionaliterator.hasNext()) {
|
|
Entry<PlayerChunk> entry = (Entry) objectbidirectionaliterator.next();
|
|
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
index 5e84bbad1ba75cb2b541a29d38d3f96832cff853..2393c8715603478142a1980a9a769de58f8bb3c4 100644
|
|
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
@@ -74,6 +74,7 @@ import net.minecraft.server.GameRules;
|
|
import net.minecraft.server.GroupDataEntity;
|
|
import net.minecraft.server.IBlockData;
|
|
import net.minecraft.server.IChunkAccess;
|
|
+import net.minecraft.server.MCUtil;
|
|
import net.minecraft.server.MinecraftKey;
|
|
import net.minecraft.server.MinecraftServer;
|
|
import net.minecraft.server.MovingObjectPosition;
|
|
@@ -291,6 +292,7 @@ public class CraftWorld implements World {
|
|
return ret;
|
|
}
|
|
public int getTileEntityCount() {
|
|
+ return MCUtil.ensureMain(() -> {
|
|
// We don't use the full world tile entity list, so we must iterate chunks
|
|
Long2ObjectLinkedOpenHashMap<PlayerChunk> chunks = world.getChunkProvider().playerChunkMap.visibleChunks;
|
|
int size = 0;
|
|
@@ -302,11 +304,13 @@ public class CraftWorld implements World {
|
|
size += chunk.tileEntities.size();
|
|
}
|
|
return size;
|
|
+ });
|
|
}
|
|
public int getTickableTileEntityCount() {
|
|
return world.tileEntityListTick.size();
|
|
}
|
|
public int getChunkCount() {
|
|
+ return MCUtil.ensureMain(() -> {
|
|
int ret = 0;
|
|
|
|
for (PlayerChunk chunkHolder : world.getChunkProvider().playerChunkMap.visibleChunks.values()) {
|
|
@@ -315,7 +319,7 @@ public class CraftWorld implements World {
|
|
}
|
|
}
|
|
|
|
- return ret;
|
|
+ return ret; });
|
|
}
|
|
public int getPlayerCount() {
|
|
return world.players.size();
|
|
@@ -435,6 +439,14 @@ public class CraftWorld implements World {
|
|
|
|
@Override
|
|
public Chunk[] getLoadedChunks() {
|
|
+ // Paper start
|
|
+ if (Thread.currentThread() != world.getMinecraftWorld().serverThread) {
|
|
+ synchronized (world.getChunkProvider().playerChunkMap.visibleChunks) {
|
|
+ Long2ObjectLinkedOpenHashMap<PlayerChunk> chunks = world.getChunkProvider().playerChunkMap.visibleChunks;
|
|
+ return chunks.values().stream().map(PlayerChunk::getFullChunk).filter(Objects::nonNull).map(net.minecraft.server.Chunk::getBukkitChunk).toArray(Chunk[]::new);
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
Long2ObjectLinkedOpenHashMap<PlayerChunk> chunks = world.getChunkProvider().playerChunkMap.visibleChunks;
|
|
return chunks.values().stream().map(PlayerChunk::getFullChunk).filter(Objects::nonNull).map(net.minecraft.server.Chunk::getBukkitChunk).toArray(Chunk[]::new);
|
|
}
|