From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Aikar Date: Sun, 12 Apr 2020 15:50:48 -0400 Subject: [PATCH] Improved Watchdog Support Forced Watchdog Crash support and Improve Async Shutdown If the request to shut down the server is received while we are in a watchdog hang, immediately treat it as a crash and begin the shutdown process. Shutdown process is now improved to also shutdown cleanly when not using restart scripts either. If a server is deadlocked, a server owner can send SIGUP (or any other signal the JVM understands to shut down as it currently does) and the watchdog will no longer need to wait until the full timeout, allowing you to trigger a close process and try to shut the server down gracefully, saving player and world data. Previously there was no way to trigger this outside of waiting for a full watchdog timeout, which may be set to a really long time... Additionally, fix everything to do with shutting the server down asynchronously. Previously, nearly everything about the process was fragile and unsafe. Main might not have actually been frozen, and might still be manipulating state. Or, some reuest might ask main to do something in the shutdown but main is dead. Or worse, other things might start closing down items such as the Console or Thread Pool before we are fully shutdown. This change tries to resolve all of these issues by moving everything into the stop method and guaranteeing only one thread is stopping the server. We then issue Thread Death to the main thread of another thread initiates the stop process. We have to ensure Thread Death propagates correctly though to stop main completely. This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save. This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they are properly accounted for and wont trip watchdog on init. diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java index e257d6b36e0e78dac5b8320017d92776171e1bb0..24641501aeae0f2c2c5ce877775c7b971fb051bf 100644 --- a/src/main/java/com/destroystokyo/paper/Metrics.java +++ b/src/main/java/com/destroystokyo/paper/Metrics.java @@ -86,6 +86,9 @@ public class Metrics { timer.scheduleAtFixedRate(new TimerTask() { @Override public void run() { + if (MinecraftServer.getServer().hasStopped()) { + return; + } submitData(); } }, 1000 * 60 * 5, 1000 * 60 * 30); diff --git a/src/main/java/net/minecraft/server/CrashReport.java b/src/main/java/net/minecraft/server/CrashReport.java index 3de19c998b749ccf74958c2412a8c9506457383e..c7dc8787cc3456c5540d6a00a6ff051533edc25a 100644 --- a/src/main/java/net/minecraft/server/CrashReport.java +++ b/src/main/java/net/minecraft/server/CrashReport.java @@ -257,6 +257,7 @@ public class CrashReport { } public static CrashReport a(Throwable throwable, String s) { + if (throwable instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(throwable); // Paper while (throwable instanceof CompletionException && throwable.getCause() != null) { throwable = throwable.getCause(); } diff --git a/src/main/java/net/minecraft/server/DedicatedServer.java b/src/main/java/net/minecraft/server/DedicatedServer.java index 1ef7890da599d13e784861035e7891efcc4cd504..b07c49f1b48cc6dedd7c2057da0ec4f6f6d446e6 100644 --- a/src/main/java/net/minecraft/server/DedicatedServer.java +++ b/src/main/java/net/minecraft/server/DedicatedServer.java @@ -299,7 +299,7 @@ public class DedicatedServer extends MinecraftServer implements IMinecraftServer long l = SystemUtils.getMonotonicNanos() - i; String s2 = String.format(Locale.ROOT, "%.3fs", (double) l / 1.0E9D); - DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s2); + //DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s2); // Paper moved to after init if (dedicatedserverproperties.announcePlayerAchievements != null) { ((GameRules.GameRuleBoolean) this.getGameRules().get(GameRules.ANNOUNCE_ADVANCEMENTS)).a(dedicatedserverproperties.announcePlayerAchievements, (MinecraftServer) this); } @@ -750,7 +750,7 @@ public class DedicatedServer extends MinecraftServer implements IMinecraftServer @Override public void stop() { super.stop(); - SystemUtils.f(); + //SystemUtils.f(); // Paper - moved into super } @Override diff --git a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java index cfe43e882e524b6ab3d9702e81269c97e6b75eba..2632c7c3ec77918be7979f2aa49209e566cafc77 100644 --- a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java +++ b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java @@ -135,6 +135,7 @@ public abstract class IAsyncTaskHandler implements Mailbox resourcePackRepository; @Nullable private ResourcePackSourceFolder resourcePackFolder; + public volatile Thread shutdownThread; // Paper public CommandDispatcher commandDispatcher; private final CraftingManager craftingManager; private final TagRegistry tagRegistry; @@ -176,7 +177,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant {}; + } + // Paper end return new TickTask(this.ticks, runnable); } diff --git a/src/main/java/net/minecraft/server/PlayerChunkMap.java b/src/main/java/net/minecraft/server/PlayerChunkMap.java index 8cd77117cdb81e3fbd196415f8ac82d3aeddcb12..00ad80bb74bd581e3fa1bf82356ee5b7bc656bfe 100644 --- a/src/main/java/net/minecraft/server/PlayerChunkMap.java +++ b/src/main/java/net/minecraft/server/PlayerChunkMap.java @@ -485,6 +485,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d { MutableBoolean mutableboolean = new MutableBoolean(); do { + boolean isShuttingDown = world.getMinecraftServer().hasStopped(); // Paper mutableboolean.setFalse(); list.stream().map((playerchunk) -> { CompletableFuture completablefuture; @@ -497,7 +498,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d { return (IChunkAccess) completablefuture.join(); }).filter((ichunkaccess) -> { return ichunkaccess instanceof ProtoChunkExtension || ichunkaccess instanceof Chunk; - }).filter(this::saveChunk).forEach((ichunkaccess) -> { + }).filter(ichunkaccess1 -> saveChunk(ichunkaccess1, !isShuttingDown)).forEach((ichunkaccess) -> { // Paper - dont save async during shutdown mutableboolean.setTrue(); }); } while (mutableboolean.isTrue()); diff --git a/src/main/java/net/minecraft/server/PlayerList.java b/src/main/java/net/minecraft/server/PlayerList.java index ab2831830ad3a4cec0671d189e0534c843b47f5e..78040e83899f1ef1a6d5c456beb9d13959307c18 100644 --- a/src/main/java/net/minecraft/server/PlayerList.java +++ b/src/main/java/net/minecraft/server/PlayerList.java @@ -400,7 +400,7 @@ public abstract class PlayerList { cserver.getPluginManager().callEvent(playerQuitEvent); entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage()); - entityplayer.playerTick();// SPIGOT-924 + if (server.isMainThread()) entityplayer.playerTick();// SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog) // CraftBukkit end // Paper start - Remove from collideRule team if needed diff --git a/src/main/java/net/minecraft/server/SystemUtils.java b/src/main/java/net/minecraft/server/SystemUtils.java index 47dbcb70a6b41438a5bf3196495cfdaf02bd2b74..0d12c3657304ccf7ab25a2614c380ee73fdad1fb 100644 --- a/src/main/java/net/minecraft/server/SystemUtils.java +++ b/src/main/java/net/minecraft/server/SystemUtils.java @@ -110,6 +110,7 @@ public class SystemUtils { return SystemUtils.c; } + public static void shutdownServerThreadPool() { f(); } // Paper - OBFHELPER public static void f() { SystemUtils.c.shutdown(); diff --git a/src/main/java/net/minecraft/server/World.java b/src/main/java/net/minecraft/server/World.java index ac8b21e69b198612427a81a16fc1a3ff0b0cb558..42be4edb36200626913fa99b1e67956933022531 100644 --- a/src/main/java/net/minecraft/server/World.java +++ b/src/main/java/net/minecraft/server/World.java @@ -793,6 +793,7 @@ public abstract class World implements GeneratorAccess, AutoCloseable { gameprofilerfiller.exit(); } catch (Throwable throwable) { + if (throwable instanceof ThreadDeath) throw throwable; // Paper // Paper start - Prevent tile entity and entity crashes String msg = "TileEntity threw exception at " + tileentity.world.getWorld().getName() + ":" + tileentity.position.getX() + "," + tileentity.position.getY() + "," + tileentity.position.getZ(); System.err.println(msg); @@ -868,6 +869,7 @@ public abstract class World implements GeneratorAccess, AutoCloseable { try { consumer.accept(entity); } catch (Throwable throwable) { + if (throwable instanceof ThreadDeath) throw throwable; // Paper // Paper start - Prevent tile entity and entity crashes String msg = "Entity threw exception at " + entity.world.getWorld().getName() + ":" + entity.locX() + "," + entity.locY() + "," + entity.locZ(); System.err.println(msg); diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java index b627180729a24a83ca383f83aee53133ea1b398e..f49193d9d7cd9655fdedf64bebdcf4e1a9b77f2c 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java +++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java @@ -1706,7 +1706,7 @@ public final class CraftServer implements Server { @Override public boolean isPrimaryThread() { - return Thread.currentThread().equals(console.serverThread); // Paper - Fix issues with detecting main thread properly + return Thread.currentThread().equals(console.serverThread) || Thread.currentThread().equals(net.minecraft.server.MinecraftServer.getServer().shutdownThread); // Paper - Fix issues with detecting main thread properly, the only time Watchdog will be used is during a crash shutdown which is a "try our best" scenario } @Override diff --git a/src/main/java/org/bukkit/craftbukkit/Main.java b/src/main/java/org/bukkit/craftbukkit/Main.java index 093dbeae2784d51dae74f66e3e1ce5bf6a370428..99e1e902d8fdf33c157416c5d6abc71b495ac48e 100644 --- a/src/main/java/org/bukkit/craftbukkit/Main.java +++ b/src/main/java/org/bukkit/craftbukkit/Main.java @@ -150,6 +150,37 @@ public class Main { OptionSet options = null; + // Paper start - preload logger classes to avoid plugins mixing versions + tryPreloadClass("com.destroystokyo.paper.log.LogFullPolicy"); + tryPreloadClass("org.apache.logging.log4j.core.Core"); + tryPreloadClass("org.apache.logging.log4j.core.Appender"); + tryPreloadClass("org.apache.logging.log4j.core.ContextDataInjector"); + tryPreloadClass("org.apache.logging.log4j.core.Filter"); + tryPreloadClass("org.apache.logging.log4j.core.ErrorHandler"); + tryPreloadClass("org.apache.logging.log4j.core.LogEvent"); + tryPreloadClass("org.apache.logging.log4j.core.Logger"); + tryPreloadClass("org.apache.logging.log4j.core.LoggerContext"); + tryPreloadClass("org.apache.logging.log4j.core.LogEventListener"); + tryPreloadClass("org.apache.logging.log4j.core.AbstractLogEvent"); + tryPreloadClass("org.apache.logging.log4j.message.AsynchronouslyFormattable"); + tryPreloadClass("org.apache.logging.log4j.message.FormattedMessage"); + tryPreloadClass("org.apache.logging.log4j.message.ParameterizedMessage"); + tryPreloadClass("org.apache.logging.log4j.message.Message"); + tryPreloadClass("org.apache.logging.log4j.message.MessageFactory"); + tryPreloadClass("org.apache.logging.log4j.message.TimestampMessage"); + tryPreloadClass("org.apache.logging.log4j.message.SimpleMessage"); + tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLogger"); + tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerContext"); + tryPreloadClass("org.apache.logging.log4j.core.async.AsyncQueueFullPolicy"); + tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerDisruptor"); + tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEvent"); + tryPreloadClass("org.apache.logging.log4j.core.async.DisruptorUtil"); + tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEventHandler"); + tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy"); + tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy$CacheEntry"); + tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedClassInfo"); + tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedStackTraceElement"); + // Paper end try { options = parser.parse(args); } catch (joptsimple.OptionException ex) { @@ -245,8 +276,61 @@ public class Main { } catch (Throwable t) { t.printStackTrace(); } + // Paper start + // load some required classes to avoid errors during shutdown if jar is replaced + // also to guarantee our version loads over plugins + tryPreloadClass("com.destroystokyo.paper.util.SneakyThrow"); + tryPreloadClass("com.google.common.collect.Iterators$PeekingImpl"); + tryPreloadClass("com.google.common.collect.MapMakerInternalMap$Values"); + tryPreloadClass("com.google.common.collect.MapMakerInternalMap$ValueIterator"); + tryPreloadClass("com.google.common.collect.MapMakerInternalMap$WriteThroughEntry"); + tryPreloadClass("com.google.common.collect.Iterables"); + for (int i = 1; i <= 15; i++) { + tryPreloadClass("com.google.common.collect.Iterables$" + i, false); + } + tryPreloadClass("org.apache.commons.lang3.mutable.MutableBoolean"); + tryPreloadClass("org.apache.commons.lang3.mutable.MutableInt"); + tryPreloadClass("org.jline.terminal.impl.MouseSupport"); + tryPreloadClass("org.jline.terminal.impl.MouseSupport$1"); + tryPreloadClass("org.jline.terminal.Terminal$MouseTracking"); + tryPreloadClass("co.aikar.timings.TimingHistory"); + tryPreloadClass("co.aikar.timings.TimingHistory$MinuteReport"); + tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext"); + tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$11"); + tryPreloadClass("io.netty.channel.AbstractChannel$AbstractUnsafe$8"); + tryPreloadClass("io.netty.util.concurrent.DefaultPromise"); + tryPreloadClass("io.netty.util.concurrent.DefaultPromise$1"); + tryPreloadClass("io.netty.util.internal.PromiseNotificationUtil"); + tryPreloadClass("io.netty.util.internal.SystemPropertyUtil"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$1"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$2"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$3"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$4"); + tryPreloadClass("org.slf4j.helpers.MessageFormatter"); + tryPreloadClass("org.slf4j.helpers.FormattingTuple"); + tryPreloadClass("org.slf4j.helpers.BasicMarker"); + tryPreloadClass("org.slf4j.helpers.Util"); + tryPreloadClass("com.destroystokyo.paper.event.player.PlayerConnectionCloseEvent"); + // Minecraft, seen during saving + tryPreloadClass("net.minecraft.server.LightEngineLayerEventListener$Void"); + tryPreloadClass("net.minecraft.server.LightEngineLayerEventListener"); + // Paper end + } + } + + // Paper start + private static void tryPreloadClass(String className) { + tryPreloadClass(className, true); + } + private static void tryPreloadClass(String className, boolean printError) { + try { + Class.forName(className); + } catch (ClassNotFoundException e) { + if (printError) System.err.println("An expected class " + className + " was not found for preloading: " + e.getMessage()); } } + // Paper end private static List asList(String... params) { return Arrays.asList(params); diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java index 449e99d1b673870ed6892f6ab2c715a2db35c35d..899a525209cfe047ce57e758c6328a130f916dc6 100644 --- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java +++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java @@ -12,12 +12,25 @@ public class ServerShutdownThread extends Thread { @Override public void run() { try { + // Paper start - try to shutdown on main + server.safeShutdown(false, false); + for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) { + Thread.sleep(100); + } + if (server.hasStopped()) { + return; + } + // Looks stalled, close async org.spigotmc.AsyncCatcher.enabled = false; // Spigot org.spigotmc.AsyncCatcher.shuttingDown = true; // Paper + server.forceTicks = true; server.close(); + } catch (InterruptedException e) { + e.printStackTrace(); + // Paper end } finally { try { - net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender + //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop } catch (Exception e) { } } diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java index aefea3a9a8b9b75c62bd20018be7cd166a213001..123de5ac9026508e21cdc225f0962f5c3c46fed5 100644 --- a/src/main/java/org/spigotmc/RestartCommand.java +++ b/src/main/java/org/spigotmc/RestartCommand.java @@ -139,7 +139,7 @@ public class RestartCommand extends Command // Paper end // Paper start - copied from above and modified to return if the hook registered - private static boolean addShutdownHook(String restartScript) + public static boolean addShutdownHook(String restartScript) { String[] split = restartScript.split( " " ); if ( split.length > 0 && new File( split[0] ).isFile() ) diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java index 5bdcdcf9e85b73086722783bff26321d03382bb9..513c1041c34ebb3ac1775674a3f4526693759c08 100644 --- a/src/main/java/org/spigotmc/WatchdogThread.java +++ b/src/main/java/org/spigotmc/WatchdogThread.java @@ -13,6 +13,7 @@ import org.bukkit.Bukkit; public class WatchdogThread extends Thread { + public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper private static WatchdogThread instance; private final long timeoutTime; private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting @@ -41,6 +42,7 @@ public class WatchdogThread extends Thread { if ( instance == null ) { + if (timeoutTime <= 0) timeoutTime = 300; // Paper instance = new WatchdogThread( timeoutTime * 1000L, restart ); instance.start(); } @@ -67,12 +69,13 @@ public class WatchdogThread extends Thread // Paper start Logger log = Bukkit.getServer().getLogger(); long currentTime = monotonicMillis(); - if ( lastTick != 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") ) + MinecraftServer server = MinecraftServer.getServer(); + if (lastTick != 0 && hasStarted && (!server.isRunning() || (currentTime > lastTick + earlyWarningEvery && !DISABLE_WATCHDOG) )) { - boolean isLongTimeout = currentTime > lastTick + timeoutTime; + boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000); // Don't spam early warning dumps if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; - if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... + if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... lastEarlyWarning = currentTime; if (isLongTimeout) { // Paper end @@ -114,7 +117,7 @@ public class WatchdogThread extends Thread log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper ChunkTaskManager.dumpAllChunkLoadInfo(); // Paper - dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log ); + dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( server.serverThread.getId(), Integer.MAX_VALUE ), log ); log.log( Level.SEVERE, "------------------------------" ); // // Paper start - Only print full dump on long timeouts @@ -135,9 +138,24 @@ public class WatchdogThread extends Thread if ( isLongTimeout ) { - if ( restart && !MinecraftServer.getServer().hasStopped() ) + if ( !server.hasStopped() ) { - RestartCommand.restart(); + AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us + AsyncCatcher.shuttingDown = true; + server.forceTicks = true; + if (restart) { + RestartCommand.addShutdownHook( SpigotConfig.restartScript ); + } + // try one last chance to safe shutdown on main incase it 'comes back' + server.safeShutdown(false, restart); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if (!server.hasStopped()) { + server.close(); + } } break; } // Paper end