From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: miclebrick Date: Wed, 8 Aug 2018 15:30:52 -0400 Subject: [PATCH] Add Early Warning Feature to WatchDog Detect when the server has been hung for a long duration, and start printing thread dumps at an interval until the point of crash. This will help diagnose what was going on in that time before the crash. diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java index bd508025b771424c942fd856c31d520b6f548082..62621562137cba4804f0465c58d25ca2786328e5 100644 --- a/src/main/java/com/destroystokyo/paper/PaperConfig.java +++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java @@ -25,6 +25,7 @@ import org.bukkit.configuration.file.YamlConfiguration; import co.aikar.timings.Timings; import co.aikar.timings.TimingsManager; import org.spigotmc.SpigotConfig; +import org.spigotmc.WatchdogThread; public class PaperConfig { @@ -297,6 +298,14 @@ public class PaperConfig { } } + public static int watchdogPrintEarlyWarningEvery = 5000; + public static int watchdogPrintEarlyWarningDelay = 10000; + private static void watchdogEarlyWarning() { + watchdogPrintEarlyWarningEvery = getInt("settings.watchdog.early-warning-every", 5000); + watchdogPrintEarlyWarningDelay = getInt("settings.watchdog.early-warning-delay", 10000); + WatchdogThread.doStart(SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash ); + } + public static int tabSpamIncrement = 1; public static int tabSpamLimit = 500; private static void tabSpamLimiters() { diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index 925dd2dc2b4af5596c9dc891f756bad66baadbea..540250a9610e2ee51685b655a7d6c0809bba64fd 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -1019,6 +1019,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant 0 && monotonicMillis() > lastTick + timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable + // Paper start + Logger log = Bukkit.getServer().getLogger(); + long currentTime = monotonicMillis(); + if ( lastTick != 0 && timeoutTime > 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") ) { - Logger log = Bukkit.getServer().getLogger(); + boolean isLongTimeout = currentTime > lastTick + timeoutTime; + // Don't spam early warning dumps + if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; + if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... + lastEarlyWarning = currentTime; + if (isLongTimeout) { + // Paper end log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" ); @@ -93,29 +108,46 @@ public class WatchdogThread extends Thread } } // Paper end + } else + { + log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---"); + log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump"); + } + // Paper end - Different message for short timeout log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log ); log.log( Level.SEVERE, "------------------------------" ); // + // Paper start - Only print full dump on long timeouts + if ( isLongTimeout ) + { log.log( Level.SEVERE, "Entire Thread Dump:" ); ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true ); for ( ThreadInfo thread : threads ) { dumpThread( thread, log ); } + } else { + log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---"); + } + + log.log( Level.SEVERE, "------------------------------" ); + if ( isLongTimeout ) + { if ( restart && !MinecraftServer.getServer().hasStopped() ) { RestartCommand.restart(); } break; + } // Paper end } try { - sleep( 10000 ); + sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout } catch ( InterruptedException ex ) { interrupt();