* Updated Upstream (Bukkit/CraftBukkit) Upstream has released updates that appears to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Bukkit Changes: 45690fe9 SPIGOT-5047: Correct slot types for 1.14 inventories CraftBukkit Changes: 4090d01f SPIGOT-5047: Correct slot types for 1.14 inventories e8c08362 SPIGOT-5046: World#getLoadedChunks returning inaccessible cached chunks. d445af3b SPIGOT-5067: Add item meta for 1.14 spawn eggs * Bring Chunk load checks in-line with spigot As of the last upstream merge spigot now checks ticket level status when returning loaded chunks for a world from api. Now our checks will respect that decision. * Fix spawn ticket levels Vanilla would keep the inner chunks of spawn available for ticking, however my changes made all chunks non-ticking. Resolve by changing ticket levels for spawn chunks inside the border to respect this behavior. * Make World#getChunkIfLoadedImmediately return only entity ticking chunks Mojang appears to be using chunks with level > 33 (non-ticking chunks) as cached chunks and not actually loaded chunks. * Bring all loaded checks in line with spigot Loaded chunks must be at least border chunks, or level <= 33
187 lines
9.2 KiB
Diff
187 lines
9.2 KiB
Diff
From 50e490ebdc5b66293d76047c4e693b038cb1a308 Mon Sep 17 00:00:00 2001
|
|
From: miclebrick <miclebrick@outlook.com>
|
|
Date: Wed, 8 Aug 2018 15:30:52 -0400
|
|
Subject: [PATCH] Add Early Warning Feature to WatchDog
|
|
|
|
Detect when the server has been hung for a long duration, and start printing
|
|
thread dumps at an interval until the point of crash.
|
|
|
|
This will help diagnose what was going on in that time before the crash.
|
|
|
|
diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java
|
|
index d773ef5b0d..833659bbb2 100644
|
|
--- a/src/main/java/com/destroystokyo/paper/PaperConfig.java
|
|
+++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java
|
|
@@ -25,6 +25,7 @@ import org.bukkit.configuration.file.YamlConfiguration;
|
|
import co.aikar.timings.Timings;
|
|
import co.aikar.timings.TimingsManager;
|
|
import org.spigotmc.SpigotConfig;
|
|
+import org.spigotmc.WatchdogThread;
|
|
|
|
public class PaperConfig {
|
|
|
|
@@ -312,6 +313,14 @@ public class PaperConfig {
|
|
}
|
|
}
|
|
|
|
+ public static int watchdogPrintEarlyWarningEvery = 5000;
|
|
+ public static int watchdogPrintEarlyWarningDelay = 10000;
|
|
+ private static void watchdogEarlyWarning() {
|
|
+ watchdogPrintEarlyWarningEvery = getInt("settings.watchdog.early-warning-every", 5000);
|
|
+ watchdogPrintEarlyWarningDelay = getInt("settings.watchdog.early-warning-delay", 10000);
|
|
+ WatchdogThread.doStart(SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash );
|
|
+ }
|
|
+
|
|
public static int tabSpamIncrement = 1;
|
|
public static int tabSpamLimit = 500;
|
|
private static void tabSpamLimiters() {
|
|
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
index 7e27f10790..3f2a3dd178 100644
|
|
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
@@ -847,6 +847,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
|
|
this.a(this.serverPing);
|
|
|
|
// Spigot start
|
|
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper
|
|
Arrays.fill( recentTps, 20 );
|
|
long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
|
|
lastTick = start - TICK_TIME; // Paper
|
|
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
index f6c5a9c364..d316652ffb 100644
|
|
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
@@ -754,6 +754,7 @@ public final class CraftServer implements Server {
|
|
|
|
@Override
|
|
public void reload() {
|
|
+ org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload
|
|
reloadCount++;
|
|
configuration = YamlConfiguration.loadConfiguration(getConfigFile());
|
|
commandsConfiguration = YamlConfiguration.loadConfiguration(getCommandsConfigFile());
|
|
@@ -852,6 +853,7 @@ public final class CraftServer implements Server {
|
|
enablePlugins(PluginLoadOrder.STARTUP);
|
|
enablePlugins(PluginLoadOrder.POSTWORLD);
|
|
getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD));
|
|
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
|
|
}
|
|
|
|
@Override
|
|
diff --git a/src/main/java/org/spigotmc/SpigotConfig.java b/src/main/java/org/spigotmc/SpigotConfig.java
|
|
index 6c5430ff94..4b8581f0ed 100644
|
|
--- a/src/main/java/org/spigotmc/SpigotConfig.java
|
|
+++ b/src/main/java/org/spigotmc/SpigotConfig.java
|
|
@@ -225,7 +225,7 @@ public class SpigotConfig
|
|
restartScript = getString( "settings.restart-script", restartScript );
|
|
restartMessage = transform( getString( "messages.restart", "Server is restarting" ) );
|
|
commands.put( "restart", new RestartCommand( "restart" ) );
|
|
- WatchdogThread.doStart( timeoutTime, restartOnCrash );
|
|
+ //WatchdogThread.doStart( timeoutTime, restartOnCrash ); // Paper - moved to PaperConfig
|
|
}
|
|
|
|
public static boolean bungee;
|
|
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
|
|
index 9532aada82..a1d93200e6 100644
|
|
--- a/src/main/java/org/spigotmc/WatchdogThread.java
|
|
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
|
|
@@ -5,6 +5,7 @@ import java.lang.management.MonitorInfo;
|
|
import java.lang.management.ThreadInfo;
|
|
import java.util.logging.Level;
|
|
import java.util.logging.Logger;
|
|
+import com.destroystokyo.paper.PaperConfig;
|
|
import net.minecraft.server.MinecraftServer;
|
|
import org.bukkit.Bukkit;
|
|
|
|
@@ -13,6 +14,10 @@ public class WatchdogThread extends Thread
|
|
|
|
private static WatchdogThread instance;
|
|
private final long timeoutTime;
|
|
+ private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
|
|
+ private final long earlyWarningDelay; // Paper
|
|
+ public static volatile boolean hasStarted; // Paper
|
|
+ private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps
|
|
private final boolean restart;
|
|
private volatile long lastTick;
|
|
private volatile boolean stopping;
|
|
@@ -22,6 +27,8 @@ public class WatchdogThread extends Thread
|
|
super( "Paper Watchdog Thread" );
|
|
this.timeoutTime = timeoutTime;
|
|
this.restart = restart;
|
|
+ earlyWarningEvery = Math.min(PaperConfig.watchdogPrintEarlyWarningEvery, timeoutTime); // Paper
|
|
+ earlyWarningDelay = Math.min(PaperConfig.watchdogPrintEarlyWarningDelay, timeoutTime); // Paper
|
|
}
|
|
|
|
private static long monotonicMillis()
|
|
@@ -56,10 +63,17 @@ public class WatchdogThread extends Thread
|
|
{
|
|
while ( !stopping )
|
|
{
|
|
- //
|
|
- if ( lastTick != 0 && monotonicMillis() > lastTick + timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
|
|
+ // Paper start
|
|
+ Logger log = Bukkit.getServer().getLogger();
|
|
+ long currentTime = monotonicMillis();
|
|
+ if ( lastTick != 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") )
|
|
{
|
|
- Logger log = Bukkit.getServer().getLogger();
|
|
+ boolean isLongTimeout = currentTime > lastTick + timeoutTime;
|
|
+ // Don't spam early warning dumps
|
|
+ if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
|
|
+ lastEarlyWarning = currentTime;
|
|
+ if (isLongTimeout) {
|
|
+ // Paper end
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper
|
|
log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
|
|
@@ -89,29 +103,46 @@ public class WatchdogThread extends Thread
|
|
}
|
|
}
|
|
// Paper end
|
|
+ } else
|
|
+ {
|
|
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---");
|
|
+ log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump");
|
|
+ }
|
|
+ // Paper end - Different message for short timeout
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
|
|
dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
//
|
|
+ // Paper start - Only print full dump on long timeouts
|
|
+ if ( isLongTimeout )
|
|
+ {
|
|
log.log( Level.SEVERE, "Entire Thread Dump:" );
|
|
ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
|
|
for ( ThreadInfo thread : threads )
|
|
{
|
|
dumpThread( thread, log );
|
|
}
|
|
+ } else {
|
|
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---");
|
|
+ }
|
|
+
|
|
+
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
|
|
+ if ( isLongTimeout )
|
|
+ {
|
|
if ( restart )
|
|
{
|
|
RestartCommand.restart();
|
|
}
|
|
break;
|
|
+ } // Paper end
|
|
}
|
|
|
|
try
|
|
{
|
|
- sleep( 10000 );
|
|
+ sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout
|
|
} catch ( InterruptedException ex )
|
|
{
|
|
interrupt();
|
|
--
|
|
2.21.0
|
|
|