diff --git a/src/main/java/baritone/cache/WorldProvider.java b/src/main/java/baritone/cache/WorldProvider.java
index 3addb1762..98ec1bbb2 100644
--- a/src/main/java/baritone/cache/WorldProvider.java
+++ b/src/main/java/baritone/cache/WorldProvider.java
@@ -145,7 +145,7 @@ public class WorldProvider implements IWorldProvider, Helper {
                 System.out.println("mc.world loaded unnoticed! Loading Baritone cache now.");
                 initWorld(mc.world.provider.getDimensionType().getId());
             }
-        } else if (currentWorld == null && mc.world != null) {
+        } else if (currentWorld == null && mc.world != null && (mc.isSingleplayer() || mc.getCurrentServerData() != null)) {
             System.out.println("Retrying to load Baritone cache");
             initWorld(mc.world.provider.getDimensionType().getId());
         }