Merge pull request #3768 from wagyourtail/1.12/fasterworldscanner

replace the world scanner with a 500x faster one

src/api/java/baritone/api/utils/BlockOptionalMeta.java

@@ -41,8 +41,8 @@ public final class BlockOptionalMeta {
     private final int meta;
     private final boolean noMeta;
     private final Set<IBlockState> blockstates;
-    private final ImmutableSet<Integer> stateHashes;
-    private final ImmutableSet<Integer> stackHashes;
+    private final Set<Integer> stateHashes;
+    private final Set<Integer> stackHashes;
     private static final Pattern pattern = Pattern.compile("^(.+?)(?::(\\d+))?$");
     private static final Map<Object, Object> normalizations;
 
@@ -315,4 +315,12 @@ public final class BlockOptionalMeta {
 
         return null;
     }
+
+    public Set<IBlockState> getAllBlockStates() {
+        return blockstates;
+    }
+
+    public Set<Integer> stackHashes() {
+        return stackHashes;
+    }
 }

src/api/java/baritone/api/utils/BlockOptionalMetaLookup.java

@@ -17,58 +17,61 @@
 
 package baritone.api.utils;
 
+import com.google.common.collect.ImmutableSet;
 import net.minecraft.block.Block;
 import net.minecraft.block.state.IBlockState;
 import net.minecraft.item.ItemStack;
 
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Stream;
 
 public class BlockOptionalMetaLookup {
-
+    private final Set<Block> blockSet;
+    private final Set<IBlockState> blockStateSet;
     private final BlockOptionalMeta[] boms;
 
     public BlockOptionalMetaLookup(BlockOptionalMeta... boms) {
         this.boms = boms;
+        Set<Block> blocks = new HashSet<>();
+        Set<IBlockState> blockStates = new HashSet<>();
+        Set<Integer> stacks = new HashSet<>();
+        for (BlockOptionalMeta bom : boms) {
+            blocks.add(bom.getBlock());
+            blockStates.addAll(bom.getAllBlockStates());
+            stacks.addAll(bom.stackHashes());
+        }
+        this.blockSet = ImmutableSet.copyOf(blocks);
+        this.blockStateSet = ImmutableSet.copyOf(blockStates);
     }
 
     public BlockOptionalMetaLookup(Block... blocks) {
-        this.boms = Stream.of(blocks)
+        this(Stream.of(blocks)
                 .map(BlockOptionalMeta::new)
-                .toArray(BlockOptionalMeta[]::new);
+                .toArray(BlockOptionalMeta[]::new));
+
     }
 
     public BlockOptionalMetaLookup(List<Block> blocks) {
-        this.boms = blocks.stream()
+        this(blocks.stream()
                 .map(BlockOptionalMeta::new)
-                .toArray(BlockOptionalMeta[]::new);
+                .toArray(BlockOptionalMeta[]::new));
     }
 
     public BlockOptionalMetaLookup(String... blocks) {
-        this.boms = Stream.of(blocks)
+        this(Stream.of(blocks)
                 .map(BlockOptionalMeta::new)
-                .toArray(BlockOptionalMeta[]::new);
+                .toArray(BlockOptionalMeta[]::new));
     }
 
     public boolean has(Block block) {
-        for (BlockOptionalMeta bom : boms) {
-            if (bom.getBlock() == block) {
-                return true;
-            }
-        }
-
-        return false;
+        return blockSet.contains(block);
     }
 
     public boolean has(IBlockState state) {
-        for (BlockOptionalMeta bom : boms) {
-            if (bom.matches(state)) {
-                return true;
-            }
-        }
-
-        return false;
+        return blockStateSet.contains(state);
     }
 
     public boolean has(ItemStack stack) {

src/launch/java/baritone/launch/mixins/MixinBitArray.java

@@ -64,4 +64,14 @@ public abstract class MixinBitArray implements IBitArray {
 
         return out;
     }
+
+    @Override
+    public long getMaxEntryValue() {
+        return maxEntryValue;
+    }
+
+    @Override
+    public int getBitsPerEntry() {
+        return bitsPerEntry;
+    }
 }

src/launch/java/baritone/launch/mixins/MixinBlockStateContainer.java

@@ -35,6 +35,16 @@ public abstract class MixinBlockStateContainer implements IBlockStateContainer {
     @Shadow
     protected IBlockStatePalette palette;
 
+    @Override
+    public IBlockStatePalette getPalette() {
+        return palette;
+    }
+
+    @Override
+    public BitArray getStorage() {
+        return storage;
+    }
+
     @Override
     public IBlockState getAtPalette(int index) {
         return palette.getBlockState(index);

src/main/java/baritone/BaritoneProvider.java

@@ -22,6 +22,7 @@ import baritone.api.IBaritoneProvider;
 import baritone.api.cache.IWorldScanner;
 import baritone.api.command.ICommandSystem;
 import baritone.api.schematic.ISchematicSystem;
+import baritone.cache.FasterWorldScanner;
 import baritone.cache.WorldScanner;
 import baritone.command.CommandSystem;
 import baritone.command.ExampleBaritoneControl;
@@ -59,7 +60,7 @@ public final class BaritoneProvider implements IBaritoneProvider {
 
     @Override
     public IWorldScanner getWorldScanner() {
-        return WorldScanner.INSTANCE;
+        return FasterWorldScanner.INSTANCE;
     }
 
     @Override

src/main/java/baritone/cache/FasterWorldScanner.java

@@ -0,0 +1,276 @@
+/*
+ * This file is part of Baritone.
+ *
+ * Baritone is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Baritone is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Baritone.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package baritone.cache;
+
+import baritone.api.cache.ICachedWorld;
+import baritone.api.cache.IWorldScanner;
+import baritone.api.utils.BetterBlockPos;
+import baritone.api.utils.BlockOptionalMetaLookup;
+import baritone.api.utils.IPlayerContext;
+import baritone.utils.accessor.IBitArray;
+import baritone.utils.accessor.IBlockStateContainer;
+import io.netty.buffer.Unpooled;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+import net.minecraft.block.Block;
+import net.minecraft.block.state.IBlockState;
+import net.minecraft.network.PacketBuffer;
+import net.minecraft.util.BitArray;
+import net.minecraft.util.ObjectIntIdentityMap;
+import net.minecraft.util.math.BlockPos;
+import net.minecraft.util.math.ChunkPos;
+import net.minecraft.world.chunk.*;
+import net.minecraft.world.chunk.storage.ExtendedBlockStorage;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BiConsumer;
+import java.util.function.IntConsumer;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public enum FasterWorldScanner implements IWorldScanner {
+    INSTANCE;
+    @Override
+    public List<BlockPos> scanChunkRadius(IPlayerContext ctx, BlockOptionalMetaLookup filter, int max, int yLevelThreshold, int maxSearchRadius) {
+        assert ctx.world() != null;
+        if (maxSearchRadius < 0) {
+            throw new IllegalArgumentException("chunkRange must be >= 0");
+        }
+        return scanChunksInternal(ctx, filter, getChunkRange(ctx.playerFeet().x >> 4, ctx.playerFeet().z >> 4, maxSearchRadius), max);
+    }
+
+    @Override
+    public List<BlockPos> scanChunk(IPlayerContext ctx, BlockOptionalMetaLookup filter, ChunkPos pos, int max, int yLevelThreshold) {
+        Stream<BlockPos> stream = scanChunkInternal(ctx, filter, pos);
+        if (max >= 0) {
+            stream = stream.limit(max);
+        }
+        return stream.collect(Collectors.toList());
+    }
+
+    @Override
+    public int repack(IPlayerContext ctx) {
+        return this.repack(ctx, 40);
+    }
+
+    @Override
+    public int repack(IPlayerContext ctx, int range) {
+        IChunkProvider chunkProvider = ctx.world().getChunkProvider();
+        ICachedWorld cachedWorld = ctx.worldData().getCachedWorld();
+
+        BetterBlockPos playerPos = ctx.playerFeet();
+
+        int playerChunkX = playerPos.getX() >> 4;
+        int playerChunkZ = playerPos.getZ() >> 4;
+
+        int minX = playerChunkX - range;
+        int minZ = playerChunkZ - range;
+        int maxX = playerChunkX + range;
+        int maxZ = playerChunkZ + range;
+
+        int queued = 0;
+        for (int x = minX; x <= maxX; x++) {
+            for (int z = minZ; z <= maxZ; z++) {
+                Chunk chunk = chunkProvider.getLoadedChunk(x, z);
+
+                if (chunk != null && !chunk.isEmpty()) {
+                    queued++;
+                    cachedWorld.queueForPacking(chunk);
+                }
+            }
+        }
+
+        return queued;
+    }
+
+    // ordered in a way that the closest blocks are generally first
+    public static List<ChunkPos> getChunkRange(int centerX, int centerZ, int chunkRadius) {
+        List<ChunkPos> chunks = new ArrayList<>();
+        // spiral out
+        chunks.add(new ChunkPos(centerX, centerZ));
+        for (int i = 1; i < chunkRadius; i++) {
+            for (int j = 0; j <= i; j++) {
+                chunks.add(new ChunkPos(centerX - j, centerZ - i));
+                if (j != 0) {
+                    chunks.add(new ChunkPos(centerX + j, centerZ - i));
+                    chunks.add(new ChunkPos(centerX - j, centerZ + i));
+                }
+                chunks.add(new ChunkPos(centerX + j, centerZ + i));
+                if (j != i) {
+                    chunks.add(new ChunkPos(centerX - i, centerZ - j));
+                    chunks.add(new ChunkPos(centerX + i, centerZ - j));
+                    if (j != 0) {
+                        chunks.add(new ChunkPos(centerX - i, centerZ + j));
+                        chunks.add(new ChunkPos(centerX + i, centerZ + j));
+                    }
+                }
+            }
+        }
+        return chunks;
+    }
+
+    private List<BlockPos> scanChunksInternal(IPlayerContext ctx, BlockOptionalMetaLookup lookup, List<ChunkPos> chunkPositions, int maxBlocks) {
+        assert ctx.world() != null;
+        try {
+            // p -> scanChunkInternal(ctx, lookup, p)
+            Stream<BlockPos> posStream = chunkPositions.parallelStream().flatMap(p -> scanChunkInternal(ctx, lookup, p));
+            if (maxBlocks >= 0) {
+                // WARNING: this can be expensive if maxBlocks is large...
+                // see limit's javadoc
+                posStream = posStream.limit(maxBlocks);
+            }
+            return posStream.collect(Collectors.toList());
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw e;
+        }
+    }
+
+    private Stream<BlockPos> scanChunkInternal(IPlayerContext ctx, BlockOptionalMetaLookup lookup, ChunkPos pos) {
+        IChunkProvider chunkProvider = ctx.world().getChunkProvider();
+        // if chunk is not loaded, return empty stream
+        if (!chunkProvider.isChunkGeneratedAt(pos.x, pos.z)) {
+            return Stream.empty();
+        }
+
+        long chunkX = (long) pos.x << 4;
+        long chunkZ = (long) pos.z << 4;
+
+        int playerSectionY = ctx.playerFeet().y >> 4;
+
+        return collectChunkSections(lookup, chunkProvider.getLoadedChunk(pos.x, pos.z), chunkX, chunkZ, playerSectionY).stream();
+    }
+
+
+
+    private List<BlockPos> collectChunkSections(BlockOptionalMetaLookup lookup, Chunk chunk, long chunkX, long chunkZ, int playerSection) {
+        // iterate over sections relative to player
+        List<BlockPos> blocks = new ArrayList<>();
+        ExtendedBlockStorage[] sections = chunk.getBlockStorageArray();
+        int l = sections.length;
+        int i = playerSection - 1;
+        int j = playerSection;
+        for (; i >= 0 || j < l; ++j, --i) {
+            if (j < l) {
+                visitSection(lookup, sections[j], blocks, chunkX, chunkZ);
+            }
+            if (i >= 0) {
+                visitSection(lookup, sections[i], blocks, chunkX, chunkZ);
+            }
+        }
+        return blocks;
+    }
+
+    private void visitSection(BlockOptionalMetaLookup lookup, ExtendedBlockStorage section, List<BlockPos> blocks, long chunkX, long chunkZ) {
+        if (section == null || section.isEmpty()) {
+            return;
+        }
+
+        BlockStateContainer sectionContainer = section.getData();
+        //this won't work if the PaletteStorage is of the type EmptyPaletteStorage
+        if (((IBlockStateContainer) sectionContainer).getStorage() == null) {
+            return;
+        }
+
+        boolean[] isInFilter = getIncludedFilterIndices(lookup, ((IBlockStateContainer) sectionContainer).getPalette());
+        if (isInFilter.length == 0) {
+            return;
+        }
+
+        BitArray array = ((IBlockStateContainer) section.getData()).getStorage();
+        long[] longArray = array.getBackingLongArray();
+        int arraySize = array.size();
+        int bitsPerEntry = ((IBitArray) array).getBitsPerEntry();
+        long maxEntryValue = ((IBitArray) array).getMaxEntryValue();
+
+
+        int yOffset = section.getYLocation();
+
+        for (int idx = 0, kl = bitsPerEntry - 1; idx < arraySize; idx++, kl += bitsPerEntry) {
+            final int i = idx * bitsPerEntry;
+            final int j = i >> 6;
+            final int l = i & 63;
+            final int k = kl >> 6;
+            final long jl = longArray[j] >>> l;
+
+            if (j == k) {
+                if (isInFilter[(int) (jl & maxEntryValue)]) {
+                    //noinspection DuplicateExpressions
+                    blocks.add(new BlockPos(
+                        chunkX + ((idx & 255) & 15),
+                        yOffset + (idx >> 8),
+                        chunkZ + ((idx & 255) >> 4)
+                    ));
+                }
+            } else {
+                if (isInFilter[(int) ((jl | longArray[k] << (64 - l)) & maxEntryValue)]) {
+                    //noinspection DuplicateExpressions
+                    blocks.add(new BlockPos(
+                        chunkX + ((idx & 255) & 15),
+                        yOffset + (idx >> 8),
+                        chunkZ + ((idx & 255) >> 4)
+                    ));
+                }
+            }
+        }
+    }
+
+    private boolean[] getIncludedFilterIndices(BlockOptionalMetaLookup lookup, IBlockStatePalette palette) {
+        boolean commonBlockFound = false;
+        ObjectIntIdentityMap<IBlockState> paletteMap = getPalette(palette);
+        int size = paletteMap.size();
+
+        boolean[] isInFilter = new boolean[size];
+
+        for (int i = 0; i < size; i++) {
+            IBlockState state = paletteMap.getByValue(i);
+            if (lookup.has(state)) {
+                isInFilter[i] = true;
+                commonBlockFound = true;
+            } else {
+                isInFilter[i] = false;
+            }
+        }
+
+        if (!commonBlockFound) {
+            return new boolean[0];
+        }
+        return isInFilter;
+    }
+
+    /**
+     * cheats to get the actual map of id -> blockstate from the various palette implementations
+     */
+    private static ObjectIntIdentityMap<IBlockState> getPalette(IBlockStatePalette palette) {
+        if (palette instanceof BlockStatePaletteRegistry) {
+            return Block.BLOCK_STATE_IDS;
+        } else {
+            PacketBuffer buf = new PacketBuffer(Unpooled.buffer());
+            palette.write(buf);
+            int size = buf.readVarInt();
+            ObjectIntIdentityMap<IBlockState> states = new ObjectIntIdentityMap<>();
+            for (int i = 0; i < size; i++) {
+                IBlockState state = Block.BLOCK_STATE_IDS.getByValue(buf.readVarInt());
+                assert state != null;
+                states.put(state, i);
+            }
+            return states;
+        }
+    }
+}

src/main/java/baritone/command/defaults/MineCommand.java

@@ -17,6 +17,7 @@
 
 package baritone.command.defaults;
 
+import baritone.api.BaritoneAPI;
 import baritone.api.IBaritone;
 import baritone.api.command.Command;
 import baritone.api.command.argument.IArgConsumer;
@@ -45,7 +46,7 @@ public class MineCommand extends Command {
         while (args.hasAny()) {
             boms.add(args.getDatatypeFor(ForBlockOptionalMeta.INSTANCE));
         }
-        WorldScanner.INSTANCE.repack(ctx);
+        BaritoneAPI.getProvider().getWorldScanner().repack(ctx);
         logDirect(String.format("Mining %s", boms.toString()));
         baritone.getMineProcess().mine(quantity, boms.toArray(new BlockOptionalMeta[0]));
     }

src/main/java/baritone/command/defaults/PathCommand.java

@@ -17,6 +17,7 @@
 
 package baritone.command.defaults;
 
+import baritone.api.BaritoneAPI;
 import baritone.api.IBaritone;
 import baritone.api.command.Command;
 import baritone.api.command.argument.IArgConsumer;
@@ -38,7 +39,7 @@ public class PathCommand extends Command {
     public void execute(String label, IArgConsumer args) throws CommandException {
         ICustomGoalProcess customGoalProcess = baritone.getCustomGoalProcess();
         args.requireMax(0);
-        WorldScanner.INSTANCE.repack(ctx);
+        BaritoneAPI.getProvider().getWorldScanner().repack(ctx);
         customGoalProcess.path();
         logDirect("Now pathing");
     }

src/main/java/baritone/command/defaults/RepackCommand.java

@@ -17,6 +17,7 @@
 
 package baritone.command.defaults;
 
+import baritone.api.BaritoneAPI;
 import baritone.api.IBaritone;
 import baritone.api.command.Command;
 import baritone.api.command.argument.IArgConsumer;
@@ -36,7 +37,7 @@ public class RepackCommand extends Command {
     @Override
     public void execute(String label, IArgConsumer args) throws CommandException {
         args.requireMax(0);
-        logDirect(String.format("Queued %d chunks for repacking", WorldScanner.INSTANCE.repack(ctx)));
+        logDirect(String.format("Queued %d chunks for repacking", BaritoneAPI.getProvider().getWorldScanner().repack(ctx)));
     }
 
     @Override

src/main/java/baritone/process/FarmProcess.java

@@ -18,6 +18,7 @@
 package baritone.process;
 
 import baritone.Baritone;
+import baritone.api.BaritoneAPI;
 import baritone.api.pathing.goals.Goal;
 import baritone.api.pathing.goals.GoalBlock;
 import baritone.api.pathing.goals.GoalComposite;
@@ -189,7 +190,7 @@ public final class FarmProcess extends BaritoneProcessHelper implements IFarmPro
         }
 
         if (Baritone.settings().mineGoalUpdateInterval.value != 0 && tickCount++ % Baritone.settings().mineGoalUpdateInterval.value == 0) {
-            Baritone.getExecutor().execute(() -> locations = WorldScanner.INSTANCE.scanChunkRadius(ctx, scan, 256, 10, 10));
+            Baritone.getExecutor().execute(() -> locations = BaritoneAPI.getProvider().getWorldScanner().scanChunkRadius(ctx, scan, 256, 10, 10));
         }
         if (locations == null) {
             return new PathingCommand(null, PathingCommandType.REQUEST_PAUSE);

src/main/java/baritone/process/MineProcess.java

@@ -18,6 +18,7 @@
 package baritone.process;
 
 import baritone.Baritone;
+import baritone.api.BaritoneAPI;
 import baritone.api.pathing.goals.*;
 import baritone.api.process.IMineProcess;
 import baritone.api.process.PathingCommand;
@@ -361,7 +362,7 @@ public final class MineProcess extends BaritoneProcessHelper implements IMinePro
         locs = prune(ctx, locs, filter, max, blacklist, dropped);
 
         if (!untracked.isEmpty() || (Baritone.settings().extendCacheOnThreshold.value && locs.size() < max)) {
-            locs.addAll(WorldScanner.INSTANCE.scanChunkRadius(
+            locs.addAll(BaritoneAPI.getProvider().getWorldScanner().scanChunkRadius(
                     ctx.getBaritone().getPlayerContext(),
                     filter,
                     max,

src/main/java/baritone/utils/accessor/IBitArray.java

@@ -3,4 +3,8 @@ package baritone.utils.accessor;
 public interface IBitArray {
 
     int[] toArray();
+
+    long getMaxEntryValue();
+
+    int getBitsPerEntry();
 }

src/main/java/baritone/utils/accessor/IBlockStateContainer.java

@@ -1,9 +1,15 @@
 package baritone.utils.accessor;
 
 import net.minecraft.block.state.IBlockState;
+import net.minecraft.util.BitArray;
+import net.minecraft.world.chunk.IBlockStatePalette;
 
 public interface IBlockStateContainer {
 
+    IBlockStatePalette getPalette();
+
+    BitArray getStorage();
+
     IBlockState getAtPalette(int index);
 
     int[] storageArray();