feat: new entity storage (#578)

CHANGELOG.md

@@ -10,7 +10,7 @@ and any changes to API will have a prefix `(API)`.
 
 Unless otherwise specified, any version comparison below is the comparison of server version, not API version.
 
-## 0.1.4 (API 0.5.0) - Unreleased
+## 0.2.0 (API 0.5.0) - Unreleased
 
 <small>[Compare with 0.1.3](https://github.com/AllayMC/Allay/compare/0.1.3...HEAD)</small>
 
@@ -45,6 +45,12 @@ Unless otherwise specified, any version comparison below is the comparison of se
 - (API) Introduced PDC (Persistent Data Container) system. The PDC is a way to store custom data on a whole range of objects, such as
   items, entities, block entities and world. More PDC types will be added in the future.
 - (API) Added `ItemType#getItemData` method which replaces the old `ItemDataComponent`.
+- (API) Introduced new option `entity-auto-save-cycle` in `ServerSettings` to control the interval of entity auto save.
+- (API) Entities are now held by `EntityService` directly, and a variety of new methods are added into `EntityService`. See the commit
+  history for more details.
+- (API) Introduced `WorldStorage#readEntities`, `WorldStorage#writeEntities` and their correspond sync methods. These methods are used
+  to read and write entities in a specified chunk area.
+- Add support for the new entity storage format used in 1.18.30+. Now entities in newer vanilla maps can be loaded correctly.
 - Implemented reeds (also called sugar cane) and cactus.
 - Implemented `UpdateSubChunkBlocksPacket` related logic, which will make client load large range block updates much quicker (e.g.
   using `/fill` command to fill a large area).
@@ -104,6 +110,11 @@ Unless otherwise specified, any version comparison below is the comparison of se
 - (API) Removed `VoxelShapes#buildStairShape` method, we now have accurate collision shape data dumped from BDS.
 - (API) Removed the old `BlockState#toNetworkBlockDefinition` method, and `BlockState#toNetworkBlockDefinitionRuntime` was renamed without `Runtime` suffix.
 - (API) Removed `ItemDataComponent`. `ItemData` is now located in `ItemType<?>`.
+- (API) Removed `Dimension#getEntityByRuntimeId`. This method is replaced by `EntityService#getEntityByRuntimeId`.
+- (API) Removed `Dimension#getEntityPhysicsService`. This method is replaced by `EntityService#getPhysicsService`.
+- (API) Removed `ChunkLoader#spawnEntity` and `ChunkLoader#despawnEntity` methods.
+- (API) Removed `UnsafeChunk#getEntity`, `UnsafeChunk#getEntities`, `UnsafeChunk#spawnEntitiesTo` and
+ `UnsafeChunk#despawnEntitiesFrom` methods. Because entity is not held by chunk now.
 - Removed `Extension#afterServerStarted` method.
 - Removed `org.allaymc.server.datastruct.collections.nb.*`, we now use the implementations provided by JCTools. Consider using `NonBlockingHashMap`
   and `NonBlockingHashMapLong` if your plugins use these classes.

api/src/main/java/org/allaymc/api/entity/component/EntityBaseComponent.java

@@ -991,4 +991,22 @@ default boolean canStandSafely(int x, int y, int z, Dimension dimension) {
         var aabb = getAABB().translate(x + 0.5, y + 0.5, z + 0.5, new AABBd());
         return dimension.getCollidingBlockStates(aabb) == null;
     }
+
+    /**
+     * Check if the entity will be saved through {@link org.allaymc.api.world.storage.WorldStorage}.
+     * If you don't want the entity to be saved, or you want to save the entity by yourself, you can
+     * override this method and return {@code false}.
+     * <p>
+     * When return {@code false}, the entity will always be loaded, and {@link org.allaymc.api.world.service.EntityService}
+     * will not remove and save the entity even if the entity is in unloaded chunk. The entity can only be removed
+     * manually in this case.
+     * <p>
+     * For example, {@link EntityPlayer} is handled by {@link org.allaymc.api.client.storage.PlayerStorage},
+     * and this method is override to return {@code false} in {@link EntityPlayer}.
+     *
+     * @return {@code true} if the entity will be saved, otherwise {@code false}.
+     */
+    default boolean willBeSaved() {
+        return true;
+    }
 }

api/src/main/java/org/allaymc/api/entity/component/player/EntityPlayerBaseComponent.java

@@ -473,4 +473,12 @@ default double getMaxInteractDistance() {
     default void regenerateEnchantmentSeed() {
         setEnchantmentSeed(ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE));
     }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    default boolean willBeSaved() {
+        return false;
+    }
 }

api/src/main/java/org/allaymc/api/server/ServerSettings.java

@@ -253,6 +253,11 @@ public static class StorageSettings extends OkaeriConfig {
         @CustomKey("chunk-auto-save-cycle")
         @Comment("Determines the cycle of chunk auto saving")
         private int chunkAutoSaveCycle = 20 * 60 * 5;
+
+        @CustomKey("entity-auto-save-cycle")
+        @Comment("Determines the cycle of entity auto saving. When entity auto saving is triggered, the")
+        @Comment("entity service will find all savable entities in unloaded chunks and save them")
+        private int entityAutoSaveCycle = 20 * 60;
     }
 
     @Getter

api/src/main/java/org/allaymc/api/utils/AllayNbtUtils.java

@@ -2,15 +2,15 @@
 
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.cloudburstmc.nbt.NBTOutputStream;
-import org.cloudburstmc.nbt.NbtMap;
-import org.cloudburstmc.nbt.NbtMapBuilder;
-import org.cloudburstmc.nbt.NbtType;
+import org.cloudburstmc.nbt.*;
 import org.joml.*;
 
+import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
 import java.util.Base64;
+import java.util.List;
 
 /**
  * @author daoge_cmd
@@ -77,6 +77,75 @@ public static NbtMap base64ToNbt(String base64) {
         return (NbtMap) org.cloudburstmc.nbt.NbtUtils.createReader(new ByteArrayInputStream(Base64.getDecoder().decode(base64))).readTag();
     }
 
+    /**
+     * Convert bytes to nbt list.
+     *
+     * @param bytes the bytes.
+     *
+     * @return the nbt list.
+     */
+    @SneakyThrows
+    public static List<NbtMap> bytesToNbtListLE(byte[] bytes) {
+        List<NbtMap> tags = new ArrayList<>();
+        try (var stream = new BufferedInputStream(new ByteArrayInputStream(bytes));
+             var readerLE = NbtUtils.createReaderLE(stream)) {
+            while (stream.available() > 0) {
+                tags.add((NbtMap) readerLE.readTag());
+            }
+        }
+
+        return tags;
+    }
+
+    /**
+     * Convert nbt list to bytes.
+     *
+     * @param tags the nbt list.
+     *
+     * @return the bytes.
+     */
+    @SneakyThrows
+    public static byte[] nbtListToBytesLE(List<NbtMap> tags) {
+        try (var stream = new ByteArrayOutputStream();
+             var writerLE = NbtUtils.createWriterLE(stream)) {
+            for (NbtMap tag : tags) {
+                writerLE.writeTag(tag);
+            }
+            return stream.toByteArray();
+        }
+    }
+
+    /**
+     * Convert bytes to nbt.
+     *
+     * @param bytes the bytes.
+     *
+     * @return the nbt.
+     */
+    @SneakyThrows
+    public static NbtMap bytesToNbtLE(byte[] bytes) {
+        try (var stream = new BufferedInputStream(new ByteArrayInputStream(bytes));
+             var readerLE = NbtUtils.createReaderLE(stream)) {
+            return (NbtMap) readerLE.readTag();
+        }
+    }
+
+    /**
+     * Convert nbt to bytes.
+     *
+     * @param nbt the nbt.
+     *
+     * @return the bytes.
+     */
+    @SneakyThrows
+    public static byte[] nbtToBytesLE(NbtMap nbt) {
+        try (var stream = new ByteArrayOutputStream();
+             var writerLE = NbtUtils.createWriterLE(stream)) {
+            writerLE.writeTag(nbt);
+            return stream.toByteArray();
+        }
+    }
+
     /**
      * Read a vector3 from NBT.
      *

api/src/main/java/org/allaymc/api/world/Dimension.java

@@ -3,7 +3,6 @@
 import com.google.common.base.Preconditions;
 import it.unimi.dsi.fastutil.ints.IntObjectImmutablePair;
 import it.unimi.dsi.fastutil.ints.IntObjectPair;
-import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
 import org.allaymc.api.block.component.BlockLiquidBaseComponent;
 import org.allaymc.api.block.data.BlockFace;
 import org.allaymc.api.block.dto.BlockStateWithPos;
@@ -24,7 +23,10 @@
 import org.allaymc.api.world.biome.BiomeId;
 import org.allaymc.api.world.biome.BiomeType;
 import org.allaymc.api.world.chunk.OperationType;
-import org.allaymc.api.world.service.*;
+import org.allaymc.api.world.service.BlockUpdateService;
+import org.allaymc.api.world.service.ChunkService;
+import org.allaymc.api.world.service.EntityService;
+import org.allaymc.api.world.service.LightService;
 import org.apache.commons.lang3.function.TriFunction;
 import org.cloudburstmc.protocol.bedrock.data.LevelEventType;
 import org.cloudburstmc.protocol.bedrock.data.ParticleType;
@@ -39,7 +41,9 @@
 import org.joml.Vector3ic;
 import org.joml.primitives.AABBdc;
 
-import java.util.*;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Predicate;
@@ -86,13 +90,6 @@ private static UpdateBlockPacket createUpdateBlockPacket(BlockState newBlockStat
      */
     ChunkService getChunkService();
 
-    /**
-     * Get the entity physics service of this dimension.
-     *
-     * @return the block base component.
-     */
-    EntityPhysicsService getEntityPhysicsService();
-
     /**
      * Get the block update service of this dimension.
      *
@@ -135,9 +132,7 @@ private static UpdateBlockPacket createUpdateBlockPacket(BlockState newBlockStat
      */
     @Unmodifiable
     default Map<Long, Entity> getEntities() {
-        var entities = new Long2ObjectOpenHashMap<Entity>();
-        getChunkService().forEachLoadedChunks(chunk -> entities.putAll(chunk.getEntities()));
-        return Collections.unmodifiableMap(entities);
+        return getEntityService().getEntities();
     }
 
     /**
@@ -146,22 +141,7 @@ default Map<Long, Entity> getEntities() {
      * @return the entity count of this dimension.
      */
     default int getEntityCount() {
-        return getChunkService().getLoadedChunks().stream().mapToInt(chunk -> chunk.getEntities().size()).sum();
-    }
-
-    /**
-     * Get the entity by its runtime id.
-     *
-     * @param runtimeId the runtime id of the entity.
-     *
-     * @return the entity with the specified runtime id, or {@code null} if not found.
-     */
-    default Entity getEntityByRuntimeId(long runtimeId) {
-        return getChunkService().getLoadedChunks().stream()
-                .map(chunk -> chunk.getEntities().get(runtimeId))
-                .filter(Objects::nonNull)
-                .findFirst()
-                .orElse(null);
+        return getEntities().size();
     }
 
     /**

api/src/main/java/org/allaymc/api/world/Explosion.java

@@ -185,7 +185,7 @@ public void explode(Dimension dimension, double x, double y, double z) {
         MathUtils.grow(aabb, 2);
 
         if (affectEntities) {
-            var affectedEntities = dimension.getEntityPhysicsService().computeCollidingEntities(aabb);
+            var affectedEntities = dimension.getEntityService().getPhysicsService().computeCollidingEntities(aabb);
             // Skip the entity that caused the explosion
             affectedEntities.remove(entity);
             var impactMap = affectedEntities.parallelStream()

api/src/main/java/org/allaymc/api/world/chunk/ChunkLoader.java

@@ -1,6 +1,5 @@
 package org.allaymc.api.world.chunk;
 
-import org.allaymc.api.entity.Entity;
 import org.allaymc.api.math.location.Location3dc;
 import org.allaymc.api.network.PacketReceiver;
 import org.jetbrains.annotations.ApiStatus;
@@ -61,22 +60,6 @@ public interface ChunkLoader extends PacketReceiver {
     @ApiStatus.OverrideOnly
     void onChunkInRangeSend(Chunk chunk);
 
-    /**
-     * Spawn an entity to this chunk loader.
-     *
-     * @param entity the entity to spawn.
-     */
-    @ApiStatus.OverrideOnly
-    void spawnEntity(Entity entity);
-
-    /**
-     * Despawn an entity from this chunk loader.
-     *
-     * @param entity the entity to despawn.
-     */
-    @ApiStatus.OverrideOnly
-    void despawnEntity(Entity entity);
-
     /**
      * A method which will be called when a chunk is out of range.
      *

api/src/main/java/org/allaymc/api/world/chunk/UnsafeChunk.java

@@ -2,7 +2,6 @@
 
 import org.allaymc.api.block.type.BlockState;
 import org.allaymc.api.blockentity.BlockEntity;
-import org.allaymc.api.entity.Entity;
 import org.allaymc.api.entity.interfaces.EntityPlayer;
 import org.allaymc.api.utils.HashUtils;
 import org.allaymc.api.world.DimensionInfo;
@@ -145,41 +144,6 @@ default int getChunkLoaderCount() {
      */
     int getZ();
 
-    /**
-     * Get the entity in the chunk by its runtime id.
-     *
-     * @param runtimeId the runtime id of the entity.
-     *
-     * @return the entity in the chunk, or {@code null} if not found.
-     */
-    Entity getEntity(long runtimeId);
-
-    /**
-     * Get all entities in the chunk.
-     *
-     * @return all entities in the chunk.
-     */
-    @UnmodifiableView
-    Map<Long, Entity> getEntities();
-
-    /**
-     * Spawn entities in this chunk to the specified player.
-     *
-     * @param player the player to spawn entities to
-     */
-    default void spawnEntitiesTo(EntityPlayer player) {
-        getEntities().values().forEach(player::spawnEntity);
-    }
-
-    /**
-     * Despawn entities in this chunk from the specified player.
-     *
-     * @param player the player to despawn entities from
-     */
-    default void despawnEntitiesFrom(EntityPlayer player) {
-        getEntities().values().forEach(player::despawnEntity);
-    }
-
     /**
      * Remove the block entity in this chunk.
      *
@@ -248,9 +212,9 @@ default void despawnEntitiesFrom(EntityPlayer player) {
     /**
      * Check if a pos in the chunk has a scheduled update.
      *
-     * @param x     the x coordinate of the pos.
-     * @param y     the y coordinate of the pos.
-     * @param z     the z coordinate of the pos.
+     * @param x the x coordinate of the pos.
+     * @param y the y coordinate of the pos.
+     * @param z the z coordinate of the pos.
      *
      * @return {@code true} if the pos has a scheduled update, otherwise {@code false}.
      *