feat(RotationSystem): Deep Learning Support

build.gradle

@@ -160,6 +160,13 @@ dependencies {
 //    includeDependency "org.graalvm.polyglot:ruby-community:24.0.2"
 //    includeDependency "org.graalvm.polyglot:llvm-native-community:24.0.2"
 
+    // Machine Learning
+    includeDependency 'ai.djl:api:0.31.1'
+    includeDependency "ai.djl.pytorch:pytorch-engine:0.31.1"
+//    runtimeOnly "ai.djl.mxnet:mxnet-engine:0.31.1"
+//    runtimeOnly "ai.djl.tensorflow:tensorflow-model-zoo:0.31.1"
+
+
     // HTTP library
     includeDependency ("com.squareup.okhttp3:okhttp:5.0.0-alpha.14") {
         exclude group: "org.jetbrains.kotlin", module: "kotlin-stdlib"

src-theme/src/routes/hud/elements/hotbar/HotBar.svelte

@@ -68,7 +68,7 @@
         {#if overlayMessage !== null}
             <div class="overlay-message" out:fade={{duration: 200}}
                  style="max-width: {slotsElement?.offsetWidth ?? 0}px">
-                <TextComponent fontSize={14} textComponent={overlayMessage.text}/>
+                <TextComponent fontSize={14} textComponent={overlayMessage.text} allowPreformatting={true} />
             </div>
         {/if}
         {#if showItemStackName && itemStackName !== null}

src/main/kotlin/net/ccbluex/liquidbounce/LiquidBounce.kt

@@ -30,6 +30,8 @@ import net.ccbluex.liquidbounce.api.services.client.ClientUpdate.update
 import net.ccbluex.liquidbounce.api.thirdparty.IpInfoApi
 import net.ccbluex.liquidbounce.config.AutoConfig.configs
 import net.ccbluex.liquidbounce.config.ConfigSystem
+import net.ccbluex.liquidbounce.deeplearn.DeepLearningEngine
+import net.ccbluex.liquidbounce.deeplearn.ModelHolster
 import net.ccbluex.liquidbounce.event.EventListener
 import net.ccbluex.liquidbounce.event.EventManager
 import net.ccbluex.liquidbounce.event.events.ClientShutdownEvent
@@ -175,15 +177,11 @@ object LiquidBounce : EventListener {
             ThemeManager
             ComponentOverlay.insertComponents()
 
-            // Load config system from disk
-            ConfigSystem.loadAll()
-
             // Netty WebSocket
             ClientInteropServer.start()
 
             // Initialize browser
             logger.info("Refresh Rate: ${mc.window.refreshRate} Hz")
-
             IntegrationListener
             BrowserManager.initBrowser()
 
@@ -215,7 +213,15 @@ object LiquidBounce : EventListener {
                         }
                     },
                     async { heads },
-                    async { configs }
+                    async { configs },
+                    async {
+                        runCatching {
+                            DeepLearningEngine.init()
+                            ModelHolster.load()
+                        }.onFailure { exception ->
+                            logger.info("Failed to initialize deep learning.", exception)
+                        }
+                    }
                 )
             }
 
@@ -286,6 +292,9 @@ object LiquidBounce : EventListener {
                 tasks?.awaitAll()
                 tasks = null
             }
+
+            // Load config system from disk
+            ConfigSystem.loadAll()
         }
     }
 

src/main/kotlin/net/ccbluex/liquidbounce/deeplearn/DeepLearningEngine.kt

@@ -0,0 +1,81 @@
+/*
+ * This file is part of LiquidBounce (https://github.com/CCBlueX/LiquidBounce)
+ *
+ * Copyright (c) 2015 - 2025 CCBlueX
+ *
+ * LiquidBounce is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LiquidBounce is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LiquidBounce. If not, see <https://www.gnu.org/licenses/>.
+ *
+ *
+ */
+package net.ccbluex.liquidbounce.deeplearn
+
+import ai.djl.engine.Engine
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import net.ccbluex.liquidbounce.config.ConfigSystem.rootFolder
+import net.ccbluex.liquidbounce.utils.client.logger
+import java.util.*
+
+object DeepLearningEngine {
+
+    var isInitialized = false
+        private set
+
+    private val deepLearningFolder = rootFolder.resolve("deeplearning").apply {
+        mkdirs()
+    }
+
+    val djlCacheFolder = deepLearningFolder.resolve("djl").apply {
+        mkdirs()
+    }
+
+    val enginesCacheFolder = deepLearningFolder.resolve("engines").apply {
+        mkdirs()
+    }
+
+    val modelsFolder = deepLearningFolder.resolve("models").apply {
+        mkdirs()
+    }
+
+    init {
+        System.setProperty("DJL_CACHE_DIR", djlCacheFolder.absolutePath)
+        System.setProperty("ENGINE_CACHE_DIR", enginesCacheFolder.absolutePath)
+
+        // Disable tracking of DJL
+        System.setProperty("OPT_OUT_TRACKING", "true")
+
+        ModelHolster
+    }
+
+    /**
+     * DJL will automatically download engine libraries, as soon we call [Engine.getInstance()],
+     * for the platform we are running on.
+     *
+     * This should be done here,
+     * as we want to make sure that the libraries are downloaded
+     * before we try to load any models.
+     */
+    suspend fun init() {
+        logger.info("[DeepLearning] Initializing engine...")
+        val engine = withContext(Dispatchers.IO) {
+            Engine.getInstance()
+        }
+        val name = engine.engineName
+        val version = engine.version
+        val deviceType = engine.defaultDevice().deviceType.uppercase(Locale.ENGLISH)
+        logger.info("[DeepLearning] Using engine $name $version on $deviceType.")
+        isInitialized = true
+    }
+
+}

src/main/kotlin/net/ccbluex/liquidbounce/deeplearn/ModelHolster.kt

@@ -0,0 +1,55 @@
+package net.ccbluex.liquidbounce.deeplearn
+
+import net.ccbluex.liquidbounce.config.types.Configurable
+import net.ccbluex.liquidbounce.deeplearn.DeepLearningEngine.modelsFolder
+import net.ccbluex.liquidbounce.deeplearn.models.MinaraiModel
+import net.ccbluex.liquidbounce.event.EventListener
+import net.ccbluex.liquidbounce.features.module.modules.render.ModuleClickGui
+import net.ccbluex.liquidbounce.utils.client.logger
+import kotlin.time.measureTimedValue
+
+object ModelHolster : EventListener, Configurable("DeepLearning") {
+
+    /**
+     * Dummy choice
+     */
+    val models = choices(this, "Model", 0) {
+        arrayOf<MinaraiModel>(MinaraiModel("Empty", it))
+    }
+
+    fun load() {
+        logger.info("[DeepLearning] Loading models...")
+
+        val choices = (modelsFolder.listFiles { file -> file.isDirectory }?.map { file ->
+            val (model, time) = measureTimedValue { MinaraiModel(file.toPath(), models) }
+            logger.info("[DeepLearning] Loaded model ${file.name} in ${time.inWholeMilliseconds}ms")
+            model
+        } ?: emptyList()).toMutableList()
+
+        // We need a new instance of [NoneChoice] in order to trigger a changed event,
+        // through [setByString] below - which is more of a hack and needs to be done properly in the future.
+        models.choices = (listOf(MinaraiModel("Empty", models)) + choices).toMutableList()
+
+        // Triggers a change event
+        models.setByString(models.activeChoice.name)
+
+        // Reload ClickGui
+        ModuleClickGui.reloadView()
+    }
+
+    fun unload() {
+        val iterator = models.choices.iterator()
+
+        while (iterator.hasNext()) {
+            val model = iterator.next()
+            model.close()
+            iterator.remove()
+        }
+    }
+
+    fun reload() {
+        unload()
+        load()
+    }
+
+}

src/main/kotlin/net/ccbluex/liquidbounce/deeplearn/data/TrainingData.kt

@@ -0,0 +1,121 @@
+package net.ccbluex.liquidbounce.deeplearn.data
+
+import com.google.gson.annotations.SerializedName
+import net.ccbluex.liquidbounce.config.gson.util.decode
+import net.ccbluex.liquidbounce.utils.aiming.data.Rotation
+import net.minecraft.util.math.Vec2f
+import net.minecraft.util.math.Vec3d
+import java.io.File
+
+/**
+ * The age defines the ticks we start tracking the entity. However, due to the fact
+ * that in simulated environments the time of combat will be much shorter,
+ * and therefore not have enough data to satisfy our whole scale of combat time.
+ *
+ * This limit will allow the model to know that we just started,
+ * until we reach the maximum training age.
+ */
+const val MAXIMUM_TRAINING_AGE = 5
+
+data class TrainingData(
+    @SerializedName(CURRENT_DIRECTION_VECTOR)
+    val currentVector: Vec3d,
+    @SerializedName(PREVIOUS_DIRECTION_VECTOR)
+    val previousVector: Vec3d,
+    @SerializedName(TARGET_DIRECTION_VECTOR)
+    val targetVector: Vec3d,
+    @SerializedName(DELTA_VECTOR)
+    val velocityDelta: Vec2f,
+
+    @SerializedName(P_DIFF)
+    val playerDiff: Vec3d,
+    @SerializedName(T_DIFF)
+    val targetDiff: Vec3d,
+
+    @SerializedName(DISTANCE)
+    val distance: Float,
+
+    @SerializedName(HURT_TIME)
+    val hurtTime: Int,
+    /**
+     * Age in this case is the Entity Age, however, we will use it later to determine
+     * the time we have been tracking this entity.
+     */
+    @SerializedName(AGE)
+    val age: Int
+) {
+
+    val currentRotation
+        get() = Rotation.fromRotationVec(currentVector)
+    val targetRotation
+        get() = Rotation.fromRotationVec(targetVector)
+    val previousRotation
+        get() = Rotation.fromRotationVec(previousVector)
+
+    /**
+     * Total delta should be in a positive direction,
+     * going from the current rotation to the target rotation.
+     */
+    val totalDelta
+        get() = currentRotation.rotationDeltaTo(targetRotation)
+
+    /**
+     * Velocity delta should be in a positive direction,
+     * going from the previous rotation to the current rotation.
+     */
+    val previousVelocityDelta
+        get() = previousRotation.rotationDeltaTo(currentRotation)
+
+    val asInput: FloatArray
+        get() = floatArrayOf(
+            // Total Delta
+            totalDelta.deltaYaw,
+            totalDelta.deltaPitch,
+
+            // Velocity Delta
+            previousVelocityDelta.deltaYaw,
+            previousVelocityDelta.deltaPitch,
+
+            // Speed
+            targetDiff.horizontalLength().toFloat() + playerDiff.horizontalLength().toFloat(),
+
+            // Distance
+            distance.toFloat()
+        )
+
+    val asOutput
+        get() = floatArrayOf(
+            velocityDelta.x,
+            velocityDelta.y
+        )
+
+    companion object {
+        const val CURRENT_DIRECTION_VECTOR = "a"
+        const val PREVIOUS_DIRECTION_VECTOR = "b"
+        const val TARGET_DIRECTION_VECTOR = "c"
+        const val DELTA_VECTOR = "d"
+        const val HURT_TIME = "e"
+        const val AGE = "f"
+        const val P_DIFF = "g"
+        const val T_DIFF = "h"
+        const val DISTANCE = "i"
+
+        fun parse(vararg file: File): List<TrainingData> {
+            val files = file.flatMap { f ->
+                if (f.isDirectory) {
+                    f.listFiles { _, name -> name.endsWith(".json") }?.toList()
+                        ?: emptyList()
+                } else {
+                    listOf(f)
+                }
+            }
+
+            return files.flatMap { file ->
+                file.inputStream().use { stream ->
+                    decode<List<TrainingData>>(stream)
+                }
+            }
+        }
+
+    }
+}