Merge pull request #12 from Mygod/brotli

Support brotli compression
Mygod · Nov 14, 2024 · e79c451 · e79c451
2 parents 303e662 + c762bd1
commit e79c451
Show file tree

Hide file tree

Showing 14 changed files with 134 additions and 30 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,7 +3,7 @@ jobs:
   build:
     working_directory: ~/code
     docker:
-      - image: cimg/android:2024.08.1
+      - image: cimg/android:2024.11.1-ndk
     environment:
       GRADLE_OPTS: -Dorg.gradle.workers.max=1 -Dorg.gradle.daemon=false -Dkotlin.compiler.execution.strategy="in-process"
     steps:

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "brotli"]
+	path = brotli
+	url = https://github.com/google/brotli.git
diff --git a/app/build.gradle.kts b/app/build.gradle.kts
@@ -56,6 +56,17 @@ android {
     kotlinOptions.jvmTarget = javaVersion.toString()
     packaging.resources.excludes += "/META-INF/{AL2.0,LGPL2.1}"
     lint.informational.add("MissingTranslation")
+
+    sourceSets.getByName("main") {
+        java.srcDirs("../brotli/java")
+        java.excludes.add("**/brotli/**/*Test.java")
+    }
+    externalNativeBuild {
+        cmake {
+            path = file("src/main/cpp/CMakeLists.txt")
+            version = "3.22.1"
+        }
+    }
 }
 
 dependencies {
@@ -66,7 +77,7 @@ dependencies {
     implementation(libs.browser)
     implementation(libs.core.ktx)
     implementation(libs.firebase.analytics)
-    implementation(libs.firebase.crashlytics)
+    implementation(libs.firebase.crashlytics.ndk)
     implementation(libs.fragment.ktx)
     implementation(libs.hiddenapibypass)
     implementation(libs.material)

diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt
@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.22.1)
+project("brotli")
+set(BROTLI_DIR "../../../../brotli")
+file(GLOB COMMON_SOURCES "${BROTLI_DIR}/c/common/*.c")
+file(GLOB ENC_SOURCES "${BROTLI_DIR}/c/enc/*.c")
+add_library(${CMAKE_PROJECT_NAME} SHARED
+    ${COMMON_SOURCES}
+    ${ENC_SOURCES}
+    ${BROTLI_DIR}/java/org/brotli/wrapper/enc/encoder_jni.cc
+)
+target_link_libraries(${CMAKE_PROJECT_NAME}
+    # List libraries link to the target library
+    #android
+    #log
+)
+include_directories(
+    ${BROTLI_DIR}/c/include
+)
diff --git a/app/src/main/java/be/mygod/reactmap/ConfigDialogFragment.kt b/app/src/main/java/be/mygod/reactmap/ConfigDialogFragment.kt
@@ -74,17 +74,18 @@ class ConfigDialogFragment : AlertDialogFragment<ConfigDialogFragment.Arg, Empty
     }
 
     override val ret get() = try {
-        val uri = urlEdit.text!!.toString().toUri().let {
-            require(BuildConfig.DEBUG || "https".equals(it.scheme, true)) { getText(R.string.error_https_only) }
-            it.host!!
-            it.toString()
-        }
+        val uri = urlEdit.text!!.toString().toUri()
+        require(BuildConfig.DEBUG || "https".equals(uri.scheme, true)) { getText(R.string.error_https_only) }
+        uri.host!!
+        val uriString = uri.toString()
         val oldApiUrl = ReactMapHttpEngine.apiUrl
+        val changing = oldApiUrl != ReactMapHttpEngine.apiUrl(uri)
         app.pref.edit {
-            putString(App.KEY_ACTIVE_URL, uri)
-            putStringSet(KEY_HISTORY_URL, historyUrl + uri)
+            putString(App.KEY_ACTIVE_URL, uriString)
+            putStringSet(KEY_HISTORY_URL, historyUrl + uriString)
+            if (changing) remove(ReactMapHttpEngine.KEY_BROTLI)
         }
-        if (oldApiUrl != ReactMapHttpEngine.apiUrl) BackgroundLocationReceiver.onApiChanged()
+        if (changing) BackgroundLocationReceiver.onApiChanged()
         Empty()
     } catch (e: Exception) {
         Toast.makeText(requireContext(), e.readableMessage, Toast.LENGTH_LONG).show()

diff --git a/app/src/main/java/be/mygod/reactmap/follower/LocationSetter.kt b/app/src/main/java/be/mygod/reactmap/follower/LocationSetter.kt
@@ -165,6 +165,10 @@ class LocationSetter(appContext: Context, workerParams: WorkerParameters) : Coro
                 }.build())
                 Result.success()
             }
+            302 -> {
+                ReactMapHttpEngine.detectBrotliError(conn)?.let { notifyError(it) }
+                Result.retry()
+            }
             else -> {
                 val error = conn.findErrorStream.bufferedReader().readText()
                 notifyErrors(error)

diff --git a/app/src/main/java/be/mygod/reactmap/webkit/BaseReactMapFragment.kt b/app/src/main/java/be/mygod/reactmap/webkit/BaseReactMapFragment.kt
@@ -29,6 +29,7 @@ import androidx.annotation.RequiresApi
 import androidx.core.net.toUri
 import androidx.core.os.bundleOf
 import androidx.fragment.app.Fragment
+import androidx.lifecycle.lifecycleScope
 import be.mygod.reactmap.App.Companion.app
 import be.mygod.reactmap.BuildConfig
 import be.mygod.reactmap.R
@@ -37,6 +38,7 @@ import be.mygod.reactmap.util.UnblockCentral
 import be.mygod.reactmap.util.findErrorStream
 import com.google.android.material.snackbar.Snackbar
 import com.google.firebase.analytics.FirebaseAnalytics
+import kotlinx.coroutines.launch
 import org.json.JSONArray
 import org.json.JSONException
 import org.json.JSONObject
@@ -123,6 +125,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {
 
     protected lateinit var web: WebView
     protected lateinit var glocation: Glocation
+    private lateinit var postInterceptor: PostInterceptor
     protected lateinit var hostname: String
 
     private var loginText: String? = null
@@ -152,6 +155,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {
                 javaScriptEnabled = true
             }
             glocation = Glocation(this, this@BaseReactMapFragment)
+            postInterceptor = PostInterceptor(this)
             webChromeClient = object : WebChromeClient() {
                 @Suppress("KotlinConstantConditions")
                 override fun onConsoleMessage(consoleMessage: ConsoleMessage) = consoleMessage.run {
@@ -194,11 +198,15 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {
 
                 override fun onPageStarted(view: WebView, url: String, favicon: Bitmap?) {
                     glocation.clear()
+                    postInterceptor.clear()
                     val uri = url.toUri()
                     if (!BuildConfig.DEBUG && "http".equals(uri.scheme, true)) {
                         web.loadUrl(uri.buildUpon().scheme("https").build().toString())
                     }
-                    if (uri.host == hostname) glocation.setupGeolocation()
+                    if (uri.host == hostname) {
+                        glocation.setupGeolocation()
+                        postInterceptor.setup()
+                    }
                     onPageStarted()
                 }
 
@@ -233,9 +241,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {
                             return handleTranslation(request)
                         }
                         if (vendorJsMatcher.matchEntire(path) != null) return handleVendorJs(request)
-                        if (path == "/graphql" && request.method == "POST") {
-                            request.requestHeaders.remove("_interceptedBody")?.let { return handleGraphql(request, it) }
-                        }
+                        postInterceptor.extractBody(request)?.let { return handleGraphql(request, it) }
                     }
                     if (ReactMapHttpEngine.isCronet && (path.substringAfterLast('.').lowercase(Locale.ENGLISH)
                                 in mediaExtensions || request.requestHeaders.any { (key, value) ->
@@ -374,7 +380,12 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {
             setupConnection(request, conn)
             ReactMapHttpEngine.writeCompressed(conn, body)
         }
-        createResponse(conn) { _ -> conn.findErrorStream }
+        if (conn.responseCode == 302) {
+            ReactMapHttpEngine.detectBrotliError(conn)?.let {
+                lifecycleScope.launch { Snackbar.make(web, it, Snackbar.LENGTH_LONG).show() }
+            }
+            null
+        } else createResponse(conn) { _ -> conn.findErrorStream }
     } catch (e: IOException) {
         Timber.d(e)
         null

diff --git a/app/src/main/java/be/mygod/reactmap/webkit/Glocation.kt b/app/src/main/java/be/mygod/reactmap/webkit/Glocation.kt
@@ -53,7 +53,7 @@ class Glocation(private val web: WebView, private val fragment: BaseReactMapFrag
         fragment.lifecycle.addObserver(this)
         it.context
     }
-    private val jsSetup = fragment.resources.openRawResource(R.raw.setup).bufferedReader().readText()
+    private val jsSetup = fragment.resources.openRawResource(R.raw.setup_glocation).bufferedReader().readText()
     private val pendingRequests = mutableSetOf<Long>()
     private var pendingWatch = false
     private val activeListeners = mutableSetOf<Long>()

diff --git a/app/src/main/java/be/mygod/reactmap/webkit/PostInterceptor.kt b/app/src/main/java/be/mygod/reactmap/webkit/PostInterceptor.kt
@@ -0,0 +1,32 @@
+package be.mygod.reactmap.webkit
+
+import android.util.LongSparseArray
+import android.webkit.JavascriptInterface
+import android.webkit.WebResourceRequest
+import android.webkit.WebView
+import be.mygod.reactmap.R
+import com.google.common.hash.Hashing
+import java.nio.charset.Charset
+
+class PostInterceptor(private val web: WebView) {
+    private val bodyLookup = LongSparseArray<String>().also {
+        web.addJavascriptInterface(this, "_postInterceptor")
+    }
+    private val jsSetup = web.resources.openRawResource(R.raw.setup_interceptor).bufferedReader().readText()
+
+    fun setup() = web.evaluateJavascript(jsSetup, null)
+
+    @JavascriptInterface
+    fun register(body: String): String {
+        val key = Hashing.sipHash24().hashString(body, Charset.defaultCharset()).asLong()
+        synchronized(bodyLookup) { bodyLookup.put(key, body) }
+        return key.toULong().toString(36)
+    }
+    fun extractBody(request: WebResourceRequest) = request.requestHeaders.remove("Body-Digest")?.let { key ->
+        synchronized(bodyLookup) {
+            val index = bodyLookup.indexOfKey(key.toULong(36).toLong())
+            if (index < 0) null else bodyLookup.valueAt(index).also { bodyLookup.removeAt(index) }
+        }
+    }
+    fun clear() = synchronized(bodyLookup) { bodyLookup.clear() }
+}
diff --git a/app/src/main/java/be/mygod/reactmap/webkit/ReactMapHttpEngine.kt b/app/src/main/java/be/mygod/reactmap/webkit/ReactMapHttpEngine.kt
@@ -1,5 +1,6 @@
 package be.mygod.reactmap.webkit
 
+import android.net.Uri
 import android.net.http.ConnectionMigrationOptions
 import android.net.http.HttpEngine
 import android.os.Build
@@ -14,6 +15,9 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.suspendCancellableCoroutine
+import org.brotli.wrapper.enc.BrotliOutputStream
+import org.brotli.wrapper.enc.Encoder
+import timber.log.Timber
 import java.io.ByteArrayOutputStream
 import java.io.File
 import java.net.HttpURLConnection
@@ -25,6 +29,7 @@ import kotlin.coroutines.resumeWithException
 
 object ReactMapHttpEngine {
     private const val KEY_COOKIE = "cookie.graphql"
+    const val KEY_BROTLI = "http.brotli"
 
     val isCronet get() = Build.VERSION.SDK_INT >= 34 || Build.VERSION.SDK_INT >= Build.VERSION_CODES.S &&
             SdkExtensions.getExtensionVersion(Build.VERSION_CODES.S) >= 7
@@ -44,9 +49,10 @@ object ReactMapHttpEngine {
         }.build()
     }
 
-    val apiUrl get() = app.activeUrl.toUri().buildUpon().apply {
+    fun apiUrl(base: Uri) = base.buildUpon().apply {
         path("/graphql")
     }.build().toString()
+    val apiUrl get() = apiUrl(app.activeUrl.toUri())
 
     private fun openConnection(url: String) = (if (isCronet) {
         engine.openConnection(URL(url))
@@ -92,16 +98,33 @@ object ReactMapHttpEngine {
         val buffer get() = buf
         val length get() = count
     }
+    private val initBrotli by lazy { System.loadLibrary("brotli") }
     fun writeCompressed(conn: HttpURLConnection, body: String) {
-        conn.setRequestProperty("Content-Encoding", "deflate")
+        val brotli = app.pref.getBoolean(KEY_BROTLI, true)
+        conn.setRequestProperty("Content-Encoding", if (brotli) {
+            initBrotli
+            "br"
+        } else "deflate")
         conn.doOutput = true
+        conn.instanceFollowRedirects = false
         val uncompressed = body.toByteArray()
         val out = ExposingBufferByteArrayOutputStream()
-        DeflaterOutputStream(out, Deflater(Deflater.BEST_COMPRESSION)).use {
-            it.write(uncompressed)
-        }
-        // Timber.tag("CompressionStat").i("${out.length}/${uncompressed.size} ~ ${out.length.toDouble() / uncompressed.size}")
+//        val time = System.nanoTime()
+        (if (brotli) BrotliOutputStream(out, Encoder.Parameters().apply {
+            setMode(Encoder.Mode.TEXT)
+            setQuality(5)
+        }) else DeflaterOutputStream(out, Deflater(Deflater.BEST_COMPRESSION))).use { it.write(uncompressed) }
+//        Timber.tag("CompressionStat").i("$brotli ${out.length}/${uncompressed.size} ~ ${out.length.toDouble() / uncompressed.size} ${(System.nanoTime() - time) * .000_001}ms")
         conn.setFixedLengthStreamingMode(out.length)
         conn.outputStream.use { it.write(out.buffer, 0, out.length) }
     }
+
+    fun detectBrotliError(conn: HttpURLConnection): String? {
+        val path = conn.getHeaderField("Location")
+        if (path.startsWith("/error/")) return Uri.decode(path.substring(7)).also {
+            if (conn.url.host == app.activeUrl.toUri().host && it == "unsupported content encoding \"br\"") app.pref.edit { putBoolean(KEY_BROTLI, false) }
+        }
+        Timber.w(Exception(path))
+        return path
+    }
 }
diff --git a/app/src/main/res/raw/setup.js → app/src/main/res/raw/setup_glocation.js b/app/src/main/res/raw/setup.js → app/src/main/res/raw/setup_glocation.js
@@ -48,10 +48,3 @@ Object.defineProperty(navigator, 'geolocation', {
         },
     },
 });
-window._fetch = window.fetch;
-window.fetch = function (input, init = {}) {
-    if (input === '/graphql' && init.method === 'POST' && init.body) {
-        init.headers['_interceptedBody'] = init.body;
-    }
-    return window._fetch(input, init);
-};
diff --git a/app/src/main/res/raw/setup_interceptor.js b/app/src/main/res/raw/setup_interceptor.js
@@ -0,0 +1,7 @@
+window._fetch = window.fetch;
+window.fetch = function (input, init = {}) {
+    if (input === '/graphql' && init.method === 'POST' && init.body) {
+        init.headers['Body-Digest'] = window._postInterceptor.register(init.body);
+    }
+    return window._fetch(input, init);
+};
diff --git a/brotli b/brotli
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -7,7 +7,7 @@ desugar = { group = "com.android.tools", name = "desugar_jdk_libs", version = "2
 espresso-core = { group = "androidx.test.espresso", name = "espresso-core", version = "3.6.1" }
 firebase-analytics = { group = "com.google.firebase", name = "firebase-analytics" }
 firebase-bom = { group = "com.google.firebase", name = "firebase-bom", version = "33.5.1" }
-firebase-crashlytics = { group = "com.google.firebase", name = "firebase-crashlytics" }
+firebase-crashlytics-ndk = { group = "com.google.firebase", name = "firebase-crashlytics-ndk" }
 fragment-ktx = { group = "androidx.fragment", name = "fragment-ktx", version = "1.8.5" }
 hiddenapibypass = { group = "org.lsposed.hiddenapibypass", name = "hiddenapibypass", version = "4.3" }
 junit = { group = "junit", name = "junit", version = "4.13.2" }