Skip to content

Commit

Permalink
JavaBridge Implementation of H5vccPlatformService (#4525)
Browse files Browse the repository at this point in the history
1. Fixed the race condition of injecting the java bridge object and
loading the url from shell.
2. The pattern to polyfill the embeded javascript in the APK binary
before Kabuki loads won't work, see context on b/379731250, we will move
the javascript file chrobalt_preload.js and its sub modules on to Kabuki
codebase. I will create a backlog bug for it when this code is
submitted, and work with Kabuki eng to integrate the code on Kabuki.
3. A lot of H5vccPlatformService implementations are on google3. We had
ClientLogInfo in the Cobalt repo, it did not cover all use cases
H5vccPlatformService provides. I expand it to cover all use cases.

b/379731250
b/379184982
b/378571210

---------

Co-authored-by: Colin Liang <[email protected]>
  • Loading branch information
zhongqiliang and Colin Liang authored Dec 5, 2024
1 parent 0116de3 commit 399c7ca
Show file tree
Hide file tree
Showing 17 changed files with 338 additions and 81 deletions.
3 changes: 1 addition & 2 deletions cobalt/android/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ android_library("cobalt_apk_java") {
"apk/app/src/main/java/dev/cobalt/coat/javabridge/AmatiDeviceInspector.java",
"apk/app/src/main/java/dev/cobalt/coat/javabridge/CobaltJavaScriptAndroidObject.java",
"apk/app/src/main/java/dev/cobalt/coat/javabridge/CobaltJavaScriptInterface.java",
"apk/app/src/main/java/dev/cobalt/coat/javabridge/H5vccPlatformService.java",
"apk/app/src/main/java/dev/cobalt/coat/javabridge/HTMLMediaElementExtension.java",

# "apk/app/src/main/java/dev/cobalt/coat/CobaltMediaSession.java",
Expand Down Expand Up @@ -115,8 +116,6 @@ android_library("cobalt_apk_java") {
android_assets("cobalt_apk_assets") {
testonly = true
sources = [
"apk/app/src/app/assets/amati_device_inspector.js",
"apk/app/src/app/assets/html_media_element_extension.js",
"apk/app/src/app/assets/not_empty.txt",
"apk/app/src/app/assets/test/not_empty.txt",
"apk/app/src/app/assets/web/cobalt_blue_splash_screen.css",
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import { initializeH5vccPlatformService } from './h5vcc_platform_service.js';
import { initializeHTMLMediaElement } from './html_media_element_extension.js';

export function chrobaltPreload() {
initializeH5vccPlatformService();
initializeHTMLMediaElement();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
function arrayBufferToBase64(buffer) {
const bytes = new Uint8Array(buffer);
let binary = '';
for (let i = 0; i < bytes.byteLength; i++) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary); // Encode binary string to Base64
}

function base64ToArrayBuffer(base64) {
const binaryString = window.atob(base64); // Decode Base64 string to binary string
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes.buffer;
}

class PlatformServiceClient {
constructor(name) {
this.name = name;
}

send(data) {
// convert the ArrayBuffer to base64 string because java bridge only takes primitive types as input.
const convertToB64 = arrayBufferToBase64(data);
const responseData = Android_H5vccPlatformService.platformServiceSend(this.name, convertToB64);
if (responseData === "") {
return null;
}

// response_data has the synchronize response data converted to base64 string.
// convert it to ArrayBuffer, and return the ArrayBuffer to client.
return base64ToArrayBuffer(responseData);
}

close() {
Android_H5vccPlatformService.closePlatformService(this.name);
}
}

export function initializeH5vccPlatformService() {
if (typeof Android_H5vccPlatformService === 'undefined') {
return;
}

// On Chrobalt, register window.H5vccPlatformService
window.H5vccPlatformService = {
// Holds the callback functions for the platform services when open() is called.
callbacks: {
},
callbackFromAndroid: (serviceID, dataFromJava) => {
const arrayBuffer = base64ToArrayBuffer(dataFromJava);
window.H5vccPlatformService.callbacks[serviceID].callback(serviceID, arrayBuffer);
},
has: (name) => {
return Android_H5vccPlatformService.hasPlatformService(name);
},
open: function(name, callback) {
if (typeof callback !== 'function') {
throw new Error("window.H5vccPlatformService.open(), missing or invalid callback function.");
}

const serviceID = Object.keys(this.callbacks).length + 1;
// Store the callback with the service ID, name, and callback
window.H5vccPlatformService.callbacks[serviceID] = {
name: name,
callback: callback
};
Android_H5vccPlatformService.openPlatformService(serviceID, name);
return new PlatformServiceClient(name);
},
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/**
* @license
* Copyright The Cobalt Authors.
* SPDX-License-Identifier: Apache-2.0
*/

export function initializeHTMLMediaElement() {
if (typeof HTMLMediaElementExtension !== 'undefined') {
HTMLMediaElement.prototype.canPlayType = HTMLMediaElementExtension.canPlayType;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@
import android.media.AudioManager;
import android.net.Uri;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.text.TextUtils;
import android.util.Pair;
import android.view.KeyEvent;
Expand All @@ -36,14 +34,13 @@
import android.widget.LinearLayout;
import android.widget.Toast;
import dev.cobalt.app.CobaltApplication;
import dev.cobalt.coat.javabridge.AmatiDeviceInspector;
import dev.cobalt.coat.javabridge.CobaltJavaScriptAndroidObject;
import dev.cobalt.coat.javabridge.CobaltJavaScriptInterface;
import dev.cobalt.coat.javabridge.H5vccPlatformService;
import dev.cobalt.coat.javabridge.HTMLMediaElementExtension;
import dev.cobalt.media.AudioOutputManager;
import dev.cobalt.media.MediaCodecCapabilitiesLogger;
import dev.cobalt.media.VideoSurfaceView;
import dev.cobalt.util.AssetLoader;
import dev.cobalt.util.DisplayUtil;
import dev.cobalt.util.Log;
import dev.cobalt.util.UsedByNative;
Expand Down Expand Up @@ -81,7 +78,6 @@ public abstract class CobaltActivity extends Activity {

private static final Pattern URL_PARAM_PATTERN = Pattern.compile("^[a-zA-Z0-9_=]*$");

public static final int JAVA_BRIDGE_INITIALIZATION_DELAY_MILLI_SECONDS = 100;
// Maintain the list of JavaScript-exposed objects as a member variable
// to prevent them from being garbage collected prematurely.
private List<CobaltJavaScriptAndroidObject> javaScriptAndroidObjectList = new ArrayList<>();
Expand Down Expand Up @@ -229,7 +225,14 @@ private void finishInitialization(Bundle savedInstanceState) {
}
// Set to overlay video mode.
mShellManager.getContentViewRenderView().setOverlayVideoMode(true);
mShellManager.launchShell(shellUrl);

// Load an empty page to let shell create WebContents.
mShellManager.launchShell("");
// Inject JavaBridge objects to the WebContents.
initializeJavaBridge();
// Load the `url` with the same shell we created above.
Log.i(TAG, "shellManager load url:" + shellUrl);
mShellManager.getActiveShell().loadUrl(shellUrl);

toggleFullscreenMode(true);
}
Expand Down Expand Up @@ -361,8 +364,6 @@ protected void onCreate(Bundle savedInstanceState) {
videoSurfaceView = new VideoSurfaceView(this);
addContentView(
videoSurfaceView, new LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT));

initializeJavaBridge();
}

/**
Expand All @@ -374,24 +375,14 @@ private void initializeJavaBridge() {

WebContents webContents = getActiveWebContents();
if (webContents == null) {
// WebContents not initialized yet, post a delayed runnable to check again
new Handler(Looper.getMainLooper())
.postDelayed(
new Runnable() {
@Override
public void run() {
initializeJavaBridge(); // Recursive call to check again
}
},
JAVA_BRIDGE_INITIALIZATION_DELAY_MILLI_SECONDS);
return;
throw new RuntimeException("webContents is null in initializeJavaBridge. This should never happen.");
}

// --- Initialize the Java Bridge ---

// 1. Gather all Java objects that need to be exposed to JavaScript.
// TODO(b/379701165): consider to refine the way to add JavaScript interfaces.
javaScriptAndroidObjectList.add(new AmatiDeviceInspector(this));
javaScriptAndroidObjectList.add(new H5vccPlatformService(this, getStarboardBridge()));
javaScriptAndroidObjectList.add(new HTMLMediaElementExtension(this));

// 2. Use JavascriptInjector to inject Java objects into the WebContents.
Expand All @@ -408,16 +399,6 @@ public void run() {
javascriptAndroidObject.getJavaScriptInterfaceName(),
CobaltJavaScriptInterface.class);
}

// 3. Load and evaluate JavaScript code that interacts with the injected Java objects.
for (CobaltJavaScriptAndroidObject javaScriptAndroidObject : javaScriptAndroidObjectList) {
String jsFileName = javaScriptAndroidObject.getJavaScriptAssetName();
if (jsFileName != null) {
Log.d(TAG, "Evaluate JavaScript from Asset:" + jsFileName);
String jsCode = AssetLoader.loadJavaScriptFromAssets(this, jsFileName);
webContents.evaluateJavaScript(jsCode, null);
}
}
}

/**
Expand Down Expand Up @@ -695,4 +676,18 @@ public void onLowMemory() {
public long getAppStartTimestamp() {
return timeInNanoseconds;
}

public void evaluateJavaScript(String jsCode) {
// evaluateJavaScript must run on UI thread.
runOnUiThread(
new Runnable() {
@Override
public void run() {
WebContents webContents = getActiveWebContents();
if (webContents != null) {
webContents.evaluateJavaScript(jsCode, null);
}
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import static dev.cobalt.util.Log.TAG;

import android.util.Base64;
import dev.cobalt.util.Log;
import dev.cobalt.util.UsedByNative;

Expand All @@ -24,6 +25,8 @@ public abstract class CobaltService {
// Indicate is the service opened, and be able to send data to client
protected boolean opened = true;
private final Object lock = new Object();
private StarboardBridge bridge;
protected CobaltActivity cobaltActivity;

/** Interface that returns an object that extends CobaltService. */
public interface Factory {
Expand All @@ -37,6 +40,10 @@ public interface Factory {
/** Take in a reference to StarboardBridge & use it as needed. Default behavior is no-op. */
public void receiveStarboardBridge(StarboardBridge bridge) {}

public void setCobaltActivity(CobaltActivity cobaltActivity) {
this.cobaltActivity = cobaltActivity;
}

// Lifecycle
/** Prepare service for start or resume. */
public abstract void beforeStartOrResume();
Expand Down Expand Up @@ -87,13 +94,16 @@ public void onClose() {

/**
* Send data from the service to the client.
*
*/
protected void sendToClient(long nativeService, byte[] data) {
// TODO(b/372558900): Implement Javascript Injection
}
if (this.cobaltActivity == null) {
Log.e(TAG, "CobaltActivity is null, can not run evaluateJavaScript()");
return;
}

private void nativeSendToClient(long nativeService, byte[] data) {
// TODO(b/372558900): Implement Javascript Injection
// Use Base64.NO_WRAP instead of Base64.DEFAULT to avoid adding a new line.
String base64Data = Base64.encodeToString(data, Base64.NO_WRAP);
String jsCode = String.format("window.H5vccPlatformService.callbackFromAndroid(%d, '%s');", nativeService, base64Data);
this.cobaltActivity.evaluateJavaScript(jsCode);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -735,15 +735,11 @@ public void registerCobaltService(CobaltService.Factory factory) {
cobaltServiceFactories.put(factory.getServiceName(), factory);
}

@SuppressWarnings("unused")
@UsedByNative
boolean hasCobaltService(String serviceName) {
public boolean hasCobaltService(String serviceName) {
return cobaltServiceFactories.get(serviceName) != null;
}

@SuppressWarnings("unused")
@UsedByNative
CobaltService openCobaltService(long nativeService, String serviceName) {
public CobaltService openCobaltService(long nativeService, String serviceName) {
if (cobaltServices.get(serviceName) != null) {
// Attempting to re-open an already open service fails.
Log.e(TAG, String.format("Cannot open already open service %s", serviceName));
Expand All @@ -758,6 +754,11 @@ CobaltService openCobaltService(long nativeService, String serviceName) {
if (service != null) {
service.receiveStarboardBridge(this);
cobaltServices.put(serviceName, service);

Activity activity = activityHolder.get();
if (activity instanceof CobaltActivity) {
service.setCobaltActivity((CobaltActivity) activity);
}
}
return service;
}
Expand All @@ -766,12 +767,25 @@ public CobaltService getOpenedCobaltService(String serviceName) {
return cobaltServices.get(serviceName);
}

@SuppressWarnings("unused")
@UsedByNative
void closeCobaltService(String serviceName) {
public void closeCobaltService(String serviceName) {
cobaltServices.remove(serviceName);
}

public byte[] sendToCobaltService(String serviceName, byte [] data) {
CobaltService service = cobaltServices.get(serviceName);
if (service == null) {
Log.e(TAG, String.format("Service not opened: %s", serviceName));
return null;
}
CobaltService.ResponseToClient response = service.receiveFromClient(data);
if (response.invalidState) {
Log.e(TAG, String.format("Service %s received invalid data, closing.", serviceName));
closeCobaltService(serviceName);
return null;
}
return response.data;
}

/** Returns the application start timestamp. */
@SuppressWarnings("unused")
@UsedByNative
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,6 @@ public String getJavaScriptInterfaceName() {
return "Android_AmatiDeviceInspector";
}

@Override
public String getJavaScriptAssetName() {
return "amati_device_inspector.js";
}

@CobaltJavaScriptInterface
public void printIsAmatiDevice() {
boolean isAmatiDevice = context.getPackageManager().hasSystemFeature("com.google.android.feature.AMATI_EXPERIENCE");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@

package dev.cobalt.coat.javabridge;

import androidx.annotation.Nullable;

/**
* Interface for Android objects that are exposed to JavaScript.
*/
Expand All @@ -28,12 +26,4 @@ public interface CobaltJavaScriptAndroidObject {
* @return The JavaScript interface name.
*/
public String getJavaScriptInterfaceName();

/**
* Gets the name of the JavaScript asset file that uses this interface.
* This allows the JavaScript code to be loaded and interact with this object.
*
* @return The name of the JavaScript asset file, or null if not applicable.
*/
public @Nullable String getJavaScriptAssetName();
}
Loading

0 comments on commit 399c7ca

Please sign in to comment.