POC for integrating Remote web server with Openbot

README.md

@@ -23,6 +23,8 @@
 OpenBot leverages smartphones as brains for low-cost robots. We have designed a small electric vehicle that costs about $50 and serves as a robot body. Our software stack for Android smartphones supports advanced robotics workloads such as person following and real-time autonomous navigation.
 
 ## Get started with OpenBot
+
+- Read the [Disclaimer](DISCLAIMER.md)
 - Build your own [Robot Body](body/README.md)
 - Flash the [Arduino Firmware](firmware/README.md)
 - Install the [Android Apps](android/README.md)

android/robot/build.gradle

@@ -1,13 +1,15 @@
 plugins {
     id 'io.github.reactivecircus.app-versioning' version "1.1.2"
+    id 'com.google.gms.google-services'
 }
 
 appVersioning {
     /**
      * Git root directory used for fetching git tags.
      * Use this to explicitly set the git root directory when the root Gradle project is not the git root directory.
      */
-    gitRootDirectory.set(rootProject.file("../")) // if the .git directory is in the root Gradle project's parent directory.
+    gitRootDirectory.set(rootProject.file("../"))
+    // if the .git directory is in the root Gradle project's parent directory.
 }
 
 apply plugin: 'com.android.application'
@@ -51,16 +53,19 @@ android {
         checkReleaseBuilds false
     }
     namespace 'org.openbot'
+    packagingOptions {
+        resources.excludes.add("META-INF/*")
+    }
 }
 
 // Download default models
 project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets'
-apply from:'download.gradle'
+apply from: 'download.gradle'
 apply plugin: 'kotlin-android'
 apply plugin: 'com.google.gms.google-services'
 
 dependencies {
-    implementation fileTree(dir: 'libs', include: ['*.jar','*.aar'])
+    implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar'])
     implementation 'com.google.android.material:material:1.4.0'
 
     // Build off of stable TensorFlow Lite
@@ -91,6 +96,9 @@ dependencies {
     implementation 'androidx.lifecycle:lifecycle-viewmodel:2.3.1'
     implementation 'androidx.legacy:legacy-support-v4:1.0.0'
     implementation 'androidx.preference:preference:1.1.1'
+    implementation 'com.google.firebase:firebase-auth'
+    implementation 'com.google.firebase:firebase-firestore:24.5.0'
+    implementation 'com.google.firebase:firebase-storage:20.1.0'
 
     def nav_version = "2.3.5"
     // Java language implementation
@@ -143,8 +151,8 @@ dependencies {
     // classpath.
     androidTestImplementation 'androidx.test.espresso:espresso-idling-resource:3.3.0'
 
-    implementation platform('com.google.firebase:firebase-bom:26.4.0')
-
+    implementation platform('com.google.firebase:firebase-bom:31.5.0')
+    implementation 'com.google.android.gms:play-services-auth:20.5.0'
     // Declare the dependencies for the Crashlytics and Analytics libraries
     // When using the BoM, you don't specify versions in Firebase library dependencies
     implementation 'com.google.firebase:firebase-crashlytics'
@@ -166,4 +174,18 @@ dependencies {
     // BLE
     implementation 'com.github.Ficat:EasyBle:v2.0.2'
     implementation 'com.github.Ficat:EasyPermissions:v2.1.0'
+
+    // Socket Server
+    implementation 'com.squareup.okhttp3:okhttp:4.9.1'
+
+    //Scan QR
+    implementation 'com.google.android.gms:play-services-vision:20.1.3'
+    implementation('com.google.api-client:google-api-client-android:1.31.0') {
+        exclude group: 'org.apache.httpcomponents'
+        exclude module: 'guava-jdk5'
+    }
+    implementation('com.google.apis:google-api-services-drive:v3-rev136-1.25.0') {
+        exclude group: 'org.apache.httpcomponents'
+        exclude module: 'guava-jdk5'
+    }
 }

android/robot/google-services.json

@@ -1,34 +1,50 @@
 {
   "project_info": {
-    "project_number": "1062508888733",
-    "project_id": "openbot-public",
-    "storage_bucket": "openbot-public.appspot.com"
+    "project_number": "955078484078",
+    "firebase_url": "https://opencode-openbot-default-rtdb.asia-southeast1.firebasedatabase.app",
+    "project_id": "opencode-openbot",
+    "storage_bucket": "opencode-openbot.appspot.com"
   },
   "client": [
     {
       "client_info": {
-        "mobilesdk_app_id": "1:1062508888733:android:0fc8c6a8842e1ff5510a74",
+        "mobilesdk_app_id": "1:955078484078:android:42efc0c06fa6583165867f",
         "android_client_info": {
           "package_name": "org.openbot"
         }
       },
       "oauth_client": [
         {
-          "client_id": "1062508888733-jlrr946jdlv59vgpt7h1oa91n9gs0hd2.apps.googleusercontent.com",
+          "client_id": "955078484078-91d6gg3p2qt9eilolpqp8o9j6nqjaqdf.apps.googleusercontent.com",
+          "client_type": 1,
+          "android_info": {
+            "package_name": "org.openbot",
+            "certificate_hash": "224d8f13596161c1d53e2008aee1650856446ce7"
+          }
+        },
+        {
+          "client_id": "955078484078-kbadvp54dljcpk0g0j5bnf5c17q63ir2.apps.googleusercontent.com",
           "client_type": 3
         }
       ],
       "api_key": [
         {
-          "current_key": "AIzaSyCD59-IMWo1ZKzp-gMoTG6eKIkza_8XVYg"
+          "current_key": "AIzaSyA7AhVQcAsZJtYIrJqcvmlxNlcPgmt5bGw"
         }
       ],
       "services": {
         "appinvite_service": {
           "other_platform_oauth_client": [
             {
-              "client_id": "1062508888733-jlrr946jdlv59vgpt7h1oa91n9gs0hd2.apps.googleusercontent.com",
+              "client_id": "955078484078-kbadvp54dljcpk0g0j5bnf5c17q63ir2.apps.googleusercontent.com",
               "client_type": 3
+            },
+            {
+              "client_id": "955078484078-4m63lascf45ranfmedahhcgo10ainnfa.apps.googleusercontent.com",
+              "client_type": 2,
+              "ios_info": {
+                "bundle_id": "org.openbot"
+              }
             }
           ]
         }

android/robot/src/main/AndroidManifest.xml

@@ -16,7 +16,8 @@
     <uses-permission android:name="android.permission.CHANGE_WIFI_STATE" />
 
     <!-- RTSP Video streaming -->
-    <uses-permission android:name="android.permission.RECORD_AUDIO"/>
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+
 
     <uses-feature android:name="android.hardware.camera" android:required="true"/>
     <uses-feature android:name="android.hardware.camera.autofocus" android:required="true"/>
@@ -28,13 +29,11 @@
     <uses-feature android:name="android.hardware.bluetooth_le" android:required="true"/>
     <uses-feature android:name="android.hardware.location.gps" android:required="true"/>
 
-
-
     <application
+        android:name=".OpenBotApplication"
         android:allowBackup="false"
         android:icon="@drawable/ic_openbot"
         android:label="@string/app_name"
-        android:name=".OpenBotApplication"
         android:requestLegacyExternalStorage="true"
         android:roundIcon="@drawable/ic_openbot"
         android:supportsRtl="true"
@@ -69,24 +68,41 @@
 
         </activity>
 
+        <activity
+            android:name=".modelManagement.BackHandlingFilePickerActivity"
+            android:exported="true"
+            android:label="@string/app_name"
+            android:theme="@style/FilePickerTheme">
+            <intent-filter>
+                <action android:name="android.intent.action.GET_CONTENT" />
+                <category android:name="android.intent.category.DEFAULT" />
+            </intent-filter>
+        </activity>
+
         <service
             android:name=".logging.SensorService"
             android:enabled="true"
             android:launchMode="singleTask" />
 
-        <service android:name="net.majorkernelpanic.streaming.rtsp.RtspServer"/>
+        <service android:name="net.majorkernelpanic.streaming.rtsp.RtspServer" />
 
         <provider
             android:name="androidx.core.content.FileProvider"
             android:authorities="${applicationId}.provider"
             android:exported="false"
-            android:grantUriPermissions="true" >
+            android:grantUriPermissions="true">
             <meta-data
                 android:name="android.support.FILE_PROVIDER_PATHS"
                 android:resource="@xml/provider_paths" />
         </provider>
 
-        <meta-data android:name="com.google.ar.core" android:value="optional" />
+        <meta-data
+            android:name="com.google.ar.core"
+            android:value="optional" />
+
+        <meta-data
+            android:name="com.google.android.gms.vision.DEPENDENCIES"
+            android:value="barcode" />
 
     </application>
 

android/robot/src/main/assets/google-services.json

@@ -0,0 +1,55 @@
+{
+  "project_info": {
+    "project_number": "955078484078",
+    "firebase_url": "https://opencode-openbot-default-rtdb.asia-southeast1.firebasedatabase.app",
+    "project_id": "opencode-openbot",
+    "storage_bucket": "opencode-openbot.appspot.com"
+  },
+  "client": [
+    {
+      "client_info": {
+        "mobilesdk_app_id": "1:955078484078:android:42efc0c06fa6583165867f",
+        "android_client_info": {
+          "package_name": "org.openbot"
+        }
+      },
+      "oauth_client": [
+        {
+          "client_id": "955078484078-91d6gg3p2qt9eilolpqp8o9j6nqjaqdf.apps.googleusercontent.com",
+          "client_type": 1,
+          "android_info": {
+            "package_name": "org.openbot",
+            "certificate_hash": "224d8f13596161c1d53e2008aee1650856446ce7"
+          }
+        },
+        {
+          "client_id": "955078484078-kbadvp54dljcpk0g0j5bnf5c17q63ir2.apps.googleusercontent.com",
+          "client_type": 3
+        }
+      ],
+      "api_key": [
+        {
+          "current_key": "AIzaSyA7AhVQcAsZJtYIrJqcvmlxNlcPgmt5bGw"
+        }
+      ],
+      "services": {
+        "appinvite_service": {
+          "other_platform_oauth_client": [
+            {
+              "client_id": "955078484078-kbadvp54dljcpk0g0j5bnf5c17q63ir2.apps.googleusercontent.com",
+              "client_type": 3
+            },
+            {
+              "client_id": "955078484078-4m63lascf45ranfmedahhcgo10ainnfa.apps.googleusercontent.com",
+              "client_type": 2,
+              "ios_info": {
+                "bundle_id": "org.openbot"
+              }
+            }
+          ]
+        }
+      }
+    }
+  ],
+  "configuration_version": "1"
+}

android/robot/src/main/java/org/openbot/OpenBotApplication.java

@@ -26,6 +26,7 @@ public void onCreate() {
     SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this);
     int baudRate = Integer.parseInt(sharedPreferences.getString("baud_rate", "115200"));
     vehicle = new Vehicle(this, baudRate);
+    vehicle.initBle();
     vehicle.connectUsb();
     vehicle.initBle();
     if (BuildConfig.DEBUG) {

android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java

@@ -75,7 +75,6 @@ public View onCreateView(
       @NotNull LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
     // Inflate the layout for this fragment
     binding = FragmentAutopilotBinding.inflate(inflater, container, false);
-
     return inflateFragment(binding, inflater, container);
   }
 
@@ -100,7 +99,6 @@ public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceStat
         getModelNames(f -> f.type.equals(Model.TYPE.CMDNAV) && f.pathType != Model.PATH_TYPE.URL);
     initModelSpinner(binding.modelSpinner, models, preferencesManager.getAutopilotModel());
     initServerSpinner(binding.serverSpinner);
-
     setAnalyserResolution(Enums.Preview.HD.getValue());
     binding.deviceSpinner.setOnItemSelectedListener(
         new AdapterView.OnItemSelectedListener() {
@@ -145,6 +143,11 @@ public void onNothingSelected(AdapterView<?> parent) {}
           Navigation.findNavController(requireView()).navigate(R.id.open_usb_fragment);
         });
 
+    binding.bleToggle.setOnClickListener(
+        v -> {
+          binding.bleToggle.setChecked(vehicle.bleConnected());
+          Navigation.findNavController(requireView()).navigate(R.id.open_bluetooth_fragment);
+        });
     binding.bleToggle.setOnClickListener(
         v -> {
           binding.bleToggle.setChecked(vehicle.bleConnected());
@@ -175,9 +178,9 @@ public void onNothingSelected(AdapterView<?> parent) {}
   }
 
   private void updateCropImageInfo() {
-    //    Timber.i("%s x %s",getPreviewSize().getWidth(), getPreviewSize().getHeight());
-    //    Timber.i("%s x %s",getMaxAnalyseImageSize().getWidth(),
-    // getMaxAnalyseImageSize().getHeight());
+    //        Timber.i("%s x %s",getPreviewSize().getWidth(), getPreviewSize().getHeight());
+    //        Timber.i("%s x %s",getMaxAnalyseImageSize().getWidth(),
+    //     getMaxAnalyseImageSize().getHeight());
     frameToCropTransform = null;
 
     sensorOrientation = 90 - ImageUtils.getScreenOrientation(requireActivity());
@@ -525,7 +528,12 @@ private void setControlMode(Enums.ControlMode controlMode) {
           if (!PermissionUtils.hasControllerPermissions(requireActivity()))
             requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
           else connectPhoneController();
-
+          break;
+        case WEBSERVER:
+          binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_server);
+          if (!PermissionUtils.hasControllerPermissions(requireActivity()))
+            requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
+          else connectWebController();
           break;
       }
       Timber.d("Updating  controlMode: %s", controlMode);
@@ -563,6 +571,16 @@ private void connectPhoneController() {
     preferencesManager.setDriveMode(oldDriveMode.getValue());
   }
 
+  private void connectWebController() {
+    phoneController.connectWebServer();
+    Enums.DriveMode oldDriveMode = currentDriveMode;
+    // Currently only dual drive mode supported
+    setDriveMode(Enums.DriveMode.GAME);
+    binding.controllerContainer.driveMode.setAlpha(0.5f);
+    binding.controllerContainer.driveMode.setEnabled(false);
+    preferencesManager.setDriveMode(oldDriveMode.getValue());
+  }
+
   private void disconnectPhoneController() {
     phoneController.disconnect();
     setDriveMode(Enums.DriveMode.getByID(preferencesManager.getDriveMode()));

android/robot/src/main/java/org/openbot/common/ControlsFragment.java

@@ -453,7 +453,7 @@ public void onAddModel(String model) {
     if (modelAdapter != null && modelAdapter.getPosition(model) == -1) {
       modelAdapter.add(model);
       masterList.add(item);
-      FileUtils.updateModelConfig(requireActivity(), masterList);
+      FileUtils.updateModelConfig(requireActivity(), requireContext(), masterList, false);
     } else {
       if (model.equals(modelSpinner.getSelectedItem())) {
         setModel(item);

android/robot/src/main/java/org/openbot/common/FeatureList.java

@@ -14,6 +14,7 @@ public class FeatureList {
   public static final String GENERAL = "General";
   public static final String LEGACY = "Legacy";
   public static final String DEFAULT = "Default";
+  public static final String PROJECTS = "Projects";
   public static final String CONTROLLER = "Controller";
   public static final String CONTROLLER_MAPPING = "Controller Mapping";
   public static final String ROBOT_INFO = "Robot Info";
@@ -71,7 +72,8 @@ public static ArrayList<Category> getCategories() {
     ArrayList<SubCategory> subCategories;
 
     subCategories = new ArrayList<>();
-    subCategories.add(new SubCategory(FREE_ROAM, R.drawable.ic_game, "#FC814A"));
+    subCategories.add(new SubCategory(FREE_ROAM, R.drawable.ic_game, "#FFFF6D00"));
+
     subCategories.add(new SubCategory(DATA_COLLECTION, R.drawable.ic_storage, "#93C47D"));
     subCategories.add(new SubCategory(CONTROLLER_MAPPING, R.drawable.ic_controller, "#7268A6"));
     subCategories.add(new SubCategory(ROBOT_INFO, R.drawable.ic_openbot_space, "#4B7BFF"));