Chrobalt on-device test work for Android

.github/actions/on_device_tests/action.yaml

@@ -1,9 +1,100 @@
-name: On Host Tests
-description: Runs on-host tests.
+name: On Device Test
+description: Runs on-device tests.
+inputs:
+  gcs_results_path:
+    description: "GCS path for the test results"
+    required: true
+  results_dir:
+    description: "Path to directory where test results are saved."
+    required: true
+  test_results_key:
+    description: "Artifact key used to store test results."
+    required: true
+
 runs:
   using: "composite"
   steps:
-    - name: Run On-Device Tests
+    - name: Install Requirements
+      # TODO (b/388329764) - set up requirements file.
+      run: |
+        pip3 install grpcio==1.38.0 grpcio-tools==1.38.0
+      shell: bash
+    - name: Generate gRPC files
+      run: |
+        python -m grpc_tools.protoc -I${GITHUB_WORKSPACE}/cobalt/tools/ --python_out=${GITHUB_WORKSPACE}/cobalt/tools/ --grpc_python_out=${GITHUB_WORKSPACE}/cobalt/tools/ ${GITHUB_WORKSPACE}/cobalt/tools/on_device_tests_gateway.proto
+      shell: bash
+    - name: Set Up Cloud SDK
+      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
+    - name: Set env vars
+      run: |
+        echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV
+        # Test results and logs
+        echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}" >> $GITHUB_ENV
+        # Dimension env
+        if [ "${{ matrix.dimension }}" != "null" ]; then
+          echo "DIMENSION=${{ matrix.dimension }}" >> $GITHUB_ENV
+        fi
+      shell: bash
+    - name: Run Tests on ${{ matrix.platform }} Platform
+      env:
+        GCS_ARTIFACTS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}
+        GCS_RESULTS_PATH: ${{ inputs.gcs_results_path }}
+        GITHUB_SHA: ${{ github.sha }}
+        GITHUB_TOKEN: ${{ github.token }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
+        GITHUB_ACTOR: ${{ github.actor }}
+        GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+        GITHUB_ACTOR_ID: ${{ github.actor_id }}
+        GITHUB_REPO: ${{ github.repository }}
+        GITHUB_PR_HEAD_USER_LOGIN: ${{ github.event.pull_request.head.user.login }}
+        GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }}
+        GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }}
+        GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }}
+        GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        GITHUB_WORKFLOW: ${{ github.workflow }} 
+      run: |
+        set -uxe
+        python3 -u cobalt/tools/on_device_tests_gateway_client.py \
+          --token ${GITHUB_TOKEN} \
+          --change_id ${GITHUB_PR_NUMBER:-postsubmit} \
+          trigger \
+          --platform_json "${GITHUB_WORKSPACE}/.github/config/${{ matrix.platform}}.json" \
+          --filter_json_dir "${GITHUB_WORKSPACE}/cobalt/testing/${{ matrix.platform}}" \
+          --label builder-${{ matrix.platform }} \
+          --label builder_url-${GITHUB_RUN_URL} \
+          --label github \
+          --label ${GITHUB_EVENT_NAME} \
+          --label ${GITHUB_WORKFLOW} \
+          --label actor-${GITHUB_ACTOR} \
+          --label actor_id-${GITHUB_ACTOR_ID} \
+          --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \
+          --label sha-${GITHUB_SHA} \
+          --label repository-${GITHUB_REPO} \
+          --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \
+          --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} \
+          ${DIMENSION:+"--dimension" "$DIMENSION"} \
+          ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \
+          --gcs_archive_path "${GCS_ARTIFACTS_PATH}" \
+          --gcs_result_path "${GCS_RESULTS_PATH}"
       shell: bash
+    - name: Download ${{ matrix.platform }} Test Results
+      if: always()
+      env:
+        GCS_RESULTS_PATH: ${{ inputs.gcs_results_path }}
+        RESULTS_DIR: ${{ inputs.results_dir }}
       run: |
-        echo "Nothing yet"
+        set -uxe
+        TEST_LOGS="${GITHUB_WORKSPACE}/${RESULTS_DIR}/"
+        mkdir -p "${TEST_LOGS}"
+        gsutil cp -r "${GCS_RESULTS_PATH}/" "${TEST_LOGS}"
+        echo "TEST_LOGS=${TEST_LOGS}" >> $GITHUB_ENV
+      shell: bash
+    - name: Archive Test Results
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: ${{ inputs.test_results_key }}
+        path: ${{ env.TEST_LOGS }}
+

.github/actions/process_test_results/action.yaml

@@ -13,7 +13,7 @@ runs:
     - name: Download Test Results
       uses: actions/download-artifact@v4
       with:
-        pattern: ${{ inputs.test_results_key }}-*
+        pattern: ${{ inputs.test_results_key }}
         path: results/**/*
 
     - name: Test Summary action

.github/config/android-arm.json

@@ -21,6 +21,10 @@
         "sql_unittests",
         "url_unittests"
     ],
+    "test_dimensions": {
+        "gtest_device": "sabrina",
+        "gtest_lab": "maneki"
+    },
     "test_on_device": true,
     "includes": [
         {

.github/config/android-arm64.json

@@ -21,7 +21,6 @@
         "sql_unittests",
         "url_unittests"
     ],
-    "test_on_device": true,
     "includes": [
         {
         "name":"arm64",

.github/workflows/main.yaml

@@ -151,7 +151,7 @@ jobs:
       matrix:
         platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
         include: ${{ fromJson(needs.initialize.outputs.includes) }}
-        config: [devel, qa, gold]
+        config: [devel]
     container: ${{ needs.docker-build-image.outputs.docker_tag }}
     env:
       TEST_ARTIFACTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_artifacts
@@ -185,18 +185,57 @@ jobs:
         if: ${{ ! (contains(matrix.platform, 'android') && matrix.config == 'debug') }}
         with:
           targets: ${{ needs.initialize.outputs.targets }} ${{ needs.initialize.outputs.gtest_targets }}
-          test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }}
       - name: Upload On Device Test Artifacts
         if: matrix.config == 'devel' && needs.initialize.outputs.test_on_device == 'true'
         uses: ./src/.github/actions/upload_test_artifacts
         with:
           test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }}
           on_host: ${{ needs.initialize.outputs.test_on_host }}
           on_device: ${{ needs.initialize.outputs.test_on_device }}
+
+  # Runs on-device integration and unit tests.
+  on-device-test:
+    needs: [initialize, build]
+    # Run ODT when on_device label is applied on PR.
+    # Also, run ODT on push and schedule if not explicitly disabled via repo vars.
+    if: needs.initialize.outputs.test_on_device == 'true' && 
+        ((
+          github.event_name == 'pull_request' &&
+          contains(github.event.pull_request.labels.*.name, 'on_device') ) || ((
+              inputs.nightly == 'true' || github.event_name == 'schedule') &&
+              vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') ||
+            ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False'
+        ))
+    runs-on: [self-hosted, odt-runner]
+    name: ${{ matrix.name }}_on_device
+    permissions: {}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
+        config: [devel]
+        include: ${{ fromJson(needs.initialize.outputs.includes) }}
+    env:
+      TEST_RESULTS_DIR: ${{ matrix.name }}_test_results
+      TEST_RESULTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_results
+    steps:
+      - name: Checkout
+        uses: kaidokert/[email protected]
+        timeout-minutes: 30
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+      - name: Run On-Device Tests (${{ matrix.shard }})
+        id: on-device-tests
+        uses: ./.github/actions/on_device_tests
+        with:
+          test_results_key: ${{ env.TEST_RESULTS_KEY }}
+          gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}
+          results_dir: ${{ env.TEST_RESULTS_DIR }}
 
-  test:
+  on-host-test:
     needs: [initialize, docker-build-image, build]
-    if: needs.initialize.outputs.test_on_host == 'true' || needs.initialize.outputs.test_on_device == 'true'
+    if: needs.initialize.outputs.test_on_host == 'true'
     permissions: {}
     # TODO(b/372303096): Should have dedicated runner?
     runs-on: [self-hosted, chrobalt-linux-runner]
@@ -220,20 +259,20 @@ jobs:
           path: src
       - name: Run On-Host Tests
         id: on-host-tests
-        if: always() && needs.initialize.outputs.test_on_host == 'true'
+        if: always()
         uses: ./src/.github/actions/on_host_tests
         with:
           test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }}
           test_results_key: ${{ env.TEST_RESULTS_KEY }}
           num_gtest_shards: ${{ needs.initialize.outputs.num_gtest_shards }}
 
   test-upload:
-    needs: [initialize, docker-build-image, build, test]
+    needs: [initialize, docker-build-image, build, on-host-test, on-device-test]
     if: always() &&
-        (
-          needs.initialize.outputs.test_on_host == 'true' ||
-          needs.initialize.outputs.test_on_device == 'true'
-        )
+      (
+        needs.initialize.outputs.test_on_host == 'true' ||
+        needs.initialize.outputs.test_on_device == 'true'
+      )
     permissions: {}
     runs-on: [self-hosted, chrobalt-linux-runner]
     name: ${{ matrix.name }}_tests_upload
@@ -260,12 +299,12 @@ jobs:
           datadog_api_key: ${{ secrets.datadog_api_key }}
         continue-on-error: true
 
+
   validate-test-result:
-    needs: [initialize, docker-build-image, build, test]
+    needs: [initialize, docker-build-image, build, on-host-test]
     if: always() &&
         (
-          needs.initialize.outputs.test_on_host == 'true' ||
-          needs.initialize.outputs.test_on_device == 'true'
+          needs.initialize.outputs.test_on_host == 'true'
         )
     permissions: {}
     runs-on: ubuntu-latest

cobalt/testing/android-arm/base_unittests_filter.json

@@ -0,0 +1,6 @@
+{
+  "failing_tests": [
+    "BreakIteratorTest.BreakCharacter",
+    "ValuesUtilTest.FilePath"
+  ]
+}

cobalt/tools/on_device_tests_gateway.proto

@@ -0,0 +1,70 @@
+// Copyright 2022 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package on_device_tests_gateway;
+
+// Interface exported by the server.
+service on_device_tests_gateway {
+  // A dumb proxy RPC service that passes user defined command line options
+  // to the on-device tests gateway and streams back output in real time.
+  rpc exec_command(OnDeviceTestsCommand)
+      returns (stream OnDeviceTestsResponse) {}
+
+  // A dumb proxy RPC service that passes user defined command line options
+  // to the on-device tests gateway and streams back output in real time.
+  rpc exec_watch_command(OnDeviceTestsWatchCommand)
+      returns (stream OnDeviceTestsResponse) {}
+}
+
+// Working directory and command line arguments to be passed to the gateway.
+message OnDeviceTestsCommand {
+  string token = 2;
+  repeated string labels = 8;
+  repeated TestRequest test_requests = 24;
+}
+
+message TestRequest {
+  // Args picked up by MH, e.g. "name1=value", "name2=value.
+  repeated string test_args = 1;
+  // Args sent to test binary/apk, e.g. "--arg1", "--arg2=value".
+  repeated string test_cmd_args = 2;
+  // Files to send to device, e.g. "build_apk=/bigstore/bucket/test.apk",
+  // "test_apk=/bigstore/bucket/test.apk",
+  repeated string files = 3;
+  // e.g. "gcs_result_path=gs://some/gcs/path"
+  repeated string params = 4;
+  // "sabrina" or "boreal"
+  string device_type = 5;
+  // "shared" or "maneki"
+  string device_pool = 6;
+}
+
+// Working directory and command line arguments to be passed to the gateway.
+message OnDeviceTestsWatchCommand {
+  // Next ID: 6
+  string workdir = 1;
+  string token = 2;
+  string session_id = 3;
+  string change_id = 4;
+  bool dry_run = 5;
+}
+
+// Response from the on-device tests.
+message OnDeviceTestsResponse {
+  // Next ID: 2
+  string response = 1;
+}
+