Add on-device tests

.github/actions/on_device_tests/action.yaml

@@ -1,9 +1,97 @@
-name: On Host Tests
-description: Runs on-host tests.
+name: On Device Test
+description: Runs on-device tests.
+inputs:
+  gcs_results_path:
+    description: "GCS path for the test results"
+    required: true
+  results_dir:
+    description: "Path to directory where test results are saved."
+    required: true
+  test_results_key:
+    description: "Artifact key used to store test results."
+    required: true
+
 runs:
   using: "composite"
   steps:
-    - name: Run On-Device Tests
+    - name: Install Requirements
+      # TODO (b/388329764) - set up requirements file.
+      run: |
+        pip3 install grpcio==1.38.0 grpcio-tools==1.38.0
+      shell: bash
+    - name: Generate gRPC files
+      run: |
+        python -m grpc_tools.protoc -I${GITHUB_WORKSPACE}/cobalt/tools/ --python_out=${GITHUB_WORKSPACE}/cobalt/tools/ --grpc_python_out=${GITHUB_WORKSPACE}/cobalt/tools/ ${GITHUB_WORKSPACE}/cobalt/tools/on_device_tests_gateway.proto
+      shell: bash
+    - name: Set Up Cloud SDK
+      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
+    - name: Set env vars
+      run: |
+        echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV
+        # Test results and logs
+        echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}" >> $GITHUB_ENV
+      shell: bash
+    - name: Run Tests on ${{ matrix.platform }} Platform
+      env:
+        # TODO(b/382508397): Replace hardcoded list with dynamically generated one.
+        TEST_TARGETS_JSON_FILE: cobalt/build/testing/targets/${{ matrix.platform }}/test_targets.json
+        GCS_ARTIFACTS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}
+        GCS_RESULTS_PATH: ${{ inputs.gcs_results_path }}
+        GITHUB_SHA: ${{ github.sha }}
+        GITHUB_TOKEN: ${{ github.token }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
+        GITHUB_ACTOR: ${{ github.actor }}
+        GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+        GITHUB_ACTOR_ID: ${{ github.actor_id }}
+        GITHUB_REPO: ${{ github.repository }}
+        GITHUB_PR_HEAD_USER_LOGIN: ${{ github.event.pull_request.head.user.login }}
+        GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }}
+        GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }}
+        GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }}
+        GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        GITHUB_WORKFLOW: ${{ github.workflow }}
+      run: |
+        set -uxe
+        python3 -u cobalt/tools/on_device_tests_gateway_client.py \
+          --token ${GITHUB_TOKEN} \
+          trigger \
+          --targets $(cat "${{ env.TEST_TARGETS_JSON_FILE }}" | jq -cr '.test_targets | join(",")') \
+          --filter_json_dir "${GITHUB_WORKSPACE}/cobalt/testing/filters/${{ matrix.platform}}" \
+          --label github_${GITHUB_PR_NUMBER:-postsubmit} \
+          --label builder-${{ matrix.platform }} \
+          --label builder_url-${GITHUB_RUN_URL} \
+          --label github \
+          --label ${GITHUB_EVENT_NAME} \
+          --label ${GITHUB_WORKFLOW} \
+          --label actor-${GITHUB_ACTOR} \
+          --label actor_id-${GITHUB_ACTOR_ID} \
+          --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \
+          --label sha-${GITHUB_SHA} \
+          --label repository-${GITHUB_REPO} \
+          --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \
+          --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} \
+          ${DIMENSION:+"--dimension" "$DIMENSION"} \
+          ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \
+          --gcs_archive_path "${GCS_ARTIFACTS_PATH}" \
+          --gcs_result_path "${GCS_RESULTS_PATH}"
       shell: bash
+    - name: Download ${{ matrix.platform }} Test Results
+      if: always()
+      env:
+        GCS_RESULTS_PATH: ${{ inputs.gcs_results_path }}
+        RESULTS_DIR: ${{ inputs.results_dir }}
       run: |
-        echo "Nothing yet"
+        set -uxe
+        TEST_LOGS="${GITHUB_WORKSPACE}/${RESULTS_DIR}/"
+        mkdir -p "${TEST_LOGS}"
+        gsutil cp -r "${GCS_RESULTS_PATH}/" "${TEST_LOGS}"
+        echo "TEST_LOGS=${TEST_LOGS}" >> $GITHUB_ENV
+      shell: bash
+    - name: Archive Test Results
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: ${{ inputs.test_results_key }}
+        path: ${{ env.TEST_LOGS }}/

.github/actions/upload_test_artifacts/action.yaml

@@ -16,22 +16,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Set up Cloud SDK
-      if: inputs.upload_on_device_test_artifacts == 'true'
-      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
-    - name: Upload Android Test Artifacts to GCS
-      if: inputs.upload_on_device_test_artifacts == 'true'
-      env:
-        WORKFLOW: ${{ github.workflow }}
-      run: |
-        set -eux
-        project_name=$(gcloud config get-value project)
-        gsutil cp "${GITHUB_WORKSPACE}/src/out/${{ matrix.platform }}_${{ matrix.config }}/**/*.apk" \
-          "gs://${project_name}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{matrix.platform}}/"
-      shell: bash
-
-    - name: Create On-Host Test Artifacts Archive
-      if: inputs.upload_on_host_test_artifacts == 'true'
+    - name: Archive Test Artifacts
       run: |
         set -x
         mkdir ${GITHUB_WORKSPACE}/artifacts
@@ -49,3 +34,17 @@ runs:
         name: ${{ inputs.test_artifacts_key }}
         path: artifacts/*
         retention-days: 3
+    - name: Set up Cloud SDK
+      if: inputs.upload_on_device_test_artifacts == 'true'
+      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
+    - name: Upload Android Test Artifacts to GCS
+      if: inputs.upload_on_device_test_artifacts == 'true'
+      env:
+        WORKFLOW: ${{ github.workflow }}
+      run: |
+        set -eux
+        project_name=$(gcloud config get-value project)
+
+        gsutil cp "${GITHUB_WORKSPACE}/artifacts/*" \
+          "gs://${project_name}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{matrix.platform}}/"
+      shell: bash

.github/config/android-arm.json

@@ -3,6 +3,11 @@
   "platforms": [
     "android-arm"
   ],
+  "test_on_device": true,
+  "test_dimensions": {
+    "gtest_device": "sabrina",
+    "gtest_lab": "maneki"
+  },
   "targets": [
     "content_shell",
     "system_webview_apk",

.github/config/chromium_android-arm.json

@@ -4,16 +4,6 @@
     "chromium_android-arm"
   ],
   "targets": [
-    "base_unittests",
-    "sql_unittests",
-    "net_unittests",
-    "url_unittests",
-    "ipc_tests",
-    "mojo_unittests",
-    "gpu_unittests",
-    "gin_unittests",
-    "blink_unittests",
-    "media_unittests",
     "content_shell",
     "system_webview_apk",
     "system_webview_shell_apk"

.github/config/chromium_android-arm64.json

@@ -4,16 +4,6 @@
     "chromium_android-arm64"
   ],
   "targets": [
-    "base_unittests",
-    "sql_unittests",
-    "net_unittests",
-    "url_unittests",
-    "ipc_tests",
-    "mojo_unittests",
-    "gpu_unittests",
-    "gin_unittests",
-    "blink_unittests",
-    "media_unittests",
     "content_shell",
     "system_webview_apk",
     "system_webview_shell_apk"

.github/config/chromium_android-x86.json

@@ -4,16 +4,6 @@
     "chromium_android-x86"
   ],
   "targets": [
-    "base_unittests",
-    "sql_unittests",
-    "net_unittests",
-    "url_unittests",
-    "ipc_tests",
-    "mojo_unittests",
-    "gpu_unittests",
-    "gin_unittests",
-    "blink_unittests",
-    "media_unittests",
     "content_shell",
     "system_webview_apk",
     "system_webview_shell_apk"

.github/workflows/main.yaml

@@ -145,7 +145,7 @@ jobs:
       matrix:
         platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
         include: ${{ fromJson(needs.initialize.outputs.includes) }}
-        config: [devel, qa, gold]
+        config: [devel]
     container: ${{ needs.docker-build-image.outputs.docker_tag }}
     env:
       TEST_ARTIFACTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_artifacts
@@ -183,9 +183,47 @@ jobs:
           upload_on_host_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_host }}
           upload_on_device_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_device }}
 
-  test:
+  # Runs on-device integration and unit tests.
+  on-device-test:
+    needs: [initialize, build]
+    # Run ODT when on_device label is applied on PR.
+    # Also, run ODT on push and schedule if not explicitly disabled via repo vars.
+    if: needs.initialize.outputs.test_on_device == 'true' &&
+        (
+          (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'on_device')) ||
+          (vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False' && (inputs.nightly == 'true' || github.event_name == 'schedule')) ||
+          (vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' && github.event_name == 'push')
+        )
+    runs-on: [self-hosted, odt-runner]
+    name: ${{ matrix.name }}_on_device
+    permissions: {}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
+        include: ${{ fromJson(needs.initialize.outputs.includes) }}
+        config: [devel]
+    env:
+      TEST_RESULTS_DIR: ${{ matrix.name }}_test_results
+      TEST_RESULTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_results
+    steps:
+      - name: Checkout
+        uses: kaidokert/[email protected]
+        timeout-minutes: 30
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+      - name: Run On-Device Tests (${{ matrix.shard }})
+        id: on-device-tests
+        uses: ./.github/actions/on_device_tests
+        with:
+          test_results_key: ${{ env.TEST_RESULTS_KEY }}
+          gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}
+          results_dir: ${{ env.TEST_RESULTS_DIR }}
+
+  on-host-test:
     needs: [initialize, docker-build-image, build]
-    if: needs.initialize.outputs.test_on_host == 'true' || needs.initialize.outputs.test_on_device == 'true'
+    if: needs.initialize.outputs.test_on_host == 'true'
     permissions: {}
     # TODO(b/372303096): Should have dedicated runner?
     runs-on: [self-hosted, chrobalt-linux-runner]
@@ -209,20 +247,20 @@ jobs:
           path: src
       - name: Run On-Host Tests
         id: on-host-tests
-        if: always() && needs.initialize.outputs.test_on_host == 'true'
+        if: always()
         uses: ./src/.github/actions/on_host_tests
         with:
           test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }}
           test_results_key: ${{ env.TEST_RESULTS_KEY }}
           num_gtest_shards: ${{ needs.initialize.outputs.num_gtest_shards }}
 
   test-upload:
-    needs: [initialize, docker-build-image, build, test]
+    needs: [initialize, docker-build-image, build, on-host-test, on-device-test]
     if: always() &&
-        (
-          needs.initialize.outputs.test_on_host == 'true' ||
-          needs.initialize.outputs.test_on_device == 'true'
-        )
+      (
+        needs.initialize.outputs.test_on_host == 'true' ||
+        needs.initialize.outputs.test_on_device == 'true'
+      )
     permissions: {}
     runs-on: [self-hosted, chrobalt-linux-runner]
     name: ${{ matrix.name }}_tests_upload
@@ -249,12 +287,12 @@ jobs:
           datadog_api_key: ${{ secrets.datadog_api_key }}
         continue-on-error: true
 
+
   validate-test-result:
-    needs: [initialize, docker-build-image, build, test]
+    needs: [initialize, docker-build-image, build, on-host-test]
     if: always() &&
         (
-          needs.initialize.outputs.test_on_host == 'true' ||
-          needs.initialize.outputs.test_on_device == 'true'
+          needs.initialize.outputs.test_on_host == 'true'
         )
     permissions: {}
     runs-on: ubuntu-latest

cobalt/build/archive_test_artifacts.py

@@ -17,6 +17,7 @@
 import argparse
 import json
 import os
+import shutil
 import subprocess
 import tempfile
 from typing import List
@@ -35,7 +36,7 @@ def _make_tar(archive_path: str, file_list: str):
 
 
 def create_archive(targets: List[str], source_dir: str, destination_dir: str,
-                   platform: str, combine: bool):
+                   platform: str, uber_archive: bool):
   """Main logic. Collects runtime dependencies from the source directory for
   each target."""
   # TODO(b/382508397): Remove when dynamically generated.
@@ -72,16 +73,27 @@ def create_archive(targets: List[str], source_dir: str, destination_dir: str,
       }
       deps |= target_deps
 
-    if not combine:
+    if not uber_archive:
       output_path = os.path.join(destination_dir, f'{target_name}_deps.tar.gz')
       _make_tar(output_path, deps)
 
-  if combine:
+  if uber_archive:
     output_path = os.path.join(destination_dir, 'test_artifacts.tar.gz')
     _make_tar(output_path, deps)
 
 
-if __name__ == '__main__':
+def copy_apks(targets: List[str], source_dir: str, destination_dir: str):
+  """Copies the target APKs from the source directory to the destination.
+  The path to the APK in the source directory (assumed here to be the out
+  directory) is defined in build/config/android/rules.gni
+  """
+  for target in targets:
+    _, target_name = target.split(':')
+    apk_path = f'{source_dir}/{target_name}_apk/{target_name}-debug.apk'
+    shutil.copy2(apk_path, destination_dir)
+
+
+def main():
   parser = argparse.ArgumentParser()
   parser.add_argument(
       '-s',
@@ -103,9 +115,17 @@ def create_archive(targets: List[str], source_dir: str, destination_dir: str,
       '--targets',
       required=True,
       type=lambda arg: arg.split(','),
-      help='The targets to package, comma-separated. Must be fully qualified '
-      'for android.')
+      help='The targets to package, comma-separated. Must be fully qualified, '
+      'e.g. path/to:target.')
   args = parser.parse_args()
 
+  uber_archive = args.platform.startswith('linux')
   create_archive(args.targets, args.source_dir, args.destination_dir,
-                 args.platform, args.platform.startswith('linux'))
+                 args.platform, uber_archive)
+
+  if args.platform.startswith('android'):
+    copy_apks(args.targets, args.source_dir, args.destination_dir)
+
+
+if __name__ == '__main__':
+  main()

cobalt/testing/filters/android-arm/base_unittests_filter.json

@@ -0,0 +1,6 @@
+{
+  "failing_tests": [
+    "BreakIteratorTest.BreakCharacter",
+    "ValuesUtilTest.FilePath"
+  ]
+}