From d290c81fb256366bfaa7cffd2ae2fbc99da2f205 Mon Sep 17 00:00:00 2001 From: Brian Ting Date: Tue, 19 Nov 2024 14:37:36 -0800 Subject: [PATCH 01/61] Set up Cobalt build acceleration Set up Cobalt build acceleration with RBE making it the default accelerator for both developers and the CI system. Remove fetching of RBE cfg files with CIPD. Instead rely on local cfg files which use public docker images rather than private ones. Taken from Chromium's trunk. b/384982606 --- .github/actions/build/action.yaml | 10 ++-------- buildtools/reclient_cfgs/fetch_reclient_cfgs.py | 4 ++++ cobalt/build/gn.py | 7 +++---- cobalt/devinfra/kokoro/bin/dind_build.sh | 4 ++-- .../chromium-browser-clang/rewrapper_linux.cfg | 8 ++++++++ cobalt/reclient_cfgs/python/rewrapper_linux.cfg | 8 ++++++++ 6 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 cobalt/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg create mode 100644 cobalt/reclient_cfgs/python/rewrapper_linux.cfg diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml index ee6157299833d..fe715dbf3adf9 100644 --- a/.github/actions/build/action.yaml +++ b/.github/actions/build/action.yaml @@ -14,7 +14,7 @@ runs: run: echo "$GITHUB_WORKSPACE/depot_tools" >> $GITHUB_PATH shell: bash - name: Generate gclient file - run: gclient config --name=src https://github.com/${{ github.repository }} + run: gclient config --name=src --custom-var=download_remoteexec_cfg=True --custom-var='rbe_instance="projects/cobalt-actions-prod/instances/default_instance"' https://github.com/${{ github.repository }} shell: bash - name: Set target OS for Android if: ${{ contains(matrix.platform, 'android') }} @@ -25,9 +25,6 @@ runs: - name: Gclient sync run: gclient sync -v --shallow --no-history -r ${{ github.sha }} shell: bash - - name: sccache check - run: sccache -s - shell: bash - name: gn gen run: | cd src @@ -44,10 +41,7 @@ runs: run: | set -x cd src - ninja -C out/${{ matrix.platform }}_${{ matrix.config }} $(echo "${TARGETS}" | tr -d '"') - shell: bash - - name: Dump sccache stats - run: sccache -s + time autoninja -C out/${{ matrix.platform }}_${{ matrix.config }} $(echo "${TARGETS}" | tr -d '"') shell: bash - name: Archive Android APKs if: startsWith(matrix.platform, 'android') && matrix.config == 'qa' diff --git a/buildtools/reclient_cfgs/fetch_reclient_cfgs.py b/buildtools/reclient_cfgs/fetch_reclient_cfgs.py index 98c8290db1898..baed7076a0417 100755 --- a/buildtools/reclient_cfgs/fetch_reclient_cfgs.py +++ b/buildtools/reclient_cfgs/fetch_reclient_cfgs.py @@ -134,6 +134,10 @@ def main(): return 1 if not GenerateReproxyCfg(args.reproxy_cfg_template, args.rbe_instance): return 1 +# COBALT Disables CIPD fetch of rewrapper configs in favor of local rewrapper configs. + return 0 + return 1 +# COBALT END if not args.rewrapper_cfg_project and not args.rbe_instance: logging.error( diff --git a/cobalt/build/gn.py b/cobalt/build/gn.py index 8e34d4e2b7c76..7060fa15529cc 100755 --- a/cobalt/build/gn.py +++ b/cobalt/build/gn.py @@ -85,10 +85,9 @@ def write_build_args(build_args_path, original_lines, dict_settings, f'The following args cannot be set in configs: {controlled_args}') gen_comment = '# Set by gn.py' with open(build_args_path, 'w', encoding='utf-8') as f: - if os.getenv('IS_CI', default='0') == '1': - f.write(f'cc_wrapper = "{CC_WRAPPER}" {gen_comment}\n') - else: - f.write(f'use_remoteexec = true {gen_comment}\n') + f.write(f'use_remoteexec = true {gen_comment}\n') + f.write( + f'rbe_cfg_dir = rebase_path("//cobalt/reclient_cfgs") {gen_comment}\n') f.write(f'build_type = "{build_type}" {gen_comment}\n') for key, value in _BUILD_TYPES[build_type].items(): f.write(f'{key} = {value} {gen_comment}\n') diff --git a/cobalt/devinfra/kokoro/bin/dind_build.sh b/cobalt/devinfra/kokoro/bin/dind_build.sh index 458b0d0205420..2b5c8247110e5 100755 --- a/cobalt/devinfra/kokoro/bin/dind_build.sh +++ b/cobalt/devinfra/kokoro/bin/dind_build.sh @@ -50,7 +50,7 @@ pipeline () { # Clone depot_tools as the GitHub action does, rather than Kokoro doing it. git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git tools/depot_tools export PATH="${PATH}:${gclient_root}/tools/depot_tools" - gclient config --name=src rpc://lbshell-internal/cobalt_src + gclient config --name=src --custom-var=download_remoteexec_cfg=True --custom-var='rbe_instance="projects/cobalt-actions-prod/instances/default_instance"' rpc://lbshell-internal/cobalt_src if [[ "${TARGET_PLATFORM}" =~ "android" ]]; then echo "target_os=['android']" >> .gclient fi @@ -61,7 +61,7 @@ pipeline () { ############################################################################## cd "${gclient_root}/src" cobalt/build/gn.py -p "${TARGET_PLATFORM}" -C "${CONFIG}" - ninja -C "out/${TARGET_PLATFORM}_${CONFIG}" ${TARGET} # TARGET may expand to multiple args + time autoninja -C "out/${TARGET_PLATFORM}_${CONFIG}" ${TARGET} # TARGET may expand to multiple args # Build bootloader config if set. if [ -n "${BOOTLOADER:-}" ]; then diff --git a/cobalt/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg b/cobalt/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg new file mode 100644 index 0000000000000..37fca98f4353e --- /dev/null +++ b/cobalt/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg @@ -0,0 +1,8 @@ +platform=container-image=docker://gcr.io/chops-public-images-prod/rbe/siso-chromium/linux@sha256:26de99218a1a8b527d4840490bcbf1690ee0b55c84316300b60776e6b3a03fe1,label:action_default=1 +server_address=unix:///tmp/reproxy.sock +labels=type=compile,compiler=clang,lang=cpp +exec_strategy=remote_local_fallback +dial_timeout=10m +exec_timeout=2m +reclient_timeout=2m +canonicalize_working_dir=true diff --git a/cobalt/reclient_cfgs/python/rewrapper_linux.cfg b/cobalt/reclient_cfgs/python/rewrapper_linux.cfg new file mode 100644 index 0000000000000..12ae5379c2fe6 --- /dev/null +++ b/cobalt/reclient_cfgs/python/rewrapper_linux.cfg @@ -0,0 +1,8 @@ +platform=container-image=docker://gcr.io/chops-public-images-prod/rbe/siso-chromium/linux@sha256:26de99218a1a8b527d4840490bcbf1690ee0b55c84316300b60776e6b3a03fe1,label:action_default=1 +server_address=unix:///tmp/reproxy.sock +labels=type=tool +exec_strategy=remote_local_fallback +dial_timeout=10m +canonicalize_working_dir=true +exec_timeout=2m +reclient_timeout=2m From 981f11c1c574bb932e42284e37c04f7bd1a0fe41 Mon Sep 17 00:00:00 2001 From: Brian Ting Date: Tue, 14 Jan 2025 13:40:54 -0800 Subject: [PATCH 02/61] Update action.yaml and dind_build.sh format and naming Disables Chromium client side build telemetry. --- .github/actions/build/action.yaml | 11 +++++++---- cobalt/devinfra/kokoro/bin/dind_build.sh | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml index fe715dbf3adf9..614ec24aedb6f 100644 --- a/.github/actions/build/action.yaml +++ b/.github/actions/build/action.yaml @@ -10,7 +10,7 @@ runs: - name: Get depot tools run: git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git shell: bash - - name: Add directory to PATH + - name: Add depot tools to PATH run: echo "$GITHUB_WORKSPACE/depot_tools" >> $GITHUB_PATH shell: bash - name: Generate gclient file @@ -22,15 +22,18 @@ runs: echo "target_os=['android']" >> .gclient gclient validate shell: bash - - name: Gclient sync + - name: gclient sync run: gclient sync -v --shallow --no-history -r ${{ github.sha }} shell: bash - - name: gn gen + - name: Disable Chromium client side build telemetry + run: build_telemetry opt-out + shell: bash + - name: GN gen run: | cd src cobalt/build/gn.py -p ${{ matrix.platform }} -C ${{ matrix.config }} shell: bash - - name: Dump GN args + - name: List GN args run: | cd src gn args --list --short --overrides-only out/${{ matrix.platform }}_${{ matrix.config }} diff --git a/cobalt/devinfra/kokoro/bin/dind_build.sh b/cobalt/devinfra/kokoro/bin/dind_build.sh index 2b5c8247110e5..f990bb9133aa8 100755 --- a/cobalt/devinfra/kokoro/bin/dind_build.sh +++ b/cobalt/devinfra/kokoro/bin/dind_build.sh @@ -47,7 +47,6 @@ pipeline () { # Set up gclient and run sync. ############################################################################## cd "${gclient_root}" - # Clone depot_tools as the GitHub action does, rather than Kokoro doing it. git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git tools/depot_tools export PATH="${PATH}:${gclient_root}/tools/depot_tools" gclient config --name=src --custom-var=download_remoteexec_cfg=True --custom-var='rbe_instance="projects/cobalt-actions-prod/instances/default_instance"' rpc://lbshell-internal/cobalt_src @@ -56,6 +55,7 @@ pipeline () { fi git config --global --add safe.directory "${gclient_root}/src" gclient sync -v --shallow --no-history -r "${KOKORO_GIT_COMMIT_src}" + build_telemtry opt-out # Run GN and Ninja. ############################################################################## From 08e544fa15eb4acaaa491037861ee9de6acc292a Mon Sep 17 00:00:00 2001 From: Dana Dahlstrom Date: Thu, 16 Jan 2025 08:00:00 -0800 Subject: [PATCH 03/61] Correct spelling of build_telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … after commit 981f11c1. Issue: 384982606 Reviewed-on: https://github.com/youtube/cobalt/pull/4707 --- cobalt/devinfra/kokoro/bin/dind_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cobalt/devinfra/kokoro/bin/dind_build.sh b/cobalt/devinfra/kokoro/bin/dind_build.sh index f990bb9133aa8..5252f838d84e1 100755 --- a/cobalt/devinfra/kokoro/bin/dind_build.sh +++ b/cobalt/devinfra/kokoro/bin/dind_build.sh @@ -55,7 +55,7 @@ pipeline () { fi git config --global --add safe.directory "${gclient_root}/src" gclient sync -v --shallow --no-history -r "${KOKORO_GIT_COMMIT_src}" - build_telemtry opt-out + build_telemetry opt-out # Run GN and Ninja. ############################################################################## From d561d947012ff98f811606e61ce1c0bc81f054b7 Mon Sep 17 00:00:00 2001 From: Dana Dahlstrom Date: Thu, 16 Jan 2025 08:00:00 -0800 Subject: [PATCH 04/61] Remove time call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … added by commit d290c81f. The command is not found. Issue: 384982606 Reviewed-on: https://github.com/youtube/cobalt/pull/4708 --- cobalt/devinfra/kokoro/bin/dind_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cobalt/devinfra/kokoro/bin/dind_build.sh b/cobalt/devinfra/kokoro/bin/dind_build.sh index 5252f838d84e1..190199b61dfda 100755 --- a/cobalt/devinfra/kokoro/bin/dind_build.sh +++ b/cobalt/devinfra/kokoro/bin/dind_build.sh @@ -61,7 +61,7 @@ pipeline () { ############################################################################## cd "${gclient_root}/src" cobalt/build/gn.py -p "${TARGET_PLATFORM}" -C "${CONFIG}" - time autoninja -C "out/${TARGET_PLATFORM}_${CONFIG}" ${TARGET} # TARGET may expand to multiple args + autoninja -C "out/${TARGET_PLATFORM}_${CONFIG}" ${TARGET} # TARGET may expand to multiple args # Build bootloader config if set. if [ -n "${BOOTLOADER:-}" ]; then From d59ac6b02a1358df91218b24aab6c4b36ff79eea Mon Sep 17 00:00:00 2001 From: Bo-Rong Chen Date: Fri, 17 Jan 2025 10:48:04 -0800 Subject: [PATCH 05/61] [android] Enable low-end-device-mode (#4663) The demuxer memory limits previously caused rebuffering in 4k playbacks, resulting in it restarts with a lower resolution. Since https://github.com/youtube/cobalt/pull/4674 fixed this issue, enable low-end-device-mode in Cobalt on ATV. b/388291268 --- .../app/src/main/java/dev/cobalt/coat/CobaltActivity.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java b/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java index ca019dd7d4b6c..1a86b211404e5 100644 --- a/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java +++ b/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java @@ -121,6 +121,12 @@ protected void createContent(final Bundle savedInstanceState) { "--enable-features=LogJsConsoleMessages", // Disable rescaling Webpage. "--force-device-scale-factor=1", + // Enable low end device mode. + "--enable-low-end-device-mode", + // Disables RGBA_4444 textures which + // causes rendering artifacts when + // low-end-device-mode is enabled. + "--disable-rgba-4444-textures", }; CommandLine.getInstance().appendSwitchesAndArguments(cobaltCommandLineParams); if (shouldSetJNIPrefix) { From d7c2bc0f23a1a7f62df2f8d86c5516cdfb9d185c Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 17 Jan 2025 11:14:35 -0800 Subject: [PATCH 06/61] Avoid calling GetMediaTime() on media thread (#4703) GetMediaTime() is mostly called on main thread. Instead of calling GetMediaTime() in StarboardRenderer::OnNeedData(), the cl now uses |last_media_time_| and |last_time_media_time_retrieved_| to estimate current media time. As HTMLMediaElement updates media time at least every 250ms, |last_media_time_| and |last_time_media_time_retrieved_| should also be updated at least every 250ms, which should be sufficient to limit audio write ahead. b/376328722 --- media/starboard/starboard_renderer.cc | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/media/starboard/starboard_renderer.cc b/media/starboard/starboard_renderer.cc index 4b5648917d579..249127c730df9 100644 --- a/media/starboard/starboard_renderer.cc +++ b/media/starboard/starboard_renderer.cc @@ -367,7 +367,8 @@ void StarboardRenderer::SetVolume(float volume) { } } -// TODO(b/376328722): Revisit playback time reporting. +// Note: Renderer::GetMediaTime() could be called on both main and media +// threads. TimeDelta StarboardRenderer::GetMediaTime() { base::AutoLock auto_lock(lock_); @@ -675,12 +676,6 @@ void StarboardRenderer::OnNeedData(DemuxerStream::Type type, return; } - // If we haven't checked the media time recently, update it now. - if (Time::Now() - last_time_media_time_retrieved_ > - kMediaTimeCheckInterval) { - GetMediaTime(); - } - // Delay reading audio more than |audio_write_duration_| ahead of playback // after the player has received enough audio for preroll, taking into // account that our estimate of playback time might be behind by @@ -701,10 +696,17 @@ void StarboardRenderer::OnNeedData(DemuxerStream::Type type, int estimated_max_buffers = max_buffers; if (!is_video_eos_written_ && time_ahead_of_playback_for_preroll > adjusted_write_duration_for_preroll) { - // The estimated time ahead of playback may be negative if no audio has - // been written. - TimeDelta time_ahead_of_playback = - timestamp_of_last_written_audio_ - last_media_time_; + TimeDelta time_ahead_of_playback; + { + base::AutoLock auto_lock(lock_); + TimeDelta time_since_last_update = + Time::Now() - last_time_media_time_retrieved_; + // The estimated time ahead of playback may be negative if no audio has + // been written. + time_ahead_of_playback = timestamp_of_last_written_audio_ - + (last_media_time_ + time_since_last_update); + } + auto adjusted_write_duration = AdjustWriteDurationForPlaybackRate( audio_write_duration_, playback_rate_); if (time_ahead_of_playback > From 782628828110eb76c42ac742ce3547c7686fa301 Mon Sep 17 00:00:00 2001 From: thorsten sideb0ard Date: Fri, 17 Jan 2025 12:34:03 -0800 Subject: [PATCH 07/61] Support WebAudio (#4577) Implement SbAudioSink based WebAudioDevice. b/340599896 Tested on a Kirkwood device by running: https://github.com/youtube/cobalt/tree/25.lts.1%2B/cobalt/demos/content/web-audio-demo and https://webaudiodemos.appspot.com/oscilloscope/index.html --- cobalt/media/audio/BUILD.gn | 26 ++ cobalt/media/audio/audio_helpers.h | 56 ++++ .../audio/cobalt_audio_device_factory.cc | 44 ++++ .../media/audio/cobalt_audio_device_factory.h | 41 +++ .../media/audio/cobalt_audio_renderer_sink.cc | 240 ++++++++++++++++++ .../media/audio/cobalt_audio_renderer_sink.h | 97 +++++++ cobalt/renderer/BUILD.gn | 1 + .../renderer/cobalt_content_renderer_client.h | 6 +- 8 files changed, 508 insertions(+), 3 deletions(-) create mode 100644 cobalt/media/audio/BUILD.gn create mode 100644 cobalt/media/audio/audio_helpers.h create mode 100644 cobalt/media/audio/cobalt_audio_device_factory.cc create mode 100644 cobalt/media/audio/cobalt_audio_device_factory.h create mode 100644 cobalt/media/audio/cobalt_audio_renderer_sink.cc create mode 100644 cobalt/media/audio/cobalt_audio_renderer_sink.h diff --git a/cobalt/media/audio/BUILD.gn b/cobalt/media/audio/BUILD.gn new file mode 100644 index 0000000000000..bdcf50c0516fd --- /dev/null +++ b/cobalt/media/audio/BUILD.gn @@ -0,0 +1,26 @@ +# Copyright 2025 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source_set("webaudio") { + sources = [ + "cobalt_audio_device_factory.cc", + "cobalt_audio_device_factory.h", + "cobalt_audio_renderer_sink.cc", + "cobalt_audio_renderer_sink.h", + ] + deps = [ + "//starboard:starboard_headers_only", + "//third_party/blink/public:blink_headers", + ] +} diff --git a/cobalt/media/audio/audio_helpers.h b/cobalt/media/audio/audio_helpers.h new file mode 100644 index 0000000000000..b45fdeb483419 --- /dev/null +++ b/cobalt/media/audio/audio_helpers.h @@ -0,0 +1,56 @@ +// Copyright 2025 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef COBALT_MEDIA_AUDIO_AUDIO_HELPERS_H_ +#define COBALT_MEDIA_AUDIO_AUDIO_HELPERS_H_ + +#include "media/base/audio_parameters.h" +#include "starboard/audio_sink.h" +#include "starboard/media.h" + +namespace media { + +const int kStandardOutputSampleRate = 48000; +const int kPreferredRenderBufferSizeFrames = 1024; + +inline media::OutputDeviceInfo GetPreferredOutputParameters() { + return media::OutputDeviceInfo( + std::string(), media::OUTPUT_DEVICE_STATUS_OK, + media::AudioParameters(media::AudioParameters::AUDIO_PCM_LOW_LATENCY, + media::ChannelLayoutConfig::Stereo(), + kStandardOutputSampleRate, + kPreferredRenderBufferSizeFrames)); +} + +inline SbMediaAudioSampleType GetPreferredOutputStarboardSampleType() { + if (SbAudioSinkIsAudioSampleTypeSupported(kSbMediaAudioSampleTypeFloat32)) { + return kSbMediaAudioSampleTypeFloat32; + } + return kSbMediaAudioSampleTypeInt16Deprecated; +} + +inline size_t GetStarboardSampleTypeSize(SbMediaAudioSampleType sample_type) { + switch (sample_type) { + case kSbMediaAudioSampleTypeFloat32: + return sizeof(float); + case kSbMediaAudioSampleTypeInt16Deprecated: + return sizeof(int16_t); + } + NOTREACHED(); + return 0u; +} + +} // namespace media + +#endif // COBALT_MEDIA_AUDIO_AUDIO_HELPERS_H_ diff --git a/cobalt/media/audio/cobalt_audio_device_factory.cc b/cobalt/media/audio/cobalt_audio_device_factory.cc new file mode 100644 index 0000000000000..9445b46eb5a89 --- /dev/null +++ b/cobalt/media/audio/cobalt_audio_device_factory.cc @@ -0,0 +1,44 @@ +// Copyright 2025 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cobalt/media/audio/cobalt_audio_device_factory.h" + +#include "base/logging.h" +#include "cobalt/media/audio/audio_helpers.h" +#include "cobalt/media/audio/cobalt_audio_renderer_sink.h" + +namespace media { + +CobaltAudioDeviceFactory::CobaltAudioDeviceFactory() { + LOG(INFO) << "Register CobaltAudioDeviceFactory"; +} + +CobaltAudioDeviceFactory::~CobaltAudioDeviceFactory() { + LOG(INFO) << "Unregister CobaltAudioDeviceFactory"; +} + +scoped_refptr +CobaltAudioDeviceFactory::NewAudioRendererSink( + blink::WebAudioDeviceSourceType source_type, + const blink::LocalFrameToken& frame_token, + const media::AudioSinkParameters& params) { + return base::MakeRefCounted(); +} + +OutputDeviceInfo CobaltAudioDeviceFactory::GetOutputDeviceInfo( + const blink::LocalFrameToken& frame_token, + const std::string& device_id) { + return GetPreferredOutputParameters(); +} +} // namespace media diff --git a/cobalt/media/audio/cobalt_audio_device_factory.h b/cobalt/media/audio/cobalt_audio_device_factory.h new file mode 100644 index 0000000000000..d7579960706d3 --- /dev/null +++ b/cobalt/media/audio/cobalt_audio_device_factory.h @@ -0,0 +1,41 @@ +// Copyright 2025 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef COBALT_MEDIA_AUDIO_COBALT_AUDIO_DEVICE_FACTORY_H +#define COBALT_MEDIA_AUDIO_COBALT_AUDIO_DEVICE_FACTORY_H + +#include "media/base/media_export.h" +#include "third_party/blink/public/web/modules/media/audio/audio_device_factory.h" + +namespace media { + +class MEDIA_EXPORT CobaltAudioDeviceFactory final + : public blink::AudioDeviceFactory { + public: + CobaltAudioDeviceFactory(); + ~CobaltAudioDeviceFactory() final; + + scoped_refptr NewAudioRendererSink( + blink::WebAudioDeviceSourceType source_type, + const blink::LocalFrameToken& frame_token, + const media::AudioSinkParameters& params) final; + + OutputDeviceInfo GetOutputDeviceInfo( + const blink::LocalFrameToken& frame_token, + const std::string& device_id) final; +}; + +} // namespace media + +#endif // COBALT_MEDIA_AUDIO_COBALT_AUDIO_DEVICE_FACTORY_H diff --git a/cobalt/media/audio/cobalt_audio_renderer_sink.cc b/cobalt/media/audio/cobalt_audio_renderer_sink.cc new file mode 100644 index 0000000000000..741af98d16c52 --- /dev/null +++ b/cobalt/media/audio/cobalt_audio_renderer_sink.cc @@ -0,0 +1,240 @@ +// Copyright 2025 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cobalt/media/audio/cobalt_audio_renderer_sink.h" + +#include "base/logging.h" +#include "base/task/bind_post_task.h" +#include "cobalt/media/audio/audio_helpers.h" +#include "media/base/audio_glitch_info.h" + +namespace media { + +namespace { +const int kDefaultFramesPerRenderBufferPadding = 2; +const int kDefaultFramesPerRenderBufferPaddingMax = 8; + +int AlignUp(int value, int alignment) { + int decremented_value = value - 1; + return decremented_value + alignment - (decremented_value % alignment); +} +} // namespace + +void CobaltAudioRendererSink::Initialize(const AudioParameters& params, + RenderCallback* callback) { + LOG(INFO) << "CobaltAudioRendererSink::Initialize - called with following " + "parameters:" + << params.AsHumanReadableString(); + params_ = params; + output_sample_type_ = GetPreferredOutputStarboardSampleType(); + + DCHECK(callback); + DCHECK(!callback_); + callback_ = callback; +} + +void CobaltAudioRendererSink::Start() { + DCHECK(callback_); + int frames_per_render_buffer = params_.frames_per_buffer(); + + frames_per_channel_ = std::max( + AlignUp( + SbAudioSinkGetMinBufferSizeInFrames( + params_.channels(), output_sample_type_, params_.sample_rate()) + + frames_per_render_buffer * kDefaultFramesPerRenderBufferPadding, + frames_per_render_buffer), + frames_per_render_buffer * kDefaultFramesPerRenderBufferPaddingMax); + + input_audio_bus_ = AudioBus::Create(params_); + + output_frame_buffer_ = std::make_unique( + frames_per_channel_ * params_.channels() * + GetStarboardSampleTypeSize(output_sample_type_)); + + DCHECK(frames_per_channel_ % frames_per_render_buffer == 0); + + int nearest_supported_sample_rate_ = + SbAudioSinkGetNearestSupportedSampleFrequency(params_.sample_rate()); + if (nearest_supported_sample_rate_ != params_.sample_rate()) { + resample_ratio_ = static_cast(params_.sample_rate() / + nearest_supported_sample_rate_); + resampler_ = std::make_unique( + params_.channels(), resample_ratio_, frames_per_render_buffer, + base::BindRepeating(&CobaltAudioRendererSink::ResamplerReadCallback, + base::Unretained(this))); + resampler_->PrimeWithSilence(); + resampled_audio_bus_ = AudioBus::Create(params_); + } + + output_frame_buffers_[0] = output_frame_buffer_.get(); + audio_sink_ = SbAudioSinkCreate( + params_.channels(), nearest_supported_sample_rate_, output_sample_type_, + kSbMediaAudioFrameStorageTypeInterleaved, &output_frame_buffers_[0], + frames_per_channel_, &CobaltAudioRendererSink::UpdateSourceStatusFunc, + &CobaltAudioRendererSink::ConsumeFramesFunc, this); + DCHECK(SbAudioSinkIsValid(audio_sink_)); +} + +void CobaltAudioRendererSink::Stop() { + is_eos_reached_.store(true); + SbAudioSinkDestroy(audio_sink_); + callback_ = nullptr; +} + +void CobaltAudioRendererSink::Play() { + is_playing_.store(true); +} + +void CobaltAudioRendererSink::Pause() { + is_playing_.store(false); +} + +void CobaltAudioRendererSink::Flush() { + if (!SbAudioSinkIsValid(audio_sink_)) { + return; + } + // TODO(b/390468794) consolidate this recreation as it duplicates code from + // Stop() and Start() + SbAudioSinkDestroy(audio_sink_); + frames_rendered_ = 0; + frames_consumed_ = 0; + audio_sink_ = SbAudioSinkCreate( + params_.channels(), + SbAudioSinkGetNearestSupportedSampleFrequency(params_.sample_rate()), + output_sample_type_, kSbMediaAudioFrameStorageTypeInterleaved, + &output_frame_buffers_[0], frames_per_channel_, + &CobaltAudioRendererSink::UpdateSourceStatusFunc, + &CobaltAudioRendererSink::ConsumeFramesFunc, this); + DCHECK(SbAudioSinkIsValid(audio_sink_)); +} + +bool CobaltAudioRendererSink::SetVolume(double volume) { + LOG(INFO) << "SetVolume - NOT IMPL"; + NOTIMPLEMENTED(); + return false; +} + +OutputDeviceInfo CobaltAudioRendererSink::GetOutputDeviceInfo() { + return GetPreferredOutputParameters(); +} + +void CobaltAudioRendererSink::GetOutputDeviceInfoAsync( + OutputDeviceInfoCB info_cb) { + base::BindPostTaskToCurrentDefault( + base::BindOnce(std::move(info_cb), GetOutputDeviceInfo())) + .Run(); +} + +bool CobaltAudioRendererSink::IsOptimizedForHardwareParameters() { + LOG(INFO) << "IsOptimizedForHardwareParameters - NOT IMPL"; + NOTIMPLEMENTED(); + return false; +} + +bool CobaltAudioRendererSink::CurrentThreadIsRenderingThread() { + LOG(INFO) << "CurrentThreadIsRenderingThread - NOT IMPL"; + NOTIMPLEMENTED(); + return true; +} + +// static +void CobaltAudioRendererSink::UpdateSourceStatusFunc(int* frames_in_buffer, + int* offset_in_frames, + bool* is_playing, + bool* is_eos_reached, + void* context) { + CobaltAudioRendererSink* sink = + static_cast(context); + DCHECK(sink); + DCHECK(frames_in_buffer); + DCHECK(offset_in_frames); + DCHECK(is_playing); + DCHECK(is_eos_reached); + sink->UpdateSourceStatus(frames_in_buffer, offset_in_frames, is_playing, + is_eos_reached); +} + +// static +void CobaltAudioRendererSink::ConsumeFramesFunc(int frames_consumed, + void* context) { + CobaltAudioRendererSink* sink = + static_cast(context); + DCHECK(sink); + sink->ConsumeFrames(frames_consumed); +} + +void CobaltAudioRendererSink::UpdateSourceStatus(int* frames_in_buffer, + int* offset_in_frames, + bool* is_playing, + bool* is_eos_reached) { + // Assert that we never consume more than we've rendered. + DCHECK_GE(frames_rendered_, frames_consumed_); + DCHECK(input_audio_bus_); + + *is_playing = is_playing_.load(); + *is_eos_reached = is_eos_reached_.load(); + + *frames_in_buffer = static_cast(frames_rendered_ - frames_consumed_); + + while ((frames_per_channel_ - *frames_in_buffer) >= + params_.frames_per_buffer()) { + int frames_filled = + callback_->Render(base::TimeDelta(), base::TimeTicks(), + /*glitch_info=*/{}, input_audio_bus_.get()); + + if (frames_filled == 0) { + break; + } + + DCHECK_EQ(params_.frames_per_buffer(), frames_filled); + FillOutputAudioBuffer(frames_filled); + + frames_rendered_ += frames_filled; + *frames_in_buffer += frames_filled; + } + *offset_in_frames = frames_consumed_ % frames_per_channel_; +} + +void CobaltAudioRendererSink::ConsumeFrames(int frames_consumed) { + frames_consumed_ += frames_consumed; +} + +void CobaltAudioRendererSink::FillOutputAudioBuffer(int num_frames) { + uint64_t channel_offset = frames_rendered_ % frames_per_channel_; + AudioBus* bus = input_audio_bus_.get(); + + if (resampler_) { + bus = resampled_audio_bus_.get(); + resampler_->Resample(num_frames, bus); + } + + if (output_sample_type_ == kSbMediaAudioSampleTypeFloat32) { + float* output_buffer = reinterpret_cast(output_frame_buffer_.get()); + output_buffer += channel_offset * params_.channels(); + bus->ToInterleaved(num_frames, + output_buffer); + } else { + int16_t* output_buffer = + reinterpret_cast(output_frame_buffer_.get()); + output_buffer += channel_offset * params_.channels(); + bus->ToInterleaved(num_frames, output_buffer); + } +} + +void CobaltAudioRendererSink::ResamplerReadCallback(int frame_delay, + AudioBus* output) { + input_audio_bus_->CopyTo(output); +} + +} // namespace media diff --git a/cobalt/media/audio/cobalt_audio_renderer_sink.h b/cobalt/media/audio/cobalt_audio_renderer_sink.h new file mode 100644 index 0000000000000..6442281e16070 --- /dev/null +++ b/cobalt/media/audio/cobalt_audio_renderer_sink.h @@ -0,0 +1,97 @@ +// Copyright 2025 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef COBALT_MEDIA_AUDIO_COBALT_AUDIO_RENDERER_SINK_H +#define COBALT_MEDIA_AUDIO_COBALT_AUDIO_RENDERER_SINK_H + +#include "media/base/audio_renderer_sink.h" + +#include + +#include "media/base/media_export.h" +#include "media/base/multi_channel_resampler.h" +#include "starboard/audio_sink.h" + +namespace media { + +class MEDIA_EXPORT CobaltAudioRendererSink final : public AudioRendererSink { + public: + CobaltAudioRendererSink() = default; + ~CobaltAudioRendererSink() final = default; + + CobaltAudioRendererSink(const CobaltAudioRendererSink&) = delete; + CobaltAudioRendererSink& operator=(const CobaltAudioRendererSink&) = delete; + + // AudioRendererSink implementation. + void Initialize(const AudioParameters& params, + RenderCallback* callback) final; + void Start() final; + void Stop() final; + void Play() final; + void Pause() final; + void Flush() final; + bool SetVolume(double volume) final; + OutputDeviceInfo GetOutputDeviceInfo() final; + void GetOutputDeviceInfoAsync(OutputDeviceInfoCB info_cb) final; + bool IsOptimizedForHardwareParameters() final; + bool CurrentThreadIsRenderingThread() final; + + private: + static void UpdateSourceStatusFunc(int* frames_in_buffer, + int* offset_in_frames, + bool* is_playing, + bool* is_eos_reached, + void* context); + + static void ConsumeFramesFunc(int frames_consumed, void* context); + + void UpdateSourceStatus(int* frames_in_buffer, + int* offset_in_frames, + bool* is_playing, + bool* is_eos_reached); + void ConsumeFrames(int frames_consumed); + + void FillOutputAudioBuffer(int num_frames); + + // only needed if we need to resample. + void ResamplerReadCallback(int frame_delay, AudioBus* output); + + SbMediaAudioSampleType output_sample_type_; + int frames_per_channel_{0}; + + AudioParameters params_; + + std::unique_ptr input_audio_bus_ = nullptr; + + std::unique_ptr output_frame_buffer_ = nullptr; + void* output_frame_buffers_[1]; + + RenderCallback* callback_ = nullptr; + + int64_t frames_rendered_ = 0; // Frames retrieved from |callback_|. + int64_t frames_consumed_ = 0; // Accumulated frames consumed by the sink. + + std::atomic is_playing_ = false; + std::atomic is_eos_reached_ = false; + + SbAudioSink audio_sink_ = kSbAudioSinkInvalid; + + double resample_ratio_ = 1.0; + std::unique_ptr resampler_ = nullptr; + std::unique_ptr resampled_audio_bus_ = nullptr; +}; + +} // namespace media + +#endif // COBALT_MEDIA_AUDIO_COBALT_AUDIO_RENDERER_SINK_H diff --git a/cobalt/renderer/BUILD.gn b/cobalt/renderer/BUILD.gn index 73d80ea2fc06c..4e32e33d7c659 100644 --- a/cobalt/renderer/BUILD.gn +++ b/cobalt/renderer/BUILD.gn @@ -19,6 +19,7 @@ source_set("renderer") { ] deps = [ + "//cobalt/media/audio:webaudio", "//components/cdm/renderer", "//components/network_hints/renderer", "//components/web_cache/renderer", diff --git a/cobalt/renderer/cobalt_content_renderer_client.h b/cobalt/renderer/cobalt_content_renderer_client.h index 855c705cdb660..eb319c8294d21 100644 --- a/cobalt/renderer/cobalt_content_renderer_client.h +++ b/cobalt/renderer/cobalt_content_renderer_client.h @@ -9,13 +9,11 @@ #include #include "build/build_config.h" +#include "cobalt/media/audio/cobalt_audio_device_factory.h" #include "content/public/common/alternative_error_page_override_info.mojom-forward.h" #include "content/public/renderer/content_renderer_client.h" #include "media/mojo/buildflags.h" -// For BUILDFLAG(USE_STARBOARD_MEDIA) -#include "build/build_config.h" - namespace blink { class URLLoaderThrottleProvider; enum class URLLoaderThrottleProviderType; @@ -72,6 +70,8 @@ class CobaltContentRendererClient : public content::ContentRendererClient { private: std::unique_ptr web_cache_impl_; + // Registers a custom content::AudioDeviceFactory + media::CobaltAudioDeviceFactory cobalt_audio_device_factory_; }; } // namespace cobalt From e84f1895eae6322acc9d02d07238e157518fc382 Mon Sep 17 00:00:00 2001 From: Hao <131711973+haozheng-cobalt@users.noreply.github.com> Date: Fri, 17 Jan 2025 16:43:33 -0500 Subject: [PATCH 08/61] JNI migration to jni_generator for file starboard/android/shared/file_internal.cc (#4684) This PR primarily does JNI migration to jni_generator for file starboard/android/shared/file_internal.cc. Manually retrieving jclass, jmethodID etc. for android.content.Context because I'm not familiar with how to generate boilerplate code using jni_generator for Android SDK in gn. This PR also migrates StarboardBridge functions requestSuspend, raisePlatformError, getUserAgentAuxField. Local testing: haozheng@haozheng:~/chromium/src$ adb logcat | grep dir: 01-15 13:57:49.181 21743 21743 I starboard: [dev.cobalt.coat/21743:0115/215749.180116(UTC):INFO:file_internal.cc(63)] Files dir: /data/user/0/dev.cobalt.coat/files 01-15 13:57:49.181 21743 21743 I starboard: [dev.cobalt.coat/21743:0115/215749.181740(UTC):INFO:file_internal.cc(69)] Cache dir: /data/user/0/dev.cobalt.coat/cache 01-15 13:57:49.182 21743 21743 I starboard: [dev.cobalt.coat/21743:0115/215749.182423(UTC):INFO:file_internal.cc(76)] Lib dir: /data/app/~~XYAjxFO9qO3w-PB__fJrIw==/dev.cobalt.coat-lopgoPVMucdrTtNnP5MMog==/lib/arm b/389117681 --- .../java/dev/cobalt/coat/StarboardBridge.java | 23 ++--- starboard/android/shared/file_internal.cc | 59 +++++------ starboard/android/shared/starboard_bridge.cc | 99 +++++++++++++++++-- starboard/android/shared/starboard_bridge.h | 28 +++++- .../android/shared/system_platform_error.cc | 11 +-- .../android/shared/system_request_conceal.cc | 9 +- 6 files changed, 151 insertions(+), 78 deletions(-) diff --git a/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java b/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java index 76eca8cde5ff7..e193d1dc1f6ed 100644 --- a/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java +++ b/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java @@ -256,10 +256,8 @@ protected void applicationStopping() { applicationStopped = true; } - // TODO: (cobalt b/372559388) remove or migrate JNI? - // Used in starboard/android/shared/system_request_conceal.cc @SuppressWarnings("unused") - @UsedByNative + @CalledByNative public void requestSuspend() { Activity activity = activityHolder.get(); if (activity != null) { @@ -281,9 +279,8 @@ public boolean onSearchRequested() { // private native boolean nativeOnSearchRequested(); - // TODO: (cobalt b/372559388) remove or migrate JNI? @SuppressWarnings("unused") - @UsedByNative + @CalledByNative public Context getApplicationContext() { if (appContext == null) { throw new IllegalArgumentException("appContext cannot be null"); @@ -291,10 +288,8 @@ public Context getApplicationContext() { return appContext; } - // TODO: (cobalt b/372559388) remove or migrate JNI? - // Used in starboard/android/shared/system_platform_error.cc @SuppressWarnings("unused") - @UsedByNative + @CalledByNative void raisePlatformError(@PlatformError.ErrorType int errorType, long data) { PlatformError error = new PlatformError(activityHolder, errorType, data); error.raise(); @@ -347,26 +342,22 @@ private void nativeHandleDeepLink(String url) { // TODO(b/374147993): Implement deep link } - // TODO: (cobalt b/372559388) remove or migrate JNI? - // Used in starboard/android/shared/file_internal.cc /** * Returns the absolute path to the directory where application specific files should be written. * May be overridden for use cases that need to segregate storage. */ @SuppressWarnings("unused") - @UsedByNative + @CalledByNative protected String getFilesAbsolutePath() { return appContext.getFilesDir().getAbsolutePath(); } - // TODO: (cobalt b/372559388) remove or migrate JNI? - // Used in starboard/android/shared/file_internal.cc /** * Returns the absolute path to the application specific cache directory on the filesystem. May be * overridden for use cases that need to segregate storage. */ @SuppressWarnings("unused") - @UsedByNative + @CalledByNative protected String getCacheAbsolutePath() { return appContext.getCacheDir().getAbsolutePath(); } @@ -586,11 +577,9 @@ public void deactivateMediaSession() { // cobaltMediaSession.deactivateMediaSession(); } - // TODO: (cobalt b/372559388) remove or migrate JNI? - // Used in starboard/android/shared/system_get_property.cc /** Returns string for kSbSystemPropertyUserAgentAuxField */ @SuppressWarnings("unused") - @UsedByNative + @CalledByNative protected String getUserAgentAuxField() { StringBuilder sb = new StringBuilder(); diff --git a/starboard/android/shared/file_internal.cc b/starboard/android/shared/file_internal.cc index 65a308aeeca10..db533b8d46d62 100644 --- a/starboard/android/shared/file_internal.cc +++ b/starboard/android/shared/file_internal.cc @@ -20,8 +20,7 @@ #include #include -#include "starboard/android/shared/jni_env_ext.h" -#include "starboard/android/shared/jni_utils.h" +#include "starboard/android/shared/starboard_bridge.h" #include "starboard/common/log.h" #include "starboard/common/string.h" @@ -35,60 +34,46 @@ const char* g_app_cache_dir = NULL; const char* g_app_lib_dir = NULL; namespace { -jobject g_java_asset_manager; +ScopedJavaGlobalRef g_java_asset_manager; AAssetManager* g_asset_manager; - -// Copies the characters from a jstring and returns a newly allocated buffer -// with the result. -const char* DuplicateJavaString(JniEnvExt* env, jstring j_string) { - SB_DCHECK(j_string); - std::string utf_str = env->GetStringStandardUTFOrAbort(j_string); - const char* result = strdup(utf_str.c_str()); - return result; -} - } // namespace void SbFileAndroidInitialize() { - JniEnvExt* env = JniEnvExt::Get(); + JNIEnv* env = base::android::AttachCurrentThread(); + StarboardBridge* starbooard_bridge = StarboardBridge::GetInstance(); - SB_DCHECK(g_java_asset_manager == NULL); + SB_DCHECK(g_java_asset_manager.is_null()); SB_DCHECK(g_asset_manager == NULL); - ScopedLocalJavaRef j_app(env->CallStarboardObjectMethodOrAbort( - "getApplicationContext", "()Landroid/content/Context;")); - g_java_asset_manager = - env->ConvertLocalRefToGlobalRef(env->CallObjectMethodOrAbort( - j_app.Get(), "getAssets", "()Landroid/content/res/AssetManager;")); - g_asset_manager = AAssetManager_fromJava(env, g_java_asset_manager); + ScopedJavaLocalRef context = + starbooard_bridge->GetApplicationContext(env); + + g_java_asset_manager = starbooard_bridge->GetAssetsFromContext(env, context); + g_asset_manager = AAssetManager_fromJava(env, g_java_asset_manager.obj()); + + std::string app_files_dir = starbooard_bridge->GetFilesAbsolutePath(env); SB_DCHECK(g_app_files_dir == NULL); - ScopedLocalJavaRef j_string(env->CallStarboardObjectMethodOrAbort( - "getFilesAbsolutePath", "()Ljava/lang/String;")); - g_app_files_dir = DuplicateJavaString(env, j_string.Get()); + g_app_files_dir = strdup(app_files_dir.c_str()); + SB_DLOG(INFO) << "Files dir: " << g_app_files_dir; + std::string app_cache_dir = starbooard_bridge->GetCacheAbsolutePath(env); SB_DCHECK(g_app_cache_dir == NULL); - j_string.Reset(env->CallStarboardObjectMethodOrAbort("getCacheAbsolutePath", - "()Ljava/lang/String;")); - g_app_cache_dir = DuplicateJavaString(env, j_string.Get()); + g_app_cache_dir = strdup(app_cache_dir.c_str()); + SB_DLOG(INFO) << "Cache dir: " << g_app_cache_dir; + std::string app_lib_dir = + starbooard_bridge->GetNativeLibraryDirFromContext(env, context); SB_DCHECK(g_app_lib_dir == NULL); - ScopedLocalJavaRef j_app_info( - env->CallObjectMethodOrAbort(j_app.Get(), "getApplicationInfo", - "()Landroid/content/pm/ApplicationInfo;")); - j_string.Reset( - env->GetStringFieldOrAbort(j_app_info.Get(), "nativeLibraryDir")); - g_app_lib_dir = DuplicateJavaString(env, j_string.Get()); + g_app_lib_dir = strdup(app_lib_dir.c_str()); + SB_DLOG(INFO) << "Lib dir: " << g_app_lib_dir; } void SbFileAndroidTeardown() { - JniEnvExt* env = JniEnvExt::Get(); - if (g_java_asset_manager) { - env->DeleteGlobalRef(g_java_asset_manager); - g_java_asset_manager = NULL; + g_java_asset_manager.Reset(); g_asset_manager = NULL; } diff --git a/starboard/android/shared/starboard_bridge.cc b/starboard/android/shared/starboard_bridge.cc index 936accf481fe4..90b874d763530 100644 --- a/starboard/android/shared/starboard_bridge.cc +++ b/starboard/android/shared/starboard_bridge.cc @@ -33,6 +33,12 @@ namespace starboard { namespace android { namespace shared { +// TODO: (cobalt b/372559388) Update namespace to jni_zero. +using base::android::AppendJavaStringArrayToStringVector; +using base::android::AttachCurrentThread; +using base::android::ConvertJavaStringToUTF8; +using base::android::GetClass; + namespace { #if SB_IS(EVERGREEN_COMPATIBLE) void StarboardThreadLaunch() { @@ -64,7 +70,7 @@ std::vector GetArgs() { // Fake program name as args[0] args.push_back("android_main"); - JNIEnv* env = base::android::AttachCurrentThread(); + JNIEnv* env = AttachCurrentThread(); StarboardBridge::GetInstance()->AppendArgs(env, &args); return args; @@ -87,10 +93,10 @@ JNI_StarboardBridge_StartNativeStarboard(JNIEnv* env) { #else auto command_line = std::make_unique(GetArgs()); LogInit(*command_line); - auto* nativeApp = new ApplicationAndroid(std::move(command_line)); + auto* native_app = new ApplicationAndroid(std::move(command_line)); // Ensure application init happens here ApplicationAndroid::Get(); - return reinterpret_cast(nativeApp); + return reinterpret_cast(native_app); #endif // SB_IS(EVERGREEN_COMPATIBLE) } @@ -130,26 +136,99 @@ void StarboardBridge::AfterStopped(JNIEnv* env) { void StarboardBridge::AppendArgs(JNIEnv* env, std::vector* args_vector) { SB_DCHECK(env); - base::android::ScopedJavaLocalRef args_java = + ScopedJavaLocalRef args_java = Java_StarboardBridge_getArgs(env, j_starboard_bridge_); - base::android::AppendJavaStringArrayToStringVector(env, args_java, - args_vector); + AppendJavaStringArrayToStringVector(env, args_java, args_vector); } std::string StarboardBridge::GetStartDeepLink(JNIEnv* env) { SB_DCHECK(env); - base::android::ScopedJavaLocalRef start_deep_link_java = + ScopedJavaLocalRef start_deep_link_java = Java_StarboardBridge_getStartDeepLink(env, j_starboard_bridge_); std::string start_deep_link = - base::android::ConvertJavaStringToUTF8(env, start_deep_link_java); + ConvertJavaStringToUTF8(env, start_deep_link_java); return start_deep_link; } -base::android::ScopedJavaLocalRef -StarboardBridge::GetSupportedHdrTypes(JNIEnv* env) { +ScopedJavaLocalRef StarboardBridge::GetSupportedHdrTypes( + JNIEnv* env) { SB_DCHECK(env); return Java_StarboardBridge_getSupportedHdrTypes(env, j_starboard_bridge_); } + +void StarboardBridge::RaisePlatformError(JNIEnv* env, + jint errorType, + jlong data) { + SB_DCHECK(env); + Java_StarboardBridge_raisePlatformError(env, j_starboard_bridge_, errorType, + data); +} + +void StarboardBridge::RequestSuspend(JNIEnv* env) { + SB_DCHECK(env); + Java_StarboardBridge_requestSuspend(env, j_starboard_bridge_); +} + +ScopedJavaLocalRef StarboardBridge::GetApplicationContext( + JNIEnv* env) { + SB_DCHECK(env); + return Java_StarboardBridge_getApplicationContext(env, j_starboard_bridge_); +} + +ScopedJavaGlobalRef StarboardBridge::GetAssetsFromContext( + JNIEnv* env, + ScopedJavaLocalRef& context) { + SB_DCHECK(env); + ScopedJavaLocalRef context_class( + GetClass(env, "android/content/Context")); + jmethodID get_assets_method = env->GetMethodID( + context_class.obj(), "getAssets", "()Landroid/content/res/AssetManager;"); + ScopedJavaLocalRef asset_manager( + env, env->CallObjectMethod(context.obj(), get_assets_method)); + ScopedJavaGlobalRef global_asset_manager; + global_asset_manager.Reset(asset_manager); + return global_asset_manager; +} + +std::string StarboardBridge::GetNativeLibraryDirFromContext( + JNIEnv* env, + ScopedJavaLocalRef& context) { + SB_DCHECK(env); + ScopedJavaLocalRef context_class( + GetClass(env, "android/content/Context")); + jmethodID get_application_info_method = + env->GetMethodID(context_class.obj(), "getApplicationInfo", + "()Landroid/content/pm/ApplicationInfo;"); + ScopedJavaLocalRef application_info( + env, env->CallObjectMethod(context.obj(), get_application_info_method)); + + ScopedJavaLocalRef application_info_class( + env, env->GetObjectClass(application_info.obj())); + jfieldID native_library_dir_field = env->GetFieldID( + application_info_class.obj(), "nativeLibraryDir", "Ljava/lang/String;"); + ScopedJavaLocalRef native_library_dir_java( + env, static_cast(env->GetObjectField(application_info.obj(), + native_library_dir_field))); + std::string native_library_dir = + ConvertJavaStringToUTF8(env, native_library_dir_java.obj()); + return native_library_dir.c_str(); +} + +std::string StarboardBridge::GetFilesAbsolutePath(JNIEnv* env) { + SB_DCHECK(env); + ScopedJavaLocalRef file_path_java = + Java_StarboardBridge_getFilesAbsolutePath(env, j_starboard_bridge_); + std::string file_path = ConvertJavaStringToUTF8(env, file_path_java); + return file_path; +} + +std::string StarboardBridge::GetCacheAbsolutePath(JNIEnv* env) { + SB_DCHECK(env); + ScopedJavaLocalRef file_path_java = + Java_StarboardBridge_getCacheAbsolutePath(env, j_starboard_bridge_); + std::string file_path = ConvertJavaStringToUTF8(env, file_path_java); + return file_path; +} } // namespace shared } // namespace android } // namespace starboard diff --git a/starboard/android/shared/starboard_bridge.h b/starboard/android/shared/starboard_bridge.h index d834c3b20543f..0aa94326c1652 100644 --- a/starboard/android/shared/starboard_bridge.h +++ b/starboard/android/shared/starboard_bridge.h @@ -25,6 +25,11 @@ namespace starboard { namespace android { namespace shared { +// TODO: (cobalt b/372559388) Update namespace to jni_zero. +using base::android::JavaParamRef; +using base::android::ScopedJavaGlobalRef; +using base::android::ScopedJavaLocalRef; + // This class serves as a bridge between the native code and Android // StarboardBridge Java class. class StarboardBridge { @@ -46,8 +51,25 @@ class StarboardBridge { std::string GetStartDeepLink(JNIEnv* env); - base::android::ScopedJavaLocalRef GetSupportedHdrTypes( - JNIEnv* env); + ScopedJavaLocalRef GetSupportedHdrTypes(JNIEnv* env); + + void RaisePlatformError(JNIEnv* env, jint errorType, jlong data); + + void RequestSuspend(JNIEnv* env); + + ScopedJavaLocalRef GetApplicationContext(JNIEnv* env); + + ScopedJavaGlobalRef GetAssetsFromContext( + JNIEnv* env, + ScopedJavaLocalRef& context); + + std::string GetNativeLibraryDirFromContext( + JNIEnv* env, + ScopedJavaLocalRef& context); + + std::string GetFilesAbsolutePath(JNIEnv* env); + + std::string GetCacheAbsolutePath(JNIEnv* env); private: StarboardBridge() = default; @@ -60,7 +82,7 @@ class StarboardBridge { friend struct base::DefaultSingletonTraits; // Java StarboardBridge instance. - base::android::ScopedJavaGlobalRef j_starboard_bridge_; + ScopedJavaGlobalRef j_starboard_bridge_; }; } // namespace shared diff --git a/starboard/android/shared/system_platform_error.cc b/starboard/android/shared/system_platform_error.cc index affd932b68d4c..f71d8a510492a 100644 --- a/starboard/android/shared/system_platform_error.cc +++ b/starboard/android/shared/system_platform_error.cc @@ -20,10 +20,9 @@ #include #include "starboard/android/shared/application_android.h" -#include "starboard/android/shared/jni_env_ext.h" +#include "starboard/android/shared/starboard_bridge.h" using starboard::android::shared::ApplicationAndroid; -using starboard::android::shared::JniEnvExt; namespace { @@ -50,7 +49,7 @@ bool SbSystemRaisePlatformError(SbSystemPlatformErrorType type, return false; } - JniEnvExt* env = JniEnvExt::Get(); + JNIEnv* env = base::android::AttachCurrentThread(); auto send_response_callback = callback ? new SendResponseCallback( @@ -60,9 +59,9 @@ bool SbSystemRaisePlatformError(SbSystemPlatformErrorType type, }) : nullptr; - env->CallStarboardVoidMethodOrAbort( - "raisePlatformError", "(IJ)V", jni_error_type, - reinterpret_cast(send_response_callback)); + starboard::android::shared::StarboardBridge::GetInstance() + ->RaisePlatformError(env, jni_error_type, + reinterpret_cast(send_response_callback)); return true; } diff --git a/starboard/android/shared/system_request_conceal.cc b/starboard/android/shared/system_request_conceal.cc index ebfeb4a5b3fb1..20a9956f548e8 100644 --- a/starboard/android/shared/system_request_conceal.cc +++ b/starboard/android/shared/system_request_conceal.cc @@ -14,11 +14,10 @@ #include "starboard/system.h" -#include "starboard/android/shared/jni_env_ext.h" - -using starboard::android::shared::JniEnvExt; +#include "starboard/android/shared/starboard_bridge.h" void SbSystemRequestConceal() { - JniEnvExt* env = JniEnvExt::Get(); - env->CallStarboardVoidMethodOrAbort("requestSuspend", "()V"); + JNIEnv* env = base::android::AttachCurrentThread(); + starboard::android::shared::StarboardBridge::GetInstance()->RequestSuspend( + env); } From 9a4ed4933ecc787ddda43462d461a361dab828d9 Mon Sep 17 00:00:00 2001 From: Holden Warriner Date: Fri, 17 Jan 2025 17:15:12 -0500 Subject: [PATCH 09/61] Refactor CrashAnnotator impl as a Cobalt service (#4709) This is in response to feedback from mcasas@, and others: since the CrashAnnotator is not a core service, it shouldn't be implemented under //content. //cobalt/services seems like a good place since the CrashAnnotator service is for use by Cobalt in particular. The use of DocumentService was recommended by dcheng@ and ortuno@ as a way to scope our Mojo impl's lifetime to the document / RenderFrameHost. b/383301493 --- cobalt/BUILD.gn | 4 ++ cobalt/android/BUILD.gn | 6 ++ cobalt/cobalt_browser_interface_binders.cc | 30 ++++++++++ .../cobalt_browser_interface_binders.h | 28 ++++----- cobalt/cobalt_content_browser_client.cc | 9 ++- cobalt/cobalt_content_browser_client.h | 13 +++++ cobalt/services/crash_annotator/BUILD.gn | 27 +++++++++ .../crash_annotator_service.cc | 44 ++++++++++++++ .../crash_annotator/crash_annotator_service.h | 57 +++++++++++++++++++ .../crash_annotator/public/mojom/BUILD.gn | 19 +++++++ .../mojom/crash_annotator_service.mojom | 6 +- content/browser/BUILD.gn | 7 --- content/browser/browser_interface_binders.cc | 10 ---- .../crash_annotator/crash_annotator_impl.h | 47 --------------- .../renderer_host/render_frame_host_impl.cc | 11 ---- .../renderer_host/render_frame_host_impl.h | 17 ------ third_party/blink/public/mojom/BUILD.gn | 4 -- .../modules/cobalt/crash_annotator/BUILD.gn | 2 + .../cobalt/crash_annotator/crash_annotator.cc | 20 +++---- .../cobalt/crash_annotator/crash_annotator.h | 4 +- .../wpt/cobalt/crash-annotator/README.md | 4 +- .../crash-annotator/resources/automation.js | 20 +++---- ...tor.js => fake-crash-annotator-service.js} | 12 ++-- 23 files changed, 258 insertions(+), 143 deletions(-) create mode 100644 cobalt/cobalt_browser_interface_binders.cc rename content/browser/cobalt/crash_annotator/crash_annotator_impl.cc => cobalt/cobalt_browser_interface_binders.h (51%) create mode 100644 cobalt/services/crash_annotator/BUILD.gn create mode 100644 cobalt/services/crash_annotator/crash_annotator_service.cc create mode 100644 cobalt/services/crash_annotator/crash_annotator_service.h create mode 100644 cobalt/services/crash_annotator/public/mojom/BUILD.gn rename third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom => cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom (86%) delete mode 100644 content/browser/cobalt/crash_annotator/crash_annotator_impl.h rename third_party/blink/web_tests/external/wpt/resources/chromium/cobalt/{fake-crash-annotator.js => fake-crash-annotator-service.js} (62%) diff --git a/cobalt/BUILD.gn b/cobalt/BUILD.gn index 8da714a12e424..f80c773450d89 100644 --- a/cobalt/BUILD.gn +++ b/cobalt/BUILD.gn @@ -36,6 +36,8 @@ if (!is_android) { sources = [ "cobalt.cc", + "cobalt_browser_interface_binders.cc", + "cobalt_browser_interface_binders.h", "cobalt_content_browser_client.cc", "cobalt_content_browser_client.h", "cobalt_main_delegate.cc", @@ -48,6 +50,8 @@ if (!is_android) { deps = [ "//cobalt/renderer:renderer", + "//cobalt/services/crash_annotator", + "//cobalt/services/crash_annotator/public/mojom", "//cobalt/user_agent", "//content/public/app", "//content/shell:content_shell_app", diff --git a/cobalt/android/BUILD.gn b/cobalt/android/BUILD.gn index e598c94595194..ab62facee1d27 100644 --- a/cobalt/android/BUILD.gn +++ b/cobalt/android/BUILD.gn @@ -242,6 +242,8 @@ shared_library("libcobalt_content_shell_content_view") { deps = [ ":content_shell_jni_headers", "//cobalt/renderer:renderer", + "//cobalt/services/crash_annotator", + "//cobalt/services/crash_annotator/public/mojom", "//cobalt/user_agent", # TODO: what can be removed in the dependencies? @@ -264,6 +266,10 @@ shared_library("libcobalt_content_shell_content_view") { ] } sources = [ + # TODO(cobalt, b/383301493): move browser-related source files under a + # //cobalt/browser directory and define a "browser" build target there. + "//cobalt/cobalt_browser_interface_binders.cc", + "//cobalt/cobalt_browser_interface_binders.h", "//cobalt/cobalt_content_browser_client.cc", "//cobalt/cobalt_content_browser_client.h", "//cobalt/cobalt_main_delegate.cc", diff --git a/cobalt/cobalt_browser_interface_binders.cc b/cobalt/cobalt_browser_interface_binders.cc new file mode 100644 index 0000000000000..0524b4c2987dc --- /dev/null +++ b/cobalt/cobalt_browser_interface_binders.cc @@ -0,0 +1,30 @@ +// Copyright 2025 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cobalt/cobalt_browser_interface_binders.h" + +#include "base/functional/bind.h" +#include "cobalt/services/crash_annotator/crash_annotator_service.h" +#include "cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom.h" + +namespace cobalt { + +void PopulateCobaltFrameBinders( + content::RenderFrameHost* render_frame_host, + mojo::BinderMapWithContext* binder_map) { + binder_map->Add( + base::BindRepeating(&crash_annotator::CrashAnnotatorService::Create)); +} + +} // namespace cobalt diff --git a/content/browser/cobalt/crash_annotator/crash_annotator_impl.cc b/cobalt/cobalt_browser_interface_binders.h similarity index 51% rename from content/browser/cobalt/crash_annotator/crash_annotator_impl.cc rename to cobalt/cobalt_browser_interface_binders.h index 72bb36a25658f..0426b0da38ed4 100644 --- a/content/browser/cobalt/crash_annotator/crash_annotator_impl.cc +++ b/cobalt/cobalt_browser_interface_binders.h @@ -1,4 +1,4 @@ -// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// Copyright 2025 The Cobalt Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,20 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "content/browser/cobalt/crash_annotator/crash_annotator_impl.h" +#ifndef COBALT_COBALT_BROWSER_INTERFACE_BINDERS_H_ +#define COBALT_COBALT_BROWSER_INTERFACE_BINDERS_H_ -#include "base/functional/bind.h" -#include "base/functional/callback.h" +#include "mojo/public/cpp/bindings/binder_map.h" namespace content { +class RenderFrameHost; +} // namespace content -void CrashAnnotatorImpl::SetString(const std::string& key, - const std::string& value, - SetStringCallback callback) { - // TODO(cobalt, b/383301493): actually implement this. - LOG(INFO) << "CrashAnnotatorImpl::SetString key=" << key << " value=" - << value; - std::move(callback).Run(false); -} +namespace cobalt { -} // namespace content +// Registers binders for Cobalt-specific, document-scoped Mojo interfaces. +void PopulateCobaltFrameBinders( + content::RenderFrameHost* render_frame_host, + mojo::BinderMapWithContext* binder_map); + +} // namespace cobalt + +#endif // COBALT_COBALT_BROWSER_INTERFACE_BINDERS_H_ diff --git a/cobalt/cobalt_content_browser_client.cc b/cobalt/cobalt_content_browser_client.cc index 9e9dd549888e4..2e01d5fb823cd 100644 --- a/cobalt/cobalt_content_browser_client.cc +++ b/cobalt/cobalt_content_browser_client.cc @@ -16,6 +16,7 @@ #include +#include "cobalt/cobalt_browser_interface_binders.h" #include "cobalt/user_agent/user_agent_platform_info.h" #include "content/public/common/user_agent.h" #include "third_party/blink/public/common/web_preferences/web_preferences.h" @@ -90,10 +91,16 @@ void CobaltContentBrowserClient::OverrideWebkitPrefs( #endif // !defined(COBALT_IS_RELEASE_BUILD) content::ShellContentBrowserClient::OverrideWebkitPrefs(web_contents, prefs); } - void CobaltContentBrowserClient::OnWebContentsCreated( content::WebContents* web_contents) { web_contents_observer_.reset(new CobaltWebContentsObserver(web_contents)); } +void CobaltContentBrowserClient::RegisterBrowserInterfaceBindersForFrame( + content::RenderFrameHost* render_frame_host, + mojo::BinderMapWithContext* map) { + PopulateCobaltFrameBinders(render_frame_host, map); + ShellContentBrowserClient::RegisterBrowserInterfaceBindersForFrame( + render_frame_host, map); +} } // namespace cobalt diff --git a/cobalt/cobalt_content_browser_client.h b/cobalt/cobalt_content_browser_client.h index 80170477dee59..12d3ba6243025 100644 --- a/cobalt/cobalt_content_browser_client.h +++ b/cobalt/cobalt_content_browser_client.h @@ -19,6 +19,15 @@ #include "content/public/browser/web_contents.h" #include "content/shell/browser/shell_content_browser_client.h" +namespace content { +class RenderFrameHost; +} + +namespace mojo { +template +class BinderMapWithContext; +} // namespace mojo + namespace cobalt { class CobaltContentBrowserClient : public content::ShellContentBrowserClient { @@ -34,6 +43,10 @@ class CobaltContentBrowserClient : public content::ShellContentBrowserClient { void OverrideWebkitPrefs(content::WebContents* web_contents, blink::web_pref::WebPreferences* prefs) override; void OnWebContentsCreated(content::WebContents* web_contents); + void RegisterBrowserInterfaceBindersForFrame( + content::RenderFrameHost* render_frame_host, + mojo::BinderMapWithContext* binder_map) + override; private: std::unique_ptr web_contents_observer_; diff --git a/cobalt/services/crash_annotator/BUILD.gn b/cobalt/services/crash_annotator/BUILD.gn new file mode 100644 index 0000000000000..cd55b4399c5df --- /dev/null +++ b/cobalt/services/crash_annotator/BUILD.gn @@ -0,0 +1,27 @@ +# Copyright 2025 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source_set("crash_annotator") { + sources = [ + "crash_annotator_service.cc", + "crash_annotator_service.h", + ] + + deps = [ + "//base", + "//cobalt/services/crash_annotator/public/mojom", + "//content/public/browser", + "//mojo/public/cpp/bindings", + ] +} diff --git a/cobalt/services/crash_annotator/crash_annotator_service.cc b/cobalt/services/crash_annotator/crash_annotator_service.cc new file mode 100644 index 0000000000000..f5c6919396d6b --- /dev/null +++ b/cobalt/services/crash_annotator/crash_annotator_service.cc @@ -0,0 +1,44 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cobalt/services/crash_annotator/crash_annotator_service.h" + +#include "base/functional/bind.h" +#include "base/functional/callback.h" + +namespace crash_annotator { + +CrashAnnotatorService::CrashAnnotatorService( + content::RenderFrameHost& render_frame_host, + mojo::PendingReceiver receiver) + : content::DocumentService( + render_frame_host, + std::move(receiver)) {} + +void CrashAnnotatorService::Create( + content::RenderFrameHost* render_frame_host, + mojo::PendingReceiver receiver) { + new CrashAnnotatorService(*render_frame_host, std::move(receiver)); +} + +void CrashAnnotatorService::SetString(const std::string& key, + const std::string& value, + SetStringCallback callback) { + // TODO(cobalt, b/383301493): actually implement this. + LOG(INFO) << "CrashAnnotatorImpl::SetString key=" << key + << " value=" << value; + std::move(callback).Run(false); +} + +} // namespace crash_annotator diff --git a/cobalt/services/crash_annotator/crash_annotator_service.h b/cobalt/services/crash_annotator/crash_annotator_service.h new file mode 100644 index 0000000000000..521385b9b8f48 --- /dev/null +++ b/cobalt/services/crash_annotator/crash_annotator_service.h @@ -0,0 +1,57 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef COBALT_SERVICES_CRASH_ANNOTATOR_CRASH_ANNOTATOR_SERVICE_H_ +#define COBALT_SERVICES_CRASH_ANNOTATOR_CRASH_ANNOTATOR_SERVICE_H_ + +#include + +#include "cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom.h" +#include "content/public/browser/document_service.h" +#include "mojo/public/cpp/bindings/pending_receiver.h" + +namespace content { +class RenderFrameHost; +} // namespace content + +namespace crash_annotator { + +// Implements the CrashAnnotatorService Mojo interface and extends +// DocumentService so that an object's lifetime is scoped to the corresponding +// document / RenderFrameHost (see DocumentService for details). +class CrashAnnotatorService + : public content::DocumentService { + public: + // Creates a CrashAnnotatorService. The CrashAnnotatorService is bound to the + // receiver and its lifetime is scoped to the render_frame_host. + static void Create( + content::RenderFrameHost* render_frame_host, + mojo::PendingReceiver receiver); + + CrashAnnotatorService(const CrashAnnotatorService&) = delete; + CrashAnnotatorService& operator=(const CrashAnnotatorService&) = delete; + + void SetString(const std::string& key, + const std::string& value, + SetStringCallback callback) override; + + private: + CrashAnnotatorService( + content::RenderFrameHost& render_frame_host, + mojo::PendingReceiver receiver); +}; + +} // namespace crash_annotator + +#endif // COBALT_SERVICES_CRASH_ANNOTATOR_CRASH_ANNOTATOR_SERVICE_H_ diff --git a/cobalt/services/crash_annotator/public/mojom/BUILD.gn b/cobalt/services/crash_annotator/public/mojom/BUILD.gn new file mode 100644 index 0000000000000..4d96bee48bb56 --- /dev/null +++ b/cobalt/services/crash_annotator/public/mojom/BUILD.gn @@ -0,0 +1,19 @@ +# Copyright 2025 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//mojo/public/tools/bindings/mojom.gni") + +mojom("mojom") { + sources = [ "crash_annotator_service.mojom" ] +} diff --git a/third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom b/cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom similarity index 86% rename from third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom rename to cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom index 84deafd8df26e..ed9d19daa489d 100644 --- a/third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom +++ b/cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -module blink.mojom; +module crash_annotator.mojom; // The browser process must provide an implementation of this interface so that // the renderer process can implement the CrashAnnotator Blink API. -interface CrashAnnotator { - // Recieves an annotation to set and responds with the result. +interface CrashAnnotatorService { + // Receives an annotation to set and responds with the result. SetString(string key, string value) => (bool result); }; diff --git a/content/browser/BUILD.gn b/content/browser/BUILD.gn index 7e224c74a6647..7824a83f0a7f7 100644 --- a/content/browser/BUILD.gn +++ b/content/browser/BUILD.gn @@ -2321,13 +2321,6 @@ source_set("browser") { "worker_host/worker_script_loader_factory.h", ] - if (is_cobalt) { - sources += [ - "cobalt/crash_annotator/crash_annotator_impl.cc", - "cobalt/crash_annotator/crash_annotator_impl.h", - ] - } - if (use_starscan) { sources += [ "starscan_load_observer.cc", diff --git a/content/browser/browser_interface_binders.cc b/content/browser/browser_interface_binders.cc index 78da69eda21c9..90bd74a313bf1 100644 --- a/content/browser/browser_interface_binders.cc +++ b/content/browser/browser_interface_binders.cc @@ -174,11 +174,6 @@ #include "third_party/blink/public/public_buildflags.h" #include "url/origin.h" -#if BUILDFLAG(IS_COBALT) -#include "content/browser/cobalt/crash_annotator/crash_annotator_impl.h" -#include "third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom.h" -#endif - #if BUILDFLAG(IS_ANDROID) #include "content/browser/android/date_time_chooser_android.h" #include "content/browser/android/text_suggestion_host_android.h" @@ -846,11 +841,6 @@ void PopulateFrameBinders(RenderFrameHostImpl* host, mojo::BinderMap* map) { map->Add(base::BindRepeating( &RenderFrameHostImpl::GetSpeechSynthesis, base::Unretained(host))); -#if BUILDFLAG(IS_COBALT) - map->Add(base::BindRepeating( - &RenderFrameHostImpl::GetCrashAnnotator, base::Unretained(host))); -#endif - #if !BUILDFLAG(IS_ANDROID) map->Add(base::BindRepeating( &RenderFrameHostImpl::GetDeviceInfoService, base::Unretained(host))); diff --git a/content/browser/cobalt/crash_annotator/crash_annotator_impl.h b/content/browser/cobalt/crash_annotator/crash_annotator_impl.h deleted file mode 100644 index 2573de92f779b..0000000000000 --- a/content/browser/cobalt/crash_annotator/crash_annotator_impl.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2024 The Cobalt Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CONTENT_BROWSER_CRASH_ANNOTATOR_IMPL_H_ -#define CONTENT_BROWSER_CRASH_ANNOTATOR_IMPL_H_ - -#include - -#include "mojo/public/cpp/bindings/pending_receiver.h" -#include "mojo/public/cpp/bindings/receiver.h" -#include "third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom.h" - -namespace content { - -// TODO(cobalt, b/383301493): consider another location for Cobalt's Mojo -// implementations, since they are not "core" services. mcasas@ suggested -// //components or //services as possible destinations. -class CrashAnnotatorImpl : public blink::mojom::CrashAnnotator { - public: - explicit CrashAnnotatorImpl( - mojo::PendingReceiver receiver) - : receiver_(this, std::move(receiver)) {} - CrashAnnotatorImpl(const CrashAnnotatorImpl&) = delete; - CrashAnnotatorImpl& operator=(const CrashAnnotatorImpl&) = delete; - - void SetString(const std::string& key, - const std::string& value, - SetStringCallback callback) override; - - private: - mojo::Receiver receiver_; -}; - -} // namespace content - -#endif // CONTENT_BROWSER_CRASH_ANNOTATOR_IMPL_H_ diff --git a/content/browser/renderer_host/render_frame_host_impl.cc b/content/browser/renderer_host/render_frame_host_impl.cc index be77a68af476f..3b35cc5d44928 100644 --- a/content/browser/renderer_host/render_frame_host_impl.cc +++ b/content/browser/renderer_host/render_frame_host_impl.cc @@ -269,10 +269,6 @@ #include "url/origin.h" #include "url/url_constants.h" -#if BUILDFLAG(IS_COBALT) -#include "content/browser/cobalt/crash_annotator/crash_annotator_impl.h" -#endif - #if BUILDFLAG(IS_ANDROID) #include "content/browser/android/content_url_loader_factory.h" #include "content/browser/android/java_interfaces_impl.h" @@ -11597,13 +11593,6 @@ void RenderFrameHostImpl::GetFileSystemManager( storage_key(), std::move(receiver))); } -#if BUILDFLAG(IS_COBALT) -void RenderFrameHostImpl::GetCrashAnnotator( - mojo::PendingReceiver receiver) { - crash_annotator_ = std::make_unique(std::move(receiver)); -} -#endif - void RenderFrameHostImpl::GetGeolocationService( mojo::PendingReceiver receiver) { if (!geolocation_service_) { diff --git a/content/browser/renderer_host/render_frame_host_impl.h b/content/browser/renderer_host/render_frame_host_impl.h index 52c56fd943335..0c61e60b8a113 100644 --- a/content/browser/renderer_host/render_frame_host_impl.h +++ b/content/browser/renderer_host/render_frame_host_impl.h @@ -172,11 +172,6 @@ #include "ui/gfx/geometry/rect.h" #include "url/gurl.h" -#if BUILDFLAG(IS_COBALT) -#include "third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom-forward.h" -#include "third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom.h" -#endif - #if BUILDFLAG(IS_ANDROID) #include "base/containers/id_map.h" #include "services/device/public/mojom/nfc.mojom.h" @@ -254,9 +249,6 @@ class DocumentServiceBase; class AgentSchedulingGroupHost; class BrowsingContextState; class CodeCacheHostImpl; -#if BUILDFLAG(IS_COBALT) -class CrashAnnotatorImpl; -#endif class CrossOriginEmbedderPolicyReporter; class CrossOriginOpenerPolicyAccessReportManager; class FeatureObserver; @@ -1950,11 +1942,6 @@ class CONTENT_EXPORT RenderFrameHostImpl void GetFileSystemAccessManager( mojo::PendingReceiver receiver); -#if BUILDFLAG(IS_COBALT) - void GetCrashAnnotator( - mojo::PendingReceiver receiver); -#endif - #if !BUILDFLAG(IS_ANDROID) void GetHidService(mojo::PendingReceiver receiver); @@ -4505,10 +4492,6 @@ class CONTENT_EXPORT RenderFrameHostImpl // Tracks the document policy which has been set on this frame. std::unique_ptr document_policy_; -#if BUILDFLAG(IS_COBALT) - std::unique_ptr crash_annotator_; -#endif - #if BUILDFLAG(IS_ANDROID) // An InterfaceProvider for Java-implemented interfaces that are scoped to // this RenderFrameHost. This provides access to interfaces implemented in diff --git a/third_party/blink/public/mojom/BUILD.gn b/third_party/blink/public/mojom/BUILD.gn index a4f61e8fca994..b1145a98a6d16 100644 --- a/third_party/blink/public/mojom/BUILD.gn +++ b/third_party/blink/public/mojom/BUILD.gn @@ -241,10 +241,6 @@ mojom("mojom_platform") { "worker/worker_options.mojom", ] - if (is_cobalt) { - sources += [ "cobalt/crash_annotator/crash_annotator.mojom" ] - } - if (is_android) { sources += [ "input/synchronous_compositor.mojom" ] } diff --git a/third_party/blink/renderer/modules/cobalt/crash_annotator/BUILD.gn b/third_party/blink/renderer/modules/cobalt/crash_annotator/BUILD.gn index cb5b0e61e73f1..b6ddae402afca 100644 --- a/third_party/blink/renderer/modules/cobalt/crash_annotator/BUILD.gn +++ b/third_party/blink/renderer/modules/cobalt/crash_annotator/BUILD.gn @@ -19,4 +19,6 @@ blink_modules_sources("crash_annotator") { "crash_annotator.cc", "crash_annotator.h", ] + + deps = [ "//cobalt/services/crash_annotator/public/mojom:mojom_blink" ] } diff --git a/third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.cc b/third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.cc index 4485eadbe71a1..20b592ac3c40f 100644 --- a/third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.cc +++ b/third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.cc @@ -14,9 +14,9 @@ #include "third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.h" +#include "cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom-blink.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/remote.h" -#include "third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom-blink.h" #include "third_party/blink/renderer/bindings/core/v8/script_promise.h" #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h" #include "third_party/blink/renderer/core/execution_context/execution_context.h" @@ -43,7 +43,7 @@ CrashAnnotator* CrashAnnotator::crashAnnotator(NavigatorBase& navigator) { CrashAnnotator::CrashAnnotator(NavigatorBase& navigator) : Supplement(navigator), ExecutionContextLifecycleObserver(navigator.GetExecutionContext()), - crash_annotator_(navigator.GetExecutionContext()) {} + service_(navigator.GetExecutionContext()) {} void CrashAnnotator::ContextDestroyed() {} @@ -56,11 +56,11 @@ ScriptPromise CrashAnnotator::setString(ScriptState* script_state, EnsureReceiverIsBound(); - crash_annotator_->SetString(key, - value, - WTF::BindOnce( - &CrashAnnotator::OnSetString, - WrapPersistent(this), WrapPersistent(resolver))); + service_->SetString(key, + value, + WTF::BindOnce( + &CrashAnnotator::OnSetString, + WrapPersistent(this), WrapPersistent(resolver))); return resolver->Promise(); } @@ -72,18 +72,18 @@ void CrashAnnotator::OnSetString(ScriptPromiseResolver* resolver, bool result) { void CrashAnnotator::EnsureReceiverIsBound() { DCHECK(GetExecutionContext()); - if (crash_annotator_.is_bound()) { + if (service_.is_bound()) { return; } auto task_runner = GetExecutionContext()->GetTaskRunner(TaskType::kMiscPlatformAPI); GetExecutionContext()->GetBrowserInterfaceBroker().GetInterface( - crash_annotator_.BindNewPipeAndPassReceiver(task_runner)); + service_.BindNewPipeAndPassReceiver(task_runner)); } void CrashAnnotator::Trace(Visitor* visitor) const { - visitor->Trace(crash_annotator_); + visitor->Trace(service_); Supplement::Trace(visitor); ExecutionContextLifecycleObserver::Trace(visitor); ScriptWrappable::Trace(visitor); diff --git a/third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.h b/third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.h index 1ea37ac1e1a00..8f61526759c65 100644 --- a/third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.h +++ b/third_party/blink/renderer/modules/cobalt/crash_annotator/crash_annotator.h @@ -15,7 +15,7 @@ #ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_COBALT_CRASH_ANNOTATOR_H_ #define THIRD_PARTY_BLINK_RENDERER_MODULES_COBALT_CRASH_ANNOTATOR_H_ -#include "third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom-blink.h" +#include "cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom-blink.h" #include "third_party/blink/renderer/bindings/core/v8/script_promise.h" #include "third_party/blink/renderer/core/execution_context/execution_context_lifecycle_observer.h" #include "third_party/blink/renderer/modules/modules_export.h" @@ -58,7 +58,7 @@ class MODULES_EXPORT CrashAnnotator final void OnSetString(ScriptPromiseResolver*, bool); void EnsureReceiverIsBound(); - HeapMojoRemote crash_annotator_; + HeapMojoRemote service_; }; } // namespace blink diff --git a/third_party/blink/web_tests/external/wpt/cobalt/crash-annotator/README.md b/third_party/blink/web_tests/external/wpt/cobalt/crash-annotator/README.md index cd1c51ca4972a..7c75fbf38dcb6 100644 --- a/third_party/blink/web_tests/external/wpt/cobalt/crash-annotator/README.md +++ b/third_party/blink/web_tests/external/wpt/cobalt/crash-annotator/README.md @@ -2,5 +2,5 @@ These automated tests rely on a test-only interface that must be provided by the browser under test. The pattern from the Serial API is followed: an -implementation of the CrashAnnotator Mojo interface is provided by -`../../resources/chromium/cobalt/fake-crash-annotator.js`. +implementation of the CrashAnnotatorService Mojo interface is provided by +`../../resources/chromium/cobalt/fake-crash-annotator-service.js`. diff --git a/third_party/blink/web_tests/external/wpt/cobalt/crash-annotator/resources/automation.js b/third_party/blink/web_tests/external/wpt/cobalt/crash-annotator/resources/automation.js index c630f663ca588..809a5f1cfa1c6 100644 --- a/third_party/blink/web_tests/external/wpt/cobalt/crash-annotator/resources/automation.js +++ b/third_party/blink/web_tests/external/wpt/cobalt/crash-annotator/resources/automation.js @@ -1,26 +1,26 @@ 'use strict'; -// These tests use a fake implementation of the CrashAnnotator Mojo interface. +// These tests use a fake implementation of the CrashAnnotatorService Mojo interface. -let fakeCrashAnnotator = undefined; +let fakeCrashAnnotatorService = undefined; function crash_annotator_test(func, name, properties) { promise_test(async (test) => { assert_implements(navigator.crashAnnotator, 'missing navigator.crashAnnotator'); - if (fakeCrashAnnotator === undefined) { + if (fakeCrashAnnotatorService === undefined) { const fakes = - await import('/resources/chromium/cobalt/fake-crash-annotator.js'); - fakeCrashAnnotator = fakes.fakeCrashAnnotator; + await import('/resources/chromium/cobalt/fake-crash-annotator-service.js'); + fakeCrashAnnotatorService = fakes.fakeCrashAnnotatorService; } - assert_implements(fakeCrashAnnotator, 'missing fakeCrashAnnotator'); + assert_implements(fakeCrashAnnotatorService, 'missing fakeCrashAnnotatorService'); - fakeCrashAnnotator.start(); + fakeCrashAnnotatorService.start(); try { - await func(test, fakeCrashAnnotator); + await func(test, fakeCrashAnnotatorService); } finally { - fakeCrashAnnotator.stop(); - fakeCrashAnnotator.reset(); + fakeCrashAnnotatorService.stop(); + fakeCrashAnnotatorService.reset(); } }, name, properties); } diff --git a/third_party/blink/web_tests/external/wpt/resources/chromium/cobalt/fake-crash-annotator.js b/third_party/blink/web_tests/external/wpt/resources/chromium/cobalt/fake-crash-annotator-service.js similarity index 62% rename from third_party/blink/web_tests/external/wpt/resources/chromium/cobalt/fake-crash-annotator.js rename to third_party/blink/web_tests/external/wpt/resources/chromium/cobalt/fake-crash-annotator-service.js index 484bcb57bf367..21764774e62c6 100644 --- a/third_party/blink/web_tests/external/wpt/resources/chromium/cobalt/fake-crash-annotator.js +++ b/third_party/blink/web_tests/external/wpt/resources/chromium/cobalt/fake-crash-annotator-service.js @@ -1,12 +1,12 @@ -import {CrashAnnotator, CrashAnnotatorReceiver} from '/gen/third_party/blink/public/mojom/cobalt/crash_annotator/crash_annotator.mojom.m.js'; +import {CrashAnnotatorService, CrashAnnotatorServiceReceiver} from '/gen/cobalt/services/crash_annotator/public/mojom/crash_annotator_service.mojom.m.js'; -// Implementation of blink.mojom.CrashAnnotator. -class FakeCrashAnnotator { +// Implementation of crash_annotator.mojom.CrashAnnotatorService. +class FakeCrashAnnotatorService { constructor() { this.interceptor_ = - new MojoInterfaceInterceptor(CrashAnnotator.$interfaceName); + new MojoInterfaceInterceptor(CrashAnnotatorService.$interfaceName); this.interceptor_.oninterfacerequest = e => this.bind(e.handle); - this.receiver_ = new CrashAnnotatorReceiver(this); + this.receiver_ = new CrashAnnotatorServiceReceiver(this); this.stub_result_ = null; this.annotations_ = new Map(); } @@ -47,4 +47,4 @@ class FakeCrashAnnotator { } -export const fakeCrashAnnotator = new FakeCrashAnnotator(); +export const fakeCrashAnnotatorService = new FakeCrashAnnotatorService(); From 1426cebb0331fc68a2148cb448d6fa2d42100c8c Mon Sep 17 00:00:00 2001 From: Ian Mindich Date: Fri, 17 Jan 2025 15:02:29 -0800 Subject: [PATCH 10/61] Enable globstar in shell to ensure Datadog upload finds all files. (#4716) b/388125345 The Datadog upload step was failing to find files because the shell's globstar option was not enabled. This change enables globstar, allowing the datadog step to locate and upload test result files. --- .github/actions/process_test_results/action.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/process_test_results/action.yaml b/.github/actions/process_test_results/action.yaml index 51274ea4785ae..73a2e16c3633f 100644 --- a/.github/actions/process_test_results/action.yaml +++ b/.github/actions/process_test_results/action.yaml @@ -53,5 +53,7 @@ runs: DD_ENV: ci run: | set -x + # Enable globstar shell option for globstar paths e.g. /**/*.xml + shopt -s globstar ./datadog-ci junit upload --service ${DD_SERVICE} results/**/*.xml shell: bash From 454cedb880c3aa30c45507c3f2973ff9b68d2d07 Mon Sep 17 00:00:00 2001 From: Niranjan Yardi Date: Fri, 17 Jan 2025 18:39:18 -0800 Subject: [PATCH 11/61] Build nplb hermetically (#4587) b/371241293 --- .github/config/linux-evergreen.json | 3 +- .pre-commit-config.yaml | 3 +- base/BUILD.gn | 87 ++++++- base/allocator/partition_allocator/BUILD.gn | 10 + .../partition_alloc_config.h | 4 +- .../shim/allocator_shim.cc | 2 + .../shim/allocator_shim_internals.h | 2 +- .../allocator_shim_override_libc_symbols.h | 2 +- base/debug/stack_trace.cc | 2 +- base/debug/stack_trace_posix.cc | 4 +- base/system/sys_info_posix.cc | 6 +- build/config/c++/BUILD.gn | 16 ++ buildtools/third_party/libc++/BUILD.gn | 14 ++ buildtools/third_party/libc++abi/BUILD.gn | 3 + cobalt/build/configs/BUILD.gn | 9 +- .../configs/linux-x64x11-evergreen/args.gn | 5 + cobalt/build/configs/modular_variables.gni | 3 + .../boringssl/src/include/openssl/thread.h | 215 ++++++++++++++++++ starboard/BUILD.gn | 82 +++---- starboard/build/config/modular/BUILD.gn | 25 +- starboard/common/condition_variable.h | 10 + starboard/common/mutex.h | 10 + starboard/configuration.h | 6 +- starboard/shared/modular/BUILD.gn | 5 +- third_party/abseil-cpp/absl/base/BUILD.gn | 6 + .../absl/base/internal/direct_mmap.h | 4 +- .../absl/base/internal/raw_logging.cc | 2 +- .../absl/base/internal/spinlock_wait.cc | 3 +- .../debugging/internal/address_is_readable.cc | 3 +- .../absl/synchronization/internal/futex.h | 3 + third_party/boringssl/BUILD.gn | 28 ++- third_party/llvm-project/compiler-rt/BUILD.gn | 1 - third_party/musl/BUILD.gn | 1 - third_party/musl/include/pthread.h | 7 - 34 files changed, 484 insertions(+), 102 deletions(-) create mode 100644 cobalt/third_party/boringssl/src/include/openssl/thread.h diff --git a/.github/config/linux-evergreen.json b/.github/config/linux-evergreen.json index 76b187f782e44..0a5074a51b667 100644 --- a/.github/config/linux-evergreen.json +++ b/.github/config/linux-evergreen.json @@ -4,7 +4,8 @@ "linux-x64x11-evergreen" ], "targets": [ - "base_unittests" + "base", + "nplb" ], "includes": [ { diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e7a98b28eaad..e527a33060122 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,8 @@ repos: (?x)^( starboard/content/ssl/certs/| starboard/shared/starboard/player/testdata/| - starboard/android/apk/app/src/app/assets/web/link_android_splash_screen.html + starboard/android/apk/app/src/app/assets/web/link_android_splash_screen.html | + cobalt/third_party/boringssl/src/include/openssl/thread.h ) - repo: local diff --git a/base/BUILD.gn b/base/BUILD.gn index 434dc918c8e39..1b81cf081fbb4 100644 --- a/base/BUILD.gn +++ b/base/BUILD.gn @@ -43,6 +43,10 @@ import("//build_overrides/build.gni") import("//testing/libfuzzer/fuzzer_test.gni") import("//testing/test.gni") +if (is_cobalt) { + import("//cobalt/build/configs/modular_variables.gni") +} + if (is_mac) { # Used to generate fuzzer corpus :base_mach_port_rendezvous_convert_corpus. import("//third_party/protobuf/proto_library.gni") @@ -101,8 +105,15 @@ if (is_apple) { "mac requires mach absolute time ticks") } +# TODO: b/384652502 - Cobalt: Fix compiler errors building hermetically. +# ../../third_party/libevent/signal.c:43:10: fatal error: 'sys/queue.h' file not found +# #include # Determines whether libevent should be dep. -dep_libevent = !is_fuchsia && !is_win && !is_mac && !is_nacl +if (is_cobalt && is_cobalt_hermetic_build) { + dep_libevent = false +} else { + dep_libevent = !is_fuchsia && !is_win && !is_mac && !is_nacl +} # Determines whether message_pump_libevent should be used. use_libevent = dep_libevent && !is_ios @@ -1111,6 +1122,10 @@ component("base") { "allocator/partition_allocator/shim/allocator_shim.h", "allocator/partition_allocator/shim/allocator_shim_internals.h", ] + + # TODO: b/384652502 - Cobalt: Fix compiler errors building hermetically. + # loc.cc -o obj/base/base/allocator_shim_default_dispatch_to_partition_alloc.o + # ../../base/allocator/partition_allocator/shim/allocator_shim_default_dispatch_to_partition_alloc.cc:748:36: error: incomplete result type 'struct mallinfo' in function definition if (use_partition_alloc) { sources += [ "allocator/partition_allocator/shim/allocator_shim_default_dispatch_to_partition_alloc.cc", @@ -1730,6 +1745,74 @@ component("base") { ] } + # TODO: b/384652502 - Cobalt: Fix compiler errors building hermetically. + if (is_cobalt && is_cobalt_hermetic_build) { + sources -= [ + # ../../base/process/process_linux.cc:8:10: fatal error: 'linux/magic.h' file not found + # #include + "debug/proc_maps_linux.cc", + "debug/proc_maps_linux.h", + "files/dir_reader_linux.h", + "files/scoped_file_linux.cc", + "process/internal_linux.cc", + "process/internal_linux.h", + "process/memory_linux.cc", + "process/process_handle_linux.cc", + "process/process_iterator_linux.cc", + "process/process_linux.cc", + "process/process_metrics_linux.cc", + "threading/platform_thread_linux.cc", + "threading/thread_type_delegate.cc", + "threading/thread_type_delegate.h", + + # ../../base/process/launch_posix.cc:145:18: error: use of undeclared identifier 'SYS_rt_sigaction' + # return syscall(SYS_rt_sigaction, sig, act, oact, sizeof(kernel_sigset_t)); + "native_library_posix.cc", + "posix/can_lower_nice_to.cc", + "posix/can_lower_nice_to.h", + "process/launch_posix.cc", + "profiler/module_cache_posix.cc", + "profiler/stack_base_address_posix.cc", + "profiler/stack_base_address_posix.h", + "profiler/stack_copier_signal.cc", + "profiler/stack_copier_signal.h", + "profiler/thread_delegate_posix.cc", + "profiler/thread_delegate_posix.h", + + # ../../base/process/process_metrics_posix.cc:119:27: error: invalid use of incomplete type 'mallinfo' + # struct mallinfo minfo = mallinfo(); + "process/process_metrics_posix.cc", + "sync_socket_posix.cc", + + # ../../base/allocator/partition_allocator/shim/allocator_shim_default_dispatch_to_partition_alloc.cc:748:36: error: incomplete result type 'struct mallinfo' in function definition + # SHIM_ALWAYS_EXPORT struct mallinfo mallinfo(void) __THROW { + "allocator/partition_allocator/shim/allocator_shim_default_dispatch_to_partition_alloc.cc", + "allocator/partition_allocator/shim/allocator_shim_default_dispatch_to_partition_alloc.h", + + # In file included from ../../base/rand_util_posix.cc:26: + # ../../third_party/lss/linux_syscall_support.h:118:10: fatal error: 'linux/unistd.h' file not found + "base_paths_posix.h", + "memory/madv_free_discardable_memory_allocator_posix.cc", + "memory/madv_free_discardable_memory_allocator_posix.h", + "memory/madv_free_discardable_memory_posix.cc", + "memory/madv_free_discardable_memory_posix.h", + "posix/unix_domain_socket.cc", + "posix/unix_domain_socket.h", + "rand_util_posix.cc", + "system/sys_info_posix.cc", + + # SHIM_ALWAYS_EXPORT size_t malloc_usable_size(void* address) __THROW { + # ^ + # ../../third_party/musl/include/malloc.h:19:8: note: previous declaration is here + # size_t malloc_usable_size(void *); + "allocator/partition_allocator/shim/allocator_shim_override_cpp_symbols.h", + "allocator/partition_allocator/shim/allocator_shim_override_glibc_weak_symbols.h", + "allocator/partition_allocator/shim/allocator_shim_override_libc_symbols.h", + ] + + deps += [ "//starboard:starboard_group" ] + } + # Windows. if (is_win) { sources += [ @@ -2230,7 +2313,7 @@ component("base") { sources += [ "system/sys_info_starboard.cc", "system/sys_info_starboard.h", - ] + ] } if (use_blink) { diff --git a/base/allocator/partition_allocator/BUILD.gn b/base/allocator/partition_allocator/BUILD.gn index 4d64e3fdb4f03..567b78c5f3afa 100644 --- a/base/allocator/partition_allocator/BUILD.gn +++ b/base/allocator/partition_allocator/BUILD.gn @@ -9,6 +9,10 @@ import("//build/config/chromeos/ui_mode.gni") import("//build/config/dcheck_always_on.gni") import("//build/config/logging.gni") +if (is_cobalt) { + import("//cobalt/build/configs/modular_variables.gni") +} + # Add partition_alloc.gni and import it for partition_alloc configs. config("partition_alloc_implementation") { @@ -228,6 +232,11 @@ component("partition_alloc") { "partition_alloc_base/time/time_conversion_posix.cc", ] + # TODO: b/384652502 - Cobalt: Fix compiler errors building hermetically. + if (is_cobalt && is_cobalt_hermetic_build) { + sources -= [ "partition_alloc_base/rand_util_posix.cc" ] + } + if (is_android || is_chromeos_ash) { sources += [ "partition_alloc_base/time/time_android.cc" ] } @@ -320,6 +329,7 @@ component("partition_alloc") { # tagging.cc requires __arm_mte_set_* functions. deps += [ "//third_party/android_ndk:cpu_features" ] } + if (is_fuchsia) { public_deps += [ "//third_party/fuchsia-sdk/sdk/pkg/fit", diff --git a/base/allocator/partition_allocator/partition_alloc_config.h b/base/allocator/partition_allocator/partition_alloc_config.h index a6671d675a6b0..9534d48cf5694 100644 --- a/base/allocator/partition_allocator/partition_alloc_config.h +++ b/base/allocator/partition_allocator/partition_alloc_config.h @@ -65,7 +65,7 @@ static_assert(sizeof(void*) != 8, ""); #endif #if BUILDFLAG(HAS_64_BIT_POINTERS) && \ - (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_ANDROID)) + (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_ANDROID)) && !defined(IS_COBALT_HERMETIC_BUILD) #include // TODO(bikineev): Enable for ChromeOS. #define PA_CONFIG_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED() \ @@ -104,7 +104,7 @@ static_assert(sizeof(void*) != 8, ""); // POSIX is not only UNIX, e.g. macOS and other OSes. We do use Linux-specific // features such as futex(2). #define PA_CONFIG_HAS_LINUX_KERNEL() \ - (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) || BUILDFLAG(IS_ANDROID)) + (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) || BUILDFLAG(IS_ANDROID)) && !defined(IS_COBALT_HERMETIC_BUILD) // On some platforms, we implement locking by spinning in userspace, then going // into the kernel only if there is contention. This requires platform support, diff --git a/base/allocator/partition_allocator/shim/allocator_shim.cc b/base/allocator/partition_allocator/shim/allocator_shim.cc index d0e23fd5ad6c9..19d28afe1111f 100644 --- a/base/allocator/partition_allocator/shim/allocator_shim.cc +++ b/base/allocator/partition_allocator/shim/allocator_shim.cc @@ -405,6 +405,8 @@ ALWAYS_INLINE void ShimAlignedFree(void* address, void* context) { #else // BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) #include "base/allocator/partition_allocator/shim/allocator_shim_override_mac_symbols.h" #endif // BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) +#elif defined(IS_COBALT_HERMETIC_BUILD) +// Don't include anything, all includes are already set up in MUSL libc #else #include "base/allocator/partition_allocator/shim/allocator_shim_override_libc_symbols.h" #endif diff --git a/base/allocator/partition_allocator/shim/allocator_shim_internals.h b/base/allocator/partition_allocator/shim/allocator_shim_internals.h index 8bddea78c3538..8bf07ce37955c 100644 --- a/base/allocator/partition_allocator/shim/allocator_shim_internals.h +++ b/base/allocator/partition_allocator/shim/allocator_shim_internals.h @@ -9,7 +9,7 @@ #if defined(__GNUC__) -#if BUILDFLAG(IS_POSIX) +#if BUILDFLAG(IS_POSIX) && !defined(IS_COBALT_HERMETIC_BUILD) #include // for __THROW #endif diff --git a/base/allocator/partition_allocator/shim/allocator_shim_override_libc_symbols.h b/base/allocator/partition_allocator/shim/allocator_shim_override_libc_symbols.h index bb07170019e52..d8ebe742f74c3 100644 --- a/base/allocator/partition_allocator/shim/allocator_shim_override_libc_symbols.h +++ b/base/allocator/partition_allocator/shim/allocator_shim_override_libc_symbols.h @@ -89,4 +89,4 @@ SHIM_ALWAYS_EXPORT size_t malloc_usable_size(void* address) __THROW { } // extern "C" -#endif // BASE_ALLOCATOR_PARTITION_ALLOCATOR_SHIM_ALLOCATOR_SHIM_OVERRIDE_LIBC_SYMBOLS_H_ +#endif // BASE_ALLOCATOR_PARTITION_ALLOCATOR_SHIM_ALLOCATOR_SHIM_OVERRIDE_LIBC_SYMBOLS_H_ \ No newline at end of file diff --git a/base/debug/stack_trace.cc b/base/debug/stack_trace.cc index 3debc8bd070cd..f7d9b08bbc794 100644 --- a/base/debug/stack_trace.cc +++ b/base/debug/stack_trace.cc @@ -281,7 +281,7 @@ std::string StackTrace::ToString() const { } std::string StackTrace::ToStringWithPrefix(const char* prefix_string) const { std::stringstream stream; -#if !defined(__UCLIBC__) && !defined(_AIX) +#if (!defined(__UCLIBC__) && !defined(_AIX)) && !defined(IS_COBALT_HERMETIC_BUILD) OutputToStreamWithPrefix(&stream, prefix_string); #endif return stream.str(); diff --git a/base/debug/stack_trace_posix.cc b/base/debug/stack_trace_posix.cc index 0eaad5ce3348c..6f25e5d1742fa 100644 --- a/base/debug/stack_trace_posix.cc +++ b/base/debug/stack_trace_posix.cc @@ -301,6 +301,7 @@ void PrintToStderr(const char* output) { std::ignore = HANDLE_EINTR(write(STDERR_FILENO, output, strlen(output))); } +#if !defined(IS_COBALT_HERMETIC_BUILD) #if BUILDFLAG(IS_LINUX) void AlarmSignalHandler(int signal, siginfo_t* info, void* void_context) { // We have seen rare cases on AMD linux where the default signal handler @@ -321,6 +322,7 @@ void AlarmSignalHandler(int signal, siginfo_t* info, void* void_context) { syscall(SYS_exit_group, EXIT_FAILURE); } #endif // BUILDFLAG(IS_LINUX) +#endif // !defined(IS_COBALT_HERMETIC_BUILD) void StackDumpSignalHandler(int signal, siginfo_t* info, void* void_context) { // NOTE: This code MUST be async-signal safe. @@ -534,7 +536,7 @@ void StackDumpSignalHandler(int signal, siginfo_t* info, void* void_context) { if (::signal(signal, SIG_DFL) == SIG_ERR) { _exit(EXIT_FAILURE); } -#elif !BUILDFLAG(IS_LINUX) +#elif !BUILDFLAG(IS_LINUX) || defined(IS_COBALT_HERMETIC_BUILD) // For all operating systems but Linux we do not reraise the signal that // brought us here but terminate the process immediately. // Otherwise various tests break on different operating systems, see diff --git a/base/system/sys_info_posix.cc b/base/system/sys_info_posix.cc index 7137b29d19cac..5b0be4af9e986 100644 --- a/base/system/sys_info_posix.cc +++ b/base/system/sys_info_posix.cc @@ -35,7 +35,7 @@ #include #endif -#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) +#if (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)) && !defined(IS_COBALT_HERMETIC_BUILD) #include #include #endif @@ -60,7 +60,7 @@ base::LazyInstance< base::internal::LazySysInfoValue>::Leaky g_lazy_virtual_memory = LAZY_INSTANCE_INITIALIZER; -#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) +#if (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)) && !defined(IS_COBALT_HERMETIC_BUILD) bool IsStatsZeroIfUnlimited(const base::FilePath& path) { struct statfs stats; @@ -90,7 +90,7 @@ bool GetDiskSpaceInfo(const base::FilePath& path, if (HANDLE_EINTR(statvfs(path.value().c_str(), &stats)) != 0) return false; -#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) +#if (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)) && !defined(IS_COBALT_HERMETIC_BUILD) const bool zero_size_means_unlimited = stats.f_blocks == 0 && IsStatsZeroIfUnlimited(path); #else diff --git a/build/config/c++/BUILD.gn b/build/config/c++/BUILD.gn index c00dcef4cd1e8..3bf425c885420 100644 --- a/build/config/c++/BUILD.gn +++ b/build/config/c++/BUILD.gn @@ -5,6 +5,10 @@ import("//build/config/compiler/compiler.gni") import("//build/config/dcheck_always_on.gni") import("//buildtools/deps_revisions.gni") +if (is_cobalt) { + import("//cobalt/build/configs/modular_variables.gni") +} + assert(use_custom_libcxx, "should only be used if use_custom_libcxx is set") # This is included by reference in the //build/config/compiler:runtime_library @@ -68,6 +72,18 @@ config("runtime_library") { "-isystem" + rebase_path("$libcxx_prefix/include", root_build_dir), "-isystem" + rebase_path("$libcxxabi_prefix/include", root_build_dir), ] + if (is_cobalt && is_cobalt_hermetic_build) { + cflags_cc += [ + "-isystem" + rebase_path("//third_party/musl/include", root_build_dir), + "-isystem" + + rebase_path("//third_party/musl/arch/generic", root_build_dir), + ] + cflags_c = [ + "-isystem" + rebase_path("//third_party/musl/include", root_build_dir), + "-isystem" + + rebase_path("//third_party/musl/arch/generic", root_build_dir), + ] + } cflags_objcc = cflags_cc diff --git a/buildtools/third_party/libc++/BUILD.gn b/buildtools/third_party/libc++/BUILD.gn index 21421aa32047d..e407a92344de7 100644 --- a/buildtools/third_party/libc++/BUILD.gn +++ b/buildtools/third_party/libc++/BUILD.gn @@ -6,6 +6,10 @@ import("//build/config/c++/c++.gni") import("//build/config/sanitizers/sanitizers.gni") import("//build/toolchain/toolchain.gni") +if (is_cobalt) { + import("//cobalt/build/configs/modular_variables.gni") +} + # Used by libc++ and libc++abi. config("config") { cflags = [ "-fstrict-aliasing" ] @@ -54,6 +58,7 @@ target(_libcxx_target_type, "libc++") { "//build/config:common_deps", "//third_party/catapult/devil:devil", ] + if (is_linux) { # This target packages libc++.so, so must have an explicit dependency on # libc++. @@ -210,4 +215,13 @@ target(_libcxx_target_type, "libc++") { deps = [ "//buildtools/third_party/libc++abi" ] } } + + if (is_cobalt && is_cobalt_hermetic_build) { + # TODO: b/384652502 - Cobalt: Fix compiler errors building hermetically. + sources -= [ "trunk/src/atomic.cpp" ] + deps += [ + "//third_party/llvm-project/compiler-rt:compiler_rt", + "//third_party/musl:c", + ] + } } diff --git a/buildtools/third_party/libc++abi/BUILD.gn b/buildtools/third_party/libc++abi/BUILD.gn index 3ab30c4f8776b..e99aa217015d4 100644 --- a/buildtools/third_party/libc++abi/BUILD.gn +++ b/buildtools/third_party/libc++abi/BUILD.gn @@ -3,6 +3,9 @@ # found in the LICENSE file. import("//build/config/c++/c++.gni") +if (is_cobalt) { + import("//cobalt/build/configs/modular_variables.gni") +} source_set("libc++abi") { if (export_libcxxabi_from_executables) { diff --git a/cobalt/build/configs/BUILD.gn b/cobalt/build/configs/BUILD.gn index c731b81109d0f..15bbb4d4fc34c 100644 --- a/cobalt/build/configs/BUILD.gn +++ b/cobalt/build/configs/BUILD.gn @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import("//cobalt/build/configs/modular_variables.gni") import("//starboard/build/buildflags.gni") config("cobalt_config") { @@ -21,8 +22,12 @@ config("cobalt_config") { ] if (current_toolchain == starboard_toolchain) { configs += [ "//${starboard_path}/platform_configuration" ] - } else if (current_toolchain == cobalt_toolchain) { - # TODO(nyardi) : Add modular/cobalt toolchain configs as needed" + } else if (is_cobalt_hermetic_build) { + configs += [ + # TODO: b/390500840 - Remove hardcoded x64 config, switch config based on platform. + "//starboard/build/config/modular/x64", + "//starboard/build/config:starboard", + ] } else { # No configs to add for any other toolchain } diff --git a/cobalt/build/configs/linux-x64x11-evergreen/args.gn b/cobalt/build/configs/linux-x64x11-evergreen/args.gn index a1107c90cfea9..6e9665ee0bf0f 100644 --- a/cobalt/build/configs/linux-x64x11-evergreen/args.gn +++ b/cobalt/build/configs/linux-x64x11-evergreen/args.gn @@ -1,3 +1,8 @@ import("//cobalt/build/configs/linux-x64x11/args.gn") use_custom_libc = true + +# TODO: b/384652502 - Cobalt: Stop overriding enable_pkeys +# ../../base/allocator/partition_allocator/pkey.cc:35:18: error: use of undeclared identifier 'SYS_pkey_mprotect' +# return syscall(SYS_pkey_mprotect, addr, len, prot, pkey); +enable_pkeys = false diff --git a/cobalt/build/configs/modular_variables.gni b/cobalt/build/configs/modular_variables.gni index 86cf8bbe13c8f..0775b2c293956 100644 --- a/cobalt/build/configs/modular_variables.gni +++ b/cobalt/build/configs/modular_variables.gni @@ -16,3 +16,6 @@ declare_args() { # TODO: b/371241293 - Remove this variable when the hermetic toolchain is setup use_custom_libc = false } + +is_cobalt_hermetic_build = + is_starboard && use_custom_libc && current_toolchain == cobalt_toolchain diff --git a/cobalt/third_party/boringssl/src/include/openssl/thread.h b/cobalt/third_party/boringssl/src/include/openssl/thread.h new file mode 100644 index 0000000000000..a745838367c7c --- /dev/null +++ b/cobalt/third_party/boringssl/src/include/openssl/thread.h @@ -0,0 +1,215 @@ +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] */ + +#ifndef OPENSSL_HEADER_THREAD_H +#define OPENSSL_HEADER_THREAD_H + +#include + +#include + +#include "build/build_config.h" + +#if defined(IS_COBALT_HERMETIC_BUILD) +#include "starboard/common/condition_variable.h" // nogncheck +#include "starboard/common/mutex.h" // nogncheck +#endif // defined(STARBOARD) && defined(_LIBCPP_HAS_MUSL_LIBC) + +#if defined(__cplusplus) +extern "C" { +#endif + +#if defined(IS_COBALT_HERMETIC_BUILD) +typedef struct crypto_mutex_st { + // It would be nice to use starboard::RWLock. However, that's a C++ class, so + // it can't even be included in this C header. It's possible to use an opaque + // pointer to it, but then an allocation would be needed or we somehow embed + // a byte array that is sizeof(starboard::RWLock) without including the + // declaration. Avoid the complication and just implement the RWMutex using + // starboard C structs. + pthread_mutex_t mutex; + pthread_cond_t condition; + size_t readers; + bool writing; +} CRYPTO_MUTEX; +#elif !defined(OPENSSL_THREADS) +typedef struct crypto_mutex_st { + char padding; // Empty structs have different sizes in C and C++. +} CRYPTO_MUTEX; +#elif defined(OPENSSL_WINDOWS) +// CRYPTO_MUTEX can appear in public header files so we really don't want to +// pull in windows.h. It's statically asserted that this structure is large +// enough to contain a Windows SRWLOCK by thread_win.c. +typedef union crypto_mutex_st { + void* handle; +} CRYPTO_MUTEX; +#elif !defined(__GLIBC__) +typedef pthread_rwlock_t CRYPTO_MUTEX; +#else +// On glibc, |pthread_rwlock_t| is hidden under feature flags, and we can't +// ensure that we'll be able to get it from a public header. It's statically +// asserted that this structure is large enough to contain a |pthread_rwlock_t| +// by thread_pthread.c. +typedef union crypto_mutex_st { + double alignment; + uint8_t padding[3 * sizeof(int) + 5 * sizeof(unsigned) + 16 + 8]; +} CRYPTO_MUTEX; +#endif // defined(IS_COBALT_HERMETIC_BUILD) + +// CRYPTO_refcount_t is the type of a reference count. +// +// Since some platforms use C11 atomics to access this, it should have the +// _Atomic qualifier. However, this header is included by C++ programs as well +// as C code that might not set -std=c11. So, in practice, it's not possible to +// do that. Instead we statically assert that the size and native alignment of +// a plain uint32_t and an _Atomic uint32_t are equal in refcount_c11.c. +typedef uint32_t CRYPTO_refcount_t; + +// Deprecated functions. +// +// Historically, OpenSSL required callers to provide locking callbacks. +// BoringSSL is thread-safe by default, but some old code calls these functions +// and so no-op implementations are provided. + +// These defines do nothing but are provided to make old code easier to +// compile. +#define CRYPTO_LOCK 1 +#define CRYPTO_UNLOCK 2 +#define CRYPTO_READ 4 +#define CRYPTO_WRITE 8 + +// CRYPTO_num_locks returns one. (This is non-zero that callers who allocate +// sizeof(lock) times this value don't get zero and then fail because malloc(0) +// returned NULL.) +OPENSSL_EXPORT int CRYPTO_num_locks(void); + +// CRYPTO_set_locking_callback does nothing. +OPENSSL_EXPORT void CRYPTO_set_locking_callback( + void (*func)(int mode, int lock_num, const char* file, int line)); + +// CRYPTO_set_add_lock_callback does nothing. +OPENSSL_EXPORT void CRYPTO_set_add_lock_callback(int ( + *func)(int* num, int amount, int lock_num, const char* file, int line)); + +// CRYPTO_get_locking_callback returns NULL. +OPENSSL_EXPORT void (*CRYPTO_get_locking_callback(void))(int mode, + int lock_num, + const char* file, + int line); + +// CRYPTO_get_lock_name returns a fixed, dummy string. +OPENSSL_EXPORT const char* CRYPTO_get_lock_name(int lock_num); + +// CRYPTO_THREADID_set_callback returns one. +OPENSSL_EXPORT int CRYPTO_THREADID_set_callback( + void (*threadid_func)(CRYPTO_THREADID* threadid)); + +// CRYPTO_THREADID_set_numeric does nothing. +OPENSSL_EXPORT void CRYPTO_THREADID_set_numeric(CRYPTO_THREADID* id, + unsigned long val); + +// CRYPTO_THREADID_set_pointer does nothing. +OPENSSL_EXPORT void CRYPTO_THREADID_set_pointer(CRYPTO_THREADID* id, void* ptr); + +// CRYPTO_THREADID_current does nothing. +OPENSSL_EXPORT void CRYPTO_THREADID_current(CRYPTO_THREADID* id); + +// CRYPTO_set_id_callback does nothing. +OPENSSL_EXPORT void CRYPTO_set_id_callback(unsigned long (*func)(void)); + +typedef struct { + int references; + struct CRYPTO_dynlock_value* data; +} CRYPTO_dynlock; + +// CRYPTO_set_dynlock_create_callback does nothing. +OPENSSL_EXPORT void CRYPTO_set_dynlock_create_callback( + struct CRYPTO_dynlock_value* (*dyn_create_function)(const char* file, + int line)); + +// CRYPTO_set_dynlock_lock_callback does nothing. +OPENSSL_EXPORT void CRYPTO_set_dynlock_lock_callback( + void (*dyn_lock_function)(int mode, + struct CRYPTO_dynlock_value* l, + const char* file, + int line)); + +// CRYPTO_set_dynlock_destroy_callback does nothing. +OPENSSL_EXPORT void CRYPTO_set_dynlock_destroy_callback( + void (*dyn_destroy_function)(struct CRYPTO_dynlock_value* l, + const char* file, + int line)); + +// CRYPTO_get_dynlock_create_callback returns NULL. +OPENSSL_EXPORT struct CRYPTO_dynlock_value* ( + *CRYPTO_get_dynlock_create_callback(void))(const char* file, int line); + +// CRYPTO_get_dynlock_lock_callback returns NULL. +OPENSSL_EXPORT void (*CRYPTO_get_dynlock_lock_callback(void))( + int mode, + struct CRYPTO_dynlock_value* l, + const char* file, + int line); + +// CRYPTO_get_dynlock_destroy_callback returns NULL. +OPENSSL_EXPORT void (*CRYPTO_get_dynlock_destroy_callback( + void))(struct CRYPTO_dynlock_value* l, const char* file, int line); + +#if defined(__cplusplus) +} // extern C +#endif + +#endif // OPENSSL_HEADER_THREAD_H diff --git a/starboard/BUILD.gn b/starboard/BUILD.gn index f4bd47c0263de..a52eef0c3f440 100644 --- a/starboard/BUILD.gn +++ b/starboard/BUILD.gn @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import("//cobalt/build/configs/modular_variables.gni") import("//starboard/build/config/os_definitions.gni") import("//starboard/build/config/starboard_target_type.gni") import("//starboard/contrib/cast/cast.gni") +# TODO: b/389791129 - Evaluate and cleanup targets in if(!is_cobalt) code block. if (!is_cobalt) { group("gn_all") { testonly = true @@ -102,7 +104,9 @@ if (!is_cobalt) { "//starboard/tools:build_app_launcher_zip", ] } +} +if (is_cobalt_hermetic_build) { group("starboard_group") { deps = [] public_deps = [ @@ -110,59 +114,37 @@ if (!is_cobalt) { "//starboard/client_porting/eztime", ] - if (sb_is_modular && current_toolchain == cobalt_toolchain) { - public_deps += [ - "//third_party/llvm-project/compiler-rt:compiler_rt", - "//third_party/llvm-project/libcxx:cxx", - "//third_party/llvm-project/libcxxabi:cxxabi", - "//third_party/musl:c", - ] - - if (!sb_is_evergreen) { - # For modular builds, we need to wrap any POSIX APIs that may not be ABI - # equivalent across the Cobalt toolchain and the platform toolchain. For - # non-Evergreen builds, this is done via the below wrapper implementations - # of the API functions, which will simply call a renamed version of the - # function that will be implemented in the Starboard shared library. The - # same behavior is achieved in Evergreen via exported_symbols.cc remapping - # the functions to the same renamed version. In both cases, implementation - # of these renamed functions will handle translating data types across the - # ABI boundary and be compiled into the Starboard shared library in - # //starboard/build/config/starboard_target_type.gni. - deps += [ "//starboard/shared/modular:cobalt_layer_posix_abi_wrappers" ] - } - - if (sb_is_evergreen) { - public_deps += [ "//starboard/elf_loader:sabi_string" ] - } else { - data_deps = [ ":starboard($starboard_toolchain)" ] - } + if (!sb_is_evergreen) { + # For modular builds, we need to wrap any POSIX APIs that may not be ABI + # equivalent across the Cobalt toolchain and the platform toolchain. For + # non-Evergreen builds, this is done via the below wrapper implementations + # of the API functions, which will simply call a renamed version of the + # function that will be implemented in the Starboard shared library. The + # same behavior is achieved in Evergreen via exported_symbols.cc remapping + # the functions to the same renamed version. In both cases, implementation + # of these renamed functions will handle translating data types across the + # ABI boundary and be compiled into the Starboard shared library in + # //starboard/build/config/starboard_target_type.gni. + deps += [ "//starboard/shared/modular:cobalt_layer_posix_abi_wrappers" ] + } - # TODO: b/295702296 Fix libunwind for modular builds. - if (sb_is_evergreen) { - public_deps += [ "//third_party/llvm-project/libunwind:unwind" ] - } + if (sb_is_evergreen) { + public_deps += [ "//starboard/elf_loader:sabi_string" ] } else { - public_deps += [ - ":starboard($starboard_toolchain)", - "//starboard/common", - ] - - if (enable_evergreen_code) { - if (!sb_is_modular || sb_is_evergreen) { - if (sb_is_evergreen_compatible && - current_toolchain == starboard_toolchain) { - public_deps += [ "//starboard/crashpad_wrapper" ] - } else { - public_deps += [ "//starboard/crashpad_wrapper:wrapper_stub" ] - } - } - } + deps += [ ":starboard($starboard_toolchain)" ] + } - if (final_executable_type == "shared_library" && - current_toolchain != default_toolchain) { - set_defaults("executable") { - sources = [ "//starboard/shared/starboard/shared_main_adapter.cc" ] + # TODO: b/295702296 Fix libunwind for modular builds. + if (sb_is_evergreen) { + public_deps += [ "//third_party/llvm-project/libunwind:unwind" ] + } + if (enable_evergreen_code) { + if (!sb_is_modular || sb_is_evergreen) { + if (sb_is_evergreen_compatible && + current_toolchain == starboard_toolchain) { + public_deps += [ "//starboard/crashpad_wrapper" ] + } else { + public_deps += [ "//starboard/crashpad_wrapper:wrapper_stub" ] } } } diff --git a/starboard/build/config/modular/BUILD.gn b/starboard/build/config/modular/BUILD.gn index aa288c5478dcd..7f792b2f56958 100644 --- a/starboard/build/config/modular/BUILD.gn +++ b/starboard/build/config/modular/BUILD.gn @@ -18,31 +18,17 @@ config("modular") { "-ffunction-sections", "-fdata-sections", "-nostdlibinc", - "-isystem" + rebase_path("//third_party/llvm-project/libcxxabi/include", - root_build_dir), - "-isystem" + rebase_path("//third_party/llvm-project/libunwind/include", - root_build_dir), - "-isystem" + rebase_path("//third_party/llvm-project/libcxx/include", - root_build_dir), - "-isystem" + rebase_path("//third_party/musl/include", root_build_dir), - "-isystem" + rebase_path("//third_party/musl/arch/generic", root_build_dir), ] cflags += [ "-fPIC" ] - cflags_cc = [ "-nostdinc++" ] - defines = [ - # Ensure that the Starboardized __external_threading file is included. - "_LIBCPP_HAS_THREAD_API_EXTERNAL", - - # Ensure that only the forward declarations and type definitions are included - # in __external_threading. - "_LIBCPP_HAS_THREAD_LIBRARY_EXTERNAL", - # Enable GNU extensions to get prototypes like ffsl. "_GNU_SOURCE=1", + # TODO: b/390675141 - Replace IS_COBALT_HERMETIC_BUILD with IS_STARBOARD. + "IS_COBALT_HERMETIC_BUILD", + "_LIBCPP_HAS_MUSL_LIBC", "__STDC_FORMAT_MACROS", # so that we get PRI* @@ -137,6 +123,11 @@ config("modular") { # Do not remove null pointer checks. "-fno-delete-null-pointer-checks", + + # ../../third_party/perfetto/src/base/unix_socket.cc:555:17: error: comparison of integers of different signs: 'unsigned long' and 'long' [-Werror,-Wsign-compare] + # cmsg = CMSG_NXTHDR(&msg_hdr, cmsg)) { + # ^~~~~~~~~~~~~~~~~~~~~~~~~~~ + "-Wno-sign-compare", ] } diff --git a/starboard/common/condition_variable.h b/starboard/common/condition_variable.h index ac533379a0e68..049efc11f383a 100644 --- a/starboard/common/condition_variable.h +++ b/starboard/common/condition_variable.h @@ -20,12 +20,19 @@ #ifndef STARBOARD_COMMON_CONDITION_VARIABLE_H_ #define STARBOARD_COMMON_CONDITION_VARIABLE_H_ +// TODO: b/390503926 - Remove the starboard condition variable API and all it's +// references. See work done in 25lts as an example. + #include #include "starboard/common/mutex.h" #include "starboard/configuration.h" #include "starboard/types.h" +#ifdef __cplusplus + +extern "C++" { + namespace starboard { // Inline class wrapper for pthread_cond_t. @@ -53,5 +60,8 @@ class ConditionVariable { }; } // namespace starboard +} + +#endif //__cplusplus #endif // STARBOARD_COMMON_CONDITION_VARIABLE_H_ diff --git a/starboard/common/mutex.h b/starboard/common/mutex.h index c9f240c594365..94693e252a8f4 100644 --- a/starboard/common/mutex.h +++ b/starboard/common/mutex.h @@ -22,6 +22,13 @@ #include "starboard/configuration.h" +// TODO: b/390503926 - Remove the starboard mutex API and all it's +// references. See work done in 25lts as an example. + +#ifdef __cplusplus + +extern "C++" { + namespace starboard { // Inline class wrapper for mutex. @@ -87,5 +94,8 @@ class ScopedTryLock { }; } // namespace starboard +} + +#endif //__cplusplus #endif // STARBOARD_COMMON_MUTEX_H_ diff --git a/starboard/configuration.h b/starboard/configuration.h index d0b1af6063a9a..379ff8165ca44 100644 --- a/starboard/configuration.h +++ b/starboard/configuration.h @@ -27,8 +27,10 @@ #ifndef STARBOARD_CONFIGURATION_H_ #define STARBOARD_CONFIGURATION_H_ -#if !defined(STARBOARD) -#error "You must define STARBOARD in Starboard builds." +#include "build/build_config.h" + +#if !BUILDFLAG(IS_COBALT) +#error "IS_COBALT should be defined while building this file" #endif // --- Common Defines -------------------------------------------------------- diff --git a/starboard/shared/modular/BUILD.gn b/starboard/shared/modular/BUILD.gn index f5648939f1b17..1da182f532645 100644 --- a/starboard/shared/modular/BUILD.gn +++ b/starboard/shared/modular/BUILD.gn @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import("//cobalt/build/configs/modular_variables.gni") + # TODO: b/315170518 - Revert to static library after fixing # symbol visibility issues for windows based modular platform builds. if ((sb_is_modular || sb_is_evergreen_compatible) && @@ -44,8 +46,7 @@ if ((sb_is_modular || sb_is_evergreen_compatible) && } } -if (sb_is_modular && !sb_is_evergreen && - current_toolchain == cobalt_toolchain) { +if (is_cobalt_hermetic_build && !sb_is_evergreen) { source_set("cobalt_layer_posix_abi_wrappers") { sources = [ "cobalt_layer_posix_directory_abi_wrappers.cc", diff --git a/third_party/abseil-cpp/absl/base/BUILD.gn b/third_party/abseil-cpp/absl/base/BUILD.gn index 3f525ac92e2e1..fb098cd94ae7c 100644 --- a/third_party/abseil-cpp/absl/base/BUILD.gn +++ b/third_party/abseil-cpp/absl/base/BUILD.gn @@ -3,6 +3,9 @@ # found in the LICENSE file. import("//third_party/abseil-cpp/absl.gni") +if (is_cobalt) { + import("//cobalt/build/configs/modular_variables.gni") +} absl_source_set("atomic_hook") { public = [ "internal/atomic_hook.h" ] @@ -39,6 +42,9 @@ absl_source_set("raw_logging_internal") { ":log_severity", ] visibility = [ "//third_party/abseil-cpp/absl/*" ] + if (is_cobalt && is_cobalt_hermetic_build) { + public_deps += [ "//starboard:starboard_headers_only" ] + } } absl_source_set("spinlock_wait") { diff --git a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h index 815b8d23ba35f..f0ef60b4cf44d 100644 --- a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +++ b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h @@ -24,8 +24,8 @@ #include -#ifdef __linux__ - +#include "build/build_config.h" +#if defined(__linux__) && !defined(IS_COBALT_HERMETIC_BUILD) #include #ifdef __BIONIC__ #include diff --git a/third_party/abseil-cpp/absl/base/internal/raw_logging.cc b/third_party/abseil-cpp/absl/base/internal/raw_logging.cc index 6273e8471bcfb..dbf95865a5f05 100644 --- a/third_party/abseil-cpp/absl/base/internal/raw_logging.cc +++ b/third_party/abseil-cpp/absl/base/internal/raw_logging.cc @@ -53,7 +53,7 @@ // syscall(SYS_write, /*int*/ fd, /*char* */ buf, /*size_t*/ len); // for low level operations that want to avoid libc. #if (defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__)) && \ - !defined(__ANDROID__) + !defined(__ANDROID__) && !defined(IS_COBALT_HERMETIC_BUILD) #include #define ABSL_HAVE_SYSCALL_WRITE 1 #define ABSL_LOW_LEVEL_WRITE_SUPPORTED 1 diff --git a/third_party/abseil-cpp/absl/base/internal/spinlock_wait.cc b/third_party/abseil-cpp/absl/base/internal/spinlock_wait.cc index fa824be1c008c..dd247a5b7a7a7 100644 --- a/third_party/abseil-cpp/absl/base/internal/spinlock_wait.cc +++ b/third_party/abseil-cpp/absl/base/internal/spinlock_wait.cc @@ -20,10 +20,11 @@ #include #include "absl/base/internal/spinlock_wait.h" +#include "build/build_config.h" #if defined(_WIN32) #include "absl/base/internal/spinlock_win32.inc" -#elif defined(__linux__) +#elif defined(__linux__) && !defined(IS_COBALT_HERMETIC_BUILD) #include "absl/base/internal/spinlock_linux.inc" #elif defined(__akaros__) #include "absl/base/internal/spinlock_akaros.inc" diff --git a/third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc b/third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc index 91eaa76f8a6c8..a348bc07645a2 100644 --- a/third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc +++ b/third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc @@ -16,8 +16,9 @@ // without faulting. #include "absl/debugging/internal/address_is_readable.h" +#include "build/build_config.h" -#if !defined(__linux__) || defined(__ANDROID__) +#if !defined(__linux__) || defined(__ANDROID__) || defined(IS_COBALT_HERMETIC_BUILD) namespace absl { ABSL_NAMESPACE_BEGIN diff --git a/third_party/abseil-cpp/absl/synchronization/internal/futex.h b/third_party/abseil-cpp/absl/synchronization/internal/futex.h index 55078f14c0e13..3774abdcda750 100644 --- a/third_party/abseil-cpp/absl/synchronization/internal/futex.h +++ b/third_party/abseil-cpp/absl/synchronization/internal/futex.h @@ -15,16 +15,19 @@ #define ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ #include "absl/base/config.h" +#include "build/build_config.h" #ifndef _WIN32 #include #include #endif +#if !defined(IS_COBALT_HERMETIC_BUILD) #ifdef __linux__ #include #include #endif +#endif // !defined(IS_COBALT_HERMETIC_BUILD) #include #include diff --git a/third_party/boringssl/BUILD.gn b/third_party/boringssl/BUILD.gn index 6fb13437c6bcc..505e53132b106 100644 --- a/third_party/boringssl/BUILD.gn +++ b/third_party/boringssl/BUILD.gn @@ -11,6 +11,10 @@ import("//testing/libfuzzer/fuzzer_test.gni") import("BUILD.generated.gni") import("BUILD.generated_tests.gni") +if (is_cobalt) { + import("//cobalt/build/configs/modular_variables.gni") +} + if (enable_rust) { import("//build/rust/cargo_crate.gni") import("//build/rust/rust_bindgen.gni") @@ -18,7 +22,14 @@ if (enable_rust) { # Config for us and everybody else depending on BoringSSL. config("external_config") { - include_dirs = [ "src/include" ] + if (is_cobalt && is_cobalt_hermetic_build) { + include_dirs = [ + "//cobalt/third_party/boringssl/src/include", + "src/include", + ] + } else { + include_dirs = [ "src/include" ] + } if (is_component_build) { defines = [ "BORINGSSL_SHARED_LIBRARY" ] } @@ -110,7 +121,14 @@ if (is_win && !is_msan && current_cpu != "arm64") { sources = [] asmflags = [] - include_dirs = [ "src/include" ] + if (is_cobalt && is_cobalt_hermetic_build) { + include_dirs = [ + "//cobalt/third_party/boringssl/src/include", + "src/include", + ] + } else { + include_dirs = [ "src/include" ] + } if (is_msan) { public_configs = [ ":no_asm_config" ] @@ -164,6 +182,12 @@ component("boringssl") { # config is forwarded to callers. In particular, boringssl_crypto_tests # requires it. public_deps = [ ":boringssl_asm" ] + if (is_cobalt && is_cobalt_hermetic_build) { + public_deps += [ + "//starboard:starboard_headers_only", + "//starboard/common", + ] + } public_configs = [ ":external_config" ] configs += [ ":component_config" ] diff --git a/third_party/llvm-project/compiler-rt/BUILD.gn b/third_party/llvm-project/compiler-rt/BUILD.gn index 2887ebfd73738..b4c8c3c7fcfb5 100644 --- a/third_party/llvm-project/compiler-rt/BUILD.gn +++ b/third_party/llvm-project/compiler-rt/BUILD.gn @@ -92,5 +92,4 @@ static_library("compiler_rt") { } configs += [ ":compiler_rt_config" ] - configs -= [ "//starboard/build/config:default_cpp_standard" ] } diff --git a/third_party/musl/BUILD.gn b/third_party/musl/BUILD.gn index 78ecc77aac79c..93541692b5f97 100644 --- a/third_party/musl/BUILD.gn +++ b/third_party/musl/BUILD.gn @@ -24,7 +24,6 @@ if (target_cpu == "x64") { config("external_settings") { include_dirs = [ - "musl/include", "arch/$musl_arch", "arch/generic", ] diff --git a/third_party/musl/include/pthread.h b/third_party/musl/include/pthread.h index 2703585ad634a..ec3123492009e 100644 --- a/third_party/musl/include/pthread.h +++ b/third_party/musl/include/pthread.h @@ -1,12 +1,6 @@ #ifndef _PTHREAD_H #define _PTHREAD_H -#if defined(STARBOARD) - -#include "third_party/musl/src/starboard/pthread/pthread.h" - -#else - #ifdef __cplusplus extern "C" { #endif @@ -249,5 +243,4 @@ __REDIR(pthread_timedjoin_np, __pthread_timedjoin_np_time64); } #endif -#endif // defined(STARBOARD) #endif From 73018f4e1c425bedd3adae4deef56386c6c2ac3d Mon Sep 17 00:00:00 2001 From: Colin Liang Date: Sun, 19 Jan 2025 08:58:58 -0800 Subject: [PATCH 12/61] Integrate js_injection and make java bridge polyfill script work (#4715) 1. Incorporate Chromium's js_injection module into the Chrobalt codebase to enable JavaScript injection capabilities. 2. Move the Kabuki polyfill scripts into cobalt/embedded_resources. 3. Utilize the embed_polyfilled_javascript GN rule to convert the JavaScript files in cobalt/embedded_resources into embedded_js.h. 4. In the CobaltWebContentsObserver constructor, load the embedded JavaScript from embedded_js.h into the GeneratedResourceMap. Use AddDocumentStartJavaScript() to inject the polyfill code into web pages. This ensures the polyfills are executed early in the page lifecycle. 5. Override the RunScriptsAtDocumentStart() method in CobaltContentRendererClient to execute the injected polyfill JavaScript code. This PR is built on top of Kaido 's draft PR https://github.com/youtube/cobalt/pull/4704. b/384742721 --------- Co-authored-by: Kaido Kert Co-authored-by: Colin Liang --- cobalt/BUILD.gn | 19 ++++++++++ cobalt/android/BUILD.gn | 2 ++ .../kabuki_polyfill/amati_device_inspector.js | 6 ---- .../src/kabuki_polyfill/chrobalt_preload.js | 7 ---- .../html_media_element_extension.js | 11 ------ cobalt/base/generated_resources_types.h | 8 ++--- cobalt/cobalt_content_browser_client.h | 2 +- cobalt/cobalt_web_contents_observer.cc | 36 +++++++++++++++++++ cobalt/cobalt_web_contents_observer.h | 10 ++++-- .../h5vcc_platform_service_on_java_bridge.js} | 4 ++- ..._media_element_extension_on_java_bridge.js | 8 +++++ cobalt/renderer/BUILD.gn | 1 + .../cobalt_content_renderer_client.cc | 11 +++++- .../renderer/cobalt_content_renderer_client.h | 4 +++ 14 files changed, 96 insertions(+), 33 deletions(-) delete mode 100644 cobalt/android/apk/app/src/kabuki_polyfill/amati_device_inspector.js delete mode 100644 cobalt/android/apk/app/src/kabuki_polyfill/chrobalt_preload.js delete mode 100644 cobalt/android/apk/app/src/kabuki_polyfill/html_media_element_extension.js rename cobalt/{android/apk/app/src/kabuki_polyfill/h5vcc_platform_service.js => embedded_resources/h5vcc_platform_service_on_java_bridge.js} (97%) create mode 100644 cobalt/embedded_resources/html_media_element_extension_on_java_bridge.js diff --git a/cobalt/BUILD.gn b/cobalt/BUILD.gn index f80c773450d89..f7f22b9ae1341 100644 --- a/cobalt/BUILD.gn +++ b/cobalt/BUILD.gn @@ -49,10 +49,12 @@ if (!is_android) { defines = [] deps = [ + ":embed_polyfilled_javascript", "//cobalt/renderer:renderer", "//cobalt/services/crash_annotator", "//cobalt/services/crash_annotator/public/mojom", "//cobalt/user_agent", + "//components/js_injection/browser:browser", "//content/public/app", "//content/shell:content_shell_app", "//content/shell:content_shell_lib", @@ -84,6 +86,23 @@ action("cobalt_build_info") { ] } +config("embed_polyfilled_javascript_config") { + include_dirs = [ root_gen_dir ] +} + +# TODO(b/390710539): migrate to GRIT +action("embed_polyfilled_javascript") { + script = "//cobalt/build/generate_data_header.py" + outputs = [ "$target_gen_dir/embedded_resources/embedded_js.h" ] + public_configs = [ ":embed_polyfilled_javascript_config" ] + + args = [ + "CobaltJavaScriptPolyfill", + rebase_path(outputs[0], root_build_dir), + rebase_path("embedded_resources", root_build_dir), + ] +} + if (is_android) { test("cobalt_unittests") { testonly = true diff --git a/cobalt/android/BUILD.gn b/cobalt/android/BUILD.gn index ab62facee1d27..ac08d7c88e899 100644 --- a/cobalt/android/BUILD.gn +++ b/cobalt/android/BUILD.gn @@ -241,6 +241,7 @@ shared_library("libcobalt_content_shell_content_view") { testonly = true deps = [ ":content_shell_jni_headers", + "//cobalt:embed_polyfilled_javascript", "//cobalt/renderer:renderer", "//cobalt/services/crash_annotator", "//cobalt/services/crash_annotator/public/mojom", @@ -248,6 +249,7 @@ shared_library("libcobalt_content_shell_content_view") { # TODO: what can be removed in the dependencies? "//components/crash/content/browser", + "//components/js_injection/browser:browser", "//content/shell:content_shell_app", "//content/shell:content_shell_lib", "//content/shell:pak", diff --git a/cobalt/android/apk/app/src/kabuki_polyfill/amati_device_inspector.js b/cobalt/android/apk/app/src/kabuki_polyfill/amati_device_inspector.js deleted file mode 100644 index 55c97fb478d35..0000000000000 --- a/cobalt/android/apk/app/src/kabuki_polyfill/amati_device_inspector.js +++ /dev/null @@ -1,6 +0,0 @@ -/** - * @license - * Copyright The Cobalt Authors. - * SPDX-License-Identifier: Apache-2.0 - */ -Android_AmatiDeviceInspector.printIsAmatiDevice(); diff --git a/cobalt/android/apk/app/src/kabuki_polyfill/chrobalt_preload.js b/cobalt/android/apk/app/src/kabuki_polyfill/chrobalt_preload.js deleted file mode 100644 index 6d529d14d28a4..0000000000000 --- a/cobalt/android/apk/app/src/kabuki_polyfill/chrobalt_preload.js +++ /dev/null @@ -1,7 +0,0 @@ -import { initializeH5vccPlatformService } from './h5vcc_platform_service.js'; -import { initializeHTMLMediaElement } from './html_media_element_extension.js'; - -export function chrobaltPreload() { - initializeH5vccPlatformService(); - initializeHTMLMediaElement(); -} diff --git a/cobalt/android/apk/app/src/kabuki_polyfill/html_media_element_extension.js b/cobalt/android/apk/app/src/kabuki_polyfill/html_media_element_extension.js deleted file mode 100644 index 2a1aef3dfe461..0000000000000 --- a/cobalt/android/apk/app/src/kabuki_polyfill/html_media_element_extension.js +++ /dev/null @@ -1,11 +0,0 @@ -/** - * @license - * Copyright The Cobalt Authors. - * SPDX-License-Identifier: Apache-2.0 - */ - -export function initializeHTMLMediaElement() { - if (typeof HTMLMediaElementExtension !== 'undefined') { - HTMLMediaElement.prototype.canPlayType = HTMLMediaElementExtension.canPlayType; - } -} diff --git a/cobalt/base/generated_resources_types.h b/cobalt/base/generated_resources_types.h index 9b807e5fc650e..a3708bd3540df 100644 --- a/cobalt/base/generated_resources_types.h +++ b/cobalt/base/generated_resources_types.h @@ -17,16 +17,16 @@ #include -#include "base/containers/hash_tables.h" +#include struct FileContents { FileContents() {} - FileContents(const unsigned char *data, int size) : data(data), size(size) {} + FileContents(const unsigned char* data, int size) : data(data), size(size) {} - const unsigned char *data; + const unsigned char* data; int size; }; -typedef base::hash_map GeneratedResourceMap; +typedef std::map GeneratedResourceMap; #endif // COBALT_BASE_GENERATED_RESOURCES_TYPES_H_ diff --git a/cobalt/cobalt_content_browser_client.h b/cobalt/cobalt_content_browser_client.h index 12d3ba6243025..2fcfcc6d7c4ad 100644 --- a/cobalt/cobalt_content_browser_client.h +++ b/cobalt/cobalt_content_browser_client.h @@ -42,7 +42,7 @@ class CobaltContentBrowserClient : public content::ShellContentBrowserClient { blink::UserAgentMetadata GetUserAgentMetadata() override; void OverrideWebkitPrefs(content::WebContents* web_contents, blink::web_pref::WebPreferences* prefs) override; - void OnWebContentsCreated(content::WebContents* web_contents); + void OnWebContentsCreated(content::WebContents* web_contents) override; void RegisterBrowserInterfaceBindersForFrame( content::RenderFrameHost* render_frame_host, mojo::BinderMapWithContext* binder_map) diff --git a/cobalt/cobalt_web_contents_observer.cc b/cobalt/cobalt_web_contents_observer.cc index e7b18bf0fab6b..ca24f044d39e6 100644 --- a/cobalt/cobalt_web_contents_observer.cc +++ b/cobalt/cobalt_web_contents_observer.cc @@ -13,9 +13,45 @@ // limitations under the License. #include "cobalt/cobalt_web_contents_observer.h" +#include "base/strings/utf_string_conversions.h" +#include "cobalt/embedded_resources/embedded_js.h" namespace cobalt { +CobaltWebContentsObserver::CobaltWebContentsObserver( + content::WebContents* web_contents) + : content::WebContentsObserver(web_contents) { + // Create browser-side mojo service component + js_communication_host_ = + std::make_unique(web_contents); + + RegisterInjectedJavaScript(); +} + +void CobaltWebContentsObserver::RegisterInjectedJavaScript() { + // Get the embedded header resource + GeneratedResourceMap resource_map; + CobaltJavaScriptPolyfill::GenerateMap(resource_map); + + for (const auto& [file_name, file_contents] : resource_map) { + LOG(INFO) << "JS injection for filename: " << file_name; + std::string js(reinterpret_cast(file_contents.data), + file_contents.size); + + // Inject a script at document start for all origins + const std::u16string script(base::UTF8ToUTF16(js)); + const std::vector allowed_origins({"*"}); + auto result = js_communication_host_->AddDocumentStartJavaScript( + script, allowed_origins); + + if (result.error_message.has_value()) { + // error_message contains a value + LOG(WARNING) << "Failed to register JS injection for:" << file_name + << ", error message: " << result.error_message.value(); + } + } +} + // Placeholder for a WebContentsObserver override void CobaltWebContentsObserver::PrimaryMainDocumentElementAvailable() { LOG(INFO) << "Cobalt::PrimaryMainDocumentElementAvailable"; diff --git a/cobalt/cobalt_web_contents_observer.h b/cobalt/cobalt_web_contents_observer.h index 82f34a4d627c6..d9d9cd08f3eaa 100644 --- a/cobalt/cobalt_web_contents_observer.h +++ b/cobalt/cobalt_web_contents_observer.h @@ -18,14 +18,20 @@ #include "content/public/browser/web_contents_observer.h" #include "content/shell/browser/shell_content_browser_client.h" +#include "components/js_injection/browser/js_communication_host.h" + namespace cobalt { class CobaltWebContentsObserver : public content::WebContentsObserver { public: - explicit CobaltWebContentsObserver(content::WebContents* web_contents) - : content::WebContentsObserver(web_contents) {} + explicit CobaltWebContentsObserver(content::WebContents* web_contents); void PrimaryMainDocumentElementAvailable() override; + + private: + void RegisterInjectedJavaScript(); + + std::unique_ptr js_communication_host_; }; } // namespace cobalt diff --git a/cobalt/android/apk/app/src/kabuki_polyfill/h5vcc_platform_service.js b/cobalt/embedded_resources/h5vcc_platform_service_on_java_bridge.js similarity index 97% rename from cobalt/android/apk/app/src/kabuki_polyfill/h5vcc_platform_service.js rename to cobalt/embedded_resources/h5vcc_platform_service_on_java_bridge.js index b8ba198a423e8..f8f7697ff637a 100644 --- a/cobalt/android/apk/app/src/kabuki_polyfill/h5vcc_platform_service.js +++ b/cobalt/embedded_resources/h5vcc_platform_service_on_java_bridge.js @@ -39,7 +39,7 @@ class PlatformServiceClient { } } -export function initializeH5vccPlatformService() { +function initializeH5vccPlatformService() { if (typeof Android_H5vccPlatformService === 'undefined') { return; } @@ -72,3 +72,5 @@ export function initializeH5vccPlatformService() { }, } } + +initializeH5vccPlatformService(); diff --git a/cobalt/embedded_resources/html_media_element_extension_on_java_bridge.js b/cobalt/embedded_resources/html_media_element_extension_on_java_bridge.js new file mode 100644 index 0000000000000..200360e0ac3a4 --- /dev/null +++ b/cobalt/embedded_resources/html_media_element_extension_on_java_bridge.js @@ -0,0 +1,8 @@ +/** + * @license + * Copyright The Cobalt Authors. + * SPDX-License-Identifier: Apache-2.0 + */ +if (typeof HTMLMediaElementExtension !== 'undefined') { + HTMLMediaElement.prototype.canPlayType = HTMLMediaElementExtension.canPlayType; +} diff --git a/cobalt/renderer/BUILD.gn b/cobalt/renderer/BUILD.gn index 4e32e33d7c659..76e29c6174e57 100644 --- a/cobalt/renderer/BUILD.gn +++ b/cobalt/renderer/BUILD.gn @@ -21,6 +21,7 @@ source_set("renderer") { deps = [ "//cobalt/media/audio:webaudio", "//components/cdm/renderer", + "//components/js_injection/renderer:renderer", "//components/network_hints/renderer", "//components/web_cache/renderer", "//content/public/common", diff --git a/cobalt/renderer/cobalt_content_renderer_client.cc b/cobalt/renderer/cobalt_content_renderer_client.cc index 13485a2b9513d..8a2a27a37bf67 100644 --- a/cobalt/renderer/cobalt_content_renderer_client.cc +++ b/cobalt/renderer/cobalt_content_renderer_client.cc @@ -105,7 +105,9 @@ void CobaltContentRendererClient::ExposeInterfacesToBrowser( } void CobaltContentRendererClient::RenderFrameCreated( - content::RenderFrame* render_frame) {} + content::RenderFrame* render_frame) { + new js_injection::JsCommunication(render_frame); +} void CobaltContentRendererClient::PrepareErrorPage( content::RenderFrame* render_frame, @@ -285,6 +287,13 @@ bool CobaltContentRendererClient::IsSupportedVideoType( } #endif // BUILDFLAG(USE_STARBOARD_MEDIA) +void CobaltContentRendererClient::RunScriptsAtDocumentStart( + content::RenderFrame* render_frame) { + js_injection::JsCommunication* communication = + js_injection::JsCommunication::Get(render_frame); + communication->RunScriptsAtDocumentStart(); +} + std::unique_ptr CobaltContentRendererClient::CreatePrescientNetworking( content::RenderFrame* render_frame) { diff --git a/cobalt/renderer/cobalt_content_renderer_client.h b/cobalt/renderer/cobalt_content_renderer_client.h index eb319c8294d21..d9e39bdd3ec01 100644 --- a/cobalt/renderer/cobalt_content_renderer_client.h +++ b/cobalt/renderer/cobalt_content_renderer_client.h @@ -10,6 +10,7 @@ #include "build/build_config.h" #include "cobalt/media/audio/cobalt_audio_device_factory.h" +#include "components/js_injection/renderer/js_communication.h" #include "content/public/common/alternative_error_page_override_info.mojom-forward.h" #include "content/public/renderer/content_renderer_client.h" #include "media/mojo/buildflags.h" @@ -65,6 +66,9 @@ class CobaltContentRendererClient : public content::ContentRendererClient { bool IsSupportedVideoType(const media::VideoType& type) override; #endif // BUILDFLAG(USE_STARBOARD_MEDIA) + // JS Injection hook + void RunScriptsAtDocumentStart(content::RenderFrame* render_frame) override; + std::unique_ptr CreatePrescientNetworking( content::RenderFrame* render_frame) override; From d78bb51bd93b8d73a5ac39f64d109ef5b5ff5a6c Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 21 Jan 2025 13:45:46 -0800 Subject: [PATCH 13/61] Update the comment of SbPlayerGetInfo() (#4713) Update the comment to explicitly mention SbPlayerGetInfo() can be called from arbitrary threads. b/390259519 --- starboard/player.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/starboard/player.h b/starboard/player.h index c69cddfa10f44..a8074b9a2897e 100644 --- a/starboard/player.h +++ b/starboard/player.h @@ -310,9 +310,6 @@ static inline bool SbPlayerIsValid(SbPlayer player) { // - The associated decoder of the returned player should be assumed to not be // in |kSbPlayerDecoderStateNeedsData| until SbPlayerSeek() has been called // on it. -// - It is expected either that the thread that calls SbPlayerCreate is the same -// thread that calls the other |SbPlayer| functions for that player, or that -// there is a mutex guarding calls into each |SbPlayer| instance. // - If there is a platform limitation on how many players can coexist // simultaneously, then calls made to this function that attempt to exceed // that limit must return |kSbPlayerInvalid|. Multiple calls to SbPlayerCreate @@ -585,8 +582,8 @@ SB_EXPORT bool SbPlayerSetPlaybackRate(SbPlayer player, double playback_rate); SB_EXPORT void SbPlayerSetVolume(SbPlayer player, double volume); // Gets a snapshot of the current player state and writes it to -// |out_player_info|. This function may be called very frequently and is -// expected to be inexpensive. +// |out_player_info|. This function may be called very frequently from arbitrary +// threads and is expected to be inexpensive and thread safe. // // |player|: The player about which information is being retrieved. Must not be // |kSbPlayerInvalid|. From e649afc3810f21146c5b488454a846f36de3bdfb Mon Sep 17 00:00:00 2001 From: Niranjan Yardi Date: Tue, 21 Jan 2025 14:40:45 -0800 Subject: [PATCH 14/61] Remove unnecessary comment (#4730) b/371241293 --- base/BUILD.gn | 3 --- 1 file changed, 3 deletions(-) diff --git a/base/BUILD.gn b/base/BUILD.gn index 1b81cf081fbb4..78533e218e880 100644 --- a/base/BUILD.gn +++ b/base/BUILD.gn @@ -1123,9 +1123,6 @@ component("base") { "allocator/partition_allocator/shim/allocator_shim_internals.h", ] - # TODO: b/384652502 - Cobalt: Fix compiler errors building hermetically. - # loc.cc -o obj/base/base/allocator_shim_default_dispatch_to_partition_alloc.o - # ../../base/allocator/partition_allocator/shim/allocator_shim_default_dispatch_to_partition_alloc.cc:748:36: error: incomplete result type 'struct mallinfo' in function definition if (use_partition_alloc) { sources += [ "allocator/partition_allocator/shim/allocator_shim_default_dispatch_to_partition_alloc.cc", From 79df06d0c4b390e51129274c9c73d7d8d3ab8e95 Mon Sep 17 00:00:00 2001 From: Oscar Vestlie Date: Tue, 21 Jan 2025 16:44:32 -0800 Subject: [PATCH 15/61] Rework test targets and artifacts (#4720) Move test targets to its own home in config json. Improve test artifacts packaging to support android. b/386420709 --- .github/actions/build/action.yaml | 30 ++++- .github/actions/on_host_tests/action.yaml | 20 +++- .../actions/upload_test_artifacts/action.yaml | 33 +++--- .github/config/android-arm.json | 43 +++---- .github/config/android-arm64.json | 43 +++---- .github/config/android-x86.json | 42 +++---- .github/config/chromium_android-arm.json | 50 ++++---- .github/config/chromium_android-arm64.json | 50 ++++---- .github/config/chromium_android-x86.json | 50 ++++---- .github/config/chromium_linux.json | 38 +++--- .github/config/linux.json | 42 +++---- .github/workflows/main.yaml | 10 +- cobalt/build/archive_test_artifacts.py | 111 ++++++++++++++++++ .../targets/android-arm/test_targets.json | 19 +++ .../targets/android-arm64/test_targets.json | 19 +++ .../targets/linux-x64x11/test_targets.json | 68 +++++++++++ .../linux-x64x11/blink_unittests_filter.json | 2 +- .../capture_unittests_filter.json | 5 + .../linux-x64x11/cc_perftests_filter.json | 23 ++++ .../command_buffer_perftests_filter.json | 6 + .../filters/linux-x64x11/gl_tests_filter.json | 8 ++ .../linux-x64x11/media_unittests_filter.json | 2 +- .../linux-x64x11/mojo_perftests_filter.json | 6 + .../linux-x64x11/mojo_unittests_filter.json | 14 +-- .../linux-x64x11/net_unittests_filter.json | 2 +- .../linux-x64x11/sql_unittests_filter.json | 4 +- .../linux-x64x11/viz_perftests_filter.json | 6 + 27 files changed, 502 insertions(+), 244 deletions(-) create mode 100755 cobalt/build/archive_test_artifacts.py create mode 100644 cobalt/build/testing/targets/android-arm/test_targets.json create mode 100644 cobalt/build/testing/targets/android-arm64/test_targets.json create mode 100644 cobalt/build/testing/targets/linux-x64x11/test_targets.json create mode 100644 cobalt/testing/filters/linux-x64x11/capture_unittests_filter.json create mode 100644 cobalt/testing/filters/linux-x64x11/cc_perftests_filter.json create mode 100644 cobalt/testing/filters/linux-x64x11/command_buffer_perftests_filter.json create mode 100644 cobalt/testing/filters/linux-x64x11/gl_tests_filter.json create mode 100644 cobalt/testing/filters/linux-x64x11/mojo_perftests_filter.json create mode 100644 cobalt/testing/filters/linux-x64x11/viz_perftests_filter.json diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml index 614ec24aedb6f..41fdc7b112b6e 100644 --- a/.github/actions/build/action.yaml +++ b/.github/actions/build/action.yaml @@ -4,6 +4,15 @@ inputs: targets: description: "List of ninja targets for Cobalt build." required: true + upload_on_host_test_artifacts: + description: "Indicates if on-host test artifacts should be uploaded." + required: true + upload_on_device_test_artifacts: + description: "Indicates if on-device test artifacts should be uploaded." + required: true + test_artifacts_key: + description: "Artifact key used to store on-host test artifacts." + required: true runs: using: "composite" steps: @@ -38,13 +47,23 @@ runs: cd src gn args --list --short --overrides-only out/${{ matrix.platform }}_${{ matrix.config }} shell: bash + - name: Ninja build test targets + if: inputs.upload_on_host_test_artifacts == 'true' || inputs.upload_on_device_test_artifacts == 'true' + env: + # TODO(b/382508397): Replace hardcoded list with dynamically generated one. + TEST_TARGETS_JSON_FILE: cobalt/build/testing/targets/${{ matrix.platform }}/test_targets.json + run: | + set -x + cd src + time autoninja -C out/${{ matrix.platform }}_${{ matrix.config }} $(cat "${TEST_TARGETS_JSON_FILE}" | jq -cr '.test_targets | join(" ")') + shell: bash - name: Ninja build env: TARGETS: ${{ inputs.targets }} run: | set -x cd src - time autoninja -C out/${{ matrix.platform }}_${{ matrix.config }} $(echo "${TARGETS}" | tr -d '"') + time autoninja -C out/${{ matrix.platform }}_${{ matrix.config }} ${TARGETS} shell: bash - name: Archive Android APKs if: startsWith(matrix.platform, 'android') && matrix.config == 'qa' @@ -55,3 +74,12 @@ runs: src/out/${{ matrix.platform }}_qa/apks/*.apk src/out/${{ matrix.platform }}_qa/*_apk/*.apk src/out/${{ matrix.platform }}_qa/gen/build_info.json + - name: Upload Test Artifacts + if: inputs.upload_on_host_test_artifacts == 'true' || inputs.upload_on_device_test_artifacts == 'true' + uses: ./src/.github/actions/upload_test_artifacts + with: + test_artifacts_key: ${{ inputs.test_artifacts_key }} + upload_on_host_test_artifacts: ${{ inputs.upload_on_host_test_artifacts }} + upload_on_device_test_artifacts: ${{ inputs.upload_on_device_test_artifacts }} + # TODO(b/382508397): Replace hardcoded list with dynamically generated one. + test_targets_json_file: cobalt/build/testing/targets/${{ matrix.platform }}/test_targets.json diff --git a/.github/actions/on_host_tests/action.yaml b/.github/actions/on_host_tests/action.yaml index df63ad34adbb4..776d05f4828d7 100644 --- a/.github/actions/on_host_tests/action.yaml +++ b/.github/actions/on_host_tests/action.yaml @@ -27,7 +27,7 @@ runs: mkdir -p ${test_dir} cd ${test_dir} - tar xvf ../test_artifacts.tar.xz + tar xvf ../test_artifacts.tar.gz - name: Run Tests id: run-tests shell: bash @@ -38,8 +38,8 @@ runs: env # Explicitly point to libraries in extracted dir. - LD_LIBRARY_PATH="${test_dir}/starboard" - LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${test_dir}" + LD_LIBRARY_PATH="${test_dir}/out/${{ matrix.platform}}_${{ matrix.config }}/starboard" + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${test_dir}/out/${{ matrix.platform}}_${{ matrix.config }}" export LD_LIBRARY_PATH # Make results dir available to the archiving step below. @@ -47,8 +47,10 @@ runs: echo "results_dir=${results_dir}" >> $GITHUB_ENV failed_suites="" - cd ${test_dir} - for test_binary in $(ls {*tests,nplb}); do + cd "${test_dir}" + + for test_binary_path in $(cat out/${{ matrix.platform}}_${{ matrix.config }}/test_targets.json | jq -cr '.executables | join(" ")'); do + test_binary=$(basename "${test_binary_path}") echo "Running tests for suite: ${test_binary}" test_filter="*" @@ -60,7 +62,13 @@ runs: echo "Test filter evaluated to: ${test_filter}" xml_path="${results_dir}/shard_${{ matrix.shard }}/${test_binary}_testoutput.xml" - /usr/bin/xvfb-run -a --server-args="${XVFB_SERVER_ARGS}" "./${test_binary}" --test-launcher-shard-index=${{ matrix.shard }} --test-launcher-total-shards=${{ inputs.num_gtest_shards }} --gtest_filter="${test_filter}" --gtest_output="xml:${xml_path}" || { + # TODO: Investigate test_runner.py + /usr/bin/xvfb-run -a --server-args="${XVFB_SERVER_ARGS}" \ + ./"${test_binary_path}" \ + --test-launcher-shard-index=${{ matrix.shard }} \ + --test-launcher-total-shards=${{ inputs.num_gtest_shards }} \ + --gtest_output="xml:${xml_path}" \ + --gtest_filter="${test_filter}" || { # Set exit code on failure. failed_suites="${failed_suites} ${test_binary}" } diff --git a/.github/actions/upload_test_artifacts/action.yaml b/.github/actions/upload_test_artifacts/action.yaml index 017950dce2fff..a2a645aea29db 100644 --- a/.github/actions/upload_test_artifacts/action.yaml +++ b/.github/actions/upload_test_artifacts/action.yaml @@ -1,23 +1,26 @@ name: Test Artifact Upload description: Uploads test archives for on-device and on-host tests. inputs: - on_host: + upload_on_host_test_artifacts: description: "Indicates if on-host test artifacts should be uploaded." required: true - on_device: + upload_on_device_test_artifacts: description: "Indicates if on-device test artifacts should be uploaded." required: true test_artifacts_key: description: "Artifact key used to store on-host test artifacts." required: true + test_targets_json_file: + description: "File containing the test target json." + required: true runs: using: "composite" steps: - name: Set up Cloud SDK - if: inputs.on_device == 'true' + if: inputs.upload_on_device_test_artifacts == 'true' uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1 - name: Upload Android Test Artifacts to GCS - if: inputs.on_device == 'true' + if: inputs.upload_on_device_test_artifacts == 'true' env: WORKFLOW: ${{ github.workflow }} run: | @@ -28,23 +31,21 @@ runs: shell: bash - name: Create On-Host Test Artifacts Archive - if: inputs.on_host == 'true' + if: inputs.upload_on_host_test_artifacts == 'true' run: | set -x - cd src/out/${{ matrix.platform }}_${{ matrix.config }} - test_deps_file="test.deps" - for test_binary in $(ls {nplb,*tests}); do - echo $test_binary - if [ -f "${test_binary}.runtime_deps" ]; then - cat "${test_binary}.runtime_deps" >> "${test_deps_file}" - fi - done - tar cvf - -T "${test_deps_file}" | xz -T0 -1 -z - > "${GITHUB_WORKSPACE}/test_artifacts.tar.xz" + mkdir ${GITHUB_WORKSPACE}/artifacts + cd src/ + ./cobalt/build/archive_test_artifacts.py \ + --source out/${{ matrix.platform }}_${{ matrix.config }}/ \ + --destination ${GITHUB_WORKSPACE}/artifacts \ + --platform ${{ matrix.platform }} \ + --targets $(cat "${{ inputs.test_targets_json_file }}" | jq -cr '.test_targets | join(",")') shell: bash - name: Upload On-Host Test Artifacts Archive - if: inputs.on_host == 'true' + if: inputs.upload_on_host_test_artifacts == 'true' uses: actions/upload-artifact@v4 with: name: ${{ inputs.test_artifacts_key }} - path: test_artifacts.tar.xz + path: artifacts/* retention-days: 3 diff --git a/.github/config/android-arm.json b/.github/config/android-arm.json index 0ece3896deb1b..fad4ee6ff895e 100644 --- a/.github/config/android-arm.json +++ b/.github/config/android-arm.json @@ -1,28 +1,19 @@ { - "docker_service": "linux", - "platforms": [ - "android-arm" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk", - "cobalt:gn_all" - ], - "includes": [ - { - "name":"arm", - "platform":"android-arm" - } - ] + "docker_service": "linux", + "platforms": [ + "android-arm" + ], + "test_on_device": true, + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk", + "cobalt:gn_all" + ], + "includes": [ + { + "name": "arm", + "platform": "android-arm" + } + ] } diff --git a/.github/config/android-arm64.json b/.github/config/android-arm64.json index 1b34e70b89756..54c04de67f944 100644 --- a/.github/config/android-arm64.json +++ b/.github/config/android-arm64.json @@ -1,28 +1,19 @@ { - "docker_service": "linux", - "platforms": [ - "android-arm64" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk", - "cobalt:gn_all" - ], - "includes": [ - { - "name":"arm64", - "platform":"android-arm64" - } - ] + "docker_service": "linux", + "platforms": [ + "android-arm64" + ], + "test_on_device": true, + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk", + "cobalt:gn_all" + ], + "includes": [ + { + "name": "arm64", + "platform": "android-arm64" + } + ] } diff --git a/.github/config/android-x86.json b/.github/config/android-x86.json index 262d0cfc8c446..c1e9fc89e8700 100644 --- a/.github/config/android-x86.json +++ b/.github/config/android-x86.json @@ -1,28 +1,18 @@ { - "docker_service": "linux", - "platforms": [ - "android-x86" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk", - "cobalt:gn_all" - ], - "includes": [ - { - "name":"x86", - "platform":"android-x86" - } - ] + "docker_service": "linux", + "platforms": [ + "android-x86" + ], + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk", + "cobalt:gn_all" + ], + "includes": [ + { + "name": "x86", + "platform": "android-x86" + } + ] } diff --git a/.github/config/chromium_android-arm.json b/.github/config/chromium_android-arm.json index 9eac89f0b668f..7e073505527ce 100644 --- a/.github/config/chromium_android-arm.json +++ b/.github/config/chromium_android-arm.json @@ -1,27 +1,27 @@ { - "docker_service": "linux", - "platforms": [ - "chromium_android-arm" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk" - ], - "includes": [ - { - "name":"arm", - "platform":"chromium_android-arm" - } - ] + "docker_service": "linux", + "platforms": [ + "chromium_android-arm" + ], + "targets": [ + "base_unittests", + "sql_unittests", + "net_unittests", + "url_unittests", + "ipc_tests", + "mojo_unittests", + "gpu_unittests", + "gin_unittests", + "blink_unittests", + "media_unittests", + "content_shell", + "system_webview_apk", + "system_webview_shell_apk" + ], + "includes": [ + { + "name": "arm", + "platform": "chromium_android-arm" + } + ] } diff --git a/.github/config/chromium_android-arm64.json b/.github/config/chromium_android-arm64.json index 1aea4037d668d..1921fdbbad913 100644 --- a/.github/config/chromium_android-arm64.json +++ b/.github/config/chromium_android-arm64.json @@ -1,27 +1,27 @@ { - "docker_service": "linux", - "platforms": [ - "chromium_android-arm64" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk" - ], - "includes": [ - { - "name":"arm64", - "platform":"chromium_android-arm64" - } - ] + "docker_service": "linux", + "platforms": [ + "chromium_android-arm64" + ], + "targets": [ + "base_unittests", + "sql_unittests", + "net_unittests", + "url_unittests", + "ipc_tests", + "mojo_unittests", + "gpu_unittests", + "gin_unittests", + "blink_unittests", + "media_unittests", + "content_shell", + "system_webview_apk", + "system_webview_shell_apk" + ], + "includes": [ + { + "name": "arm64", + "platform": "chromium_android-arm64" + } + ] } diff --git a/.github/config/chromium_android-x86.json b/.github/config/chromium_android-x86.json index c97a39f99ec87..8498733285ed5 100644 --- a/.github/config/chromium_android-x86.json +++ b/.github/config/chromium_android-x86.json @@ -1,27 +1,27 @@ { - "docker_service": "linux", - "platforms": [ - "chromium_android-x86" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk" - ], - "includes": [ - { - "name":"x86", - "platform":"chromium_android-x86" - } - ] + "docker_service": "linux", + "platforms": [ + "chromium_android-x86" + ], + "targets": [ + "base_unittests", + "sql_unittests", + "net_unittests", + "url_unittests", + "ipc_tests", + "mojo_unittests", + "gpu_unittests", + "gin_unittests", + "blink_unittests", + "media_unittests", + "content_shell", + "system_webview_apk", + "system_webview_shell_apk" + ], + "includes": [ + { + "name": "x86", + "platform": "chromium_android-x86" + } + ] } diff --git a/.github/config/chromium_linux.json b/.github/config/chromium_linux.json index 01114cd966a66..9bf7409f3fea4 100644 --- a/.github/config/chromium_linux.json +++ b/.github/config/chromium_linux.json @@ -1,25 +1,15 @@ { - "docker_service": "linux", - "platforms": [ - "chromium_linux-x64x11" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell" - ], - "includes": [ - { - "name":"x64", - "platform":"chromium_linux-x64x11" - } - ] - } + "docker_service": "linux", + "platforms": [ + "chromium_linux-x64x11" + ], + "targets": [ + "content_shell" + ], + "includes": [ + { + "name": "x64", + "platform": "chromium_linux-x64x11" + } + ] +} diff --git a/.github/config/linux.json b/.github/config/linux.json index a6259f06bddef..bf861faf61c70 100644 --- a/.github/config/linux.json +++ b/.github/config/linux.json @@ -1,26 +1,18 @@ { - "docker_service": "linux", - "test_on_host": true, - "num_gtest_shards": 10, - "platforms": [ - "linux-x64x11" - ], - "targets": [ - "base_unittests", - "cobalt:gn_all", - "content_shell", - "gin_unittests", - "gpu_unittests", - "ipc_tests", - "media_unittests", - "mojo_unittests", - "sql_unittests", - "url_unittests" - ], - "includes": [ - { - "name":"x64", - "platform":"linux-x64x11" - } - ] - } + "docker_service": "linux", + "platforms": [ + "linux-x64x11" + ], + "test_on_host": true, + "num_gtest_shards": 10, + "targets": [ + "cobalt:gn_all", + "content_shell" + ], + "includes": [ + { + "name": "x64", + "platform": "linux-x64x11" + } + ] +} diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9f3cf8e1125f5..d521ed65099fb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -63,7 +63,7 @@ jobs: - id: set-targets shell: bash run: | - targets=$(cat ${GITHUB_WORKSPACE}/.github/config/${{ inputs.platform }}.json | jq -c '.targets | join(" ")') + targets=$(cat ${GITHUB_WORKSPACE}/.github/config/${{ inputs.platform }}.json | jq -cr '.targets | join(" ")') echo "targets=${targets}" >> $GITHUB_ENV - id: set-includes shell: bash @@ -179,13 +179,9 @@ jobs: if: ${{ ! (contains(matrix.platform, 'android') && matrix.config == 'debug') }} with: targets: ${{ needs.initialize.outputs.targets }} - - name: Upload Test Artifacts - if: matrix.config == 'devel' - uses: ./src/.github/actions/upload_test_artifacts - with: test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }} - on_host: ${{ needs.initialize.outputs.test_on_host }} - on_device: ${{ needs.initialize.outputs.test_on_device }} + upload_on_host_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_host }} + upload_on_device_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_device }} test: needs: [initialize, docker-build-image, build] diff --git a/cobalt/build/archive_test_artifacts.py b/cobalt/build/archive_test_artifacts.py new file mode 100755 index 0000000000000..711f7bb2734cb --- /dev/null +++ b/cobalt/build/archive_test_artifacts.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# Copyright 2025 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Creates test artifacts tar with runtime dependencies.""" + +import argparse +import json +import os +import subprocess +import tempfile +from typing import List + + +def _make_tar(archive_path: str, file_list: str): + """Creates the tar file. Uses tar command instead of tarfile for performance. + """ + print(f'Creating {os.path.basename(archive_path)}') + # Create temporary file to hold file list to not blow out the commandline. + with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8') as temp_file: + temp_file.write('\n'.join(sorted(file_list))) + temp_file.flush() + tar_cmd = ['tar', '-I gzip -1', '-cvf', archive_path, '-T', temp_file.name] + subprocess.check_call(tar_cmd) + + +def create_archive(targets: List[str], source_dir: str, destination_dir: str, + platform: str, combine: bool): + """Main logic. Collects runtime dependencies from the source directory for + each target.""" + # TODO(b/382508397): Remove when dynamically generated. + # Put the test targets in a json file in the archive. + test_target_names = [target.split(':')[1] for target in targets] + test_targets_json = os.path.join(source_dir, 'test_targets.json') + with open(test_targets_json, 'w', encoding='utf-8') as test_targets_file: + test_targets_file.write( + json.dumps({ + 'test_targets': + test_target_names, + 'executables': [ + os.path.join(source_dir, target_name) + for target_name in test_target_names + ] + })) + + deps = set([test_targets_json]) + for target in targets: + target_path, target_name = target.split(':') + # These paths are configured in test.gni: + # https://github.com/youtube/cobalt/blob/main/testing/test.gni + if platform.startswith('android'): + deps_file = os.path.join( + source_dir, 'gen.runtime', target_path, + f'{target_name}__test_runner_script.runtime_deps') + else: + deps_file = os.path.join(source_dir, f'{target_name}.runtime_deps') + + with open(deps_file, 'r', encoding='utf-8') as runtime_deps_file: + target_deps = { + os.path.relpath(os.path.join(source_dir, line.strip())) + for line in runtime_deps_file + } + deps |= target_deps + + if not combine: + output_path = os.path.join(destination_dir, f'{target_name}_deps.tar.gz') + _make_tar(output_path, deps) + + if combine: + output_path = os.path.join(destination_dir, 'test_artifacts.tar.gz') + _make_tar(output_path, deps) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-s', + '--source-dir', + required=True, + help='The path to where the artifacts are stored. ' + 'Typically the out folder.') + parser.add_argument( + '-d', + '--destination-dir', + required=True, + help='The output directory. For linux test_artifacts.tar.gz is created ' + 'with all test artifacts, else a `_runtime_deps.tar.gz` is ' + 'created for each target passed.') + parser.add_argument( + '-p', '--platform', required=True, help='The platform getting packaged.') + parser.add_argument( + '-t', + '--targets', + required=True, + type=lambda arg: arg.split(','), + help='The targets to package, comma-separated. Must be fully qualified ' + 'for android.') + args = parser.parse_args() + + create_archive(args.targets, args.source_dir, args.destination_dir, + args.platform, args.platform.startswith('linux')) diff --git a/cobalt/build/testing/targets/android-arm/test_targets.json b/cobalt/build/testing/targets/android-arm/test_targets.json new file mode 100644 index 0000000000000..c67c6a0c5b16b --- /dev/null +++ b/cobalt/build/testing/targets/android-arm/test_targets.json @@ -0,0 +1,19 @@ +{ + "TODO(b/382508397)": "Remove this when list is dynamically generated.", + "test_targets": [ + "base:base_unittests", + "cc:cc_perftests", + "cc:cc_unittests", + "ipc:ipc_tests", + "media/capture:capture_unittests", + "media/midi:midi_unittests", + "media:media_unittests", + "mojo:mojo_perftests", + "net:net_unittests", + "services/service_manager/tests:service_manager_unittests", + "skia:skia_unittests", + "sql:sql_unittests", + "url:url_perftests", + "url:url_unittests" + ] +} diff --git a/cobalt/build/testing/targets/android-arm64/test_targets.json b/cobalt/build/testing/targets/android-arm64/test_targets.json new file mode 100644 index 0000000000000..c67c6a0c5b16b --- /dev/null +++ b/cobalt/build/testing/targets/android-arm64/test_targets.json @@ -0,0 +1,19 @@ +{ + "TODO(b/382508397)": "Remove this when list is dynamically generated.", + "test_targets": [ + "base:base_unittests", + "cc:cc_perftests", + "cc:cc_unittests", + "ipc:ipc_tests", + "media/capture:capture_unittests", + "media/midi:midi_unittests", + "media:media_unittests", + "mojo:mojo_perftests", + "net:net_unittests", + "services/service_manager/tests:service_manager_unittests", + "skia:skia_unittests", + "sql:sql_unittests", + "url:url_perftests", + "url:url_unittests" + ] +} diff --git a/cobalt/build/testing/targets/linux-x64x11/test_targets.json b/cobalt/build/testing/targets/linux-x64x11/test_targets.json new file mode 100644 index 0000000000000..b8fc38f99400f --- /dev/null +++ b/cobalt/build/testing/targets/linux-x64x11/test_targets.json @@ -0,0 +1,68 @@ +{ + "TODO(b/382508397)": "Remove this when list is dynamically generated.", + "test_targets": [ + "base:base_unittests", + "cc:cc_perftests", + "cc:cc_unittests", + "components/viz:viz_perftests", + "components/viz:viz_unittests", + "crypto:crypto_unittests", + "device:device_unittests", + "google_apis/gcm:gcm_unit_tests", + "google_apis:google_apis_unittests", + "gpu/gles2_conform_support:gles2_conform_test", + "gpu:command_buffer_perftests", + "gpu:gl_tests", + "gpu:gpu_benchmark", + "gpu:gpu_perftests", + "gpu:gpu_unittests", + "ipc:ipc_tests", + "media/capture:capture_unittests", + "media:media_unittests", + "media/midi:midi_unittests", + "mojo:mojo_perftests", + "ppapi:ppapi_perftests", + "ppapi:ppapi_unittests", + "services/service_manager/tests:service_manager_unittests", + "skia:skia_unittests", + "sql:sql_unittests", + "ui/wm:wm_unittests", + "url:url_perftests", + "url:url_unittests" + ], + "executables": [ + "out/linux-x64x11_devel/base_nocompile_tests", + "out/linux-x64x11_devel/base_unittests", + "out/linux-x64x11_devel/cc_perftests", + "out/linux-x64x11_devel/cc_unittests", + "out/linux-x64x11_devel/viz_perftests", + "out/linux-x64x11_devel/viz_unittests", + "out/linux-x64x11_devel/content_browsertests", + "out/linux-x64x11_devel/content_nocompile_tests", + "out/linux-x64x11_devel/content_unittests", + "out/linux-x64x11_devel/crypto_unittests", + "out/linux-x64x11_devel/device_unittests", + "out/linux-x64x11_devel/gcm_unit_tests", + "out/linux-x64x11_devel/google_apis_unittests", + "out/linux-x64x11_devel/gles2_conform_test", + "out/linux-x64x11_devel/command_buffer_perftests", + "out/linux-x64x11_devel/gl_tests", + "out/linux-x64x11_devel/gpu_benchmark", + "out/linux-x64x11_devel/gpu_perftests", + "out/linux-x64x11_devel/gpu_unittests", + "out/linux-x64x11_devel/ipc_tests", + "out/linux-x64x11_devel/capture_unittests", + "out/linux-x64x11_devel/media_unittests", + "out/linux-x64x11_devel/midi_unittests", + "out/linux-x64x11_devel/mojo_perftests", + "out/linux-x64x11_devel/net_unittests", + "out/linux-x64x11_devel/ppapi_perftests", + "out/linux-x64x11_devel/ppapi_unittests", + "out/linux-x64x11_devel/service_manager_unittests", + "out/linux-x64x11_devel/skia_unittests", + "out/linux-x64x11_devel/sql_unittests", + "out/linux-x64x11_devel/wm_unittests", + "out/linux-x64x11_devel/url_perftests", + "out/linux-x64x11_devel/url_unittest" + ] +} diff --git a/cobalt/testing/filters/linux-x64x11/blink_unittests_filter.json b/cobalt/testing/filters/linux-x64x11/blink_unittests_filter.json index f872d21542541..8f78a1aeddc6f 100644 --- a/cobalt/testing/filters/linux-x64x11/blink_unittests_filter.json +++ b/cobalt/testing/filters/linux-x64x11/blink_unittests_filter.json @@ -1,5 +1,5 @@ { "failing_tests": [ - "TextFragmentGenerationNavigationTest.*" + "TextFragmentGenerationNavigationTest.*" ] } diff --git a/cobalt/testing/filters/linux-x64x11/capture_unittests_filter.json b/cobalt/testing/filters/linux-x64x11/capture_unittests_filter.json new file mode 100644 index 0000000000000..81ef3fefe7def --- /dev/null +++ b/cobalt/testing/filters/linux-x64x11/capture_unittests_filter.json @@ -0,0 +1,5 @@ +{ + "failing_tests": [ + "VideoCaptureDeviceTests/VideoCaptureDeviceTest.*" + ] +} diff --git a/cobalt/testing/filters/linux-x64x11/cc_perftests_filter.json b/cobalt/testing/filters/linux-x64x11/cc_perftests_filter.json new file mode 100644 index 0000000000000..dcc81ac9d3ce8 --- /dev/null +++ b/cobalt/testing/filters/linux-x64x11/cc_perftests_filter.json @@ -0,0 +1,23 @@ +{ + "failing_tests": [ + "AnimationHostPerfTest.Push10TimelinesPropertiesTo", + "AnimationHostPerfTest.Push1000AnimationsPropertiesTo", + "AnimationHostPerfTest.Push1000TimelinesPropertiesTo", + "BrowserCompositorInvalidateLayerTreePerfTest.DenseBrowserUIThreaded", + "LayerPerfTest.PushPropertiesTo", + "LayerTreeHostPerfTestJsonReader.HeavyPageThreaded", + "LayerTreeHostPerfTestJsonReader.TenTenThreaded", + "LayerTreeHostPerfTestJsonReader.TenTenThreaded_FullDamageEachFrame", + "LayerTreeHostPerfTestLeafInvalidates.TenTenThreaded", + "PaintOpPerfTest.ManyFlagsOps", + "PictureLayerImplPerfTest.TilingSetEvictionQueueConstruct", + "PictureLayerImplPerfTest.TilingSetEvictionQueueConstructAndIterate", + "PictureLayerImplPerfTest.TilingSetRasterQueueConstructAndIterate", + "RasterBufferProviderCommonPerfTest.BuildTileTaskGraph", + "RasterBufferProviderPerfTests/RasterBufferProviderPerfTest.ScheduleAlternateTasks/3", + "RasterBufferProviderPerfTests/RasterBufferProviderPerfTest.ScheduleAndExecuteTasks/3", + "RasterBufferProviderPerfTests/RasterBufferProviderPerfTest.ScheduleTasks/3", + "ScrollingLayerTreePerfTest.LongScrollablePageSingleThread", + "ScrollingLayerTreePerfTest.LongScrollablePageThreaded" + ] +} diff --git a/cobalt/testing/filters/linux-x64x11/command_buffer_perftests_filter.json b/cobalt/testing/filters/linux-x64x11/command_buffer_perftests_filter.json new file mode 100644 index 0000000000000..8968ff665a662 --- /dev/null +++ b/cobalt/testing/filters/linux-x64x11/command_buffer_perftests_filter.json @@ -0,0 +1,6 @@ +{ + "failing_tests": [ + "DecoderPerfTest.*", + "TextFragmentGenerationNavigationTest.*" + ] +} diff --git a/cobalt/testing/filters/linux-x64x11/gl_tests_filter.json b/cobalt/testing/filters/linux-x64x11/gl_tests_filter.json new file mode 100644 index 0000000000000..563897587116e --- /dev/null +++ b/cobalt/testing/filters/linux-x64x11/gl_tests_filter.json @@ -0,0 +1,8 @@ +{ + "failing_tests": [ + "SharedImageGLBackingProduceDawnTest.Basic", + "TranslatorVariants/EXTBlendFuncExtendedES3DrawTest.*", + "WebGPUMailboxTest.*", + "WebGPUTest.*" + ] +} diff --git a/cobalt/testing/filters/linux-x64x11/media_unittests_filter.json b/cobalt/testing/filters/linux-x64x11/media_unittests_filter.json index 4a7dc4f11e3ff..d6a1314f6ea58 100644 --- a/cobalt/testing/filters/linux-x64x11/media_unittests_filter.json +++ b/cobalt/testing/filters/linux-x64x11/media_unittests_filter.json @@ -1,5 +1,5 @@ { "failing_tests": [ - "*" + "*" ] } diff --git a/cobalt/testing/filters/linux-x64x11/mojo_perftests_filter.json b/cobalt/testing/filters/linux-x64x11/mojo_perftests_filter.json new file mode 100644 index 0000000000000..45f9a00937b84 --- /dev/null +++ b/cobalt/testing/filters/linux-x64x11/mojo_perftests_filter.json @@ -0,0 +1,6 @@ +{ + "failing_tests": [ + "CorePerftest.MessagePipe_Threaded", + "MojoBindingsPerftest.*" + ] +} diff --git a/cobalt/testing/filters/linux-x64x11/mojo_unittests_filter.json b/cobalt/testing/filters/linux-x64x11/mojo_unittests_filter.json index c9018994c6530..3904e3afec0ca 100644 --- a/cobalt/testing/filters/linux-x64x11/mojo_unittests_filter.json +++ b/cobalt/testing/filters/linux-x64x11/mojo_unittests_filter.json @@ -1,11 +1,11 @@ { "failing_tests": [ - "ValidationIntegrationTest.Receiver", - "ValidationIntegrationTest.Remote", - "ValidationTest.AssociatedConformace", - "ValidationTest.BoundsCheck", - "ValidationTest.Conformance", - "ValidationTest.ResponseBoundsCheck", - "ValidationTest.ResponseConformance" + "ValidationIntegrationTest.Receiver", + "ValidationIntegrationTest.Remote", + "ValidationTest.AssociatedConformace", + "ValidationTest.BoundsCheck", + "ValidationTest.Conformance", + "ValidationTest.ResponseBoundsCheck", + "ValidationTest.ResponseConformance" ] } diff --git a/cobalt/testing/filters/linux-x64x11/net_unittests_filter.json b/cobalt/testing/filters/linux-x64x11/net_unittests_filter.json index 4a7dc4f11e3ff..d6a1314f6ea58 100644 --- a/cobalt/testing/filters/linux-x64x11/net_unittests_filter.json +++ b/cobalt/testing/filters/linux-x64x11/net_unittests_filter.json @@ -1,5 +1,5 @@ { "failing_tests": [ - "*" + "*" ] } diff --git a/cobalt/testing/filters/linux-x64x11/sql_unittests_filter.json b/cobalt/testing/filters/linux-x64x11/sql_unittests_filter.json index fd2f9bd5bd4d9..2d6f843ed8ff1 100644 --- a/cobalt/testing/filters/linux-x64x11/sql_unittests_filter.json +++ b/cobalt/testing/filters/linux-x64x11/sql_unittests_filter.json @@ -1,6 +1,6 @@ { "failing_tests": [ - "SQLRecoveryTest.Bug387868", - "SQLiteFeaturesTest.FTS3_Prefix" + "SQLRecoveryTest.Bug387868", + "SQLiteFeaturesTest.FTS3_Prefix" ] } diff --git a/cobalt/testing/filters/linux-x64x11/viz_perftests_filter.json b/cobalt/testing/filters/linux-x64x11/viz_perftests_filter.json new file mode 100644 index 0000000000000..ee4d6ed5e6e27 --- /dev/null +++ b/cobalt/testing/filters/linux-x64x11/viz_perftests_filter.json @@ -0,0 +1,6 @@ +{ + "failing_tests": [ + "RendererPerfTest.*", + "SurfaceAggregatorPerfTest.*" + ] +} From 6698790118fb0da73ea35432bbdc2ce3be252bd8 Mon Sep 17 00:00:00 2001 From: Bo-Rong Chen Date: Tue, 21 Jan 2025 18:09:41 -0800 Subject: [PATCH 16/61] [media] Enable blink feature for MediaSource.duration (#4734) A blink runtime feature for new abort and duration behavior was added by this commit (https://codereview.chromium.org/2102323002), but it is not enabled by default. - Enabling this feature to allow to pass YTS tests on MediaSource.duration. b/388666609 --- .../apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java b/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java index 1a86b211404e5..24ff6af6789fd 100644 --- a/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java +++ b/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/CobaltActivity.java @@ -127,6 +127,8 @@ protected void createContent(final Bundle savedInstanceState) { // causes rendering artifacts when // low-end-device-mode is enabled. "--disable-rgba-4444-textures", + // Align with MSE spec for MediaSource.duration. + "--enable-blink-features=MediaSourceNewAbortAndDuration", }; CommandLine.getInstance().appendSwitchesAndArguments(cobaltCommandLineParams); if (shouldSetJNIPrefix) { From 2bf716e9f293db1d2aa8223eb55127f3ef3c90b7 Mon Sep 17 00:00:00 2001 From: Brian Ting Date: Tue, 21 Jan 2025 18:55:37 -0800 Subject: [PATCH 17/61] Add --no-rbe flag to gn.py to disable RBE. This allows Remote Build Execution to be situationally disabled. Disables RBE for Kokoro builds until RBE is fully supported. b/384982606 --- cobalt/build/gn.py | 22 +++++++++++++++------- cobalt/devinfra/kokoro/bin/dind_build.sh | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cobalt/build/gn.py b/cobalt/build/gn.py index 7060fa15529cc..19e47cb9dc20f 100755 --- a/cobalt/build/gn.py +++ b/cobalt/build/gn.py @@ -74,8 +74,8 @@ def get_build_args(build_args_path): ] -def write_build_args(build_args_path, original_lines, dict_settings, - build_type): +def write_build_args(build_args_path, original_lines, dict_settings, build_type, + use_rbe): """ Write args file, modifying settings for config""" controlled_args = [ (k, dict_settings[k]) for k in CONTROLLED_ARGS if k in dict_settings @@ -85,7 +85,8 @@ def write_build_args(build_args_path, original_lines, dict_settings, f'The following args cannot be set in configs: {controlled_args}') gen_comment = '# Set by gn.py' with open(build_args_path, 'w', encoding='utf-8') as f: - f.write(f'use_remoteexec = true {gen_comment}\n') + if use_rbe: + f.write(f'use_remoteexec = true {gen_comment}\n') f.write( f'rbe_cfg_dir = rebase_path("//cobalt/reclient_cfgs") {gen_comment}\n') f.write(f'build_type = "{build_type}" {gen_comment}\n') @@ -95,12 +96,12 @@ def write_build_args(build_args_path, original_lines, dict_settings, f.write(line) -def main(out_directory: str, platform: str, build_type: str, +def main(out_directory: str, platform: str, build_type: str, use_rbe: bool, gn_gen_args: List[str]): + Path(out_directory).mkdir(parents=True, exist_ok=True) platform_path = f'cobalt/build/configs/{platform}' dst_args_gn_file = os.path.join(out_directory, 'args.gn') src_args_gn_file = os.path.join(platform_path, 'args.gn') - Path(out_directory).mkdir(parents=True, exist_ok=True) if os.path.exists(dst_args_gn_file): # Copy the stale args.gn into stale_args.gn @@ -111,7 +112,9 @@ def main(out_directory: str, platform: str, build_type: str, 'In general, if the file exists, you should run' ' `gn args ` to edit it instead.') build_args = get_build_args(src_args_gn_file) - write_build_args(dst_args_gn_file, build_args[0], build_args[1], build_type) + write_build_args(dst_args_gn_file, build_args[0], build_args[1], build_type, + use_rbe) + gn_command = ['gn', 'gen', out_directory] + gn_gen_args print(' '.join(gn_command)) subprocess.check_call(gn_command) @@ -154,6 +157,11 @@ def main(out_directory: str, platform: str, build_type: str, default=False, action='store_true', help='Pass this flag to disable the header dependency gn check.') + parser.add_argument( + '--no-rbe', + default=False, + action='store_true', + help='Pass this flag to disable Remote Build Execution.') script_args, gen_args = parser.parse_known_args() if script_args.platform == 'linux': @@ -169,4 +177,4 @@ def main(out_directory: str, platform: str, build_type: str, builds_out_directory = os.path.join( BUILDS_DIRECTORY, f'{script_args.platform}_{script_args.build_type}') main(builds_out_directory, script_args.platform, script_args.build_type, - gen_args) + not script_args.no_rbe, gen_args) diff --git a/cobalt/devinfra/kokoro/bin/dind_build.sh b/cobalt/devinfra/kokoro/bin/dind_build.sh index 190199b61dfda..ff0c35ce587ba 100755 --- a/cobalt/devinfra/kokoro/bin/dind_build.sh +++ b/cobalt/devinfra/kokoro/bin/dind_build.sh @@ -60,7 +60,7 @@ pipeline () { # Run GN and Ninja. ############################################################################## cd "${gclient_root}/src" - cobalt/build/gn.py -p "${TARGET_PLATFORM}" -C "${CONFIG}" + cobalt/build/gn.py -p "${TARGET_PLATFORM}" -C "${CONFIG}" --no-rbe autoninja -C "out/${TARGET_PLATFORM}_${CONFIG}" ${TARGET} # TARGET may expand to multiple args # Build bootloader config if set. From 498c246308d93b054c4b6ab5cdb25f337e8a74b2 Mon Sep 17 00:00:00 2001 From: Dana Dahlstrom Date: Wed, 15 Jan 2025 15:00:00 -0800 Subject: [PATCH 18/61] Remove files for Kokoro on-device tests Issue: 365150653 Reviewed-on: https://github.com/youtube/cobalt/pull/4722 --- cobalt/devinfra/kokoro/bin/on_device_test.sh | 35 ------------------- .../kokoro/bin/on_device_test_disable.sh | 25 ------------- cobalt/devinfra/kokoro/odt/common.gcl | 24 ------------- 3 files changed, 84 deletions(-) delete mode 100755 cobalt/devinfra/kokoro/bin/on_device_test.sh delete mode 100755 cobalt/devinfra/kokoro/bin/on_device_test_disable.sh delete mode 100644 cobalt/devinfra/kokoro/odt/common.gcl diff --git a/cobalt/devinfra/kokoro/bin/on_device_test.sh b/cobalt/devinfra/kokoro/bin/on_device_test.sh deleted file mode 100755 index c912c5b571b86..0000000000000 --- a/cobalt/devinfra/kokoro/bin/on_device_test.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -uex - -# This script runs the Mobile Harness test trigger. - -WORKSPACE_COBALT="${KOKORO_ARTIFACTS_DIR}/git/src" - -. $(dirname "$0")/common.sh -configure_environment - -# Tests are always run on the 'devel' config. -CONFIG=devel - -# Get GCS path from build job artifact. -GCS_ARCHIVE_PATH="$(<${KOKORO_GFILE_DIR}/gcs_archive_path)" - -# The path is determined by the name of the blaze action in the build config -# followed by the path to the target in blaze-bin. -BLAZE_PATH="${KOKORO_BLAZE_DIR}/mh_trigger/blaze-bin/video/youtube/testing/maneki/cobalt/mh_client" - -TEST_ATTEMPTS=1 -if is_release_build; then - # Increase test attempts for nightly jobs. - TEST_ATTEMPTS=3 -fi - -"${BLAZE_PATH}/trigger_tests.par" \ - --enforce_kernel_ipv6_support=false --gid= --uid= \ - --platform=${PLATFORM} \ - --config=${CONFIG} \ - --branch=${KOKORO_GOB_BRANCH} \ - --remote_archive_path=${GCS_ARCHIVE_PATH} \ - --test_type=${TEST_TYPE} \ - --test_attempts=${TEST_ATTEMPTS} \ - --unittest_shard_index=${TEST_SHARD} diff --git a/cobalt/devinfra/kokoro/bin/on_device_test_disable.sh b/cobalt/devinfra/kokoro/bin/on_device_test_disable.sh deleted file mode 100755 index fff4f7fce55e1..0000000000000 --- a/cobalt/devinfra/kokoro/bin/on_device_test_disable.sh +++ /dev/null @@ -1,25 +0,0 @@ -set -ex - -# This script determines if ODT should run or not. -# Exits gracefully (0) if ODT should run, otherwise -# exit 1 which prevents downstream ODT from running. - -# src/ maps to cobalt_src/. -readonly WORKSPACE_COBALT="${KOKORO_ARTIFACTS_DIR}/git/src" -readonly ODT_TRAILER="Test-On-Device" - -# Load common routines. -. $(dirname "$0")/common.sh - -configure_environment - -# Check the commit message for the 'Test-On-Device' trailer. -value=$(git -C ${WORKSPACE_COBALT} log --pretty=format:%B -n1 | grep "${ODT_TRAILER}" || true) -if [[ -n "${value}" ]]; then - exit 0 -fi - -# Check PRESUBMIT_ON_DEVICE_TEST environment variable. -if [[ "${PRESUBMIT_ON_DEVICE_TEST}" == "false" ]]; then - exit 1 -fi diff --git a/cobalt/devinfra/kokoro/odt/common.gcl b/cobalt/devinfra/kokoro/odt/common.gcl deleted file mode 100644 index 005db1646a1ad..0000000000000 --- a/cobalt/devinfra/kokoro/odt/common.gcl +++ /dev/null @@ -1,24 +0,0 @@ -// -*- protobuffer -*- -// proto-file: google3/devtools/kokoro/config/proto/build.proto -// proto-message: BuildConfig - -build = { - build_file = 'src/cobalt/devinfra/kokoro/bin/on_device_test.sh' - before_action = [ - { - invoke_blaze = { - name = 'mh_trigger' - command = 'BUILD' - targets = [ - '//video/youtube/testing/maneki/cobalt/mh_client:trigger_tests.par', - ] - } - }, - ] - env_vars = [ - { - key = 'TEST_TYPE' - value = 'unit_tests' - }, - ] -} From 1e393e2faced18c78fcf2bed303fbba791f61d6f Mon Sep 17 00:00:00 2001 From: johnx Date: Wed, 22 Jan 2025 15:55:03 +0000 Subject: [PATCH 19/61] Remove one line of setCrashContext log (#4723) The production content was considered pii. b/370167229 --- .../apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java | 1 - 1 file changed, 1 deletion(-) diff --git a/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java b/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java index e193d1dc1f6ed..3bb91abbc5803 100644 --- a/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java +++ b/cobalt/android/apk/app/src/main/java/dev/cobalt/coat/StarboardBridge.java @@ -779,7 +779,6 @@ void reportFullyDrawn() { @SuppressWarnings("unused") @UsedByNative public void setCrashContext(String key, String value) { - Log.i(TAG, "setCrashContext Called: " + key + ", " + value); crashContext.put(key, value); if (this.crashContextUpdateHandler != null) { this.crashContextUpdateHandler.onCrashContextUpdate(); From 8c6474acf9449dfd4c7d69338b14f6859661fe60 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 22 Jan 2025 10:34:42 -0800 Subject: [PATCH 20/61] Set default color depth to 8 in nplb tests (#4729) (#4733) 1. Set the default |color_metadata.bits_per_channel| to 8 for nplb tests. 2. Re-enable SbPlayerTest.MaxVideoCapabilities tests. b/292319097 (cherry picked from commit a1277c5e8234911c28a61b6687d0c45d058d9997) --- starboard/android/shared/test_filters.py | 3 --- starboard/nplb/player_creation_param_helpers.cc | 1 + starboard/shared/starboard/media/media_util.cc | 2 ++ starboard/shared/starboard/media/media_util.h | 14 +++++++------- .../starboard/player/filter/testing/test_util.cc | 1 + 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/starboard/android/shared/test_filters.py b/starboard/android/shared/test_filters.py index 53617abe9f1ad..0c710ff2a3a17 100644 --- a/starboard/android/shared/test_filters.py +++ b/starboard/android/shared/test_filters.py @@ -92,9 +92,6 @@ # TODO: Filter this test on a per-device basis. 'SbMediaCanPlayMimeAndKeySystem.MinimumSupport', - # TODO: b/292319097 Make this test work on lab devices consistently. - 'SbPlayerTest.MaxVideoCapabilities', - # TODO: b/280432564 Make this test work on lab devices consistently. 'SbAudioSinkTest.ContinuousAppend', diff --git a/starboard/nplb/player_creation_param_helpers.cc b/starboard/nplb/player_creation_param_helpers.cc index fc41c27c443a9..f7782519c35b1 100644 --- a/starboard/nplb/player_creation_param_helpers.cc +++ b/starboard/nplb/player_creation_param_helpers.cc @@ -109,6 +109,7 @@ VideoStreamInfo CreateVideoStreamInfo(SbMediaVideoCodec codec) { video_stream_info.mime = ""; video_stream_info.max_video_capabilities = ""; + video_stream_info.color_metadata.bits_per_channel = 8; video_stream_info.color_metadata.primaries = kSbMediaPrimaryIdBt709; video_stream_info.color_metadata.transfer = kSbMediaTransferIdBt709; video_stream_info.color_metadata.matrix = kSbMediaMatrixIdBt709; diff --git a/starboard/shared/starboard/media/media_util.cc b/starboard/shared/starboard/media/media_util.cc index 3452b2508cec6..d775730797b76 100644 --- a/starboard/shared/starboard/media/media_util.cc +++ b/starboard/shared/starboard/media/media_util.cc @@ -295,6 +295,8 @@ bool IsSDRVideo(int bit_depth, SbMediaPrimaryId primary_id, SbMediaTransferId transfer_id, SbMediaMatrixId matrix_id) { + SB_DCHECK(bit_depth == 8 || bit_depth == 10); + if (bit_depth != 8) { return false; } diff --git a/starboard/shared/starboard/media/media_util.h b/starboard/shared/starboard/media/media_util.h index 2df64fd2fb130..ceed7b6b1b7cf 100644 --- a/starboard/shared/starboard/media/media_util.h +++ b/starboard/shared/starboard/media/media_util.h @@ -52,9 +52,9 @@ struct AudioStreamInfo { // of `SbMediaAudioStreamInfo` for more details. SbMediaAudioCodec codec = kSbMediaAudioCodecNone; std::string mime; - uint16_t number_of_channels; - uint32_t samples_per_second; - uint16_t bits_per_sample; + uint16_t number_of_channels = 0; + uint32_t samples_per_second = 0; + uint16_t bits_per_sample = 0; std::vector audio_specific_config; }; @@ -111,9 +111,9 @@ struct VideoStreamInfo { SbMediaVideoCodec codec = kSbMediaVideoCodecNone; std::string mime; std::string max_video_capabilities; - int frame_width; - int frame_height; - SbMediaColorMetadata color_metadata; + int frame_width = 0; + int frame_height = 0; + SbMediaColorMetadata color_metadata = {}; }; bool operator==(const VideoStreamInfo& left, const VideoStreamInfo& right); @@ -141,7 +141,7 @@ struct VideoSampleInfo { // `SbMediaVideoSampleInfo` defined in `media.h`. Please refer to the comment // of `SbMediaVideoSampleInfo` for more details. VideoStreamInfo stream_info; - bool is_key_frame; + bool is_key_frame = false; }; std::ostream& operator<<(std::ostream& os, const VideoSampleInfo& stream_info); diff --git a/starboard/shared/starboard/player/filter/testing/test_util.cc b/starboard/shared/starboard/player/filter/testing/test_util.cc index 05585ac0614be..04946d18d8dcc 100644 --- a/starboard/shared/starboard/player/filter/testing/test_util.cc +++ b/starboard/shared/starboard/player/filter/testing/test_util.cc @@ -269,6 +269,7 @@ media::VideoStreamInfo CreateVideoStreamInfo(SbMediaVideoCodec codec) { video_stream_info.mime = ""; video_stream_info.max_video_capabilities = ""; + video_stream_info.color_metadata.bits_per_channel = 8; video_stream_info.color_metadata.primaries = kSbMediaPrimaryIdBt709; video_stream_info.color_metadata.transfer = kSbMediaTransferIdBt709; video_stream_info.color_metadata.matrix = kSbMediaMatrixIdBt709; From 654b491fa8d528ac240865c6d243eb2b1cbf3cf6 Mon Sep 17 00:00:00 2001 From: xiaomings Date: Wed, 22 Jan 2025 11:47:07 -0800 Subject: [PATCH 21/61] Fix typo in HTMLMediaElement::GetSupportsType() (#4732) Replace "interupt" with "interrupt". b/276483058 --- .../blink/renderer/core/html/media/html_media_element.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/blink/renderer/core/html/media/html_media_element.cc b/third_party/blink/renderer/core/html/media/html_media_element.cc index e565c5c81d777..551c443be9077 100644 --- a/third_party/blink/renderer/core/html/media/html_media_element.cc +++ b/third_party/blink/renderer/core/html/media/html_media_element.cc @@ -432,7 +432,7 @@ bool IsProgressiveFormat(const ContentType& content_type) { MIMETypeRegistry::SupportsType HTMLMediaElement::GetSupportsType( const ContentType& content_type) { #if BUILDFLAG(USE_STARBOARD_MEDIA) - // Interupt Chromium's IsTypeSupported() from here for better performance. + // Interrupt Chromium's IsTypeSupported() from here for better performance. MIMETypeRegistry::SupportsType result; if (!base::FeatureList::IsEnabled(media::kCobaltProgressivePlayback) && IsProgressiveFormat(content_type)) { From bb771fcda4979dd6941e4d152b010db21bec0da3 Mon Sep 17 00:00:00 2001 From: Bo-Rong Chen Date: Wed, 22 Jan 2025 12:50:14 -0800 Subject: [PATCH 22/61] [media] Enable e/ac3 support (#4738) Enable e/ac3 support for Cobalt. This allows to pass YTS format support tests for e/ac3 formats. b/390198194 --- cobalt/build/configs/common.gn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cobalt/build/configs/common.gn b/cobalt/build/configs/common.gn index d9ed0045e1f17..86dcf70a099c6 100644 --- a/cobalt/build/configs/common.gn +++ b/cobalt/build/configs/common.gn @@ -15,3 +15,6 @@ ffmpeg_branding = "Chrome" # Removing these flags causes a lot of compiler errors in starboard code. clang_use_chrome_plugins = false enable_check_raw_ptr_fields = false + +# Enable e/ac3 support for Cobalt. +enable_platform_ac3_eac3_audio = true From 505f753b8560f8b2b9054044e3dd1d74803d3640 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Fri, 13 Dec 2024 17:27:11 -0800 Subject: [PATCH 23/61] Break out test targets into a separate config b/354062546 --- .github/config/android-arm.json | 46 ++++++++++++------- .github/config/android-arm64.json | 46 ++++++++++++------- .github/config/android-x86.json | 44 +++++++++++------- .github/config/chromium_android-arm.json | 52 +++++++++++----------- .github/config/chromium_android-arm64.json | 52 +++++++++++----------- .github/config/chromium_android-x86.json | 52 +++++++++++----------- .github/workflows/main.yaml | 12 ++++- docker-compose.yaml | 4 +- 8 files changed, 180 insertions(+), 128 deletions(-) diff --git a/.github/config/android-arm.json b/.github/config/android-arm.json index fad4ee6ff895e..e8be887ccd2aa 100644 --- a/.github/config/android-arm.json +++ b/.github/config/android-arm.json @@ -1,19 +1,31 @@ { - "docker_service": "linux", - "platforms": [ - "android-arm" - ], - "test_on_device": true, - "targets": [ - "content_shell", - "system_webview_apk", - "system_webview_shell_apk", - "cobalt:gn_all" - ], - "includes": [ - { - "name": "arm", - "platform": "android-arm" - } - ] + "docker_service": "linux", + "platforms": [ + "android-arm" + ], + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk", + "cobalt:gn_all" + ], + "gtest_targets": [ + "base_unittests", + "blink_unittests", + "gin_unittests", + "gpu_unittests", + "ipc_tests", + "media_unittests", + "mojo_unittests", + "net_unittests", + "sql_unittests", + "url_unittests" + ], + "test_on_device": true, + "includes": [ + { + "name":"arm", + "platform":"android-arm" + } + ] } diff --git a/.github/config/android-arm64.json b/.github/config/android-arm64.json index 54c04de67f944..e90a3d101158a 100644 --- a/.github/config/android-arm64.json +++ b/.github/config/android-arm64.json @@ -1,19 +1,31 @@ { - "docker_service": "linux", - "platforms": [ - "android-arm64" - ], - "test_on_device": true, - "targets": [ - "content_shell", - "system_webview_apk", - "system_webview_shell_apk", - "cobalt:gn_all" - ], - "includes": [ - { - "name": "arm64", - "platform": "android-arm64" - } - ] + "docker_service": "linux", + "platforms": [ + "android-arm64" + ], + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk", + "cobalt:gn_all" + ], + "gtest_targets": [ + "base_unittests", + "blink_unittests", + "gin_unittests", + "gpu_unittests", + "ipc_tests", + "media_unittests", + "mojo_unittests", + "net_unittests", + "sql_unittests", + "url_unittests" + ], + "test_on_device": true, + "includes": [ + { + "name":"arm64", + "platform":"android-arm64" + } + ] } diff --git a/.github/config/android-x86.json b/.github/config/android-x86.json index c1e9fc89e8700..a96814f41ba48 100644 --- a/.github/config/android-x86.json +++ b/.github/config/android-x86.json @@ -1,18 +1,30 @@ { - "docker_service": "linux", - "platforms": [ - "android-x86" - ], - "targets": [ - "content_shell", - "system_webview_apk", - "system_webview_shell_apk", - "cobalt:gn_all" - ], - "includes": [ - { - "name": "x86", - "platform": "android-x86" - } - ] + "docker_service": "linux", + "platforms": [ + "android-x86" + ], + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk", + "cobalt:gn_all" + ], + "gtest_targets": [ + "base_unittests", + "blink_unittests", + "gin_unittests", + "gpu_unittests", + "ipc_tests", + "media_unittests", + "mojo_unittests", + "net_unittests", + "sql_unittests", + "url_unittests" + ], + "includes": [ + { + "name":"x86", + "platform":"android-x86" + } + ] } diff --git a/.github/config/chromium_android-arm.json b/.github/config/chromium_android-arm.json index 7e073505527ce..52a8e509c1d1b 100644 --- a/.github/config/chromium_android-arm.json +++ b/.github/config/chromium_android-arm.json @@ -1,27 +1,29 @@ { - "docker_service": "linux", - "platforms": [ - "chromium_android-arm" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk" - ], - "includes": [ - { - "name": "arm", - "platform": "chromium_android-arm" - } - ] + "docker_service": "linux", + "platforms": [ + "chromium_android-arm" + ], + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk" + ], + "gtest_targets": [ + "base_unittests", + "blink_unittests", + "gin_unittests", + "gpu_unittests", + "ipc_tests", + "media_unittests", + "mojo_unittests", + "net_unittests", + "sql_unittests", + "url_unittests" + ], + "includes": [ + { + "name":"arm", + "platform":"chromium_android-arm" + } + ] } diff --git a/.github/config/chromium_android-arm64.json b/.github/config/chromium_android-arm64.json index 1921fdbbad913..77a3541cadb1b 100644 --- a/.github/config/chromium_android-arm64.json +++ b/.github/config/chromium_android-arm64.json @@ -1,27 +1,29 @@ { - "docker_service": "linux", - "platforms": [ - "chromium_android-arm64" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk" - ], - "includes": [ - { - "name": "arm64", - "platform": "chromium_android-arm64" - } - ] + "docker_service": "linux", + "platforms": [ + "chromium_android-arm64" + ], + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk" + ], + "gtest_targets": [ + "base_unittests", + "blink_unittests", + "gin_unittests", + "gpu_unittests", + "ipc_tests", + "media_unittests", + "mojo_unittests", + "net_unittests", + "sql_unittests", + "url_unittests" + ], + "includes": [ + { + "name":"arm64", + "platform":"chromium_android-arm64" + } + ] } diff --git a/.github/config/chromium_android-x86.json b/.github/config/chromium_android-x86.json index 8498733285ed5..68a5ecab872ab 100644 --- a/.github/config/chromium_android-x86.json +++ b/.github/config/chromium_android-x86.json @@ -1,27 +1,29 @@ { - "docker_service": "linux", - "platforms": [ - "chromium_android-x86" - ], - "targets": [ - "base_unittests", - "sql_unittests", - "net_unittests", - "url_unittests", - "ipc_tests", - "mojo_unittests", - "gpu_unittests", - "gin_unittests", - "blink_unittests", - "media_unittests", - "content_shell", - "system_webview_apk", - "system_webview_shell_apk" - ], - "includes": [ - { - "name": "x86", - "platform": "chromium_android-x86" - } - ] + "docker_service": "linux", + "platforms": [ + "chromium_android-x86" + ], + "targets": [ + "content_shell", + "system_webview_apk", + "system_webview_shell_apk" + ], + "gtest_targets": [ + "base_unittests", + "blink_unittests", + "gin_unittests", + "gpu_unittests", + "ipc_tests", + "media_unittests", + "mojo_unittests", + "net_unittests", + "sql_unittests", + "url_unittests" + ], + "includes": [ + { + "name":"x86", + "platform":"chromium_android-x86" + } + ] } diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d521ed65099fb..75306d1b1d805 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,4 +1,4 @@ -# Reusable Cobalt CI workflow. +vi # Reusable Cobalt CI workflow. name: main @@ -65,6 +65,11 @@ jobs: run: | targets=$(cat ${GITHUB_WORKSPACE}/.github/config/${{ inputs.platform }}.json | jq -cr '.targets | join(" ")') echo "targets=${targets}" >> $GITHUB_ENV + - id: set-test-targets + shell: bash + run: | + gtest_targets=$(cat ${GITHUB_WORKSPACE}/.github/config/${{ inputs.platform }}.json | jq -c '.gtest_targets | join(" ")') + echo "gtest_targets=${gtest_targets}" >> $GITHUB_ENV - id: set-includes shell: bash run: | @@ -98,6 +103,7 @@ jobs: outputs: platforms: ${{ env.platforms }} targets: ${{ env.targets }} + gtest_targets: ${{ env.gtest_targets }} includes: ${{ env.includes }} docker_service: ${{ env.docker_service }} num_gtest_shards: ${{ env.num_gtest_shards }} @@ -179,6 +185,10 @@ jobs: if: ${{ ! (contains(matrix.platform, 'android') && matrix.config == 'debug') }} with: targets: ${{ needs.initialize.outputs.targets }} + - name: Upload Test Artifacts + if: matrix.config == 'devel' + uses: ./src/.github/actions/upload_test_artifacts + with: test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }} upload_on_host_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_host }} upload_on_device_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_device }} diff --git a/docker-compose.yaml b/docker-compose.yaml index 656d567a5ec33..1bd9a0de3d133 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,8 +4,8 @@ services: build: context: cobalt/docker/linux cache_from: - - ghcr.io/youtube/cobalt/linux:latest - image: ghcr.io/youtube/cobalt/linux:latest + - ghcr.io/youtube/cobalt_sandbox/linux:latest + image: ghcr.io/youtube/cobalt_sandbox/linux:latest platform: linux/amd64 environment: - DEPOT_TOOLS_UPDATE=0 From d66148073e8df76eda2c3182ef21c2f71ae300f9 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Wed, 18 Dec 2024 21:59:46 -0800 Subject: [PATCH 24/61] Chrobalt on-device test work for Android b/354062546 --- .github/actions/on_device_tests/action.yaml | 156 +++++++++++++++++++- .github/config/android-arm.json | 2 + .github/workflows/main.yaml | 41 ++++- 3 files changed, 190 insertions(+), 9 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index df6b4bf39d626..2d8b8c42a02f7 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -1,9 +1,157 @@ -name: On Host Tests -description: Runs on-host tests. +name: On Device Test +description: Runs on-device tests. + runs: using: "composite" steps: - - name: Run On-Device Tests + - name: Install Requirements + run: | + pip3 install grpcio==1.38.0 grpcio-tools==1.38.0 + shell: bash + - name: Generate gRPC files + run: | + python -m grpc_tools.protoc -Itools/ --python_out=tools/ --grpc_python_out=tools/ tools/on_device_tests_gateway.proto + shell: bash + - name: Set Up Cloud SDK + uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1 + - name: Set env vars + run: | + echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV + + # Test results and logs + echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}" >> $GITHUB_ENV + + # Dimension env + if [ "${{ matrix.dimension }}" != "null" ]; then + echo "DIMENSION=${{ matrix.dimension }}" >> $GITHUB_ENV + fi + + echo "TEST_TYPE=unit_test" >> $GITHUB_ENV shell: bash + - name: Run ${{ env.SHARD_NAME }} Tests on ${{ matrix.platform }} Platform + env: + GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_TOKEN: ${{ github.token }} + GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} + GITHUB_ACTOR_ID: ${{ github.actor_id }} + GITHUB_REPO: ${{ github.repository }} + GITHUB_PR_HEAD_USER_LOGIN: ${{ github.event.pull_request.head.user.login }} + GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }} + GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }} + GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_WORKFLOW: ${{ github.workflow }} run: | - echo "Nothing yet" + set -uxe + python3 -u tools/on_device_tests_gateway_client.py \ + --github_workspace ${GITHUB_WORKSPACE} \ + --token ${GITHUB_TOKEN} \ + --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ + trigger \ + --test_type ${TEST_TYPE} \ + --platform ${{ matrix.target_platform }} \ + --config ${{ matrix.config }} \ + --tag cobalt_github_${GITHUB_EVENT_NAME} \ + --builder_name github_${{ matrix.platform }}_tests \ + --build_number ${GITHUB_RUN_NUMBER} \ + --builder_url ${GITHUB_RUN_URL} \ + ${LOADER_PLATFORM:+"--loader_config" "$LOADER_CONFIG"} \ + ${LOADER_PLATFORM:+"--loader_platform" "$LOADER_PLATFORM"} \ + ${DIMENSION:+"--dimension" "$DIMENSION"} \ + ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \ + ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ + --archive_path "${GCS_PATH}/artifacts.tar" \ + --gcs_result_path "${GCS_RESULTS_PATH}" \ + --label github \ + --label ${GITHUB_EVENT_NAME} \ + --label ${GITHUB_WORKFLOW} \ + --label actor-${GITHUB_ACTOR} \ + --label actor_id-${GITHUB_ACTOR_ID} \ + --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ + --label sha-${GITHUB_SHA} \ + --label repository-${GITHUB_REPO} \ + --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ + --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} + shell: bash + - name: Download ${{ matrix.platform }} Test Results + if: always() && env.TEST_TYPE == 'unit_test' + run: | + # Don't break on error (-e), some commands are expected to fail. + set -ux + + COBALT_LOGS_DIR="${GITHUB_WORKSPACE}/cobalt_logs" + UNIT_TEST_RESULT_PATH="${GITHUB_WORKSPACE}/unit-test-results" + COBALT_XMLS_FILENAME="cobalt_xmls.zip" + + # Forward environment variables for uploading artifacts in later steps. + echo "UNIT_TEST_RESULT_PATH=${UNIT_TEST_RESULT_PATH}" >> $GITHUB_ENV + echo "COBALT_LOGS_DIR=${COBALT_LOGS_DIR}" >> $GITHUB_ENV + + mkdir -p "${GITHUB_WORKSPACE}/test_results" + cd "${GITHUB_WORKSPACE}/test_results" + + i=0 + # Try downloading the results for 6x 10 seconds before giving up. + while [ $i -lt 6 ]; do + # The results are uploaded after the test has completed. + sleep 10 + + # The log files are named by the device lab test driver. + COBALT_ERROR_LOG_FILENAME="webDriverTestLog.ERROR" + COBALT_INFO_LOG_FILENAME="webDriverTestLog.INFO" + + # This command will fail until the results have been uploaded. + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_ERROR_LOG_FILENAME}" . + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_INFO_LOG_FILENAME}" . + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_XMLS_FILENAME}" . + + # Break if all files were downloaded. + if [[ -f "${COBALT_XMLS_FILENAME}" && -f "${COBALT_ERROR_LOG_FILENAME}" && -f "${COBALT_INFO_LOG_FILENAME}" ]]; then + break + fi + + i=$(( ${i} + 1 )) + done + + # Rename log files for archiving to not expose legacy weirdness. + mkdir -p "${COBALT_LOGS_DIR}/${{ matrix.platform }}/" + cp "${COBALT_ERROR_LOG_FILENAME}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stderr_${{ matrix.shard }}.log" + cp "${COBALT_INFO_LOG_FILENAME}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stdout_${{ matrix.shard }}.log" + + # Prepare unit test results for DataDog upload. + RESULT_PATH=${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/${{ matrix.shard }}/ + mkdir -p ${RESULT_PATH} + + # Set tags for test differentiation. + tags="os.platform:${{ matrix.platform }}" + echo $tags > ${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/TAGS + + unzip ${COBALT_XMLS_FILENAME} -d ${RESULT_PATH} + shell: bash + - name: Archive Unit Test Logs + uses: actions/upload-artifact@v3 + if: always() && env.TEST_TYPE == 'unit_test' + with: + name: Device logs + path: ${{ env.COBALT_LOGS_DIR }}/ + - name: Archive Unit Test Results + uses: actions/upload-artifact@v3 + if: always() && env.TEST_TYPE == 'unit_test' + with: + name: unit-test-results + path: ${{ env.UNIT_TEST_RESULT_PATH }}/ + - name: Print device logs + if: always() + run: | + if ls ${COBALT_LOGS_DIR}/**/*.log 1> /dev/null 2>&1; then + cat ${COBALT_LOGS_DIR}/**/*.log + else + echo "No device logs found" + fi + shell: bash diff --git a/.github/config/android-arm.json b/.github/config/android-arm.json index e8be887ccd2aa..684725c6f75e2 100644 --- a/.github/config/android-arm.json +++ b/.github/config/android-arm.json @@ -21,6 +21,8 @@ "sql_unittests", "url_unittests" ], + "gtest_device": "sabrina", + "gtest_lab": "maneki", "test_on_device": true, "includes": [ { diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 75306d1b1d805..42dd44083757d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -193,9 +193,9 @@ jobs: upload_on_host_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_host }} upload_on_device_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_device }} - test: + on-host-test: needs: [initialize, docker-build-image, build] - if: needs.initialize.outputs.test_on_host == 'true' || needs.initialize.outputs.test_on_device == 'true' + if: needs.initialize.outputs.test_on_host == 'true' permissions: {} # TODO(b/372303096): Should have dedicated runner? runs-on: [self-hosted, chrobalt-linux-runner] @@ -259,12 +259,12 @@ jobs: datadog_api_key: ${{ secrets.datadog_api_key }} continue-on-error: true + validate-test-result: - needs: [initialize, docker-build-image, build, test] + needs: [initialize, docker-build-image, build, on-host-test] if: always() && ( - needs.initialize.outputs.test_on_host == 'true' || - needs.initialize.outputs.test_on_device == 'true' + needs.initialize.outputs.test_on_host == 'true' ) permissions: {} runs-on: ubuntu-latest @@ -280,3 +280,34 @@ jobs: run: | echo "Failing because at least one test shard had errors." exit 1 + + # Runs on-device integration and unit tests. + on-device-test: + needs: [initialize, build] + # Run ODT when on_device label is applied on PR. + # Also, run ODT on push and schedule if not explicitly disabled via repo vars. + if: | + needs.initialize.outputs.test_on_device == 'true' && (( + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( + inputs.nightly == 'true' || github.event_name == 'schedule') && + vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || + ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) ) + runs-on: [self-hosted, odt-runner] + name: ${{ matrix.name }}_on_device + permissions: {} + strategy: + fail-fast: false + matrix: + platform: ${{ fromJson(needs.initialize.outputs.platforms) }} + config: [devel] + include: ${{ fromJson(needs.initialize.outputs.includes) }} + steps: + - name: Checkout + uses: kaidokert/checkout@v3.5.999 + timeout-minutes: 30 + with: + fetch-depth: 1 + persist-credentials: false + - name: Run Tests (${{ matrix.shard }}) + uses: ./.github/actions/on_device_tests From ded1c7ae58f9ef51d9d19935dee8b42953821237 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Thu, 19 Dec 2024 14:30:59 -0800 Subject: [PATCH 25/61] Update params passed to ODT gateway --- .github/actions/on_device_tests/action.yaml | 44 ++------------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 2d8b8c42a02f7..b03c0772535c8 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -25,59 +25,21 @@ runs: if [ "${{ matrix.dimension }}" != "null" ]; then echo "DIMENSION=${{ matrix.dimension }}" >> $GITHUB_ENV fi - - echo "TEST_TYPE=unit_test" >> $GITHUB_ENV shell: bash - name: Run ${{ env.SHARD_NAME }} Tests on ${{ matrix.platform }} Platform env: GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} - GITHUB_SHA: ${{ github.sha }} - GITHUB_TOKEN: ${{ github.token }} - GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - GITHUB_EVENT_NAME: ${{ github.event_name }} - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} - GITHUB_ACTOR_ID: ${{ github.actor_id }} - GITHUB_REPO: ${{ github.repository }} - GITHUB_PR_HEAD_USER_LOGIN: ${{ github.event.pull_request.head.user.login }} - GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }} - GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }} - GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }} - GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GITHUB_WORKFLOW: ${{ github.workflow }} run: | set -uxe python3 -u tools/on_device_tests_gateway_client.py \ - --github_workspace ${GITHUB_WORKSPACE} \ + --platform_json "${GITHUB_WORKSPACE}/src/.github/config/${{ matrix.platform}}.json" \ + --filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/${{ matrix.platform}}" \ --token ${GITHUB_TOKEN} \ - --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ - trigger \ - --test_type ${TEST_TYPE} \ - --platform ${{ matrix.target_platform }} \ - --config ${{ matrix.config }} \ - --tag cobalt_github_${GITHUB_EVENT_NAME} \ - --builder_name github_${{ matrix.platform }}_tests \ - --build_number ${GITHUB_RUN_NUMBER} \ - --builder_url ${GITHUB_RUN_URL} \ - ${LOADER_PLATFORM:+"--loader_config" "$LOADER_CONFIG"} \ - ${LOADER_PLATFORM:+"--loader_platform" "$LOADER_PLATFORM"} \ ${DIMENSION:+"--dimension" "$DIMENSION"} \ - ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \ ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ --archive_path "${GCS_PATH}/artifacts.tar" \ --gcs_result_path "${GCS_RESULTS_PATH}" \ - --label github \ - --label ${GITHUB_EVENT_NAME} \ - --label ${GITHUB_WORKFLOW} \ - --label actor-${GITHUB_ACTOR} \ - --label actor_id-${GITHUB_ACTOR_ID} \ - --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ - --label sha-${GITHUB_SHA} \ - --label repository-${GITHUB_REPO} \ - --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ - --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} + trigger \ shell: bash - name: Download ${{ matrix.platform }} Test Results if: always() && env.TEST_TYPE == 'unit_test' From 9b6f0e0c0cb1c98ecc74a80c7a4f43f187e22cae Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Fri, 20 Dec 2024 23:36:40 +0000 Subject: [PATCH 26/61] ODT Client code --- .../android-arm/base_unittests_filter.json | 10 + tools/on_device_tests_gateway.proto | 81 ++++ tools/on_device_tests_gateway_client.py | 305 ++++++++++++ tools/on_device_tests_gateway_pb2.py | 437 ++++++++++++++++++ tools/on_device_tests_gateway_pb2_grpc.py | 64 +++ 5 files changed, 897 insertions(+) create mode 100644 cobalt/testing/android-arm/base_unittests_filter.json create mode 100644 tools/on_device_tests_gateway.proto create mode 100644 tools/on_device_tests_gateway_client.py create mode 100644 tools/on_device_tests_gateway_pb2.py create mode 100644 tools/on_device_tests_gateway_pb2_grpc.py diff --git a/cobalt/testing/android-arm/base_unittests_filter.json b/cobalt/testing/android-arm/base_unittests_filter.json new file mode 100644 index 0000000000000..bb161c725dc43 --- /dev/null +++ b/cobalt/testing/android-arm/base_unittests_filter.json @@ -0,0 +1,10 @@ +{ + "failing_tests": [ + "BreakIteratorTest.BreakCharacter", + "ValuesUtilTest.FilePath" + ], + "passing_tests": [ + "test1", + "test2" + ] +} diff --git a/tools/on_device_tests_gateway.proto b/tools/on_device_tests_gateway.proto new file mode 100644 index 0000000000000..50cc02d478f7e --- /dev/null +++ b/tools/on_device_tests_gateway.proto @@ -0,0 +1,81 @@ +// Copyright 2022 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package on_device_tests_gateway; + +// Interface exported by the server. +service on_device_tests_gateway { + // A dumb proxy RPC service that passes user defined command line options + // to the on-device tests gateway and streams back output in real time. + rpc exec_command (OnDeviceTestsCommand) returns (stream OnDeviceTestsResponse) { + } + + rpc exec_watch_command (OnDeviceTestsWatchCommand) returns (stream OnDeviceTestsResponse) { + } +} + +// Working directory and command line arguments to be passed to the gateway. +message OnDeviceTestsCommand { + // Next ID: 23 + string workdir = 1; + string token = 2; + string test_type = 3; + string platform = 4; + string archive_path = 5; + string config = 6; + string tag = 7; + repeated string labels = 8; + string builder_name = 9; + string change_id = 10; + string build_number = 11; + string loader_platform = 12; + string loader_config = 13; + string version = 14; + bool dry_run = 15; + repeated string dimension = 16; + string unittest_shard_index = 17; + string test_attempts = 18; + string retry_level = 19; + string start_timeout = 20; + string test_timeout = 21; + string builder_url = 22; + string gcs_result_path = 23; + repeated ApkTest apk_tests = 24; +} + +message ApkTest { + string test_target = 1; + string apk_path = 2; + string device_model = 3; + string device_pool = 4; + string gtest_filters = 5; +} + +// Working directory and command line arguments to be passed to the gateway. +message OnDeviceTestsWatchCommand { + // Next ID: 6 + string workdir = 1; + string token = 2; + string session_id = 3; + string change_id = 4; + bool dry_run = 5; +} + +// Response from the on-device tests. +message OnDeviceTestsResponse { + // Next ID: 2 + string response = 1; +} diff --git a/tools/on_device_tests_gateway_client.py b/tools/on_device_tests_gateway_client.py new file mode 100644 index 0000000000000..99cdbb5f7166a --- /dev/null +++ b/tools/on_device_tests_gateway_client.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""gRPC On-device Tests Gateway client.""" + +import argparse +import logging +import sys +import os +import json + +import grpc + +import on_device_tests_gateway_pb2 +import on_device_tests_gateway_pb2_grpc + +# All tests On-Device tests support +_TEST_TYPES = [ + 'black_box_test', + 'evergreen_test', + 'unit_test', +] + +# All test configs On-Device tests support +_TEST_CONFIGS = [ + 'devel', + 'staging', + 'production', +] + +_WORK_DIR = '/on_device_tests_gateway' +_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( + #'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') + 'localhost') +#_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' +_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '12345' + + +class OnDeviceTestsGatewayClient(): + """On-device tests Gateway Client class.""" + + def __init__(self): + self.channel = grpc.insecure_channel( + target=f'{_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST}:{_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT}', # pylint:disable=line-too-long + # These options need to match server settings. + options=[('grpc.keepalive_time_ms', 10000), + ('grpc.keepalive_timeout_ms', 5000), + ('grpc.keepalive_permit_without_calls', 1), + ('grpc.http2.max_pings_without_data', 0), + ('grpc.http2.min_time_between_pings_ms', 10000), + ('grpc.http2.min_ping_interval_without_data_ms', 5000)]) + self.stub = on_device_tests_gateway_pb2_grpc.on_device_tests_gatewayStub( + self.channel) + + def run_trigger_command(self, workdir: str, args: argparse.Namespace, apk_tests = None): + """Calls On-Device Tests service and passing given parameters to it. + + Args: + workdir (str): Current script workdir. + args (Namespace): Arguments passed in command line. + """ + for response_line in self.stub.exec_command( + on_device_tests_gateway_pb2.OnDeviceTestsCommand( + workdir=workdir, + token=args.token, + #test_type=args.test_type, + platform=args.platform, + archive_path=args.archive_path, + #config=args.config, + #tag=args.tag, + labels=args.label, + gcs_result_path=args.gcs_result_path, + #builder_name=args.builder_name, + #builder_url=args.builder_url, + change_id=args.change_id, + #build_number=args.build_number, + #loader_platform=args.loader_platform, + #loader_config=args.loader_config, + #version=args.version, + dry_run=args.dry_run, + dimension=args.dimension or [], + #unittest_shard_index=args.unittest_shard_index, + test_attempts=args.test_attempts, + retry_level=args.retry_level, + apk_tests=apk_tests, + )): + + print(response_line.response) + + def run_watch_command(self, workdir: str, args: argparse.Namespace): + """Calls On-Device Tests watch service and passing given parameters to it. + + Args: + workdir (str): Current script workdir. + args (Namespace): Arguments passed in command line. + """ + for response_line in self.stub.exec_watch_command( + on_device_tests_gateway_pb2.OnDeviceTestsWatchCommand( + workdir=workdir, + token=args.token, + change_id=args.change_id, + session_id=args.session_id, + )): + + print(response_line.response) + +def _read_json_config(filename): + """ + Reads and parses data from a JSON configuration file. + + Args: + filename: The name of the JSON configuration file. + + Returns: + A list of dictionaries, where each dictionary represents a test configuration. + """ + try: + with open(filename, 'r') as f: + data = json.load(f) + return data + except FileNotFoundError: + #print(f" Config file '{filename}' not found.") + return None + except json.JSONDecodeError: + #print(f" Invalid JSON format in '{filename}'.") + return None + +def _default_platform_json_file(platform): + current_dir = os.path.dirname(os.path.abspath(__file__)) + relative_path = os.path.join("..", ".github", "config", platform + ".json") + return os.path.join(current_dir, relative_path) + +def _get_tests_filter_json_file(platform, gtest_target): + current_dir = os.path.dirname(os.path.abspath(__file__)) + relative_path = os.path.join("..", "cobalt", "testing", platform, gtest_target + "_filter.json") + return os.path.join(current_dir, relative_path) + +def _process_apk_tests(args): + apk_tests = [] + platform_json_file = args.platform_json if args.platform_json else _default_platform_json_file(args.platform) + #print(' The platform_json_file is ' + platform_json_file) + platform_json_file_data = _read_json_config(platform_json_file) + gtest_device = platform_json_file_data["gtest_device"] + gtest_lab = platform_json_file_data["gtest_lab"] + gtest_blaze_target = "//experimental/cobalt/chrobalt_poc:custom_chrobalt_unit_tests_" + gtest_device + default_gtest_filters = "GURL*" + + #print(' processing gtest_targets in json file') + for gtest_target in platform_json_file_data["gtest_targets"]: + apk_test = { + "test_target": gtest_blaze_target, + "device_model": gtest_device, + "device_pool": gtest_lab + } + print(f" gtest_target: {gtest_target}") + gtest_filter_json_file = _get_tests_filter_json_file(args.platform, gtest_target) + print(f" gtest_filter_json_file = {gtest_filter_json_file}") + gtest_filter_json_file_data = _read_json_config(gtest_filter_json_file) + gtest_filters = default_gtest_filters + if gtest_filter_json_file_data is None: + print(" This gtest_target does not have gtest_filters_specified"); + else: + failing_tests_list = gtest_filter_json_file_data["failing_tests"] + passing_tests_list = gtest_filter_json_file_data["passing_tests"] + #print(f" failing tests in json: {failing_tests_list}") + #print(f" passing tests in json: {passing_tests_list}") + failing_tests_string = ":".join(failing_tests_list) + passing_tests_string = ":".join(passing_tests_list) + #print(f" failing_tests_string = '{failing_tests_string}'") + #print(f" passing_tests_string = '{passing_tests_string}'") + if passing_tests_string: + gtest_filters += ":" + passing_tests_string + if failing_tests_string: + gtest_filters += ":-" + failing_tests_string + #print(f" gtest_filters = {gtest_filters}"); + apk_test["apk_path"] = "/bigstore/yt-temp/" + gtest_target + "-debug.apk" + apk_test["gtest_filters"] = gtest_filters + apk_tests.append(apk_test) + + print(f" apk_tests: {apk_tests}") + return apk_tests + +def main(): + """Main routine.""" + print('Starting main routine') + + logging.basicConfig( + level=logging.INFO, format='[%(filename)s:%(lineno)s] %(message)s') + + parser = argparse.ArgumentParser( + epilog=('Example: ./on_device_tests_gateway_client.py ' + '--test_type=unit_test ' + '--remote_archive_path=gs://my_bucket/path/to/artifacts.tar ' + '--platform=raspi-2 ' + '--config=devel'), + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '-t', + '--token', + type=str, + required=True, + help='On Device Tests authentication token') + parser.add_argument( + '--dry_run', + action='store_true', + help='Specifies to show what would be done without actually doing it.') + parser.add_argument( + '-i', + '--change_id', + type=str, + help='ChangeId that triggered this test, if any. ' + 'Saved with performance test results.') + + subparsers = parser.add_subparsers( + dest='action', help='On-Device tests commands', required=True) + trigger_parser = subparsers.add_parser( + 'trigger', help='Trigger On-Device tests') + + trigger_parser.add_argument( + '-p', + '--platform', + type=str, + required=True, + help='Platform this test was built for.') + trigger_parser.add_argument( + '-pf', + '--platform_json', + type=str, + help='Platform-specific json file containing the list of target tests.') + trigger_parser.add_argument( + '-a', + '--archive_path', + type=str, + required=True, + help='Path to Chrobalt archive to be tested. Must be on gcs.') + trigger_parser.add_argument( + '-l', + '--label', + type=str, + default=[], + action='append', + help='Additional labels to assign to the test.') + trigger_parser.add_argument( + '--gcs_result_path', + type=str, + help='GCS url where test result files should be uploaded.') + trigger_parser.add_argument( + '--dimension', + type=str, + action='append', + help='On-Device Tests dimension used to select a device. ' + 'Must have the following form: =.' + ' E.G. "release_version=regex:10.*') + trigger_parser.add_argument( + '--test_attempts', + type=str, + default='1', + required=False, + help='The maximum number of times a test could retry.') + trigger_parser.add_argument( + '--retry_level', + type=str, + default='ERROR', + required=False, + help='The retry level of Mobile harness job. Either ERROR (to retry for ' + 'MH errors) or FAIL (to retry for failing tests). Setting retry_level to ' + 'FAIL will also retry for MH errors.') + + watch_parser = subparsers.add_parser('watch', help='Trigger On-Device tests') + watch_parser.add_argument( + 'session_id', + type=str, + help='Session id of a previously triggered mobile ' + 'harness test. If passed, the test will not be ' + 'triggered, but will be watched until the exit ' + 'status is reached.') + + args = parser.parse_args() + apk_tests = _process_apk_tests(args) + + client = OnDeviceTestsGatewayClient() + try: + if args.action == 'trigger': + client.run_trigger_command(workdir=_WORK_DIR, args=args, apk_tests=apk_tests) + else: + client.run_watch_command(workdir=_WORK_DIR, args=args) + except grpc.RpcError as e: + print(e) + return e.code().value + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/on_device_tests_gateway_pb2.py b/tools/on_device_tests_gateway_pb2.py new file mode 100644 index 0000000000000..f6c8947b850fa --- /dev/null +++ b/tools/on_device_tests_gateway_pb2.py @@ -0,0 +1,437 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: on_device_tests_gateway.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='on_device_tests_gateway.proto', + package='on_device_tests_gateway', + syntax='proto3', + serialized_pb=_b('\n\x1don_device_tests_gateway.proto\x12\x17on_device_tests_gateway\"\x9c\x04\n\x14OnDeviceTestsCommand\x12\x0f\n\x07workdir\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\t\x12\x11\n\ttest_type\x18\x03 \x01(\t\x12\x10\n\x08platform\x18\x04 \x01(\t\x12\x14\n\x0c\x61rchive_path\x18\x05 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x06 \x01(\t\x12\x0b\n\x03tag\x18\x07 \x01(\t\x12\x0e\n\x06labels\x18\x08 \x03(\t\x12\x14\n\x0c\x62uilder_name\x18\t \x01(\t\x12\x11\n\tchange_id\x18\n \x01(\t\x12\x14\n\x0c\x62uild_number\x18\x0b \x01(\t\x12\x17\n\x0floader_platform\x18\x0c \x01(\t\x12\x15\n\rloader_config\x18\r \x01(\t\x12\x0f\n\x07version\x18\x0e \x01(\t\x12\x0f\n\x07\x64ry_run\x18\x0f \x01(\x08\x12\x11\n\tdimension\x18\x10 \x03(\t\x12\x1c\n\x14unittest_shard_index\x18\x11 \x01(\t\x12\x15\n\rtest_attempts\x18\x12 \x01(\t\x12\x13\n\x0bretry_level\x18\x13 \x01(\t\x12\x15\n\rstart_timeout\x18\x14 \x01(\t\x12\x14\n\x0ctest_timeout\x18\x15 \x01(\t\x12\x13\n\x0b\x62uilder_url\x18\x16 \x01(\t\x12\x17\n\x0fgcs_result_path\x18\x17 \x01(\t\x12\x33\n\tapk_tests\x18\x18 \x03(\x0b\x32 .on_device_tests_gateway.ApkTest\"r\n\x07\x41pkTest\x12\x13\n\x0btest_target\x18\x01 \x01(\t\x12\x10\n\x08\x61pk_path\x18\x02 \x01(\t\x12\x14\n\x0c\x64\x65vice_model\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x65vice_pool\x18\x04 \x01(\t\x12\x15\n\rgtest_filters\x18\x05 \x01(\t\"s\n\x19OnDeviceTestsWatchCommand\x12\x0f\n\x07workdir\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\t\x12\x12\n\nsession_id\x18\x03 \x01(\t\x12\x11\n\tchange_id\x18\x04 \x01(\t\x12\x0f\n\x07\x64ry_run\x18\x05 \x01(\x08\")\n\x15OnDeviceTestsResponse\x12\x10\n\x08response\x18\x01 \x01(\t2\x8a\x02\n\x17on_device_tests_gateway\x12q\n\x0c\x65xec_command\x12-.on_device_tests_gateway.OnDeviceTestsCommand\x1a..on_device_tests_gateway.OnDeviceTestsResponse\"\x00\x30\x01\x12|\n\x12\x65xec_watch_command\x12\x32.on_device_tests_gateway.OnDeviceTestsWatchCommand\x1a..on_device_tests_gateway.OnDeviceTestsResponse\"\x00\x30\x01\x62\x06proto3') +) + + + + +_ONDEVICETESTSCOMMAND = _descriptor.Descriptor( + name='OnDeviceTestsCommand', + full_name='on_device_tests_gateway.OnDeviceTestsCommand', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='workdir', full_name='on_device_tests_gateway.OnDeviceTestsCommand.workdir', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='token', full_name='on_device_tests_gateway.OnDeviceTestsCommand.token', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='test_type', full_name='on_device_tests_gateway.OnDeviceTestsCommand.test_type', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='platform', full_name='on_device_tests_gateway.OnDeviceTestsCommand.platform', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='archive_path', full_name='on_device_tests_gateway.OnDeviceTestsCommand.archive_path', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='config', full_name='on_device_tests_gateway.OnDeviceTestsCommand.config', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tag', full_name='on_device_tests_gateway.OnDeviceTestsCommand.tag', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='labels', full_name='on_device_tests_gateway.OnDeviceTestsCommand.labels', index=7, + number=8, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='builder_name', full_name='on_device_tests_gateway.OnDeviceTestsCommand.builder_name', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='change_id', full_name='on_device_tests_gateway.OnDeviceTestsCommand.change_id', index=9, + number=10, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='build_number', full_name='on_device_tests_gateway.OnDeviceTestsCommand.build_number', index=10, + number=11, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='loader_platform', full_name='on_device_tests_gateway.OnDeviceTestsCommand.loader_platform', index=11, + number=12, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='loader_config', full_name='on_device_tests_gateway.OnDeviceTestsCommand.loader_config', index=12, + number=13, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='version', full_name='on_device_tests_gateway.OnDeviceTestsCommand.version', index=13, + number=14, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='dry_run', full_name='on_device_tests_gateway.OnDeviceTestsCommand.dry_run', index=14, + number=15, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='dimension', full_name='on_device_tests_gateway.OnDeviceTestsCommand.dimension', index=15, + number=16, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='unittest_shard_index', full_name='on_device_tests_gateway.OnDeviceTestsCommand.unittest_shard_index', index=16, + number=17, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='test_attempts', full_name='on_device_tests_gateway.OnDeviceTestsCommand.test_attempts', index=17, + number=18, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='retry_level', full_name='on_device_tests_gateway.OnDeviceTestsCommand.retry_level', index=18, + number=19, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='start_timeout', full_name='on_device_tests_gateway.OnDeviceTestsCommand.start_timeout', index=19, + number=20, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='test_timeout', full_name='on_device_tests_gateway.OnDeviceTestsCommand.test_timeout', index=20, + number=21, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='builder_url', full_name='on_device_tests_gateway.OnDeviceTestsCommand.builder_url', index=21, + number=22, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='gcs_result_path', full_name='on_device_tests_gateway.OnDeviceTestsCommand.gcs_result_path', index=22, + number=23, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='apk_tests', full_name='on_device_tests_gateway.OnDeviceTestsCommand.apk_tests', index=23, + number=24, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=59, + serialized_end=599, +) + + +_APKTEST = _descriptor.Descriptor( + name='ApkTest', + full_name='on_device_tests_gateway.ApkTest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='test_target', full_name='on_device_tests_gateway.ApkTest.test_target', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='apk_path', full_name='on_device_tests_gateway.ApkTest.apk_path', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device_model', full_name='on_device_tests_gateway.ApkTest.device_model', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device_pool', full_name='on_device_tests_gateway.ApkTest.device_pool', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='gtest_filters', full_name='on_device_tests_gateway.ApkTest.gtest_filters', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=601, + serialized_end=715, +) + + +_ONDEVICETESTSWATCHCOMMAND = _descriptor.Descriptor( + name='OnDeviceTestsWatchCommand', + full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='workdir', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.workdir', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='token', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.token', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='session_id', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.session_id', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='change_id', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.change_id', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='dry_run', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.dry_run', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=717, + serialized_end=832, +) + + +_ONDEVICETESTSRESPONSE = _descriptor.Descriptor( + name='OnDeviceTestsResponse', + full_name='on_device_tests_gateway.OnDeviceTestsResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='response', full_name='on_device_tests_gateway.OnDeviceTestsResponse.response', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=834, + serialized_end=875, +) + +_ONDEVICETESTSCOMMAND.fields_by_name['apk_tests'].message_type = _APKTEST +DESCRIPTOR.message_types_by_name['OnDeviceTestsCommand'] = _ONDEVICETESTSCOMMAND +DESCRIPTOR.message_types_by_name['ApkTest'] = _APKTEST +DESCRIPTOR.message_types_by_name['OnDeviceTestsWatchCommand'] = _ONDEVICETESTSWATCHCOMMAND +DESCRIPTOR.message_types_by_name['OnDeviceTestsResponse'] = _ONDEVICETESTSRESPONSE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +OnDeviceTestsCommand = _reflection.GeneratedProtocolMessageType('OnDeviceTestsCommand', (_message.Message,), dict( + DESCRIPTOR = _ONDEVICETESTSCOMMAND, + __module__ = 'on_device_tests_gateway_pb2' + # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsCommand) + )) +_sym_db.RegisterMessage(OnDeviceTestsCommand) + +ApkTest = _reflection.GeneratedProtocolMessageType('ApkTest', (_message.Message,), dict( + DESCRIPTOR = _APKTEST, + __module__ = 'on_device_tests_gateway_pb2' + # @@protoc_insertion_point(class_scope:on_device_tests_gateway.ApkTest) + )) +_sym_db.RegisterMessage(ApkTest) + +OnDeviceTestsWatchCommand = _reflection.GeneratedProtocolMessageType('OnDeviceTestsWatchCommand', (_message.Message,), dict( + DESCRIPTOR = _ONDEVICETESTSWATCHCOMMAND, + __module__ = 'on_device_tests_gateway_pb2' + # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsWatchCommand) + )) +_sym_db.RegisterMessage(OnDeviceTestsWatchCommand) + +OnDeviceTestsResponse = _reflection.GeneratedProtocolMessageType('OnDeviceTestsResponse', (_message.Message,), dict( + DESCRIPTOR = _ONDEVICETESTSRESPONSE, + __module__ = 'on_device_tests_gateway_pb2' + # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsResponse) + )) +_sym_db.RegisterMessage(OnDeviceTestsResponse) + + + +_ON_DEVICE_TESTS_GATEWAY = _descriptor.ServiceDescriptor( + name='on_device_tests_gateway', + full_name='on_device_tests_gateway.on_device_tests_gateway', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=878, + serialized_end=1144, + methods=[ + _descriptor.MethodDescriptor( + name='exec_command', + full_name='on_device_tests_gateway.on_device_tests_gateway.exec_command', + index=0, + containing_service=None, + input_type=_ONDEVICETESTSCOMMAND, + output_type=_ONDEVICETESTSRESPONSE, + options=None, + ), + _descriptor.MethodDescriptor( + name='exec_watch_command', + full_name='on_device_tests_gateway.on_device_tests_gateway.exec_watch_command', + index=1, + containing_service=None, + input_type=_ONDEVICETESTSWATCHCOMMAND, + output_type=_ONDEVICETESTSRESPONSE, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_ON_DEVICE_TESTS_GATEWAY) + +DESCRIPTOR.services_by_name['on_device_tests_gateway'] = _ON_DEVICE_TESTS_GATEWAY + +# @@protoc_insertion_point(module_scope) diff --git a/tools/on_device_tests_gateway_pb2_grpc.py b/tools/on_device_tests_gateway_pb2_grpc.py new file mode 100644 index 0000000000000..fdbfa59ddf188 --- /dev/null +++ b/tools/on_device_tests_gateway_pb2_grpc.py @@ -0,0 +1,64 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import on_device_tests_gateway_pb2 as on__device__tests__gateway__pb2 + + +class on_device_tests_gatewayStub(object): + """Interface exported by the server. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.exec_command = channel.unary_stream( + '/on_device_tests_gateway.on_device_tests_gateway/exec_command', + request_serializer=on__device__tests__gateway__pb2.OnDeviceTestsCommand.SerializeToString, + response_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.FromString, + ) + self.exec_watch_command = channel.unary_stream( + '/on_device_tests_gateway.on_device_tests_gateway/exec_watch_command', + request_serializer=on__device__tests__gateway__pb2.OnDeviceTestsWatchCommand.SerializeToString, + response_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.FromString, + ) + + +class on_device_tests_gatewayServicer(object): + """Interface exported by the server. + """ + + def exec_command(self, request, context): + """A dumb proxy RPC service that passes user defined command line options + to the on-device tests gateway and streams back output in real time. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def exec_watch_command(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_on_device_tests_gatewayServicer_to_server(servicer, server): + rpc_method_handlers = { + 'exec_command': grpc.unary_stream_rpc_method_handler( + servicer.exec_command, + request_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsCommand.FromString, + response_serializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.SerializeToString, + ), + 'exec_watch_command': grpc.unary_stream_rpc_method_handler( + servicer.exec_watch_command, + request_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsWatchCommand.FromString, + response_serializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'on_device_tests_gateway.on_device_tests_gateway', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) From 79de31b4e3b76029eabcedd82a089194aea01d3d Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Mon, 23 Dec 2024 14:32:50 -0800 Subject: [PATCH 27/61] Incremental commit based on review comments --- .github/actions/on_device_tests/action.yaml | 78 +-------------------- 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index b03c0772535c8..9040a36d95ad7 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -26,7 +26,7 @@ runs: echo "DIMENSION=${{ matrix.dimension }}" >> $GITHUB_ENV fi shell: bash - - name: Run ${{ env.SHARD_NAME }} Tests on ${{ matrix.platform }} Platform + - name: Run Tests on ${{ matrix.platform }} Platform env: GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} run: | @@ -41,79 +41,3 @@ runs: --gcs_result_path "${GCS_RESULTS_PATH}" \ trigger \ shell: bash - - name: Download ${{ matrix.platform }} Test Results - if: always() && env.TEST_TYPE == 'unit_test' - run: | - # Don't break on error (-e), some commands are expected to fail. - set -ux - - COBALT_LOGS_DIR="${GITHUB_WORKSPACE}/cobalt_logs" - UNIT_TEST_RESULT_PATH="${GITHUB_WORKSPACE}/unit-test-results" - COBALT_XMLS_FILENAME="cobalt_xmls.zip" - - # Forward environment variables for uploading artifacts in later steps. - echo "UNIT_TEST_RESULT_PATH=${UNIT_TEST_RESULT_PATH}" >> $GITHUB_ENV - echo "COBALT_LOGS_DIR=${COBALT_LOGS_DIR}" >> $GITHUB_ENV - - mkdir -p "${GITHUB_WORKSPACE}/test_results" - cd "${GITHUB_WORKSPACE}/test_results" - - i=0 - # Try downloading the results for 6x 10 seconds before giving up. - while [ $i -lt 6 ]; do - # The results are uploaded after the test has completed. - sleep 10 - - # The log files are named by the device lab test driver. - COBALT_ERROR_LOG_FILENAME="webDriverTestLog.ERROR" - COBALT_INFO_LOG_FILENAME="webDriverTestLog.INFO" - - # This command will fail until the results have been uploaded. - gsutil cp "${GCS_RESULTS_PATH}/${COBALT_ERROR_LOG_FILENAME}" . - gsutil cp "${GCS_RESULTS_PATH}/${COBALT_INFO_LOG_FILENAME}" . - gsutil cp "${GCS_RESULTS_PATH}/${COBALT_XMLS_FILENAME}" . - - # Break if all files were downloaded. - if [[ -f "${COBALT_XMLS_FILENAME}" && -f "${COBALT_ERROR_LOG_FILENAME}" && -f "${COBALT_INFO_LOG_FILENAME}" ]]; then - break - fi - - i=$(( ${i} + 1 )) - done - - # Rename log files for archiving to not expose legacy weirdness. - mkdir -p "${COBALT_LOGS_DIR}/${{ matrix.platform }}/" - cp "${COBALT_ERROR_LOG_FILENAME}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stderr_${{ matrix.shard }}.log" - cp "${COBALT_INFO_LOG_FILENAME}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stdout_${{ matrix.shard }}.log" - - # Prepare unit test results for DataDog upload. - RESULT_PATH=${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/${{ matrix.shard }}/ - mkdir -p ${RESULT_PATH} - - # Set tags for test differentiation. - tags="os.platform:${{ matrix.platform }}" - echo $tags > ${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/TAGS - - unzip ${COBALT_XMLS_FILENAME} -d ${RESULT_PATH} - shell: bash - - name: Archive Unit Test Logs - uses: actions/upload-artifact@v3 - if: always() && env.TEST_TYPE == 'unit_test' - with: - name: Device logs - path: ${{ env.COBALT_LOGS_DIR }}/ - - name: Archive Unit Test Results - uses: actions/upload-artifact@v3 - if: always() && env.TEST_TYPE == 'unit_test' - with: - name: unit-test-results - path: ${{ env.UNIT_TEST_RESULT_PATH }}/ - - name: Print device logs - if: always() - run: | - if ls ${COBALT_LOGS_DIR}/**/*.log 1> /dev/null 2>&1; then - cat ${COBALT_LOGS_DIR}/**/*.log - else - echo "No device logs found" - fi - shell: bash From 45d996991e78813b8c735261128965d6ea9ce41a Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Thu, 2 Jan 2025 22:46:03 +0000 Subject: [PATCH 28/61] new changes for odt gateway --- .../android-arm/base_unittests_filter.json | 4 -- tools/on_device_tests_gateway.proto | 36 +++++-------- tools/on_device_tests_gateway_client.py | 54 ++++++------------- 3 files changed, 29 insertions(+), 65 deletions(-) diff --git a/cobalt/testing/android-arm/base_unittests_filter.json b/cobalt/testing/android-arm/base_unittests_filter.json index bb161c725dc43..fb7886542530b 100644 --- a/cobalt/testing/android-arm/base_unittests_filter.json +++ b/cobalt/testing/android-arm/base_unittests_filter.json @@ -2,9 +2,5 @@ "failing_tests": [ "BreakIteratorTest.BreakCharacter", "ValuesUtilTest.FilePath" - ], - "passing_tests": [ - "test1", - "test2" ] } diff --git a/tools/on_device_tests_gateway.proto b/tools/on_device_tests_gateway.proto index 50cc02d478f7e..933b5818faa4e 100644 --- a/tools/on_device_tests_gateway.proto +++ b/tools/on_device_tests_gateway.proto @@ -29,33 +29,23 @@ service on_device_tests_gateway { // Working directory and command line arguments to be passed to the gateway. message OnDeviceTestsCommand { - // Next ID: 23 string workdir = 1; string token = 2; - string test_type = 3; - string platform = 4; - string archive_path = 5; - string config = 6; - string tag = 7; - repeated string labels = 8; - string builder_name = 9; - string change_id = 10; - string build_number = 11; - string loader_platform = 12; - string loader_config = 13; - string version = 14; - bool dry_run = 15; - repeated string dimension = 16; - string unittest_shard_index = 17; - string test_attempts = 18; - string retry_level = 19; - string start_timeout = 20; - string test_timeout = 21; - string builder_url = 22; - string gcs_result_path = 23; - repeated ApkTest apk_tests = 24; + string platform = 3; + string archive_path = 4; + repeated string labels = 5; + string change_id = 6; + bool dry_run = 7; + repeated string dimension = 8; + string test_attempts = 9; + string retry_level = 10; + string start_timeout = 11; + string test_timeout = 12; + string gcs_result_path = 13; + repeated ApkTest apk_tests = 14; } +// apk_test details message ApkTest { string test_target = 1; string apk_path = 2; diff --git a/tools/on_device_tests_gateway_client.py b/tools/on_device_tests_gateway_client.py index 99cdbb5f7166a..2ff477bd00875 100644 --- a/tools/on_device_tests_gateway_client.py +++ b/tools/on_device_tests_gateway_client.py @@ -26,25 +26,15 @@ import on_device_tests_gateway_pb2 import on_device_tests_gateway_pb2_grpc -# All tests On-Device tests support -_TEST_TYPES = [ - 'black_box_test', - 'evergreen_test', - 'unit_test', -] - -# All test configs On-Device tests support -_TEST_CONFIGS = [ - 'devel', - 'staging', - 'production', -] - _WORK_DIR = '/on_device_tests_gateway' + +# Comment out thw next three lines for local testing _ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( - #'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') - 'localhost') -#_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' + 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') +_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' + +# Uncomment the next two lines for local testing +_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') _ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '12345' @@ -75,23 +65,13 @@ def run_trigger_command(self, workdir: str, args: argparse.Namespace, apk_tests on_device_tests_gateway_pb2.OnDeviceTestsCommand( workdir=workdir, token=args.token, - #test_type=args.test_type, platform=args.platform, archive_path=args.archive_path, - #config=args.config, - #tag=args.tag, labels=args.label, gcs_result_path=args.gcs_result_path, - #builder_name=args.builder_name, - #builder_url=args.builder_url, change_id=args.change_id, - #build_number=args.build_number, - #loader_platform=args.loader_platform, - #loader_config=args.loader_config, - #version=args.version, dry_run=args.dry_run, dimension=args.dimension or [], - #unittest_shard_index=args.unittest_shard_index, test_attempts=args.test_attempts, retry_level=args.retry_level, apk_tests=apk_tests, @@ -131,10 +111,10 @@ def _read_json_config(filename): data = json.load(f) return data except FileNotFoundError: - #print(f" Config file '{filename}' not found.") + print(f" Config file '{filename}' not found.") return None except json.JSONDecodeError: - #print(f" Invalid JSON format in '{filename}'.") + print(f" Invalid JSON format in '{filename}'.") return None def _default_platform_json_file(platform): @@ -150,12 +130,12 @@ def _get_tests_filter_json_file(platform, gtest_target): def _process_apk_tests(args): apk_tests = [] platform_json_file = args.platform_json if args.platform_json else _default_platform_json_file(args.platform) - #print(' The platform_json_file is ' + platform_json_file) + print(f" The platform_json_file is '{platform_json_file}'") platform_json_file_data = _read_json_config(platform_json_file) gtest_device = platform_json_file_data["gtest_device"] gtest_lab = platform_json_file_data["gtest_lab"] gtest_blaze_target = "//experimental/cobalt/chrobalt_poc:custom_chrobalt_unit_tests_" + gtest_device - default_gtest_filters = "GURL*" + default_gtest_filters = "*" #print(' processing gtest_targets in json file') for gtest_target in platform_json_file_data["gtest_targets"]: @@ -170,22 +150,20 @@ def _process_apk_tests(args): gtest_filter_json_file_data = _read_json_config(gtest_filter_json_file) gtest_filters = default_gtest_filters if gtest_filter_json_file_data is None: - print(" This gtest_target does not have gtest_filters_specified"); + print(" This gtest_target does not have gtest_filters specified"); else: failing_tests_list = gtest_filter_json_file_data["failing_tests"] passing_tests_list = gtest_filter_json_file_data["passing_tests"] - #print(f" failing tests in json: {failing_tests_list}") - #print(f" passing tests in json: {passing_tests_list}") failing_tests_string = ":".join(failing_tests_list) passing_tests_string = ":".join(passing_tests_list) - #print(f" failing_tests_string = '{failing_tests_string}'") - #print(f" passing_tests_string = '{passing_tests_string}'") if passing_tests_string: gtest_filters += ":" + passing_tests_string if failing_tests_string: gtest_filters += ":-" + failing_tests_string - #print(f" gtest_filters = {gtest_filters}"); - apk_test["apk_path"] = "/bigstore/yt-temp/" + gtest_target + "-debug.apk" + print(f" gtest_filters = {gtest_filters}"); + + #apk_test["apk_path"] = "/bigstore/yt-temp/" + gtest_target + "-debug.apk" + apk_test["apk_path"] = "/bigstore/yt-temp/base_unittests-debug.apk" apk_test["gtest_filters"] = gtest_filters apk_tests.append(apk_test) From f74f774d148c2817b74d76cd5d7e8e988b4855bb Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Thu, 2 Jan 2025 23:54:04 +0000 Subject: [PATCH 29/61] additional changes to odt gateway code --- .github/actions/on_device_tests/action.yaml | 2 +- tools/on_device_tests_gateway_client.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 9040a36d95ad7..1e61b0892b405 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -28,7 +28,7 @@ runs: shell: bash - name: Run Tests on ${{ matrix.platform }} Platform env: - GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} + GCS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} run: | set -uxe python3 -u tools/on_device_tests_gateway_client.py \ diff --git a/tools/on_device_tests_gateway_client.py b/tools/on_device_tests_gateway_client.py index 2ff477bd00875..55937fd4b39ca 100644 --- a/tools/on_device_tests_gateway_client.py +++ b/tools/on_device_tests_gateway_client.py @@ -28,10 +28,10 @@ _WORK_DIR = '/on_device_tests_gateway' -# Comment out thw next three lines for local testing -_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( - 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') -_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' +# Comment out the next three lines for local testing +#_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( +# 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') +#_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' # Uncomment the next two lines for local testing _ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') @@ -153,11 +153,11 @@ def _process_apk_tests(args): print(" This gtest_target does not have gtest_filters specified"); else: failing_tests_list = gtest_filter_json_file_data["failing_tests"] - passing_tests_list = gtest_filter_json_file_data["passing_tests"] + #passing_tests_list = gtest_filter_json_file_data["passing_tests"] failing_tests_string = ":".join(failing_tests_list) - passing_tests_string = ":".join(passing_tests_list) - if passing_tests_string: - gtest_filters += ":" + passing_tests_string + #passing_tests_string = ":".join(passing_tests_list) + #if passing_tests_string: + # gtest_filters += ":" + passing_tests_string if failing_tests_string: gtest_filters += ":-" + failing_tests_string print(f" gtest_filters = {gtest_filters}"); From 5b674ac758c7d6d5c91dd1cdee802061cc6929e1 Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Fri, 3 Jan 2025 00:36:28 +0000 Subject: [PATCH 30/61] changing --archive_path in action.yaml --- .github/actions/on_device_tests/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 1e61b0892b405..a1e7fbe2542ed 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -37,7 +37,7 @@ runs: --token ${GITHUB_TOKEN} \ ${DIMENSION:+"--dimension" "$DIMENSION"} \ ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ - --archive_path "${GCS_PATH}/artifacts.tar" \ + --archive_path "${GCS_PATH}" \ --gcs_result_path "${GCS_RESULTS_PATH}" \ trigger \ shell: bash From 318c12146ab5f2aef7d4a1b9be05fa9dd36b8d3a Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Tue, 7 Jan 2025 00:50:48 +0000 Subject: [PATCH 31/61] Additional changes for odt gateway --- tools/on_device_tests_gateway_client.py | 30 +++++++++++-------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/tools/on_device_tests_gateway_client.py b/tools/on_device_tests_gateway_client.py index 55937fd4b39ca..78e91763bf02e 100644 --- a/tools/on_device_tests_gateway_client.py +++ b/tools/on_device_tests_gateway_client.py @@ -29,13 +29,13 @@ _WORK_DIR = '/on_device_tests_gateway' # Comment out the next three lines for local testing -#_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( -# 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') -#_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' +_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( + 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') +_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' # Uncomment the next two lines for local testing -_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') -_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '12345' +#_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') +#_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '12345' class OnDeviceTestsGatewayClient(): @@ -137,7 +137,6 @@ def _process_apk_tests(args): gtest_blaze_target = "//experimental/cobalt/chrobalt_poc:custom_chrobalt_unit_tests_" + gtest_device default_gtest_filters = "*" - #print(' processing gtest_targets in json file') for gtest_target in platform_json_file_data["gtest_targets"]: apk_test = { "test_target": gtest_blaze_target, @@ -153,17 +152,13 @@ def _process_apk_tests(args): print(" This gtest_target does not have gtest_filters specified"); else: failing_tests_list = gtest_filter_json_file_data["failing_tests"] - #passing_tests_list = gtest_filter_json_file_data["passing_tests"] failing_tests_string = ":".join(failing_tests_list) - #passing_tests_string = ":".join(passing_tests_list) - #if passing_tests_string: - # gtest_filters += ":" + passing_tests_string if failing_tests_string: gtest_filters += ":-" + failing_tests_string print(f" gtest_filters = {gtest_filters}"); - #apk_test["apk_path"] = "/bigstore/yt-temp/" + gtest_target + "-debug.apk" - apk_test["apk_path"] = "/bigstore/yt-temp/base_unittests-debug.apk" + apk_test["apk_path"] = f"{args.archive_path}/{gtest_target}-debug.apk" + #apk_test["apk_path"] = "/bigstore/yt-temp/base_unittests-debug.apk" apk_test["gtest_filters"] = gtest_filters apk_tests.append(apk_test) @@ -178,11 +173,12 @@ def main(): level=logging.INFO, format='[%(filename)s:%(lineno)s] %(message)s') parser = argparse.ArgumentParser( - epilog=('Example: ./on_device_tests_gateway_client.py ' - '--test_type=unit_test ' - '--remote_archive_path=gs://my_bucket/path/to/artifacts.tar ' - '--platform=raspi-2 ' - '--config=devel'), + epilog=('Example: ./on_device_tests_gateway_client.py' + 'trigger' + '--token token1' + '--platform android-arm' + '--archive_path /bigstore/yt-temp' + '--dimension host_name=regex:maneki-mhserver-05.*'), formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '-t', From f6cfd7e8bc2d1d850bb824d7475c47ecd35ce807 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Tue, 7 Jan 2025 11:21:00 -0800 Subject: [PATCH 32/61] Add additional argument for ODT gateway client --- .github/actions/on_device_tests/action.yaml | 32 ++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index a1e7fbe2542ed..d4f26042e2614 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -29,15 +29,45 @@ runs: - name: Run Tests on ${{ matrix.platform }} Platform env: GCS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_TOKEN: ${{ github.token }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} + GITHUB_ACTOR_ID: ${{ github.actor_id }} + GITHUB_REPO: ${{ github.repository }} + GITHUB_PR_HEAD_USER_LOGIN: ${{ github.event.pull_request.head.user.login }} + GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }} + GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }} + GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_WORKFLOW: ${{ github.workflow }} run: | set -uxe python3 -u tools/on_device_tests_gateway_client.py \ --platform_json "${GITHUB_WORKSPACE}/src/.github/config/${{ matrix.platform}}.json" \ --filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/${{ matrix.platform}}" \ --token ${GITHUB_TOKEN} \ + --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ + --tag cobalt_github_${GITHUB_EVENT_NAME} \ + --builder_name github_${{ matrix.platform }}_tests \ + --build_number ${GITHUB_RUN_NUMBER} \ + --builder_url ${GITHUB_RUN_URL} \ + --label github \ + --label ${GITHUB_EVENT_NAME} \ + --label ${GITHUB_WORKFLOW} \ + --label actor-${GITHUB_ACTOR} \ + --label actor_id-${GITHUB_ACTOR_ID} \ + --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ + --label sha-${GITHUB_SHA} \ + --label repository-${GITHUB_REPO} \ + --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ + --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} ${DIMENSION:+"--dimension" "$DIMENSION"} \ ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ --archive_path "${GCS_PATH}" \ --gcs_result_path "${GCS_RESULTS_PATH}" \ - trigger \ + trigger shell: bash From 82f228e7100e68d38a9d6ddcdb3efa411bdf5e4f Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Tue, 7 Jan 2025 19:20:46 +0000 Subject: [PATCH 33/61] Additional improvements for odt gateway code --- tools/on_device_tests_gateway_client.py | 392 +++++++++++++++--------- 1 file changed, 244 insertions(+), 148 deletions(-) diff --git a/tools/on_device_tests_gateway_client.py b/tools/on_device_tests_gateway_client.py index 78e91763bf02e..2cfad00758fce 100644 --- a/tools/on_device_tests_gateway_client.py +++ b/tools/on_device_tests_gateway_client.py @@ -117,162 +117,258 @@ def _read_json_config(filename): print(f" Invalid JSON format in '{filename}'.") return None -def _default_platform_json_file(platform): - current_dir = os.path.dirname(os.path.abspath(__file__)) - relative_path = os.path.join("..", ".github", "config", platform + ".json") - return os.path.join(current_dir, relative_path) +def _get_platform_json_file(platform): + """Constructs the path to the platform JSON configuration file. + + Args: + platform: The name of the platform. + + Returns: + The absolute path to the platform JSON file. + """ + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + ".github", + "config", + f"{platform}.json" + ) def _get_tests_filter_json_file(platform, gtest_target): - current_dir = os.path.dirname(os.path.abspath(__file__)) - relative_path = os.path.join("..", "cobalt", "testing", platform, gtest_target + "_filter.json") - return os.path.join(current_dir, relative_path) + """Constructs the path to the gtest filter JSON file. + + Args: + platform: The name of the platform. + gtest_target: The name of the gtest target. + + Returns: + The absolute path to the gtest filter JSON file. + """ + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "cobalt", + "testing", + platform, + f"{gtest_target}_filter.json" + ) def _process_apk_tests(args): - apk_tests = [] - platform_json_file = args.platform_json if args.platform_json else _default_platform_json_file(args.platform) - print(f" The platform_json_file is '{platform_json_file}'") - platform_json_file_data = _read_json_config(platform_json_file) - gtest_device = platform_json_file_data["gtest_device"] - gtest_lab = platform_json_file_data["gtest_lab"] - gtest_blaze_target = "//experimental/cobalt/chrobalt_poc:custom_chrobalt_unit_tests_" + gtest_device - default_gtest_filters = "*" - - for gtest_target in platform_json_file_data["gtest_targets"]: - apk_test = { - "test_target": gtest_blaze_target, - "device_model": gtest_device, - "device_pool": gtest_lab - } - print(f" gtest_target: {gtest_target}") - gtest_filter_json_file = _get_tests_filter_json_file(args.platform, gtest_target) - print(f" gtest_filter_json_file = {gtest_filter_json_file}") - gtest_filter_json_file_data = _read_json_config(gtest_filter_json_file) - gtest_filters = default_gtest_filters - if gtest_filter_json_file_data is None: - print(" This gtest_target does not have gtest_filters specified"); - else: - failing_tests_list = gtest_filter_json_file_data["failing_tests"] - failing_tests_string = ":".join(failing_tests_list) - if failing_tests_string: - gtest_filters += ":-" + failing_tests_string - print(f" gtest_filters = {gtest_filters}"); + """Processes APK tests based on provided arguments. - apk_test["apk_path"] = f"{args.archive_path}/{gtest_target}-debug.apk" - #apk_test["apk_path"] = "/bigstore/yt-temp/base_unittests-debug.apk" - apk_test["gtest_filters"] = gtest_filters - apk_tests.append(apk_test) + Args: + args: An argparse.Namespace object containing the following attributes: + platform: The platform for the tests. + platform_json: Optional path to a platform JSON file. + blaze_targets: A list of Blaze targets. + archive_path: Path to the archive containing APKs. + + Returns: + A list of dictionaries, where each dictionary represents an APK test + with keys like "test_target", "device_model", "device_pool", + "apk_path", and "gtest_filters". + """ - print(f" apk_tests: {apk_tests}") - return apk_tests + apk_tests = [] + platform_json_file = args.platform_json or _default_platform_json_file(args.platform) + print(f"The platform_json_file is '{platform_json_file}'") + platform_data = _read_json_config(platform_json_file) + print(f"Loaded platform data: {platform_data}") # Added verbosity + + for target in args.blaze_targets: + print(f"Processing Blaze target: {target}") # Added verbosity + for gtest_target in platform_data["gtest_targets"]: + print(f" Processing gtest_target: {gtest_target}") # Added verbosity + apk_test = { + "test_target": target, + "device_model": platform_data["gtest_device"], + "device_pool": platform_data["gtest_lab"], + "apk_path": f"{args.archive_path}/{gtest_target}-debug.apk", + "gtest_filters": _get_gtest_filters(args.platform, gtest_target) + } + apk_tests.append(apk_test) + print(f" Created apk_test: {apk_test}") # Added verbosity + + print(f"apk_tests: {apk_tests}") + return apk_tests + + +def _get_gtest_filters(platform, gtest_target): + """Retrieves gtest filters for a given target. -def main(): - """Main routine.""" - print('Starting main routine') - - logging.basicConfig( - level=logging.INFO, format='[%(filename)s:%(lineno)s] %(message)s') - - parser = argparse.ArgumentParser( - epilog=('Example: ./on_device_tests_gateway_client.py' - 'trigger' - '--token token1' - '--platform android-arm' - '--archive_path /bigstore/yt-temp' - '--dimension host_name=regex:maneki-mhserver-05.*'), - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '-t', - '--token', - type=str, - required=True, - help='On Device Tests authentication token') - parser.add_argument( - '--dry_run', - action='store_true', - help='Specifies to show what would be done without actually doing it.') - parser.add_argument( - '-i', - '--change_id', - type=str, - help='ChangeId that triggered this test, if any. ' - 'Saved with performance test results.') - - subparsers = parser.add_subparsers( - dest='action', help='On-Device tests commands', required=True) - trigger_parser = subparsers.add_parser( - 'trigger', help='Trigger On-Device tests') - - trigger_parser.add_argument( - '-p', - '--platform', - type=str, - required=True, - help='Platform this test was built for.') - trigger_parser.add_argument( - '-pf', - '--platform_json', - type=str, - help='Platform-specific json file containing the list of target tests.') - trigger_parser.add_argument( - '-a', - '--archive_path', - type=str, - required=True, - help='Path to Chrobalt archive to be tested. Must be on gcs.') - trigger_parser.add_argument( - '-l', - '--label', - type=str, - default=[], - action='append', - help='Additional labels to assign to the test.') - trigger_parser.add_argument( - '--gcs_result_path', - type=str, - help='GCS url where test result files should be uploaded.') - trigger_parser.add_argument( - '--dimension', - type=str, - action='append', - help='On-Device Tests dimension used to select a device. ' - 'Must have the following form: =.' - ' E.G. "release_version=regex:10.*') - trigger_parser.add_argument( - '--test_attempts', - type=str, - default='1', - required=False, - help='The maximum number of times a test could retry.') - trigger_parser.add_argument( - '--retry_level', - type=str, - default='ERROR', - required=False, - help='The retry level of Mobile harness job. Either ERROR (to retry for ' - 'MH errors) or FAIL (to retry for failing tests). Setting retry_level to ' - 'FAIL will also retry for MH errors.') - - watch_parser = subparsers.add_parser('watch', help='Trigger On-Device tests') - watch_parser.add_argument( - 'session_id', - type=str, - help='Session id of a previously triggered mobile ' - 'harness test. If passed, the test will not be ' - 'triggered, but will be watched until the exit ' - 'status is reached.') - - args = parser.parse_args() - apk_tests = _process_apk_tests(args) - - client = OnDeviceTestsGatewayClient() - try: - if args.action == 'trigger': - client.run_trigger_command(workdir=_WORK_DIR, args=args, apk_tests=apk_tests) + Args: + platform: The platform for the tests. + gtest_target: The name of the gtest target. + + Returns: + A string representing the gtest filters. + """ + + gtest_filters = "*" + filter_json_file = _get_tests_filter_json_file(platform, gtest_target) + print(f" gtest_filter_json_file = {filter_json_file}") + filter_data = _read_json_config(filter_json_file) + if filter_data: + print(f" Loaded filter data: {filter_data}") # Added verbosity + failing_tests = ":".join(filter_data.get("failing_tests", [])) + if failing_tests: + gtest_filters += ":-" + failing_tests + print(f" gtest_filters = {gtest_filters}") else: - client.run_watch_command(workdir=_WORK_DIR, args=args) - except grpc.RpcError as e: - print(e) - return e.code().value + print(f" This gtest_target does not have gtest_filters specified") + return gtest_filters + + +def main(): + """Main routine for the on-device tests gateway client.""" + + logging.basicConfig( + level=logging.INFO, + format='[%(filename)s:%(lineno)s] %(message)s' + ) + print('Starting main routine') + + parser = argparse.ArgumentParser( + description="Client for interacting with the On-Device Tests gateway.", + epilog=( + 'Example: ./on_device_tests_gateway_client.py trigger ' + '--token token1 ' + '--platform android-arm ' + '--archive_path /bigstore/yt-temp ' + '--blaze_targets //experimental/cobalt/chrobalt_poc:chrobalt_unit_tests_maneki_sabrina //experimental/cobalt/chrobalt_poc:chrobalt_unit_tests_shared_boreal ' + '--dimension host_name=regex:maneki-mhserver-05.*' + ), + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + # Authentication + parser.add_argument( + '-t', + '--token', + type=str, + required=True, + help='On Device Tests authentication token' + ) + + # General options + parser.add_argument( + '--dry_run', + action='store_true', + help='Show what would be done without actually doing it.' + ) + parser.add_argument( + '-i', + '--change_id', + type=str, + help='ChangeId that triggered this test, if any. Saved with performance test results.' + ) + + subparsers = parser.add_subparsers( + dest='action', + help='On-Device tests commands', + required=True + ) + + # Trigger command + trigger_parser = subparsers.add_parser( + 'trigger', + help='Trigger On-Device tests', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + trigger_parser.add_argument( + '-p', + '--platform', + type=str, + required=True, + help='Platform this test was built for.' + ) + trigger_parser.add_argument( + '-pf', + '--platform_json', + type=str, + help='Platform-specific JSON file containing the list of target tests.' + ) + trigger_parser.add_argument( + '-a', + '--archive_path', + type=str, + required=True, + help='Path to Chrobalt archive to be tested. Must be on GCS.' + ) + trigger_parser.add_argument( + '-l', + '--label', + type=str, + action='append', + default=[], + help='Additional labels to assign to the test.' + ) + trigger_parser.add_argument( + '--gcs_result_path', + type=str, + help='GCS URL where test result files should be uploaded.' + ) + trigger_parser.add_argument( + '--dimension', + type=str, + action='append', + help=( + 'On-Device Tests dimension used to select a device. ' + 'Must have the following form: =. ' + 'E.G. "release_version=regex:10.*"' + ) + ) + trigger_parser.add_argument( + '--test_attempts', + type=str, + default='1', + help='The maximum number of times a test could retry.' + ) + trigger_parser.add_argument( + '--blaze_targets', + nargs="+", + type=str, + required=True, + help='A list of Blaze targets to run.' + ) + trigger_parser.add_argument( + '--retry_level', + type=str, + default='ERROR', + choices=['ERROR', 'FAIL'], + help=( + 'The retry level of Mobile Harness job. ' + 'ERROR to retry for MH errors, ' + 'FAIL to retry for failing tests (and MH errors).' + ) + ) + + # Watch command + watch_parser = subparsers.add_parser( + 'watch', + help='Watch a previously triggered On-Device test' + ) + watch_parser.add_argument( + 'session_id', + type=str, + help=( + 'Session ID of a previously triggered Mobile Harness test. ' + 'The test will be watched until it completes.' + ) + ) + + args = parser.parse_args() + apk_tests = _process_apk_tests(args) + + client = OnDeviceTestsGatewayClient() + try: + if args.action == 'trigger': + client.run_trigger_command(workdir=_WORK_DIR, args=args, apk_tests=apk_tests) + else: + client.run_watch_command(workdir=_WORK_DIR, args=args) + except grpc.RpcError as e: + print(e) + return e.code().value if __name__ == '__main__': From 02209ecde9fb0c59bdaf67a5f8cf4fff4171d5af Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Tue, 7 Jan 2025 19:37:40 +0000 Subject: [PATCH 34/61] removing autogenerated files from pr --- tools/on_device_tests_gateway_pb2.py | 437 ---------------------- tools/on_device_tests_gateway_pb2_grpc.py | 64 ---- 2 files changed, 501 deletions(-) delete mode 100644 tools/on_device_tests_gateway_pb2.py delete mode 100644 tools/on_device_tests_gateway_pb2_grpc.py diff --git a/tools/on_device_tests_gateway_pb2.py b/tools/on_device_tests_gateway_pb2.py deleted file mode 100644 index f6c8947b850fa..0000000000000 --- a/tools/on_device_tests_gateway_pb2.py +++ /dev/null @@ -1,437 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: on_device_tests_gateway.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='on_device_tests_gateway.proto', - package='on_device_tests_gateway', - syntax='proto3', - serialized_pb=_b('\n\x1don_device_tests_gateway.proto\x12\x17on_device_tests_gateway\"\x9c\x04\n\x14OnDeviceTestsCommand\x12\x0f\n\x07workdir\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\t\x12\x11\n\ttest_type\x18\x03 \x01(\t\x12\x10\n\x08platform\x18\x04 \x01(\t\x12\x14\n\x0c\x61rchive_path\x18\x05 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x06 \x01(\t\x12\x0b\n\x03tag\x18\x07 \x01(\t\x12\x0e\n\x06labels\x18\x08 \x03(\t\x12\x14\n\x0c\x62uilder_name\x18\t \x01(\t\x12\x11\n\tchange_id\x18\n \x01(\t\x12\x14\n\x0c\x62uild_number\x18\x0b \x01(\t\x12\x17\n\x0floader_platform\x18\x0c \x01(\t\x12\x15\n\rloader_config\x18\r \x01(\t\x12\x0f\n\x07version\x18\x0e \x01(\t\x12\x0f\n\x07\x64ry_run\x18\x0f \x01(\x08\x12\x11\n\tdimension\x18\x10 \x03(\t\x12\x1c\n\x14unittest_shard_index\x18\x11 \x01(\t\x12\x15\n\rtest_attempts\x18\x12 \x01(\t\x12\x13\n\x0bretry_level\x18\x13 \x01(\t\x12\x15\n\rstart_timeout\x18\x14 \x01(\t\x12\x14\n\x0ctest_timeout\x18\x15 \x01(\t\x12\x13\n\x0b\x62uilder_url\x18\x16 \x01(\t\x12\x17\n\x0fgcs_result_path\x18\x17 \x01(\t\x12\x33\n\tapk_tests\x18\x18 \x03(\x0b\x32 .on_device_tests_gateway.ApkTest\"r\n\x07\x41pkTest\x12\x13\n\x0btest_target\x18\x01 \x01(\t\x12\x10\n\x08\x61pk_path\x18\x02 \x01(\t\x12\x14\n\x0c\x64\x65vice_model\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x65vice_pool\x18\x04 \x01(\t\x12\x15\n\rgtest_filters\x18\x05 \x01(\t\"s\n\x19OnDeviceTestsWatchCommand\x12\x0f\n\x07workdir\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\t\x12\x12\n\nsession_id\x18\x03 \x01(\t\x12\x11\n\tchange_id\x18\x04 \x01(\t\x12\x0f\n\x07\x64ry_run\x18\x05 \x01(\x08\")\n\x15OnDeviceTestsResponse\x12\x10\n\x08response\x18\x01 \x01(\t2\x8a\x02\n\x17on_device_tests_gateway\x12q\n\x0c\x65xec_command\x12-.on_device_tests_gateway.OnDeviceTestsCommand\x1a..on_device_tests_gateway.OnDeviceTestsResponse\"\x00\x30\x01\x12|\n\x12\x65xec_watch_command\x12\x32.on_device_tests_gateway.OnDeviceTestsWatchCommand\x1a..on_device_tests_gateway.OnDeviceTestsResponse\"\x00\x30\x01\x62\x06proto3') -) - - - - -_ONDEVICETESTSCOMMAND = _descriptor.Descriptor( - name='OnDeviceTestsCommand', - full_name='on_device_tests_gateway.OnDeviceTestsCommand', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='workdir', full_name='on_device_tests_gateway.OnDeviceTestsCommand.workdir', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='token', full_name='on_device_tests_gateway.OnDeviceTestsCommand.token', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='test_type', full_name='on_device_tests_gateway.OnDeviceTestsCommand.test_type', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='platform', full_name='on_device_tests_gateway.OnDeviceTestsCommand.platform', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='archive_path', full_name='on_device_tests_gateway.OnDeviceTestsCommand.archive_path', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='config', full_name='on_device_tests_gateway.OnDeviceTestsCommand.config', index=5, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tag', full_name='on_device_tests_gateway.OnDeviceTestsCommand.tag', index=6, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='labels', full_name='on_device_tests_gateway.OnDeviceTestsCommand.labels', index=7, - number=8, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='builder_name', full_name='on_device_tests_gateway.OnDeviceTestsCommand.builder_name', index=8, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='change_id', full_name='on_device_tests_gateway.OnDeviceTestsCommand.change_id', index=9, - number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='build_number', full_name='on_device_tests_gateway.OnDeviceTestsCommand.build_number', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='loader_platform', full_name='on_device_tests_gateway.OnDeviceTestsCommand.loader_platform', index=11, - number=12, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='loader_config', full_name='on_device_tests_gateway.OnDeviceTestsCommand.loader_config', index=12, - number=13, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='version', full_name='on_device_tests_gateway.OnDeviceTestsCommand.version', index=13, - number=14, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='dry_run', full_name='on_device_tests_gateway.OnDeviceTestsCommand.dry_run', index=14, - number=15, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='dimension', full_name='on_device_tests_gateway.OnDeviceTestsCommand.dimension', index=15, - number=16, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unittest_shard_index', full_name='on_device_tests_gateway.OnDeviceTestsCommand.unittest_shard_index', index=16, - number=17, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='test_attempts', full_name='on_device_tests_gateway.OnDeviceTestsCommand.test_attempts', index=17, - number=18, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='retry_level', full_name='on_device_tests_gateway.OnDeviceTestsCommand.retry_level', index=18, - number=19, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start_timeout', full_name='on_device_tests_gateway.OnDeviceTestsCommand.start_timeout', index=19, - number=20, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='test_timeout', full_name='on_device_tests_gateway.OnDeviceTestsCommand.test_timeout', index=20, - number=21, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='builder_url', full_name='on_device_tests_gateway.OnDeviceTestsCommand.builder_url', index=21, - number=22, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='gcs_result_path', full_name='on_device_tests_gateway.OnDeviceTestsCommand.gcs_result_path', index=22, - number=23, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='apk_tests', full_name='on_device_tests_gateway.OnDeviceTestsCommand.apk_tests', index=23, - number=24, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=59, - serialized_end=599, -) - - -_APKTEST = _descriptor.Descriptor( - name='ApkTest', - full_name='on_device_tests_gateway.ApkTest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='test_target', full_name='on_device_tests_gateway.ApkTest.test_target', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='apk_path', full_name='on_device_tests_gateway.ApkTest.apk_path', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='device_model', full_name='on_device_tests_gateway.ApkTest.device_model', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='device_pool', full_name='on_device_tests_gateway.ApkTest.device_pool', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='gtest_filters', full_name='on_device_tests_gateway.ApkTest.gtest_filters', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=601, - serialized_end=715, -) - - -_ONDEVICETESTSWATCHCOMMAND = _descriptor.Descriptor( - name='OnDeviceTestsWatchCommand', - full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='workdir', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.workdir', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='token', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.token', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='session_id', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.session_id', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='change_id', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.change_id', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='dry_run', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.dry_run', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=717, - serialized_end=832, -) - - -_ONDEVICETESTSRESPONSE = _descriptor.Descriptor( - name='OnDeviceTestsResponse', - full_name='on_device_tests_gateway.OnDeviceTestsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='response', full_name='on_device_tests_gateway.OnDeviceTestsResponse.response', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=834, - serialized_end=875, -) - -_ONDEVICETESTSCOMMAND.fields_by_name['apk_tests'].message_type = _APKTEST -DESCRIPTOR.message_types_by_name['OnDeviceTestsCommand'] = _ONDEVICETESTSCOMMAND -DESCRIPTOR.message_types_by_name['ApkTest'] = _APKTEST -DESCRIPTOR.message_types_by_name['OnDeviceTestsWatchCommand'] = _ONDEVICETESTSWATCHCOMMAND -DESCRIPTOR.message_types_by_name['OnDeviceTestsResponse'] = _ONDEVICETESTSRESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -OnDeviceTestsCommand = _reflection.GeneratedProtocolMessageType('OnDeviceTestsCommand', (_message.Message,), dict( - DESCRIPTOR = _ONDEVICETESTSCOMMAND, - __module__ = 'on_device_tests_gateway_pb2' - # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsCommand) - )) -_sym_db.RegisterMessage(OnDeviceTestsCommand) - -ApkTest = _reflection.GeneratedProtocolMessageType('ApkTest', (_message.Message,), dict( - DESCRIPTOR = _APKTEST, - __module__ = 'on_device_tests_gateway_pb2' - # @@protoc_insertion_point(class_scope:on_device_tests_gateway.ApkTest) - )) -_sym_db.RegisterMessage(ApkTest) - -OnDeviceTestsWatchCommand = _reflection.GeneratedProtocolMessageType('OnDeviceTestsWatchCommand', (_message.Message,), dict( - DESCRIPTOR = _ONDEVICETESTSWATCHCOMMAND, - __module__ = 'on_device_tests_gateway_pb2' - # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsWatchCommand) - )) -_sym_db.RegisterMessage(OnDeviceTestsWatchCommand) - -OnDeviceTestsResponse = _reflection.GeneratedProtocolMessageType('OnDeviceTestsResponse', (_message.Message,), dict( - DESCRIPTOR = _ONDEVICETESTSRESPONSE, - __module__ = 'on_device_tests_gateway_pb2' - # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsResponse) - )) -_sym_db.RegisterMessage(OnDeviceTestsResponse) - - - -_ON_DEVICE_TESTS_GATEWAY = _descriptor.ServiceDescriptor( - name='on_device_tests_gateway', - full_name='on_device_tests_gateway.on_device_tests_gateway', - file=DESCRIPTOR, - index=0, - options=None, - serialized_start=878, - serialized_end=1144, - methods=[ - _descriptor.MethodDescriptor( - name='exec_command', - full_name='on_device_tests_gateway.on_device_tests_gateway.exec_command', - index=0, - containing_service=None, - input_type=_ONDEVICETESTSCOMMAND, - output_type=_ONDEVICETESTSRESPONSE, - options=None, - ), - _descriptor.MethodDescriptor( - name='exec_watch_command', - full_name='on_device_tests_gateway.on_device_tests_gateway.exec_watch_command', - index=1, - containing_service=None, - input_type=_ONDEVICETESTSWATCHCOMMAND, - output_type=_ONDEVICETESTSRESPONSE, - options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_ON_DEVICE_TESTS_GATEWAY) - -DESCRIPTOR.services_by_name['on_device_tests_gateway'] = _ON_DEVICE_TESTS_GATEWAY - -# @@protoc_insertion_point(module_scope) diff --git a/tools/on_device_tests_gateway_pb2_grpc.py b/tools/on_device_tests_gateway_pb2_grpc.py deleted file mode 100644 index fdbfa59ddf188..0000000000000 --- a/tools/on_device_tests_gateway_pb2_grpc.py +++ /dev/null @@ -1,64 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -import grpc - -import on_device_tests_gateway_pb2 as on__device__tests__gateway__pb2 - - -class on_device_tests_gatewayStub(object): - """Interface exported by the server. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.exec_command = channel.unary_stream( - '/on_device_tests_gateway.on_device_tests_gateway/exec_command', - request_serializer=on__device__tests__gateway__pb2.OnDeviceTestsCommand.SerializeToString, - response_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.FromString, - ) - self.exec_watch_command = channel.unary_stream( - '/on_device_tests_gateway.on_device_tests_gateway/exec_watch_command', - request_serializer=on__device__tests__gateway__pb2.OnDeviceTestsWatchCommand.SerializeToString, - response_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.FromString, - ) - - -class on_device_tests_gatewayServicer(object): - """Interface exported by the server. - """ - - def exec_command(self, request, context): - """A dumb proxy RPC service that passes user defined command line options - to the on-device tests gateway and streams back output in real time. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def exec_watch_command(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_on_device_tests_gatewayServicer_to_server(servicer, server): - rpc_method_handlers = { - 'exec_command': grpc.unary_stream_rpc_method_handler( - servicer.exec_command, - request_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsCommand.FromString, - response_serializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.SerializeToString, - ), - 'exec_watch_command': grpc.unary_stream_rpc_method_handler( - servicer.exec_watch_command, - request_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsWatchCommand.FromString, - response_serializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'on_device_tests_gateway.on_device_tests_gateway', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) From ac128ff86c416ced52be74a078d78f9c09795c90 Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Tue, 7 Jan 2025 19:58:14 +0000 Subject: [PATCH 35/61] moving odt gateway code from tools folder to cobalt/tools --- .github/actions/on_device_tests/action.yaml | 4 ++-- {tools => cobalt/tools}/on_device_tests_gateway.proto | 0 {tools => cobalt/tools}/on_device_tests_gateway_client.py | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {tools => cobalt/tools}/on_device_tests_gateway.proto (100%) rename {tools => cobalt/tools}/on_device_tests_gateway_client.py (100%) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index d4f26042e2614..478772a450ea8 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -10,7 +10,7 @@ runs: shell: bash - name: Generate gRPC files run: | - python -m grpc_tools.protoc -Itools/ --python_out=tools/ --grpc_python_out=tools/ tools/on_device_tests_gateway.proto + python -m grpc_tools.protoc -Itools/ --python_out=cobalt/tools/ --grpc_python_out=cobalt/tools/ cobalt/tools/on_device_tests_gateway.proto shell: bash - name: Set Up Cloud SDK uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1 @@ -46,7 +46,7 @@ runs: GITHUB_WORKFLOW: ${{ github.workflow }} run: | set -uxe - python3 -u tools/on_device_tests_gateway_client.py \ + python3 -u cobalt/tools/on_device_tests_gateway_client.py \ --platform_json "${GITHUB_WORKSPACE}/src/.github/config/${{ matrix.platform}}.json" \ --filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/${{ matrix.platform}}" \ --token ${GITHUB_TOKEN} \ diff --git a/tools/on_device_tests_gateway.proto b/cobalt/tools/on_device_tests_gateway.proto similarity index 100% rename from tools/on_device_tests_gateway.proto rename to cobalt/tools/on_device_tests_gateway.proto diff --git a/tools/on_device_tests_gateway_client.py b/cobalt/tools/on_device_tests_gateway_client.py similarity index 100% rename from tools/on_device_tests_gateway_client.py rename to cobalt/tools/on_device_tests_gateway_client.py From 9fb00070847a03f877d19952e4fe05f1a0c83a22 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Tue, 7 Jan 2025 14:36:08 -0800 Subject: [PATCH 36/61] Introduce "test_dimension" key --- .github/actions/on_device_tests/action.yaml | 1 + .github/config/android-arm.json | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 478772a450ea8..816277766ebc0 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -5,6 +5,7 @@ runs: using: "composite" steps: - name: Install Requirements + # TODO (b/388329764) - set up requirements file. run: | pip3 install grpcio==1.38.0 grpcio-tools==1.38.0 shell: bash diff --git a/.github/config/android-arm.json b/.github/config/android-arm.json index 684725c6f75e2..6bfc85207a7e0 100644 --- a/.github/config/android-arm.json +++ b/.github/config/android-arm.json @@ -21,8 +21,10 @@ "sql_unittests", "url_unittests" ], - "gtest_device": "sabrina", - "gtest_lab": "maneki", + "test_dimensions": { + "gtest_device": "sabrina", + "gtest_lab": "maneki" + }, "test_on_device": true, "includes": [ { From b6f6786a31f81879b355a107003caec6d162778e Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Wed, 8 Jan 2025 16:20:34 -0800 Subject: [PATCH 37/61] Log and XML file processing --- .github/actions/on_device_tests/action.yaml | 24 +++++++++++++++++++-- .github/workflows/main.yaml | 20 ++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 816277766ebc0..7bb7d04a6fb54 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -1,5 +1,9 @@ name: On Device Test description: Runs on-device tests. +inputs: + results_dir: + description: "Path to directory where test results are saved." + required: true runs: using: "composite" @@ -18,10 +22,8 @@ runs: - name: Set env vars run: | echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV - # Test results and logs echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}" >> $GITHUB_ENV - # Dimension env if [ "${{ matrix.dimension }}" != "null" ]; then echo "DIMENSION=${{ matrix.dimension }}" >> $GITHUB_ENV @@ -72,3 +74,21 @@ runs: --gcs_result_path "${GCS_RESULTS_PATH}" \ trigger shell: bash + - name: Download ${{ matrix.platform }} Test Results + if: always() + env: + RESULTS_DIR: ${{ inputs.results_dir }} + run: | + set -uxe + mkdir -p "${GITHUB_WORKSPACE}/${RESULTS_DIR}" + cd "${GITHUB_WORKSPACE}/${RESULTS_DIR}" + gsutil cp "${GCS_RESULTS_PATH}/" . + echo "TEST_LOG=${GITHUB_WORKSPACE}/${RESULTS_DIR}/test_results.txt" >> $GITHUB_ENV + shell: bash + - name: Archive Test Logs + uses: actions/upload-artifact@v3 + if: always() + with: + name: Test log + path: ${{ env.TEST_LOG }}/ + diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 42dd44083757d..2511b4a68ad05 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -302,6 +302,8 @@ jobs: platform: ${{ fromJson(needs.initialize.outputs.platforms) }} config: [devel] include: ${{ fromJson(needs.initialize.outputs.includes) }} + env: + TEST_RESULTS_DIR: ${{ matrix.name }}_test_results steps: - name: Checkout uses: kaidokert/checkout@v3.5.999 @@ -309,5 +311,21 @@ jobs: with: fetch-depth: 1 persist-credentials: false - - name: Run Tests (${{ matrix.shard }}) + - name: Run On-Device Tests (${{ matrix.shard }}) + id: on-device-tests uses: ./.github/actions/on_device_tests + with: + results_dir: ${{ env.TEST_RESULTS_DIR }} + - name: Process Test Results + if: | + always() && + ( + steps.on-device-tests.outcome == 'success' || + steps.on-device-tests.outcome == 'failure' + ) + uses: ./src/.github/actions/process_test_results + with: + results_dir: ${{ env.TEST_RESULTS_DIR }} + datadog_api_key: ${{ secrets.DD_API_KEY }} + is_postsubmit: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} + continue-on-error: true From 94e115a90cf7418c087d5a6ec178fb72cbf674df Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Mon, 13 Jan 2025 21:14:10 -0800 Subject: [PATCH 38/61] Update gateway client parameters --- .github/actions/on_device_tests/action.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 7bb7d04a6fb54..baf057b7bde27 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -55,9 +55,8 @@ runs: --token ${GITHUB_TOKEN} \ --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ --tag cobalt_github_${GITHUB_EVENT_NAME} \ - --builder_name github_${{ matrix.platform }}_tests \ - --build_number ${GITHUB_RUN_NUMBER} \ - --builder_url ${GITHUB_RUN_URL} \ + --label builder-${{ matrix.platform }} \ + --label builder_url-${GITHUB_RUN_URL} \ --label github \ --label ${GITHUB_EVENT_NAME} \ --label ${GITHUB_WORKFLOW} \ @@ -91,4 +90,3 @@ runs: with: name: Test log path: ${{ env.TEST_LOG }}/ - From de4e8171bc60b212113006578770cc44afc477dc Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Tue, 14 Jan 2025 11:11:24 -0800 Subject: [PATCH 39/61] remove tag --- .github/actions/on_device_tests/action.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index baf057b7bde27..e31e34c17d2a3 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -54,7 +54,6 @@ runs: --filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/${{ matrix.platform}}" \ --token ${GITHUB_TOKEN} \ --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ - --tag cobalt_github_${GITHUB_EVENT_NAME} \ --label builder-${{ matrix.platform }} \ --label builder_url-${GITHUB_RUN_URL} \ --label github \ From d0d5c8d061aacce39170027e5694b15c6c409eee Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Tue, 14 Jan 2025 15:30:40 -0800 Subject: [PATCH 40/61] Parameter work --- .github/actions/on_device_tests/action.yaml | 9 +++++++-- .github/workflows/main.yaml | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index e31e34c17d2a3..86bf7a2d940a9 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -1,6 +1,9 @@ name: On Device Test description: Runs on-device tests. inputs: + gcs_results_path: + description: "GCS path for the test results" + required: true results_dir: description: "Path to directory where test results are saved." required: true @@ -31,7 +34,8 @@ runs: shell: bash - name: Run Tests on ${{ matrix.platform }} Platform env: - GCS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} + GCS_ARTIFACTS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} + GCS_RESULTS_PATH: ${{ inputs.gcs_results_path }} GITHUB_SHA: ${{ github.sha }} GITHUB_TOKEN: ${{ github.token }} GITHUB_EVENT_NAME: ${{ github.event_name }} @@ -68,13 +72,14 @@ runs: --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} ${DIMENSION:+"--dimension" "$DIMENSION"} \ ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ - --archive_path "${GCS_PATH}" \ + --gcs_archive_path "${GCS_ARTIFACTS_PATH}" \ --gcs_result_path "${GCS_RESULTS_PATH}" \ trigger shell: bash - name: Download ${{ matrix.platform }} Test Results if: always() env: + GCS_RESULTS_PATH: ${{ inputs.gcs_results_path }} RESULTS_DIR: ${{ inputs.results_dir }} run: | set -uxe diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2511b4a68ad05..ab049aac270ce 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -315,6 +315,7 @@ jobs: id: on-device-tests uses: ./.github/actions/on_device_tests with: + gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }} results_dir: ${{ env.TEST_RESULTS_DIR }} - name: Process Test Results if: | From 7b88ba75c8f37dcd296700305a6a73a924bd2db7 Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Thu, 16 Jan 2025 00:04:18 +0000 Subject: [PATCH 41/61] Updated ODT Gateway Client code --- "cobalt/tools/\\" | 413 ++++++++++++++ cobalt/tools/on_device_tests_gateway.proto | 47 +- .../tools/on_device_tests_gateway_client.py | 535 ++++++++++-------- cobalt/tools/on_device_tests_gateway_pb2.py | 297 ++++++++++ .../tools/on_device_tests_gateway_pb2_grpc.py | 65 +++ 5 files changed, 1085 insertions(+), 272 deletions(-) create mode 100644 "cobalt/tools/\\" create mode 100644 cobalt/tools/on_device_tests_gateway_pb2.py create mode 100644 cobalt/tools/on_device_tests_gateway_pb2_grpc.py diff --git "a/cobalt/tools/\\" "b/cobalt/tools/\\" new file mode 100644 index 0000000000000..ea587638e0876 --- /dev/null +++ "b/cobalt/tools/\\" @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""gRPC On-device Tests Gateway client.""" + +import argparse +import json +import logging +import sys + +import grpc +import on_device_tests_gateway_pb2 +import on_device_tests_gateway_pb2_grpc + + +_WORK_DIR = '/on_device_tests_gateway' + +# Comment out the next three lines for local testing +# _ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( +# 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') +_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' + +# Uncomment the next two lines for local testing +_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') +# _ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '12345' + +_ON_DEVICE_OUTPUT_FILES_DIR = '/sdcard/Download' +_CHROMIUM_TEST_ROOT = '/sdcard/chromium_tests_root' + +class OnDeviceTestsGatewayClient(): + """On-device tests Gateway Client class.""" + + def __init__(self): + self.channel = grpc.insecure_channel( + target=f'{_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST}:{_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT}', # pylint:disable=line-too-long + # These options need to match server settings. + options=[('grpc.keepalive_time_ms', 10000), + ('grpc.keepalive_timeout_ms', 5000), + ('grpc.keepalive_permit_without_calls', 1), + ('grpc.http2.max_pings_without_data', 0), + ('grpc.http2.min_time_between_pings_ms', 10000), + ('grpc.http2.min_ping_interval_without_data_ms', 5000)]) + self.stub = on_device_tests_gateway_pb2_grpc.on_device_tests_gatewayStub( + self.channel) + + def run_trigger_command(self, args: argparse.Namespace, test_requests): + """Calls On-Device Tests service and passing given parameters to it. + + Args: + args (Namespace): Arguments passed in command line. + test_requests (list): A list of test requests. + """ + for response_line in self.stub.exec_command( + on_device_tests_gateway_pb2.OnDeviceTestsCommand( + token=args.token, + labels=args.label, + test_requests=test_requests, + )): + + print(response_line.response) + + def run_watch_command(self, workdir: str, args: argparse.Namespace): + """Calls On-Device Tests watch service and passing given parameters to it. + + Args: + workdir (str): Current script workdir. + args (Namespace): Arguments passed in command line. + """ + for response_line in self.stub.exec_watch_command( + on_device_tests_gateway_pb2.OnDeviceTestsWatchCommand( + workdir=workdir, + token=args.token, + change_id=args.change_id, + session_id=args.session_id, + )): + + print(response_line.response) + + +def _read_json_config(filename): + """Reads and parses data from a JSON configuration file. + + Args: + filename: The name of the JSON configuration file. + + Returns: + A list of dictionaries, where each dictionary represents a test + configuration. + """ + try: + with open(filename, 'r') as f: + data = json.load(f) + return data + except FileNotFoundError: + print(f" Config file '{filename}' not found.") + return None + except json.JSONDecodeError: + print(f" Invalid JSON format in '{filename}'.") + return None + + +def _get_gtest_filters(filter_json_dir, gtest_target): + """Retrieves gtest filters for a given target. + + Args: + filter_json_dir: Directory containing filter JSON files. + gtest_target: The name of the gtest target. + + Returns: + A string containing the gtest filters. + """ + gtest_filters = '*' + filter_json_file = f'{filter_json_dir}/{gtest_target}_filter.json' + print(f" gtest_filter_json_file = {filter_json_file}") + filter_data = _read_json_config(filter_json_file) + if filter_data: + print(f" Loaded filter data: {filter_data}") + failing_tests = ':'.join(filter_data.get('failing_tests', [])) + if failing_tests: + gtest_filters += ':-' + failing_tests + print(f" gtest_filters = {gtest_filters}") + else: + print(' This gtest_target does not have gtest_filters specified') + return gtest_filters + + +def _process_test_requests(args): + """Processes test requests from the given arguments. + + Constructs a list of test requests based on the provided arguments, + including test arguments, command arguments, files, parameters, + and device information. + + Args: + args: The parsed command-line arguments. + + Returns: + A list of test request dictionaries. + """ + test_requests = [] + platform_data = _read_json_config(args.platform_json) + + tests_args = [ + f'job_timeout_secs={args.job_timeout_secs}', + f'test_timeout_secs={args.test_timeout_secs}', + f'start_timeout_secs={args.start_timeout_secs}' + ] + + if args.change_id: + tests_args.append(f'change_id={args.change_id}') + if args.test_attempts: + tests_args.append(f'test_attempts={args.test_attempts}') + if args.dimension: + tests_args.extend( + f'dimension_{dimension}' for dimension in args.dimension + ) + + base_params = [f'push_files=test_runtime_deps:{_CHROMIUM_TEST_ROOT}'] + + if args.gcs_result_path: + base_params.append(f'gcs_result_path={args.gcs_result_path}') + + for gtest_target in platform_data['gtest_targets']: + print(f" Processing gtest_target: {gtest_target}") + + apk_file = f"{args.gcs_archive_path}/{gtest_target}-debug.apk" + test_runtime_deps = f"{args.gcs_archive_path}/{gtest_target}_deps.tar.gz" + + files = [ + f'test_apk={apk_file}', + f'build_apk={apk_file}', + f'test_runtime_deps={test_runtime_deps}' + ] + + gtest_params = [ + f'gtest_xml_file_on_device={_ON_DEVICE_OUTPUT_FILES_DIR}/{gtest_target}_result.xml', + f'gcs_result_filename={gtest_target}_result.xml', + f'gcs_log_filename={gtest_target}_log.txt' + ] + + gtest_filter = _get_gtest_filters(args.filter_json_dir, gtest_target) + + test_cmd_args = [ + f'command_line_args=--gtest_output=xml:{_ON_DEVICE_OUTPUT_FILES_DIR}/{gtest_target}_result.xml', + f'--gtest_filter={gtest_filter}' + ] + + test_request = { + 'test_args': tests_args, + 'test_cmd_args': test_cmd_args, + 'files': files, + 'params': base_params + gtest_params, + 'device_type': platform_data.get('test_dimensions', {}).get( + 'gtest_device' + ), + 'device_pool': platform_data.get('test_dimensions', {}).get( + 'gtest_lab' + ), + } + + test_requests.append(test_request) + print(f' Created test_request: {test_request}') + + print(f'test_requests: {test_requests}') + return test_requests + + +def main() -> int: # Add a return type hint + """Main routine for the on-device tests gateway client.""" + + logging.basicConfig( + level=logging.INFO, format='[%(filename)s:%(lineno)s] %(message)s' + ) + print('Starting main routine') + + parser = argparse.ArgumentParser( + description='Client for interacting with the On-Device Tests gateway.', + epilog=( + 'Example:' + 'python3 -u cobalt/tools/on_device_tests_gateway_client.py' + '--platform_json "${GITHUB_WORKSPACE}/src/.github/config/' + '${{ matrix.platform}}.json"' + '--filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/' + '${{ matrix.platform}}"' + '--token ${GITHUB_TOKEN}' + '--change_id ${GITHUB_PR_NUMBER:-postsubmit}' + '--label builder-${{ matrix.platform }}' + '--label builder_url-${GITHUB_RUN_URL}' + '--label github' + '--label ${GITHUB_EVENT_NAME}' + '--label ${GITHUB_WORKFLOW}' + '--label actor-${GITHUB_ACTOR}' + '--label actor_id-${GITHUB_ACTOR_ID}' + '--label triggering_actor-${GITHUB_TRIGGERING_ACTOR}' + '--label sha-${GITHUB_SHA}' + '--label repository-${GITHUB_REPO}' + '--label author-${GITHUB_PR_HEAD_USER_LOGIN:-' + '$GITHUB_COMMIT_AUTHOR_USERNAME}' + '--label author_id-${GITHUB_PR_HEAD_USER_ID:-' + '$GITHUB_COMMIT_AUTHOR_EMAIL}' + '--dimension host_name=regex:maneki-mhserver-05.*' + '${DIMENSION:+"--dimension" "$DIMENSION"}' + '${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" ' + '"$ON_DEVICE_TEST_ATTEMPTS"}' + '--gcs_archive_path "${GCS_ARTIFACTS_PATH}"' + '--gcs_result_path "${GCS_RESULTS_PATH}"' + 'trigger' + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + # Authentication + parser.add_argument( + '-t', + '--token', + type=str, + required=True, + help='On Device Tests authentication token', + ) + + # General options + parser.add_argument( + '-i', + '--change_id', + type=str, + help=( + 'ChangeId that triggered this test, if any. Saved with performance' + ' test results.' + ), + ) + + subparsers = parser.add_subparsers( + dest='action', help='On-Device tests commands', required=True + ) + + # Trigger command + trigger_parser = subparsers.add_parser( + 'trigger', + help='Trigger On-Device tests', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Group trigger arguments + trigger_args = trigger_parser.add_argument_group('Trigger Arguments') + trigger_args.add_argument( + '-pf', + '--platform_json', + type=str, + help='Platform-specific JSON file containing the list of target tests.', + ) + trigger_parser.add_argument( + '--filter_json_dir', + type=str, + help='Directory containing filter JSON files for test selection.', + ) + trigger_parser.add_argument( + '-l', + '--label', + type=str, + action='append', + help='Additional labels to assign to the test.', + ) + trigger_parser.add_argument( + '--dimension', + type=str, + action='append', + help=( + 'On-Device Tests dimension used to select a device. ' + 'Must have the following form: =. ' + 'E.G. "release_version=regex:10.*"' + ), + ) + trigger_parser.add_argument( + '--test_attempts', + type=str, + default='1', + help='The maximum number of times a test can retry.', + ) + trigger_args.add_argument( + '-a', + '--archive_path', + type=str, + required=True, + help='Path to Chrobalt archive to be tested. Must be on GCS.', + ) + trigger_parser.add_argument( + '--gcs_result_path', + type=str, + help='GCS URL where test result files should be uploaded.', + ) + trigger_parser.add_argument( + '--job_timeout_sec', + type=str, + default='1800', + help='Timeout in seconds for the job (default: 1800 seconds).', + ) + trigger_parser.add_argument( + '--test_timeout_sec', + type=str, + default='1800', + help='Timeout in seconds for the test (default: 1800 seconds).', + ) + trigger_parser.add_argument( + '--start_timeout_sec', + type=str, + default='180', + help='Timeout in seconds for the test to start (default: 180 seconds).', + ) + + # Watch command + watch_parser = subparsers.add_parser( + 'watch', help='Watch a previously triggered On-Device test' + ) + watch_parser.add_argument( + 'session_id', + type=str, + help=( + 'Session ID of a previously triggered Mobile Harness test. ' + 'The test will be watched until it completes.' + ), + ) + + args = parser.parse_args() + + if not args.platform_json: + print( + "Error: The '--platform_json' argument is required. " + 'Please provide the path to the Platform JSON file.' + ) + exit(1) # Exit with an error code + if not args.filter_json_dir: + print( + "Error: The '--filter_json_dir' argument is required. " + 'Please provide the directory containing filter JSON files.' + ) + exit(1) # Exit with an error code + if not args.gcs_archive_path: + print( + "Error: The '--gcs_archive_path' argument is required. " + 'Please provide the GCS archive path info.' + ) + exit(1) # Exit with an error code + + test_requests = _process_test_requests(args) + client = OnDeviceTestsGatewayClient() + + try: + if args.action == 'trigger': + client.run_trigger_command(args, test_requests) + else: + client.run_watch_command(workdir=_WORK_DIR, args=args) + except grpc.RpcError as e: + logging.exception('gRPC error occurred:') # Log the full traceback + print(f'Error: {e}') + return e.code().value # Return the error code + + return 0 # Indicate successful execution + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/cobalt/tools/on_device_tests_gateway.proto b/cobalt/tools/on_device_tests_gateway.proto index 933b5818faa4e..61f3da41f9599 100644 --- a/cobalt/tools/on_device_tests_gateway.proto +++ b/cobalt/tools/on_device_tests_gateway.proto @@ -20,38 +20,36 @@ package on_device_tests_gateway; service on_device_tests_gateway { // A dumb proxy RPC service that passes user defined command line options // to the on-device tests gateway and streams back output in real time. - rpc exec_command (OnDeviceTestsCommand) returns (stream OnDeviceTestsResponse) { - } + rpc exec_command(OnDeviceTestsCommand) + returns (stream OnDeviceTestsResponse) {} - rpc exec_watch_command (OnDeviceTestsWatchCommand) returns (stream OnDeviceTestsResponse) { - } + // A dumb proxy RPC service that passes user defined command line options + // to the on-device tests gateway and streams back output in real time. + rpc exec_watch_command(OnDeviceTestsWatchCommand) + returns (stream OnDeviceTestsResponse) {} } // Working directory and command line arguments to be passed to the gateway. message OnDeviceTestsCommand { - string workdir = 1; string token = 2; - string platform = 3; - string archive_path = 4; - repeated string labels = 5; - string change_id = 6; - bool dry_run = 7; - repeated string dimension = 8; - string test_attempts = 9; - string retry_level = 10; - string start_timeout = 11; - string test_timeout = 12; - string gcs_result_path = 13; - repeated ApkTest apk_tests = 14; + repeated string labels = 8; + repeated TestRequest test_requests = 24; } -// apk_test details -message ApkTest { - string test_target = 1; - string apk_path = 2; - string device_model = 3; - string device_pool = 4; - string gtest_filters = 5; +message TestRequest { + // Args picked up by MH, e.g. "name1=value", "name2=value. + repeated string test_args = 1; + // Args sent to test binary/apk, e.g. "--arg1", "--arg2=value". + repeated string test_cmd_args = 2; + // Files to send to device, e.g. "build_apk=/bigstore/bucket/test.apk", + // "test_apk=/bigstore/bucket/test.apk", + repeated string files = 3; + // e.g. "gcs_result_path=gs://some/gcs/path" + repeated string params = 4; + // "sabrina" or "boreal" + string device_type = 5; + // "shared" or "maneki" + string device_pool = 6; } // Working directory and command line arguments to be passed to the gateway. @@ -69,3 +67,4 @@ message OnDeviceTestsResponse { // Next ID: 2 string response = 1; } + diff --git a/cobalt/tools/on_device_tests_gateway_client.py b/cobalt/tools/on_device_tests_gateway_client.py index 2cfad00758fce..fb258896eab7b 100644 --- a/cobalt/tools/on_device_tests_gateway_client.py +++ b/cobalt/tools/on_device_tests_gateway_client.py @@ -16,27 +16,25 @@ """gRPC On-device Tests Gateway client.""" import argparse +import json import logging import sys -import os -import json import grpc - import on_device_tests_gateway_pb2 import on_device_tests_gateway_pb2_grpc + _WORK_DIR = '/on_device_tests_gateway' -# Comment out the next three lines for local testing -_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( - 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') +# For local testing, set: _ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') +# _ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( +# 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') +_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') _ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' -# Uncomment the next two lines for local testing -#_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') -#_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '12345' - +_ON_DEVICE_OUTPUT_FILES_DIR = '/sdcard/Download' +_CHROMIUM_TEST_ROOT = '/sdcard/chromium_tests_root' class OnDeviceTestsGatewayClient(): """On-device tests Gateway Client class.""" @@ -54,27 +52,18 @@ def __init__(self): self.stub = on_device_tests_gateway_pb2_grpc.on_device_tests_gatewayStub( self.channel) - def run_trigger_command(self, workdir: str, args: argparse.Namespace, apk_tests = None): + def run_trigger_command(self, args: argparse.Namespace, test_requests): """Calls On-Device Tests service and passing given parameters to it. Args: - workdir (str): Current script workdir. args (Namespace): Arguments passed in command line. + test_requests (list): A list of test requests. """ for response_line in self.stub.exec_command( on_device_tests_gateway_pb2.OnDeviceTestsCommand( - workdir=workdir, token=args.token, - platform=args.platform, - archive_path=args.archive_path, labels=args.label, - gcs_result_path=args.gcs_result_path, - change_id=args.change_id, - dry_run=args.dry_run, - dimension=args.dimension or [], - test_attempts=args.test_attempts, - retry_level=args.retry_level, - apk_tests=apk_tests, + test_requests=test_requests, )): print(response_line.response) @@ -96,15 +85,16 @@ def run_watch_command(self, workdir: str, args: argparse.Namespace): print(response_line.response) + def _read_json_config(filename): - """ - Reads and parses data from a JSON configuration file. + """Reads and parses data from a JSON configuration file. Args: filename: The name of the JSON configuration file. Returns: - A list of dictionaries, where each dictionary represents a test configuration. + A list of dictionaries, where each dictionary represents a test + configuration. """ try: with open(filename, 'r') as f: @@ -117,258 +107,307 @@ def _read_json_config(filename): print(f" Invalid JSON format in '{filename}'.") return None -def _get_platform_json_file(platform): - """Constructs the path to the platform JSON configuration file. + +def _get_gtest_filters(filter_json_dir, gtest_target): + """Retrieves gtest filters for a given target. Args: - platform: The name of the platform. + filter_json_dir: Directory containing filter JSON files. + gtest_target: The name of the gtest target. Returns: - The absolute path to the platform JSON file. + A string containing the gtest filters. """ - return os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - ".github", - "config", - f"{platform}.json" - ) - -def _get_tests_filter_json_file(platform, gtest_target): - """Constructs the path to the gtest filter JSON file. + gtest_filters = '*' + filter_json_file = f'{filter_json_dir}/{gtest_target}_filter.json' + print(f" gtest_filter_json_file = {filter_json_file}") + filter_data = _read_json_config(filter_json_file) + if filter_data: + print(f" Loaded filter data: {filter_data}") + failing_tests = ':'.join(filter_data.get('failing_tests', [])) + if failing_tests: + gtest_filters += ':-' + failing_tests + print(f" gtest_filters = {gtest_filters}") + else: + print(' This gtest_target does not have gtest_filters specified') + return gtest_filters + + +def _process_test_requests(args): + """Processes test requests from the given arguments. + + Constructs a list of test requests based on the provided arguments, + including test arguments, command arguments, files, parameters, + and device information. Args: - platform: The name of the platform. - gtest_target: The name of the gtest target. + args: The parsed command-line arguments. Returns: - The absolute path to the gtest filter JSON file. + A list of test request dictionaries. """ - return os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "cobalt", - "testing", - platform, - f"{gtest_target}_filter.json" - ) + test_requests = [] + platform_data = _read_json_config(args.platform_json) + + tests_args = [ + f'job_timeout_secs={args.job_timeout_secs}', + f'test_timeout_secs={args.test_timeout_secs}', + f'start_timeout_secs={args.start_timeout_secs}' + ] + + if args.change_id: + tests_args.append(f'change_id={args.change_id}') + if args.test_attempts: + tests_args.append(f'test_attempts={args.test_attempts}') + if args.dimension: + tests_args.extend( + f'dimension_{dimension}' for dimension in args.dimension + ) -def _process_apk_tests(args): - """Processes APK tests based on provided arguments. + base_params = [f'push_files=test_runtime_deps:{_CHROMIUM_TEST_ROOT}'] - Args: - args: An argparse.Namespace object containing the following attributes: - platform: The platform for the tests. - platform_json: Optional path to a platform JSON file. - blaze_targets: A list of Blaze targets. - archive_path: Path to the archive containing APKs. - - Returns: - A list of dictionaries, where each dictionary represents an APK test - with keys like "test_target", "device_model", "device_pool", - "apk_path", and "gtest_filters". - """ + if args.gcs_result_path: + base_params.append(f'gcs_result_path={args.gcs_result_path}') - apk_tests = [] - platform_json_file = args.platform_json or _default_platform_json_file(args.platform) - print(f"The platform_json_file is '{platform_json_file}'") - platform_data = _read_json_config(platform_json_file) - print(f"Loaded platform data: {platform_data}") # Added verbosity - - for target in args.blaze_targets: - print(f"Processing Blaze target: {target}") # Added verbosity - for gtest_target in platform_data["gtest_targets"]: - print(f" Processing gtest_target: {gtest_target}") # Added verbosity - apk_test = { - "test_target": target, - "device_model": platform_data["gtest_device"], - "device_pool": platform_data["gtest_lab"], - "apk_path": f"{args.archive_path}/{gtest_target}-debug.apk", - "gtest_filters": _get_gtest_filters(args.platform, gtest_target) - } - apk_tests.append(apk_test) - print(f" Created apk_test: {apk_test}") # Added verbosity - - print(f"apk_tests: {apk_tests}") - return apk_tests - - -def _get_gtest_filters(platform, gtest_target): - """Retrieves gtest filters for a given target. + for gtest_target in platform_data['gtest_targets']: + print(f" Processing gtest_target: {gtest_target}") - Args: - platform: The platform for the tests. - gtest_target: The name of the gtest target. + apk_file = f"{args.gcs_archive_path}/{gtest_target}-debug.apk" + test_runtime_deps = f"{args.gcs_archive_path}/{gtest_target}_deps.tar.gz" - Returns: - A string representing the gtest filters. - """ + files = [ + f'test_apk={apk_file}', + f'build_apk={apk_file}', + f'test_runtime_deps={test_runtime_deps}' + ] - gtest_filters = "*" - filter_json_file = _get_tests_filter_json_file(platform, gtest_target) - print(f" gtest_filter_json_file = {filter_json_file}") - filter_data = _read_json_config(filter_json_file) - if filter_data: - print(f" Loaded filter data: {filter_data}") # Added verbosity - failing_tests = ":".join(filter_data.get("failing_tests", [])) - if failing_tests: - gtest_filters += ":-" + failing_tests - print(f" gtest_filters = {gtest_filters}") - else: - print(f" This gtest_target does not have gtest_filters specified") - return gtest_filters + gtest_params = [ + f'gtest_xml_file_on_device={_ON_DEVICE_OUTPUT_FILES_DIR}/{gtest_target}_result.xml', + f'gcs_result_filename={gtest_target}_result.xml', + f'gcs_log_filename={gtest_target}_log.txt' + ] + gtest_filter = _get_gtest_filters(args.filter_json_dir, gtest_target) -def main(): - """Main routine for the on-device tests gateway client.""" + test_cmd_args = [ + f'command_line_args=--gtest_output=xml:{_ON_DEVICE_OUTPUT_FILES_DIR}/{gtest_target}_result.xml', + f'--gtest_filter={gtest_filter}' + ] - logging.basicConfig( - level=logging.INFO, - format='[%(filename)s:%(lineno)s] %(message)s' - ) - print('Starting main routine') - - parser = argparse.ArgumentParser( - description="Client for interacting with the On-Device Tests gateway.", - epilog=( - 'Example: ./on_device_tests_gateway_client.py trigger ' - '--token token1 ' - '--platform android-arm ' - '--archive_path /bigstore/yt-temp ' - '--blaze_targets //experimental/cobalt/chrobalt_poc:chrobalt_unit_tests_maneki_sabrina //experimental/cobalt/chrobalt_poc:chrobalt_unit_tests_shared_boreal ' - '--dimension host_name=regex:maneki-mhserver-05.*' + print(f" test dimensions is: {platform_data.get('test_dimensions', {})}") + + test_request = { + 'test_args': tests_args, + 'test_cmd_args': test_cmd_args, + 'files': files, + 'params': base_params + gtest_params, + 'device_type': platform_data.get('test_dimensions', {}).get( + 'gtest_device' ), - formatter_class=argparse.RawDescriptionHelpFormatter - ) + 'device_pool': platform_data.get('test_dimensions', {}).get( + 'gtest_lab' + ), + } - # Authentication - parser.add_argument( - '-t', - '--token', - type=str, - required=True, - help='On Device Tests authentication token' - ) + test_requests.append(test_request) + print(f' Created test_request: {test_request}') - # General options - parser.add_argument( - '--dry_run', - action='store_true', - help='Show what would be done without actually doing it.' - ) - parser.add_argument( - '-i', - '--change_id', - type=str, - help='ChangeId that triggered this test, if any. Saved with performance test results.' - ) + print(f'test_requests: {test_requests}') + return test_requests - subparsers = parser.add_subparsers( - dest='action', - help='On-Device tests commands', - required=True - ) - # Trigger command - trigger_parser = subparsers.add_parser( - 'trigger', - help='Trigger On-Device tests', - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - trigger_parser.add_argument( - '-p', - '--platform', - type=str, - required=True, - help='Platform this test was built for.' - ) - trigger_parser.add_argument( - '-pf', - '--platform_json', - type=str, - help='Platform-specific JSON file containing the list of target tests.' - ) - trigger_parser.add_argument( - '-a', - '--archive_path', - type=str, - required=True, - help='Path to Chrobalt archive to be tested. Must be on GCS.' - ) - trigger_parser.add_argument( - '-l', - '--label', - type=str, - action='append', - default=[], - help='Additional labels to assign to the test.' - ) - trigger_parser.add_argument( - '--gcs_result_path', - type=str, - help='GCS URL where test result files should be uploaded.' - ) - trigger_parser.add_argument( - '--dimension', - type=str, - action='append', - help=( - 'On-Device Tests dimension used to select a device. ' - 'Must have the following form: =. ' - 'E.G. "release_version=regex:10.*"' - ) - ) - trigger_parser.add_argument( - '--test_attempts', - type=str, - default='1', - help='The maximum number of times a test could retry.' +def main() -> int: # Add a return type hint + """Main routine for the on-device tests gateway client.""" + + logging.basicConfig( + level=logging.INFO, format='[%(filename)s:%(lineno)s] %(message)s' + ) + print('Starting main routine') + + parser = argparse.ArgumentParser( + description='Client for interacting with the On-Device Tests gateway.', + epilog=( + 'Example:' + 'python3 -u cobalt/tools/on_device_tests_gateway_client.py' + '--platform_json "${GITHUB_WORKSPACE}/src/.github/config/' + '${{ matrix.platform}}.json"' + '--filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/' + '${{ matrix.platform}}"' + '--token ${GITHUB_TOKEN}' + '--change_id ${GITHUB_PR_NUMBER:-postsubmit}' + '--label builder-${{ matrix.platform }}' + '--label builder_url-${GITHUB_RUN_URL}' + '--label github' + '--label ${GITHUB_EVENT_NAME}' + '--label ${GITHUB_WORKFLOW}' + '--label actor-${GITHUB_ACTOR}' + '--label actor_id-${GITHUB_ACTOR_ID}' + '--label triggering_actor-${GITHUB_TRIGGERING_ACTOR}' + '--label sha-${GITHUB_SHA}' + '--label repository-${GITHUB_REPO}' + '--label author-${GITHUB_PR_HEAD_USER_LOGIN:-' + '$GITHUB_COMMIT_AUTHOR_USERNAME}' + '--label author_id-${GITHUB_PR_HEAD_USER_ID:-' + '$GITHUB_COMMIT_AUTHOR_EMAIL}' + '--dimension host_name=regex:maneki-mhserver-05.*' + '${DIMENSION:+"--dimension" "$DIMENSION"}' + '${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" ' + '"$ON_DEVICE_TEST_ATTEMPTS"}' + '--gcs_archive_path "${GCS_ARTIFACTS_PATH}"' + '--gcs_result_path "${GCS_RESULTS_PATH}"' + 'trigger' + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + # Authentication + parser.add_argument( + '-t', + '--token', + type=str, + required=True, + help='On Device Tests authentication token', + ) + + # General options + parser.add_argument( + '-i', + '--change_id', + type=str, + help=( + 'ChangeId that triggered this test, if any. Saved with performance' + ' test results.' + ), + ) + + subparsers = parser.add_subparsers( + dest='action', help='On-Device tests commands', required=True + ) + + # Trigger command + trigger_parser = subparsers.add_parser( + 'trigger', + help='Trigger On-Device tests', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Group trigger arguments + trigger_args = trigger_parser.add_argument_group('Trigger Arguments') + trigger_args.add_argument( + '-pf', + '--platform_json', + type=str, + required=True, + help='Platform-specific JSON file containing the list of target tests.', + ) + trigger_parser.add_argument( + '--filter_json_dir', + type=str, + required=True, + help='Directory containing filter JSON files for test selection.', + ) + trigger_parser.add_argument( + '-l', + '--label', + type=str, + action='append', + help='Additional labels to assign to the test.', + ) + trigger_parser.add_argument( + '--dimension', + type=str, + action='append', + help=( + 'On-Device Tests dimension used to select a device. ' + 'Must have the following form: =. ' + 'E.G. "release_version=regex:10.*"' + ), + ) + trigger_parser.add_argument( + '--test_attempts', + type=str, + default='1', + help='The maximum number of times a test can retry.', + ) + trigger_args.add_argument( + '-a', + '--gcs_archive_path', + type=str, + required=True, + help='Path to Chrobalt archive to be tested. Must be on GCS.', + ) + trigger_parser.add_argument( + '--gcs_result_path', + type=str, + help='GCS URL where test result files should be uploaded.', + ) + trigger_parser.add_argument( + '--job_timeout_secs', + type=str, + default='1800', + help='Timeout in seconds for the job (default: 1800 seconds).', + ) + trigger_parser.add_argument( + '--test_timeout_secs', + type=str, + default='1800', + help='Timeout in seconds for the test (default: 1800 seconds).', + ) + trigger_parser.add_argument( + '--start_timeout_secs', + type=str, + default='180', + help='Timeout in seconds for the test to start (default: 180 seconds).', + ) + + # Watch command + watch_parser = subparsers.add_parser( + 'watch', help='Watch a previously triggered On-Device test' + ) + watch_parser.add_argument( + 'session_id', + type=str, + help=( + 'Session ID of a previously triggered Mobile Harness test. ' + 'The test will be watched until it completes.' + ), + ) + + args = parser.parse_args() + + if not args.platform_json: + print( + "Error: The '--platform_json' argument is required. " + 'Please provide the path to the Platform JSON file.' ) - trigger_parser.add_argument( - '--blaze_targets', - nargs="+", - type=str, - required=True, - help='A list of Blaze targets to run.' + exit(1) # Exit with an error code + if not args.filter_json_dir: + print( + "Error: The '--filter_json_dir' argument is required. " + 'Please provide the directory containing filter JSON files.' ) - trigger_parser.add_argument( - '--retry_level', - type=str, - default='ERROR', - choices=['ERROR', 'FAIL'], - help=( - 'The retry level of Mobile Harness job. ' - 'ERROR to retry for MH errors, ' - 'FAIL to retry for failing tests (and MH errors).' - ) + exit(1) # Exit with an error code + if not args.gcs_archive_path: + print( + "Error: The '--gcs_archive_path' argument is required. " + 'Please provide the GCS archive path info.' ) + exit(1) # Exit with an error code - # Watch command - watch_parser = subparsers.add_parser( - 'watch', - help='Watch a previously triggered On-Device test' - ) - watch_parser.add_argument( - 'session_id', - type=str, - help=( - 'Session ID of a previously triggered Mobile Harness test. ' - 'The test will be watched until it completes.' - ) - ) + test_requests = _process_test_requests(args) + client = OnDeviceTestsGatewayClient() + + try: + if args.action == 'trigger': + client.run_trigger_command(args, test_requests) + else: + client.run_watch_command(workdir=_WORK_DIR, args=args) + except grpc.RpcError as e: + logging.exception('gRPC error occurred:') # Log the full traceback + print(f'Error: {e}') + return e.code().value # Return the error code - args = parser.parse_args() - apk_tests = _process_apk_tests(args) - - client = OnDeviceTestsGatewayClient() - try: - if args.action == 'trigger': - client.run_trigger_command(workdir=_WORK_DIR, args=args, apk_tests=apk_tests) - else: - client.run_watch_command(workdir=_WORK_DIR, args=args) - except grpc.RpcError as e: - print(e) - return e.code().value + return 0 # Indicate successful execution if __name__ == '__main__': diff --git a/cobalt/tools/on_device_tests_gateway_pb2.py b/cobalt/tools/on_device_tests_gateway_pb2.py new file mode 100644 index 0000000000000..ca42ce7d76f74 --- /dev/null +++ b/cobalt/tools/on_device_tests_gateway_pb2.py @@ -0,0 +1,297 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: on_device_tests_gateway.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='on_device_tests_gateway.proto', + package='on_device_tests_gateway', + syntax='proto3', + serialized_pb=_b('\n\x1don_device_tests_gateway.proto\x12\x17on_device_tests_gateway\"r\n\x14OnDeviceTestsCommand\x12\r\n\x05token\x18\x02 \x01(\t\x12\x0e\n\x06labels\x18\x08 \x03(\t\x12;\n\rtest_requests\x18\x18 \x03(\x0b\x32$.on_device_tests_gateway.TestRequest\"\x80\x01\n\x0bTestRequest\x12\x11\n\ttest_args\x18\x01 \x03(\t\x12\x15\n\rtest_cmd_args\x18\x02 \x03(\t\x12\r\n\x05\x66iles\x18\x03 \x03(\t\x12\x0e\n\x06params\x18\x04 \x03(\t\x12\x13\n\x0b\x64\x65vice_type\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65vice_pool\x18\x06 \x01(\t\"s\n\x19OnDeviceTestsWatchCommand\x12\x0f\n\x07workdir\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\t\x12\x12\n\nsession_id\x18\x03 \x01(\t\x12\x11\n\tchange_id\x18\x04 \x01(\t\x12\x0f\n\x07\x64ry_run\x18\x05 \x01(\x08\")\n\x15OnDeviceTestsResponse\x12\x10\n\x08response\x18\x01 \x01(\t2\x8a\x02\n\x17on_device_tests_gateway\x12q\n\x0c\x65xec_command\x12-.on_device_tests_gateway.OnDeviceTestsCommand\x1a..on_device_tests_gateway.OnDeviceTestsResponse\"\x00\x30\x01\x12|\n\x12\x65xec_watch_command\x12\x32.on_device_tests_gateway.OnDeviceTestsWatchCommand\x1a..on_device_tests_gateway.OnDeviceTestsResponse\"\x00\x30\x01\x62\x06proto3') +) + + + + +_ONDEVICETESTSCOMMAND = _descriptor.Descriptor( + name='OnDeviceTestsCommand', + full_name='on_device_tests_gateway.OnDeviceTestsCommand', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='token', full_name='on_device_tests_gateway.OnDeviceTestsCommand.token', index=0, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='labels', full_name='on_device_tests_gateway.OnDeviceTestsCommand.labels', index=1, + number=8, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='test_requests', full_name='on_device_tests_gateway.OnDeviceTestsCommand.test_requests', index=2, + number=24, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=58, + serialized_end=172, +) + + +_TESTREQUEST = _descriptor.Descriptor( + name='TestRequest', + full_name='on_device_tests_gateway.TestRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='test_args', full_name='on_device_tests_gateway.TestRequest.test_args', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='test_cmd_args', full_name='on_device_tests_gateway.TestRequest.test_cmd_args', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='files', full_name='on_device_tests_gateway.TestRequest.files', index=2, + number=3, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='params', full_name='on_device_tests_gateway.TestRequest.params', index=3, + number=4, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device_type', full_name='on_device_tests_gateway.TestRequest.device_type', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device_pool', full_name='on_device_tests_gateway.TestRequest.device_pool', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=175, + serialized_end=303, +) + + +_ONDEVICETESTSWATCHCOMMAND = _descriptor.Descriptor( + name='OnDeviceTestsWatchCommand', + full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='workdir', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.workdir', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='token', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.token', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='session_id', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.session_id', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='change_id', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.change_id', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='dry_run', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.dry_run', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=305, + serialized_end=420, +) + + +_ONDEVICETESTSRESPONSE = _descriptor.Descriptor( + name='OnDeviceTestsResponse', + full_name='on_device_tests_gateway.OnDeviceTestsResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='response', full_name='on_device_tests_gateway.OnDeviceTestsResponse.response', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=422, + serialized_end=463, +) + +_ONDEVICETESTSCOMMAND.fields_by_name['test_requests'].message_type = _TESTREQUEST +DESCRIPTOR.message_types_by_name['OnDeviceTestsCommand'] = _ONDEVICETESTSCOMMAND +DESCRIPTOR.message_types_by_name['TestRequest'] = _TESTREQUEST +DESCRIPTOR.message_types_by_name['OnDeviceTestsWatchCommand'] = _ONDEVICETESTSWATCHCOMMAND +DESCRIPTOR.message_types_by_name['OnDeviceTestsResponse'] = _ONDEVICETESTSRESPONSE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +OnDeviceTestsCommand = _reflection.GeneratedProtocolMessageType('OnDeviceTestsCommand', (_message.Message,), dict( + DESCRIPTOR = _ONDEVICETESTSCOMMAND, + __module__ = 'on_device_tests_gateway_pb2' + # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsCommand) + )) +_sym_db.RegisterMessage(OnDeviceTestsCommand) + +TestRequest = _reflection.GeneratedProtocolMessageType('TestRequest', (_message.Message,), dict( + DESCRIPTOR = _TESTREQUEST, + __module__ = 'on_device_tests_gateway_pb2' + # @@protoc_insertion_point(class_scope:on_device_tests_gateway.TestRequest) + )) +_sym_db.RegisterMessage(TestRequest) + +OnDeviceTestsWatchCommand = _reflection.GeneratedProtocolMessageType('OnDeviceTestsWatchCommand', (_message.Message,), dict( + DESCRIPTOR = _ONDEVICETESTSWATCHCOMMAND, + __module__ = 'on_device_tests_gateway_pb2' + # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsWatchCommand) + )) +_sym_db.RegisterMessage(OnDeviceTestsWatchCommand) + +OnDeviceTestsResponse = _reflection.GeneratedProtocolMessageType('OnDeviceTestsResponse', (_message.Message,), dict( + DESCRIPTOR = _ONDEVICETESTSRESPONSE, + __module__ = 'on_device_tests_gateway_pb2' + # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsResponse) + )) +_sym_db.RegisterMessage(OnDeviceTestsResponse) + + + +_ON_DEVICE_TESTS_GATEWAY = _descriptor.ServiceDescriptor( + name='on_device_tests_gateway', + full_name='on_device_tests_gateway.on_device_tests_gateway', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=466, + serialized_end=732, + methods=[ + _descriptor.MethodDescriptor( + name='exec_command', + full_name='on_device_tests_gateway.on_device_tests_gateway.exec_command', + index=0, + containing_service=None, + input_type=_ONDEVICETESTSCOMMAND, + output_type=_ONDEVICETESTSRESPONSE, + options=None, + ), + _descriptor.MethodDescriptor( + name='exec_watch_command', + full_name='on_device_tests_gateway.on_device_tests_gateway.exec_watch_command', + index=1, + containing_service=None, + input_type=_ONDEVICETESTSWATCHCOMMAND, + output_type=_ONDEVICETESTSRESPONSE, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_ON_DEVICE_TESTS_GATEWAY) + +DESCRIPTOR.services_by_name['on_device_tests_gateway'] = _ON_DEVICE_TESTS_GATEWAY + +# @@protoc_insertion_point(module_scope) diff --git a/cobalt/tools/on_device_tests_gateway_pb2_grpc.py b/cobalt/tools/on_device_tests_gateway_pb2_grpc.py new file mode 100644 index 0000000000000..284d31d4b2cac --- /dev/null +++ b/cobalt/tools/on_device_tests_gateway_pb2_grpc.py @@ -0,0 +1,65 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import on_device_tests_gateway_pb2 as on__device__tests__gateway__pb2 + + +class on_device_tests_gatewayStub(object): + """Interface exported by the server. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.exec_command = channel.unary_stream( + '/on_device_tests_gateway.on_device_tests_gateway/exec_command', + request_serializer=on__device__tests__gateway__pb2.OnDeviceTestsCommand.SerializeToString, + response_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.FromString, + ) + self.exec_watch_command = channel.unary_stream( + '/on_device_tests_gateway.on_device_tests_gateway/exec_watch_command', + request_serializer=on__device__tests__gateway__pb2.OnDeviceTestsWatchCommand.SerializeToString, + response_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.FromString, + ) + + +class on_device_tests_gatewayServicer(object): + """Interface exported by the server. + """ + + def exec_command(self, request, context): + """A dumb proxy RPC service that passes user defined command line options + to the on-device tests gateway and streams back output in real time. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def exec_watch_command(self, request, context): + """A dumb proxy RPC service that passes user defined command line options + to the on-device tests gateway and streams back output in real time. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_on_device_tests_gatewayServicer_to_server(servicer, server): + rpc_method_handlers = { + 'exec_command': grpc.unary_stream_rpc_method_handler( + servicer.exec_command, + request_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsCommand.FromString, + response_serializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.SerializeToString, + ), + 'exec_watch_command': grpc.unary_stream_rpc_method_handler( + servicer.exec_watch_command, + request_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsWatchCommand.FromString, + response_serializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'on_device_tests_gateway.on_device_tests_gateway', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) From fbd00a824f6d550da47afa420c221951d53e8b51 Mon Sep 17 00:00:00 2001 From: Felipe Mira Date: Thu, 16 Jan 2025 21:50:15 +0000 Subject: [PATCH 42/61] New Modifications to ODT Gateway code --- .../tools/on_device_tests_gateway_client.py | 8 +- cobalt/tools/on_device_tests_gateway_pb2.py | 297 ------------------ .../tools/on_device_tests_gateway_pb2_grpc.py | 65 ---- 3 files changed, 2 insertions(+), 368 deletions(-) delete mode 100644 cobalt/tools/on_device_tests_gateway_pb2.py delete mode 100644 cobalt/tools/on_device_tests_gateway_pb2_grpc.py diff --git a/cobalt/tools/on_device_tests_gateway_client.py b/cobalt/tools/on_device_tests_gateway_client.py index fb258896eab7b..4d0777bd3491a 100644 --- a/cobalt/tools/on_device_tests_gateway_client.py +++ b/cobalt/tools/on_device_tests_gateway_client.py @@ -28,9 +28,8 @@ _WORK_DIR = '/on_device_tests_gateway' # For local testing, set: _ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') -# _ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( -# 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') -_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') +_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( + 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') _ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' _ON_DEVICE_OUTPUT_FILES_DIR = '/sdcard/Download' @@ -194,8 +193,6 @@ def _process_test_requests(args): f'--gtest_filter={gtest_filter}' ] - print(f" test dimensions is: {platform_data.get('test_dimensions', {})}") - test_request = { 'test_args': tests_args, 'test_cmd_args': test_cmd_args, @@ -404,7 +401,6 @@ def main() -> int: # Add a return type hint client.run_watch_command(workdir=_WORK_DIR, args=args) except grpc.RpcError as e: logging.exception('gRPC error occurred:') # Log the full traceback - print(f'Error: {e}') return e.code().value # Return the error code return 0 # Indicate successful execution diff --git a/cobalt/tools/on_device_tests_gateway_pb2.py b/cobalt/tools/on_device_tests_gateway_pb2.py deleted file mode 100644 index ca42ce7d76f74..0000000000000 --- a/cobalt/tools/on_device_tests_gateway_pb2.py +++ /dev/null @@ -1,297 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: on_device_tests_gateway.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='on_device_tests_gateway.proto', - package='on_device_tests_gateway', - syntax='proto3', - serialized_pb=_b('\n\x1don_device_tests_gateway.proto\x12\x17on_device_tests_gateway\"r\n\x14OnDeviceTestsCommand\x12\r\n\x05token\x18\x02 \x01(\t\x12\x0e\n\x06labels\x18\x08 \x03(\t\x12;\n\rtest_requests\x18\x18 \x03(\x0b\x32$.on_device_tests_gateway.TestRequest\"\x80\x01\n\x0bTestRequest\x12\x11\n\ttest_args\x18\x01 \x03(\t\x12\x15\n\rtest_cmd_args\x18\x02 \x03(\t\x12\r\n\x05\x66iles\x18\x03 \x03(\t\x12\x0e\n\x06params\x18\x04 \x03(\t\x12\x13\n\x0b\x64\x65vice_type\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65vice_pool\x18\x06 \x01(\t\"s\n\x19OnDeviceTestsWatchCommand\x12\x0f\n\x07workdir\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\t\x12\x12\n\nsession_id\x18\x03 \x01(\t\x12\x11\n\tchange_id\x18\x04 \x01(\t\x12\x0f\n\x07\x64ry_run\x18\x05 \x01(\x08\")\n\x15OnDeviceTestsResponse\x12\x10\n\x08response\x18\x01 \x01(\t2\x8a\x02\n\x17on_device_tests_gateway\x12q\n\x0c\x65xec_command\x12-.on_device_tests_gateway.OnDeviceTestsCommand\x1a..on_device_tests_gateway.OnDeviceTestsResponse\"\x00\x30\x01\x12|\n\x12\x65xec_watch_command\x12\x32.on_device_tests_gateway.OnDeviceTestsWatchCommand\x1a..on_device_tests_gateway.OnDeviceTestsResponse\"\x00\x30\x01\x62\x06proto3') -) - - - - -_ONDEVICETESTSCOMMAND = _descriptor.Descriptor( - name='OnDeviceTestsCommand', - full_name='on_device_tests_gateway.OnDeviceTestsCommand', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='token', full_name='on_device_tests_gateway.OnDeviceTestsCommand.token', index=0, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='labels', full_name='on_device_tests_gateway.OnDeviceTestsCommand.labels', index=1, - number=8, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='test_requests', full_name='on_device_tests_gateway.OnDeviceTestsCommand.test_requests', index=2, - number=24, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=58, - serialized_end=172, -) - - -_TESTREQUEST = _descriptor.Descriptor( - name='TestRequest', - full_name='on_device_tests_gateway.TestRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='test_args', full_name='on_device_tests_gateway.TestRequest.test_args', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='test_cmd_args', full_name='on_device_tests_gateway.TestRequest.test_cmd_args', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='files', full_name='on_device_tests_gateway.TestRequest.files', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='params', full_name='on_device_tests_gateway.TestRequest.params', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='device_type', full_name='on_device_tests_gateway.TestRequest.device_type', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='device_pool', full_name='on_device_tests_gateway.TestRequest.device_pool', index=5, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=175, - serialized_end=303, -) - - -_ONDEVICETESTSWATCHCOMMAND = _descriptor.Descriptor( - name='OnDeviceTestsWatchCommand', - full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='workdir', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.workdir', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='token', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.token', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='session_id', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.session_id', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='change_id', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.change_id', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='dry_run', full_name='on_device_tests_gateway.OnDeviceTestsWatchCommand.dry_run', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=305, - serialized_end=420, -) - - -_ONDEVICETESTSRESPONSE = _descriptor.Descriptor( - name='OnDeviceTestsResponse', - full_name='on_device_tests_gateway.OnDeviceTestsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='response', full_name='on_device_tests_gateway.OnDeviceTestsResponse.response', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=422, - serialized_end=463, -) - -_ONDEVICETESTSCOMMAND.fields_by_name['test_requests'].message_type = _TESTREQUEST -DESCRIPTOR.message_types_by_name['OnDeviceTestsCommand'] = _ONDEVICETESTSCOMMAND -DESCRIPTOR.message_types_by_name['TestRequest'] = _TESTREQUEST -DESCRIPTOR.message_types_by_name['OnDeviceTestsWatchCommand'] = _ONDEVICETESTSWATCHCOMMAND -DESCRIPTOR.message_types_by_name['OnDeviceTestsResponse'] = _ONDEVICETESTSRESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -OnDeviceTestsCommand = _reflection.GeneratedProtocolMessageType('OnDeviceTestsCommand', (_message.Message,), dict( - DESCRIPTOR = _ONDEVICETESTSCOMMAND, - __module__ = 'on_device_tests_gateway_pb2' - # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsCommand) - )) -_sym_db.RegisterMessage(OnDeviceTestsCommand) - -TestRequest = _reflection.GeneratedProtocolMessageType('TestRequest', (_message.Message,), dict( - DESCRIPTOR = _TESTREQUEST, - __module__ = 'on_device_tests_gateway_pb2' - # @@protoc_insertion_point(class_scope:on_device_tests_gateway.TestRequest) - )) -_sym_db.RegisterMessage(TestRequest) - -OnDeviceTestsWatchCommand = _reflection.GeneratedProtocolMessageType('OnDeviceTestsWatchCommand', (_message.Message,), dict( - DESCRIPTOR = _ONDEVICETESTSWATCHCOMMAND, - __module__ = 'on_device_tests_gateway_pb2' - # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsWatchCommand) - )) -_sym_db.RegisterMessage(OnDeviceTestsWatchCommand) - -OnDeviceTestsResponse = _reflection.GeneratedProtocolMessageType('OnDeviceTestsResponse', (_message.Message,), dict( - DESCRIPTOR = _ONDEVICETESTSRESPONSE, - __module__ = 'on_device_tests_gateway_pb2' - # @@protoc_insertion_point(class_scope:on_device_tests_gateway.OnDeviceTestsResponse) - )) -_sym_db.RegisterMessage(OnDeviceTestsResponse) - - - -_ON_DEVICE_TESTS_GATEWAY = _descriptor.ServiceDescriptor( - name='on_device_tests_gateway', - full_name='on_device_tests_gateway.on_device_tests_gateway', - file=DESCRIPTOR, - index=0, - options=None, - serialized_start=466, - serialized_end=732, - methods=[ - _descriptor.MethodDescriptor( - name='exec_command', - full_name='on_device_tests_gateway.on_device_tests_gateway.exec_command', - index=0, - containing_service=None, - input_type=_ONDEVICETESTSCOMMAND, - output_type=_ONDEVICETESTSRESPONSE, - options=None, - ), - _descriptor.MethodDescriptor( - name='exec_watch_command', - full_name='on_device_tests_gateway.on_device_tests_gateway.exec_watch_command', - index=1, - containing_service=None, - input_type=_ONDEVICETESTSWATCHCOMMAND, - output_type=_ONDEVICETESTSRESPONSE, - options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_ON_DEVICE_TESTS_GATEWAY) - -DESCRIPTOR.services_by_name['on_device_tests_gateway'] = _ON_DEVICE_TESTS_GATEWAY - -# @@protoc_insertion_point(module_scope) diff --git a/cobalt/tools/on_device_tests_gateway_pb2_grpc.py b/cobalt/tools/on_device_tests_gateway_pb2_grpc.py deleted file mode 100644 index 284d31d4b2cac..0000000000000 --- a/cobalt/tools/on_device_tests_gateway_pb2_grpc.py +++ /dev/null @@ -1,65 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -import grpc - -import on_device_tests_gateway_pb2 as on__device__tests__gateway__pb2 - - -class on_device_tests_gatewayStub(object): - """Interface exported by the server. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.exec_command = channel.unary_stream( - '/on_device_tests_gateway.on_device_tests_gateway/exec_command', - request_serializer=on__device__tests__gateway__pb2.OnDeviceTestsCommand.SerializeToString, - response_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.FromString, - ) - self.exec_watch_command = channel.unary_stream( - '/on_device_tests_gateway.on_device_tests_gateway/exec_watch_command', - request_serializer=on__device__tests__gateway__pb2.OnDeviceTestsWatchCommand.SerializeToString, - response_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.FromString, - ) - - -class on_device_tests_gatewayServicer(object): - """Interface exported by the server. - """ - - def exec_command(self, request, context): - """A dumb proxy RPC service that passes user defined command line options - to the on-device tests gateway and streams back output in real time. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def exec_watch_command(self, request, context): - """A dumb proxy RPC service that passes user defined command line options - to the on-device tests gateway and streams back output in real time. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_on_device_tests_gatewayServicer_to_server(servicer, server): - rpc_method_handlers = { - 'exec_command': grpc.unary_stream_rpc_method_handler( - servicer.exec_command, - request_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsCommand.FromString, - response_serializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.SerializeToString, - ), - 'exec_watch_command': grpc.unary_stream_rpc_method_handler( - servicer.exec_watch_command, - request_deserializer=on__device__tests__gateway__pb2.OnDeviceTestsWatchCommand.FromString, - response_serializer=on__device__tests__gateway__pb2.OnDeviceTestsResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'on_device_tests_gateway.on_device_tests_gateway', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) From 4c07925763a3da757f1fd4380c36b0d0325847d2 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Thu, 16 Jan 2025 14:27:28 -0800 Subject: [PATCH 43/61] Deleting erroneous file --- "cobalt/tools/\\" | 413 ---------------------------------------------- 1 file changed, 413 deletions(-) delete mode 100644 "cobalt/tools/\\" diff --git "a/cobalt/tools/\\" "b/cobalt/tools/\\" deleted file mode 100644 index ea587638e0876..0000000000000 --- "a/cobalt/tools/\\" +++ /dev/null @@ -1,413 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2022 The Cobalt Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""gRPC On-device Tests Gateway client.""" - -import argparse -import json -import logging -import sys - -import grpc -import on_device_tests_gateway_pb2 -import on_device_tests_gateway_pb2_grpc - - -_WORK_DIR = '/on_device_tests_gateway' - -# Comment out the next three lines for local testing -# _ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( -# 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') -_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' - -# Uncomment the next two lines for local testing -_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') -# _ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '12345' - -_ON_DEVICE_OUTPUT_FILES_DIR = '/sdcard/Download' -_CHROMIUM_TEST_ROOT = '/sdcard/chromium_tests_root' - -class OnDeviceTestsGatewayClient(): - """On-device tests Gateway Client class.""" - - def __init__(self): - self.channel = grpc.insecure_channel( - target=f'{_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST}:{_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT}', # pylint:disable=line-too-long - # These options need to match server settings. - options=[('grpc.keepalive_time_ms', 10000), - ('grpc.keepalive_timeout_ms', 5000), - ('grpc.keepalive_permit_without_calls', 1), - ('grpc.http2.max_pings_without_data', 0), - ('grpc.http2.min_time_between_pings_ms', 10000), - ('grpc.http2.min_ping_interval_without_data_ms', 5000)]) - self.stub = on_device_tests_gateway_pb2_grpc.on_device_tests_gatewayStub( - self.channel) - - def run_trigger_command(self, args: argparse.Namespace, test_requests): - """Calls On-Device Tests service and passing given parameters to it. - - Args: - args (Namespace): Arguments passed in command line. - test_requests (list): A list of test requests. - """ - for response_line in self.stub.exec_command( - on_device_tests_gateway_pb2.OnDeviceTestsCommand( - token=args.token, - labels=args.label, - test_requests=test_requests, - )): - - print(response_line.response) - - def run_watch_command(self, workdir: str, args: argparse.Namespace): - """Calls On-Device Tests watch service and passing given parameters to it. - - Args: - workdir (str): Current script workdir. - args (Namespace): Arguments passed in command line. - """ - for response_line in self.stub.exec_watch_command( - on_device_tests_gateway_pb2.OnDeviceTestsWatchCommand( - workdir=workdir, - token=args.token, - change_id=args.change_id, - session_id=args.session_id, - )): - - print(response_line.response) - - -def _read_json_config(filename): - """Reads and parses data from a JSON configuration file. - - Args: - filename: The name of the JSON configuration file. - - Returns: - A list of dictionaries, where each dictionary represents a test - configuration. - """ - try: - with open(filename, 'r') as f: - data = json.load(f) - return data - except FileNotFoundError: - print(f" Config file '{filename}' not found.") - return None - except json.JSONDecodeError: - print(f" Invalid JSON format in '{filename}'.") - return None - - -def _get_gtest_filters(filter_json_dir, gtest_target): - """Retrieves gtest filters for a given target. - - Args: - filter_json_dir: Directory containing filter JSON files. - gtest_target: The name of the gtest target. - - Returns: - A string containing the gtest filters. - """ - gtest_filters = '*' - filter_json_file = f'{filter_json_dir}/{gtest_target}_filter.json' - print(f" gtest_filter_json_file = {filter_json_file}") - filter_data = _read_json_config(filter_json_file) - if filter_data: - print(f" Loaded filter data: {filter_data}") - failing_tests = ':'.join(filter_data.get('failing_tests', [])) - if failing_tests: - gtest_filters += ':-' + failing_tests - print(f" gtest_filters = {gtest_filters}") - else: - print(' This gtest_target does not have gtest_filters specified') - return gtest_filters - - -def _process_test_requests(args): - """Processes test requests from the given arguments. - - Constructs a list of test requests based on the provided arguments, - including test arguments, command arguments, files, parameters, - and device information. - - Args: - args: The parsed command-line arguments. - - Returns: - A list of test request dictionaries. - """ - test_requests = [] - platform_data = _read_json_config(args.platform_json) - - tests_args = [ - f'job_timeout_secs={args.job_timeout_secs}', - f'test_timeout_secs={args.test_timeout_secs}', - f'start_timeout_secs={args.start_timeout_secs}' - ] - - if args.change_id: - tests_args.append(f'change_id={args.change_id}') - if args.test_attempts: - tests_args.append(f'test_attempts={args.test_attempts}') - if args.dimension: - tests_args.extend( - f'dimension_{dimension}' for dimension in args.dimension - ) - - base_params = [f'push_files=test_runtime_deps:{_CHROMIUM_TEST_ROOT}'] - - if args.gcs_result_path: - base_params.append(f'gcs_result_path={args.gcs_result_path}') - - for gtest_target in platform_data['gtest_targets']: - print(f" Processing gtest_target: {gtest_target}") - - apk_file = f"{args.gcs_archive_path}/{gtest_target}-debug.apk" - test_runtime_deps = f"{args.gcs_archive_path}/{gtest_target}_deps.tar.gz" - - files = [ - f'test_apk={apk_file}', - f'build_apk={apk_file}', - f'test_runtime_deps={test_runtime_deps}' - ] - - gtest_params = [ - f'gtest_xml_file_on_device={_ON_DEVICE_OUTPUT_FILES_DIR}/{gtest_target}_result.xml', - f'gcs_result_filename={gtest_target}_result.xml', - f'gcs_log_filename={gtest_target}_log.txt' - ] - - gtest_filter = _get_gtest_filters(args.filter_json_dir, gtest_target) - - test_cmd_args = [ - f'command_line_args=--gtest_output=xml:{_ON_DEVICE_OUTPUT_FILES_DIR}/{gtest_target}_result.xml', - f'--gtest_filter={gtest_filter}' - ] - - test_request = { - 'test_args': tests_args, - 'test_cmd_args': test_cmd_args, - 'files': files, - 'params': base_params + gtest_params, - 'device_type': platform_data.get('test_dimensions', {}).get( - 'gtest_device' - ), - 'device_pool': platform_data.get('test_dimensions', {}).get( - 'gtest_lab' - ), - } - - test_requests.append(test_request) - print(f' Created test_request: {test_request}') - - print(f'test_requests: {test_requests}') - return test_requests - - -def main() -> int: # Add a return type hint - """Main routine for the on-device tests gateway client.""" - - logging.basicConfig( - level=logging.INFO, format='[%(filename)s:%(lineno)s] %(message)s' - ) - print('Starting main routine') - - parser = argparse.ArgumentParser( - description='Client for interacting with the On-Device Tests gateway.', - epilog=( - 'Example:' - 'python3 -u cobalt/tools/on_device_tests_gateway_client.py' - '--platform_json "${GITHUB_WORKSPACE}/src/.github/config/' - '${{ matrix.platform}}.json"' - '--filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/' - '${{ matrix.platform}}"' - '--token ${GITHUB_TOKEN}' - '--change_id ${GITHUB_PR_NUMBER:-postsubmit}' - '--label builder-${{ matrix.platform }}' - '--label builder_url-${GITHUB_RUN_URL}' - '--label github' - '--label ${GITHUB_EVENT_NAME}' - '--label ${GITHUB_WORKFLOW}' - '--label actor-${GITHUB_ACTOR}' - '--label actor_id-${GITHUB_ACTOR_ID}' - '--label triggering_actor-${GITHUB_TRIGGERING_ACTOR}' - '--label sha-${GITHUB_SHA}' - '--label repository-${GITHUB_REPO}' - '--label author-${GITHUB_PR_HEAD_USER_LOGIN:-' - '$GITHUB_COMMIT_AUTHOR_USERNAME}' - '--label author_id-${GITHUB_PR_HEAD_USER_ID:-' - '$GITHUB_COMMIT_AUTHOR_EMAIL}' - '--dimension host_name=regex:maneki-mhserver-05.*' - '${DIMENSION:+"--dimension" "$DIMENSION"}' - '${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" ' - '"$ON_DEVICE_TEST_ATTEMPTS"}' - '--gcs_archive_path "${GCS_ARTIFACTS_PATH}"' - '--gcs_result_path "${GCS_RESULTS_PATH}"' - 'trigger' - ), - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - # Authentication - parser.add_argument( - '-t', - '--token', - type=str, - required=True, - help='On Device Tests authentication token', - ) - - # General options - parser.add_argument( - '-i', - '--change_id', - type=str, - help=( - 'ChangeId that triggered this test, if any. Saved with performance' - ' test results.' - ), - ) - - subparsers = parser.add_subparsers( - dest='action', help='On-Device tests commands', required=True - ) - - # Trigger command - trigger_parser = subparsers.add_parser( - 'trigger', - help='Trigger On-Device tests', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - - # Group trigger arguments - trigger_args = trigger_parser.add_argument_group('Trigger Arguments') - trigger_args.add_argument( - '-pf', - '--platform_json', - type=str, - help='Platform-specific JSON file containing the list of target tests.', - ) - trigger_parser.add_argument( - '--filter_json_dir', - type=str, - help='Directory containing filter JSON files for test selection.', - ) - trigger_parser.add_argument( - '-l', - '--label', - type=str, - action='append', - help='Additional labels to assign to the test.', - ) - trigger_parser.add_argument( - '--dimension', - type=str, - action='append', - help=( - 'On-Device Tests dimension used to select a device. ' - 'Must have the following form: =. ' - 'E.G. "release_version=regex:10.*"' - ), - ) - trigger_parser.add_argument( - '--test_attempts', - type=str, - default='1', - help='The maximum number of times a test can retry.', - ) - trigger_args.add_argument( - '-a', - '--archive_path', - type=str, - required=True, - help='Path to Chrobalt archive to be tested. Must be on GCS.', - ) - trigger_parser.add_argument( - '--gcs_result_path', - type=str, - help='GCS URL where test result files should be uploaded.', - ) - trigger_parser.add_argument( - '--job_timeout_sec', - type=str, - default='1800', - help='Timeout in seconds for the job (default: 1800 seconds).', - ) - trigger_parser.add_argument( - '--test_timeout_sec', - type=str, - default='1800', - help='Timeout in seconds for the test (default: 1800 seconds).', - ) - trigger_parser.add_argument( - '--start_timeout_sec', - type=str, - default='180', - help='Timeout in seconds for the test to start (default: 180 seconds).', - ) - - # Watch command - watch_parser = subparsers.add_parser( - 'watch', help='Watch a previously triggered On-Device test' - ) - watch_parser.add_argument( - 'session_id', - type=str, - help=( - 'Session ID of a previously triggered Mobile Harness test. ' - 'The test will be watched until it completes.' - ), - ) - - args = parser.parse_args() - - if not args.platform_json: - print( - "Error: The '--platform_json' argument is required. " - 'Please provide the path to the Platform JSON file.' - ) - exit(1) # Exit with an error code - if not args.filter_json_dir: - print( - "Error: The '--filter_json_dir' argument is required. " - 'Please provide the directory containing filter JSON files.' - ) - exit(1) # Exit with an error code - if not args.gcs_archive_path: - print( - "Error: The '--gcs_archive_path' argument is required. " - 'Please provide the GCS archive path info.' - ) - exit(1) # Exit with an error code - - test_requests = _process_test_requests(args) - client = OnDeviceTestsGatewayClient() - - try: - if args.action == 'trigger': - client.run_trigger_command(args, test_requests) - else: - client.run_watch_command(workdir=_WORK_DIR, args=args) - except grpc.RpcError as e: - logging.exception('gRPC error occurred:') # Log the full traceback - print(f'Error: {e}') - return e.code().value # Return the error code - - return 0 # Indicate successful execution - - -if __name__ == '__main__': - sys.exit(main()) From 42c6bbfc43145bd7aa10780c1ffa6aa7bc8ebb86 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Thu, 16 Jan 2025 14:32:15 -0800 Subject: [PATCH 44/61] Fix build dependency problem --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index ab049aac270ce..5761aa03a4556 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -227,7 +227,7 @@ jobs: num_gtest_shards: ${{ needs.initialize.outputs.num_gtest_shards }} test-upload: - needs: [initialize, docker-build-image, build, test] + needs: [initialize, docker-build-image, build, on-host-test] if: always() && ( needs.initialize.outputs.test_on_host == 'true' || From be98ad03138c8d8789d3004e6f4afa28a25fbf36 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Fri, 17 Jan 2025 11:06:17 -0800 Subject: [PATCH 45/61] Testing ODT trigger --- .github/config/android-arm.json | 9 --------- .github/config/android-arm64.json | 1 - .github/workflows/main.yaml | 18 ++++++++++++++++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/config/android-arm.json b/.github/config/android-arm.json index 6bfc85207a7e0..51b99d238b9f6 100644 --- a/.github/config/android-arm.json +++ b/.github/config/android-arm.json @@ -10,15 +10,6 @@ "cobalt:gn_all" ], "gtest_targets": [ - "base_unittests", - "blink_unittests", - "gin_unittests", - "gpu_unittests", - "ipc_tests", - "media_unittests", - "mojo_unittests", - "net_unittests", - "sql_unittests", "url_unittests" ], "test_dimensions": { diff --git a/.github/config/android-arm64.json b/.github/config/android-arm64.json index e90a3d101158a..1856609020d5c 100644 --- a/.github/config/android-arm64.json +++ b/.github/config/android-arm64.json @@ -21,7 +21,6 @@ "sql_unittests", "url_unittests" ], - "test_on_device": true, "includes": [ { "name":"arm64", diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5761aa03a4556..7ca8afbf563e5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -111,6 +111,24 @@ jobs: test_on_host: ${{ env.test_on_host }} test_on_device: ${{ env.test_on_device }} + foo: + needs: [initialize] + + if: | + ${{ needs.initialize.outputs.test_on_device == 'true' && (( + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( + inputs.nightly == 'true' || github.event_name == 'schedule') && + vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || + ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) ) }} + runs-on: [self-hosted, odt-runner] + name: foo-bar + permissions: {} + steps: + - name: Run Foo Bar + shell: bash + run: echo "HELLO WORLD" + # Builds, tags, and pushes Cobalt docker build images to ghr. docker-build-image: needs: [initialize] From ee7c0c41c54b422437f3605421465cc48e56d661 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Fri, 17 Jan 2025 12:55:08 -0800 Subject: [PATCH 46/61] Fix github actions syntax error --- .github/workflows/main.yaml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7ca8afbf563e5..7edbb7b7a541f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -110,24 +110,6 @@ jobs: gtest_shards: ${{ env.gtest_shards }} test_on_host: ${{ env.test_on_host }} test_on_device: ${{ env.test_on_device }} - - foo: - needs: [initialize] - - if: | - ${{ needs.initialize.outputs.test_on_device == 'true' && (( - github.event_name == 'pull_request' && - contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( - inputs.nightly == 'true' || github.event_name == 'schedule') && - vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || - ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) ) }} - runs-on: [self-hosted, odt-runner] - name: foo-bar - permissions: {} - steps: - - name: Run Foo Bar - shell: bash - run: echo "HELLO WORLD" # Builds, tags, and pushes Cobalt docker build images to ghr. docker-build-image: @@ -305,12 +287,12 @@ jobs: # Run ODT when on_device label is applied on PR. # Also, run ODT on push and schedule if not explicitly disabled via repo vars. if: | - needs.initialize.outputs.test_on_device == 'true' && (( + ${{ needs.initialize.outputs.test_on_device == 'true' && (( github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( inputs.nightly == 'true' || github.event_name == 'schedule') && vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || - ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) ) + ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) ) }} runs-on: [self-hosted, odt-runner] name: ${{ matrix.name }}_on_device permissions: {} From da7f17c6f65e75abaf4e5efd4d0e67096c9366a2 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Fri, 17 Jan 2025 14:50:39 -0800 Subject: [PATCH 47/61] Fix grpc proto paths --- .github/actions/on_device_tests/action.yaml | 6 +++--- .github/workflows/main.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 86bf7a2d940a9..437bd837f70fc 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -18,7 +18,7 @@ runs: shell: bash - name: Generate gRPC files run: | - python -m grpc_tools.protoc -Itools/ --python_out=cobalt/tools/ --grpc_python_out=cobalt/tools/ cobalt/tools/on_device_tests_gateway.proto + python -m grpc_tools.protoc -I${GITHUB_WORKSPACE}/cobalt/tools/ --python_out=${GITHUB_WORKSPACE}/cobalt/tools/ --grpc_python_out=${GITHUB_WORKSPACE}/cobalt/tools/ ${GITHUB_WORKSPACE}/cobalt/tools/on_device_tests_gateway.proto shell: bash - name: Set Up Cloud SDK uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1 @@ -54,8 +54,8 @@ runs: run: | set -uxe python3 -u cobalt/tools/on_device_tests_gateway_client.py \ - --platform_json "${GITHUB_WORKSPACE}/src/.github/config/${{ matrix.platform}}.json" \ - --filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/${{ matrix.platform}}" \ + --platform_json "${GITHUB_WORKSPACE}/.github/config/${{ matrix.platform}}.json" \ + --filter_json_dir "${GITHUB_WORKSPACE}/cobalt/testing/${{ matrix.platform}}" \ --token ${GITHUB_TOKEN} \ --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ --label builder-${{ matrix.platform }} \ diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7edbb7b7a541f..a6590c517acaa 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -151,7 +151,7 @@ jobs: matrix: platform: ${{ fromJson(needs.initialize.outputs.platforms) }} include: ${{ fromJson(needs.initialize.outputs.includes) }} - config: [devel, qa, gold] + config: [devel] container: ${{ needs.docker-build-image.outputs.docker_tag }} env: TEST_ARTIFACTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_artifacts From e04f20309e914de0a36b3ada85aa771c1f7b334d Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Sat, 18 Jan 2025 08:45:47 -0800 Subject: [PATCH 48/61] Fix command line arguments --- .github/actions/on_device_tests/action.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 437bd837f70fc..d74ec80c774e3 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -54,10 +54,11 @@ runs: run: | set -uxe python3 -u cobalt/tools/on_device_tests_gateway_client.py \ - --platform_json "${GITHUB_WORKSPACE}/.github/config/${{ matrix.platform}}.json" \ - --filter_json_dir "${GITHUB_WORKSPACE}/cobalt/testing/${{ matrix.platform}}" \ --token ${GITHUB_TOKEN} \ --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ + trigger \ + --platform_json "${GITHUB_WORKSPACE}/.github/config/${{ matrix.platform}}.json" \ + --filter_json_dir "${GITHUB_WORKSPACE}/cobalt/testing/${{ matrix.platform}}" \ --label builder-${{ matrix.platform }} \ --label builder_url-${GITHUB_RUN_URL} \ --label github \ @@ -73,8 +74,7 @@ runs: ${DIMENSION:+"--dimension" "$DIMENSION"} \ ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ --gcs_archive_path "${GCS_ARTIFACTS_PATH}" \ - --gcs_result_path "${GCS_RESULTS_PATH}" \ - trigger + --gcs_result_path "${GCS_RESULTS_PATH}" shell: bash - name: Download ${{ matrix.platform }} Test Results if: always() From 5b47adfe570a4d12206ccca0e331db08ed7ccc3a Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Sat, 18 Jan 2025 10:17:39 -0800 Subject: [PATCH 49/61] Testing ODT trigger --- .github/actions/on_device_tests/action.yaml | 4 ++-- .github/workflows/main.yaml | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index d74ec80c774e3..089464d02ffc5 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -34,7 +34,7 @@ runs: shell: bash - name: Run Tests on ${{ matrix.platform }} Platform env: - GCS_ARTIFACTS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} + GCS_ARTIFACTS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }} GCS_RESULTS_PATH: ${{ inputs.gcs_results_path }} GITHUB_SHA: ${{ github.sha }} GITHUB_TOKEN: ${{ github.token }} @@ -70,7 +70,7 @@ runs: --label sha-${GITHUB_SHA} \ --label repository-${GITHUB_REPO} \ --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ - --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} + --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} \ ${DIMENSION:+"--dimension" "$DIMENSION"} \ ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ --gcs_archive_path "${GCS_ARTIFACTS_PATH}" \ diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a6590c517acaa..b206bf45ad35d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -110,7 +110,7 @@ jobs: gtest_shards: ${{ env.gtest_shards }} test_on_host: ${{ env.test_on_host }} test_on_device: ${{ env.test_on_device }} - + # Builds, tags, and pushes Cobalt docker build images to ghr. docker-build-image: needs: [initialize] @@ -286,13 +286,14 @@ jobs: needs: [initialize, build] # Run ODT when on_device label is applied on PR. # Also, run ODT on push and schedule if not explicitly disabled via repo vars. - if: | - ${{ needs.initialize.outputs.test_on_device == 'true' && (( - github.event_name == 'pull_request' && - contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( - inputs.nightly == 'true' || github.event_name == 'schedule') && - vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || - ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) ) }} + if: needs.initialize.outputs.test_on_device == 'true' && + (( + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( + inputs.nightly == 'true' || github.event_name == 'schedule') && + vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || + ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' + )) runs-on: [self-hosted, odt-runner] name: ${{ matrix.name }}_on_device permissions: {} From 54bf26e4bcff7614a94da4054ec03392b1227c33 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Sun, 19 Jan 2025 16:39:20 -0800 Subject: [PATCH 50/61] Hardcode runtime deps file --- cobalt/tools/on_device_tests_gateway_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cobalt/tools/on_device_tests_gateway_client.py b/cobalt/tools/on_device_tests_gateway_client.py index 4d0777bd3491a..ec2135da502d5 100644 --- a/cobalt/tools/on_device_tests_gateway_client.py +++ b/cobalt/tools/on_device_tests_gateway_client.py @@ -172,7 +172,8 @@ def _process_test_requests(args): print(f" Processing gtest_target: {gtest_target}") apk_file = f"{args.gcs_archive_path}/{gtest_target}-debug.apk" - test_runtime_deps = f"{args.gcs_archive_path}/{gtest_target}_deps.tar.gz" + # test_runtime_deps = f"{args.gcs_archive_path}/{gtest_target}_deps.tar.gz" + test_runtime_deps = "/bigstore/yt-temp/odt-test/url_unittests_deps.tar.gz" files = [ f'test_apk={apk_file}', From 87290dc2983ce02792e0a690c3c09fff6b1c8676 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Sun, 19 Jan 2025 18:26:59 -0800 Subject: [PATCH 51/61] Update deps archive path on device --- cobalt/tools/on_device_tests_gateway_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cobalt/tools/on_device_tests_gateway_client.py b/cobalt/tools/on_device_tests_gateway_client.py index ec2135da502d5..cabac2bb627b8 100644 --- a/cobalt/tools/on_device_tests_gateway_client.py +++ b/cobalt/tools/on_device_tests_gateway_client.py @@ -33,7 +33,7 @@ _ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' _ON_DEVICE_OUTPUT_FILES_DIR = '/sdcard/Download' -_CHROMIUM_TEST_ROOT = '/sdcard/chromium_tests_root' +_DEPS_ARCHIVE = '/sdcard/chromium_tests_root/deps.tar.gz' class OnDeviceTestsGatewayClient(): """On-device tests Gateway Client class.""" @@ -163,7 +163,7 @@ def _process_test_requests(args): f'dimension_{dimension}' for dimension in args.dimension ) - base_params = [f'push_files=test_runtime_deps:{_CHROMIUM_TEST_ROOT}'] + base_params = [f'push_files=test_runtime_deps:{_DEPS_ARCHIVE}'] if args.gcs_result_path: base_params.append(f'gcs_result_path={args.gcs_result_path}') From 1348a702d12b6a52d5fa84ca06d1646f925e9954 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Sun, 19 Jan 2025 20:20:05 -0800 Subject: [PATCH 52/61] Fix gsutil copy command --- .github/actions/on_device_tests/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 089464d02ffc5..22fb38845b9b2 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -85,7 +85,7 @@ runs: set -uxe mkdir -p "${GITHUB_WORKSPACE}/${RESULTS_DIR}" cd "${GITHUB_WORKSPACE}/${RESULTS_DIR}" - gsutil cp "${GCS_RESULTS_PATH}/" . + gsutil cp -r "${GCS_RESULTS_PATH}/" . echo "TEST_LOG=${GITHUB_WORKSPACE}/${RESULTS_DIR}/test_results.txt" >> $GITHUB_ENV shell: bash - name: Archive Test Logs From d35036649f208ea2888208abe97937b772de4e10 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Sun, 19 Jan 2025 21:21:06 -0800 Subject: [PATCH 53/61] Fix test logs archival mechanism --- .github/actions/on_device_tests/action.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 22fb38845b9b2..4b69d75472be9 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -83,14 +83,14 @@ runs: RESULTS_DIR: ${{ inputs.results_dir }} run: | set -uxe - mkdir -p "${GITHUB_WORKSPACE}/${RESULTS_DIR}" - cd "${GITHUB_WORKSPACE}/${RESULTS_DIR}" - gsutil cp -r "${GCS_RESULTS_PATH}/" . - echo "TEST_LOG=${GITHUB_WORKSPACE}/${RESULTS_DIR}/test_results.txt" >> $GITHUB_ENV + TEST_LOGS="${GITHUB_WORKSPACE}/${RESULTS_DIR}/" + mkdir -p "${TEST_LOGS}" + gsutil cp -r "${GCS_RESULTS_PATH}/" "${TEST_LOGS}" + echo "TEST_LOGS=${TEST_LOGS}" >> $GITHUB_ENV shell: bash - name: Archive Test Logs uses: actions/upload-artifact@v3 if: always() with: name: Test log - path: ${{ env.TEST_LOG }}/ + path: ${{ env.TEST_LOGS }} From 4120de83b50a756f6e8b748025ea9635b0dfb147 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Sun, 19 Jan 2025 22:18:59 -0800 Subject: [PATCH 54/61] Fix test result processing mechanism --- .github/workflows/main.yaml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b206bf45ad35d..a7c4dc3183fad 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -219,7 +219,7 @@ jobs: path: src - name: Run On-Host Tests id: on-host-tests - if: always() && needs.initialize.outputs.test_on_host == 'true' + if: always() uses: ./src/.github/actions/on_host_tests with: test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }} @@ -228,11 +228,7 @@ jobs: test-upload: needs: [initialize, docker-build-image, build, on-host-test] - if: always() && - ( - needs.initialize.outputs.test_on_host == 'true' || - needs.initialize.outputs.test_on_device == 'true' - ) + if: always() && needs.initialize.outputs.test_on_host == 'true' permissions: {} runs-on: [self-hosted, chrobalt-linux-runner] name: ${{ matrix.name }}_tests_upload @@ -319,13 +315,8 @@ jobs: gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }} results_dir: ${{ env.TEST_RESULTS_DIR }} - name: Process Test Results - if: | - always() && - ( - steps.on-device-tests.outcome == 'success' || - steps.on-device-tests.outcome == 'failure' - ) - uses: ./src/.github/actions/process_test_results + if: always() + uses: ./.github/actions/process_test_results with: results_dir: ${{ env.TEST_RESULTS_DIR }} datadog_api_key: ${{ secrets.DD_API_KEY }} From a98fe4263510096f7f82b238c18a5497d8da1f52 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Mon, 20 Jan 2025 10:50:16 -0800 Subject: [PATCH 55/61] Add removed unit tests back --- .github/config/android-arm.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/config/android-arm.json b/.github/config/android-arm.json index 51b99d238b9f6..6bfc85207a7e0 100644 --- a/.github/config/android-arm.json +++ b/.github/config/android-arm.json @@ -10,6 +10,15 @@ "cobalt:gn_all" ], "gtest_targets": [ + "base_unittests", + "blink_unittests", + "gin_unittests", + "gpu_unittests", + "ipc_tests", + "media_unittests", + "mojo_unittests", + "net_unittests", + "sql_unittests", "url_unittests" ], "test_dimensions": { From 49e90eb466c1d74f94982c5618e4e0ac00438df0 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Mon, 20 Jan 2025 18:21:36 -0800 Subject: [PATCH 56/61] Artifact upload work for ODT --- .github/actions/on_device_tests/action.yaml | 10 +- .github/workflows/main.yaml | 104 +++++++++++--------- 2 files changed, 63 insertions(+), 51 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 4b69d75472be9..a0e53033c4c9b 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -7,6 +7,9 @@ inputs: results_dir: description: "Path to directory where test results are saved." required: true + test_results_key: + description: "Artifact key used to store test results." + required: true runs: using: "composite" @@ -88,9 +91,10 @@ runs: gsutil cp -r "${GCS_RESULTS_PATH}/" "${TEST_LOGS}" echo "TEST_LOGS=${TEST_LOGS}" >> $GITHUB_ENV shell: bash - - name: Archive Test Logs + - name: Archive Test Results uses: actions/upload-artifact@v3 if: always() with: - name: Test log - path: ${{ env.TEST_LOGS }} + name: ${{ inputs.test_results_key }} + path: ${{ env.TEST_LOGS }}/ + diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a7c4dc3183fad..b41a43f952bfd 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -192,6 +192,56 @@ jobs: test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }} upload_on_host_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_host }} upload_on_device_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_device }} + on_host: ${{ needs.initialize.outputs.test_on_host }} + on_device: ${{ needs.initialize.outputs.test_on_device }} + + # Runs on-device integration and unit tests. + on-device-test: + needs: [initialize, build] + # Run ODT when on_device label is applied on PR. + # Also, run ODT on push and schedule if not explicitly disabled via repo vars. + if: needs.initialize.outputs.test_on_device == 'true' && + (( + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( + inputs.nightly == 'true' || github.event_name == 'schedule') && + vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || + ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' + )) + runs-on: [self-hosted, odt-runner] + name: ${{ matrix.name }}_on_device + permissions: {} + strategy: + fail-fast: false + matrix: + platform: ${{ fromJson(needs.initialize.outputs.platforms) }} + config: [devel] + include: ${{ fromJson(needs.initialize.outputs.includes) }} + env: + TEST_RESULTS_DIR: ${{ matrix.name }}_test_results + TEST_RESULTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_results + steps: + - name: Checkout + uses: kaidokert/checkout@v3.5.999 + timeout-minutes: 30 + with: + fetch-depth: 1 + persist-credentials: false + - name: Run On-Device Tests (${{ matrix.shard }}) + id: on-device-tests + uses: ./.github/actions/on_device_tests + with: + test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }} + gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }} + results_dir: ${{ env.TEST_RESULTS_DIR }} + - name: Process Test Results + if: always() + uses: ./.github/actions/process_test_results + with: + results_dir: ${{ env.TEST_RESULTS_DIR }} + datadog_api_key: ${{ secrets.DD_API_KEY }} + is_postsubmit: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} + continue-on-error: true on-host-test: needs: [initialize, docker-build-image, build] @@ -227,8 +277,12 @@ jobs: num_gtest_shards: ${{ needs.initialize.outputs.num_gtest_shards }} test-upload: - needs: [initialize, docker-build-image, build, on-host-test] - if: always() && needs.initialize.outputs.test_on_host == 'true' + needs: [initialize, docker-build-image, build, on-host-test, on-device-test] + if: always() && + ( + needs.initialize.outputs.test_on_host == 'true' || + needs.initialize.outputs.test_on_device == 'true' + ) permissions: {} runs-on: [self-hosted, chrobalt-linux-runner] name: ${{ matrix.name }}_tests_upload @@ -276,49 +330,3 @@ jobs: run: | echo "Failing because at least one test shard had errors." exit 1 - - # Runs on-device integration and unit tests. - on-device-test: - needs: [initialize, build] - # Run ODT when on_device label is applied on PR. - # Also, run ODT on push and schedule if not explicitly disabled via repo vars. - if: needs.initialize.outputs.test_on_device == 'true' && - (( - github.event_name == 'pull_request' && - contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( - inputs.nightly == 'true' || github.event_name == 'schedule') && - vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || - ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' - )) - runs-on: [self-hosted, odt-runner] - name: ${{ matrix.name }}_on_device - permissions: {} - strategy: - fail-fast: false - matrix: - platform: ${{ fromJson(needs.initialize.outputs.platforms) }} - config: [devel] - include: ${{ fromJson(needs.initialize.outputs.includes) }} - env: - TEST_RESULTS_DIR: ${{ matrix.name }}_test_results - steps: - - name: Checkout - uses: kaidokert/checkout@v3.5.999 - timeout-minutes: 30 - with: - fetch-depth: 1 - persist-credentials: false - - name: Run On-Device Tests (${{ matrix.shard }}) - id: on-device-tests - uses: ./.github/actions/on_device_tests - with: - gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }} - results_dir: ${{ env.TEST_RESULTS_DIR }} - - name: Process Test Results - if: always() - uses: ./.github/actions/process_test_results - with: - results_dir: ${{ env.TEST_RESULTS_DIR }} - datadog_api_key: ${{ secrets.DD_API_KEY }} - is_postsubmit: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} - continue-on-error: true From fa5d3fa6df6abbcdd9af8f5b0d3925b8781453f3 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Wed, 22 Jan 2025 09:49:41 -0800 Subject: [PATCH 57/61] Fix artifact upload mechanism --- .github/actions/on_device_tests/action.yaml | 2 +- .github/workflows/main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index a0e53033c4c9b..7f70ec9251ecc 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -96,5 +96,5 @@ runs: if: always() with: name: ${{ inputs.test_results_key }} - path: ${{ env.TEST_LOGS }}/ + path: ${{ env.TEST_LOGS }} diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b41a43f952bfd..4239edb9d97aa 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -231,7 +231,7 @@ jobs: id: on-device-tests uses: ./.github/actions/on_device_tests with: - test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }} + test_results_key: ${{ env.TEST_RESULTS_KEY }} gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }} results_dir: ${{ env.TEST_RESULTS_DIR }} - name: Process Test Results From 6239a4734c4cdc86f2316105cbcd93a18c09fd8f Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Wed, 22 Jan 2025 14:36:15 -0800 Subject: [PATCH 58/61] Fix wildcard for process test --- .github/actions/process_test_results/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/process_test_results/action.yaml b/.github/actions/process_test_results/action.yaml index 73a2e16c3633f..90c5da98ccda4 100644 --- a/.github/actions/process_test_results/action.yaml +++ b/.github/actions/process_test_results/action.yaml @@ -13,7 +13,7 @@ runs: - name: Download Test Results uses: actions/download-artifact@v4 with: - pattern: ${{ inputs.test_results_key }}-* + pattern: ${{ inputs.test_results_key }}* path: results/**/* - name: Test Summary action From 6e19a0090d441ef5a8be86227fc094c3d0c76687 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Wed, 22 Jan 2025 16:07:30 -0800 Subject: [PATCH 59/61] Testing artifact download --- .github/actions/on_device_tests/action.yaml | 2 +- .github/actions/process_test_results/action.yaml | 2 +- .github/workflows/main.yaml | 8 -------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index 7f70ec9251ecc..08bb1d25c07df 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -92,7 +92,7 @@ runs: echo "TEST_LOGS=${TEST_LOGS}" >> $GITHUB_ENV shell: bash - name: Archive Test Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: ${{ inputs.test_results_key }} diff --git a/.github/actions/process_test_results/action.yaml b/.github/actions/process_test_results/action.yaml index 90c5da98ccda4..c56a46b05a353 100644 --- a/.github/actions/process_test_results/action.yaml +++ b/.github/actions/process_test_results/action.yaml @@ -13,7 +13,7 @@ runs: - name: Download Test Results uses: actions/download-artifact@v4 with: - pattern: ${{ inputs.test_results_key }}* + pattern: ${{ inputs.test_results_key }} path: results/**/* - name: Test Summary action diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4239edb9d97aa..16231e63a55c3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -234,14 +234,6 @@ jobs: test_results_key: ${{ env.TEST_RESULTS_KEY }} gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }} results_dir: ${{ env.TEST_RESULTS_DIR }} - - name: Process Test Results - if: always() - uses: ./.github/actions/process_test_results - with: - results_dir: ${{ env.TEST_RESULTS_DIR }} - datadog_api_key: ${{ secrets.DD_API_KEY }} - is_postsubmit: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} - continue-on-error: true on-host-test: needs: [initialize, docker-build-image, build] From c2b5458c95f135ff16323f8eb51972ef76109943 Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Wed, 22 Jan 2025 19:08:39 -0800 Subject: [PATCH 60/61] Remove erroneous input to build action --- .github/workflows/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 16231e63a55c3..3488814d9a18e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -322,3 +322,4 @@ jobs: run: | echo "Failing because at least one test shard had errors." exit 1 + From 90fde43d41b62759ccf0a5ae1319401477367b6f Mon Sep 17 00:00:00 2001 From: Igor Sarkisov Date: Wed, 22 Jan 2025 22:04:22 -0800 Subject: [PATCH 61/61] Remove extra gtest targets from android-arm4/x86 --- .github/config/android-arm64.json | 12 ------------ .github/config/android-x86.json | 12 ------------ .github/workflows/main.yaml | 3 +-- 3 files changed, 1 insertion(+), 26 deletions(-) diff --git a/.github/config/android-arm64.json b/.github/config/android-arm64.json index 1856609020d5c..c636baaa86f50 100644 --- a/.github/config/android-arm64.json +++ b/.github/config/android-arm64.json @@ -9,18 +9,6 @@ "system_webview_shell_apk", "cobalt:gn_all" ], - "gtest_targets": [ - "base_unittests", - "blink_unittests", - "gin_unittests", - "gpu_unittests", - "ipc_tests", - "media_unittests", - "mojo_unittests", - "net_unittests", - "sql_unittests", - "url_unittests" - ], "includes": [ { "name":"arm64", diff --git a/.github/config/android-x86.json b/.github/config/android-x86.json index a96814f41ba48..5e6d96f33b96f 100644 --- a/.github/config/android-x86.json +++ b/.github/config/android-x86.json @@ -9,18 +9,6 @@ "system_webview_shell_apk", "cobalt:gn_all" ], - "gtest_targets": [ - "base_unittests", - "blink_unittests", - "gin_unittests", - "gpu_unittests", - "ipc_tests", - "media_unittests", - "mojo_unittests", - "net_unittests", - "sql_unittests", - "url_unittests" - ], "includes": [ { "name":"x86", diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3488814d9a18e..fe49d5acc560c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,4 +1,4 @@ -vi # Reusable Cobalt CI workflow. +# Reusable Cobalt CI workflow. name: main @@ -322,4 +322,3 @@ jobs: run: | echo "Failing because at least one test shard had errors." exit 1 -