From f0d86d25ddacf948896ce560793452ec42e93c45 Mon Sep 17 00:00:00 2001
From: Amaury Pouly <amaury.pouly@lowrisc.org>
Date: Mon, 2 Sep 2024 10:56:47 +0200
Subject: [PATCH] [ci] Add a lint to check for unrunnable tests

We have discovered that some unittest have become unrunnable because
we added my mistake dependencies that are marked as only compatible
with the OT platforms. This means that those unittests are then skipped
when running all tests using //...
This commit adds a lint to make sure that all tests are compatible with
the host platform. For efficiency reasons, this lint is run in the sw_test
step. The reason is that running a cquery is quite expensive as it requires
building the graph and even compiling things for certain repositories. By
running it after building the tests, we are sure not to duplicate this work.

Signed-off-by: Amaury Pouly <amaury.pouly@lowrisc.org>
---
 azure-pipelines.yml                    |  3 +++
 ci/scripts/check-unrunnable-tests.sh   | 23 +++++++++++++++++++++++
 ci/scripts/incompatible_targets.cquery |  6 ++++++
 3 files changed, 32 insertions(+)
 create mode 100755 ci/scripts/check-unrunnable-tests.sh
 create mode 100644 ci/scripts/incompatible_targets.cquery

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index e55daa9f6b25af..7c95ce1e4f3756 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -278,6 +278,9 @@ jobs:
         --target_pattern_file="${TARGET_PATTERN_FILE}"
     displayName: Build & test SW
   - template: ci/publish-bazel-test-results.yml
+  - bash: ci/scripts/check-unrunnable-tests.sh
+    displayName: Check for unrunnable tests
+    continueOnError: succeededOrFailed()
 
 - job: chip_englishbreakfast_verilator
   displayName: Verilated English Breakfast (Build)
diff --git a/ci/scripts/check-unrunnable-tests.sh b/ci/scripts/check-unrunnable-tests.sh
new file mode 100755
index 00000000000000..82ef4c4da6f0a2
--- /dev/null
+++ b/ci/scripts/check-unrunnable-tests.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+# Look for all tests are incompatible with host, i.e. unrunnable.
+# The cquery will output an empty string for compatible tests and
+# a non-empty string (test name) for the incompatible ones. Therefore
+# we filter out the empty lines.
+ci/bazelisk.sh cquery 'tests(//...)' \
+    --noinclude_aspects \
+    --define DISABLE_VERILATOR_BUILD=true \
+    --output=starlark \
+    --starlark:file=ci/scripts/incompatible_targets.cquery \
+    | sed '/^$/d' > output.txt
+if [ ! -s output.txt ]; then
+    echo "The following tests are incompatible with the host platform:"
+    cat output.txt
+    exit 1
+fi
diff --git a/ci/scripts/incompatible_targets.cquery b/ci/scripts/incompatible_targets.cquery
new file mode 100644
index 00000000000000..c3648f54ff3242
--- /dev/null
+++ b/ci/scripts/incompatible_targets.cquery
@@ -0,0 +1,6 @@
+# Find all targets that are incompatible with the specified platform
+# (i.e. host by default).
+def format(target):
+  if "IncompatiblePlatformProvider" in providers(target):
+    return target.label
+  return ""