miri: Enable tests using epoll_wait

.config/nextest.toml

@@ -2,7 +2,7 @@
 slow-timeout = { period = "60s", terminate-after = 2 }
 
 [profile.default-miri]
-slow-timeout = { period = "600s", terminate-after = 2 }
+slow-timeout = { period = "1200s", terminate-after = 2 }
 
 # For a given configuration parameter, the first override to match wins. Keep
 # these sorted in order from most specific to least specific.

bin/ci-builder

@@ -180,6 +180,13 @@ case "$cmd" in
             --env NIGHTLY_CANARY_CONFLUENT_CLOUD_API_SECRET
             --env NO_COLOR
             --env PYPI_TOKEN
+            # For Miri with nightly Rust
+            --env ZOOKEEPER_ADDR
+            --env KAFKA_ADDRS
+            --env SCHEMA_REGISTRY_URL
+            --env POSTGRES_URL
+            --env COCKROACH_URL
+            --env MZ_SOFT_ASSERTIONS
         )
 
         if [[ $detach_container == "true" ]]; then

ci/nightly/pipeline.yml

@@ -17,6 +17,7 @@ steps:
       - select: Tests
         key: tests
         options:
+          - { value: miri-test }
           - { value: kafka-matrix }
           - { value: kafka-multi-broker }
           - { value: redpanda-testdrive }
@@ -92,6 +93,18 @@ steps:
 
   - wait: ~
 
+  - id: miri-test
+    label: Miri test (full)
+    timeout_in_minutes: 600
+    artifact_paths: [junit_*.xml, target/nextest/ci/junit_cargo-test.xml]
+    plugins:
+      - ./ci/plugins/scratch-aws-access: ~
+      - ./ci/plugins/mzcompose:
+          composition: cargo-test
+          args: [--miri-full]
+    agents:
+      queue: builder-linux-x86_64
+
   - id: feature-benchmark
     label: "Feature benchmark against 'latest'"
     timeout_in_minutes: 360

ci/test/cargo-test-miri-fast.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+#
+# cargo-test-miri.sh — runs subset of unit tests under miri to check for
+# undefined behaviour.
+
+set -euo pipefail
+
+# miri artifacts are thoroughly incompatible with normal build artifacts,
+# so keep them away from the `target` directory.
+export CARGO_TARGET_DIR="$PWD/miri-target"
+export MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-strict-provenance"
+
+PARTITION=$((${BUILDKITE_PARALLEL_JOB:-0}+1))
+TOTAL=${BUILDKITE_PARALLEL_JOB_COUNT:-1}
+
+# exclude network-based and more complex tests which run out of memory
+cargo miri nextest run -j"$(nproc)" --partition "count:$PARTITION/$TOTAL" --no-fail-fast --workspace --exclude 'mz-adapter*' --exclude 'mz-environmentd*' --exclude 'mz-expr*' --exclude 'mz-compute-client*' --exclude 'mz-persist-client*' --exclude 'mz-ssh-util*' --exclude 'mz-rocksdb*' --exclude 'mz-sqllogictest*'

ci/test/cargo-test-miri.sh

@@ -22,5 +22,5 @@ export MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-strict-provenance"
 PARTITION=$((${BUILDKITE_PARALLEL_JOB:-0}+1))
 TOTAL=${BUILDKITE_PARALLEL_JOB_COUNT:-1}
 
-# exclude netwrok based tests, they mostly fail on epoll_wait
-cargo miri nextest run -j"$(nproc)" --partition "count:$PARTITION/$TOTAL" --no-fail-fast --workspace --exclude 'mz-adapter*' --exclude 'mz-environmentd*' --exclude 'mz-expr*' --exclude 'mz-compute-client*' --exclude 'mz-persist-client*' --exclude 'mz-ssh-util*' --exclude 'mz-rocksdb*' --exclude 'mz-sqllogictest*'
+# exclude network-based and more complex tests which run out of memory
+cargo miri nextest run -j"$(nproc)" --partition "count:$PARTITION/$TOTAL" --no-fail-fast --workspace --exclude 'mz-environmentd*' --exclude 'mz-compute-client*' --exclude 'mz-ssh-util*' --exclude 'mz-rocksdb*'

ci/test/cargo-test/mzcompose.py

@@ -38,6 +38,8 @@
 
 
 def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
+    parser.add_argument("--miri-full", action="store_true")
+    parser.add_argument("--miri-fast", action="store_true")
     parser.add_argument("args", nargs="*")
     args = parser.parse_args()
     c.up("zookeeper", "kafka", "schema-registry", "postgres", "cockroach")
@@ -107,27 +109,38 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
                 ["buildkite-agent", "artifact", "upload", "coverage/cargotest.lcov.xz"]
             )
     else:
-        spawn.runv(
-            [
-                "cargo",
-                "build",
-                "--bin",
-                "clusterd",
-            ],
-            env=env,
-        )
-        cpu_count = os.cpu_count()
-        assert cpu_count
-        spawn.runv(
-            [
-                "cargo",
-                "nextest",
-                "run",
-                "--profile=ci",
-                # Most tests don't use 100% of a CPU core, so run two tests per CPU.
-                # TODO(def-): Reenable when #19931 is fixed
-                # f"--test-threads={cpu_count * 2}",
-                *args.args,
-            ],
-            env=env,
-        )
+        if args.miri_full:
+            spawn.runv(
+                ["bin/ci-builder", "run", "nightly", "ci/test/cargo-test-miri.sh"],
+                env=env,
+            )
+        elif args.miri_fast:
+            spawn.runv(
+                ["bin/ci-builder", "run", "nightly", "ci/test/cargo-test-miri-fast.sh"],
+                env=env,
+            )
+        else:
+            spawn.runv(
+                [
+                    "cargo",
+                    "build",
+                    "--bin",
+                    "clusterd",
+                ],
+                env=env,
+            )
+            cpu_count = os.cpu_count()
+            assert cpu_count
+            spawn.runv(
+                [
+                    "cargo",
+                    "nextest",
+                    "run",
+                    "--profile=ci",
+                    # Most tests don't use 100% of a CPU core, so run two tests per CPU.
+                    # TODO(def-): Reenable when #19931 is fixed
+                    # f"--test-threads={cpu_count * 2}",
+                    *args.args,
+                ],
+                env=env,
+            )

ci/test/pipeline.template.yml

@@ -155,11 +155,16 @@ steps:
       queue: builder-linux-x86_64
 
   - id: miri-test
-    label: Miri test %n
-    command: bin/ci-builder run nightly ci/test/cargo-test-miri.sh
+    label: Miri test (fast) %n
     inputs: [src]
     parallelism: 2
     timeout_in_minutes: 30
+    artifact_paths: [junit_*.xml, target/nextest/ci/junit_cargo-test.xml]
+    plugins:
+      - ./ci/plugins/scratch-aws-access: ~
+      - ./ci/plugins/mzcompose:
+          composition: cargo-test
+          args: [--miri-fast]
     agents:
       queue: builder-linux-x86_64
     coverage: skip

src/adapter/src/catalog.rs

@@ -8791,6 +8791,7 @@ mod tests {
     /// search paths, so do not require schema qualification on system objects such
     /// as types.
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_minimal_qualification() {
         Catalog::with_debug(NOW_ZERO.clone(), |catalog| async move {
             struct TestCase {
@@ -8863,6 +8864,7 @@ mod tests {
     }
 
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_catalog_revision() {
         let debug_stash_factory = DebugStashFactory::new().await;
         {
@@ -8898,6 +8900,7 @@ mod tests {
     }
 
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_effective_search_path() {
         Catalog::with_debug(NOW_ZERO.clone(), |catalog| async move {
             let mz_catalog_schema = (
@@ -9043,6 +9046,7 @@ mod tests {
     }
 
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_builtin_migration() {
         enum ItemNamespace {
             System,
@@ -9697,6 +9701,7 @@ mod tests {
     }
 
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_normalized_create() {
         Catalog::with_debug(NOW_ZERO.clone(), |catalog| {
             let catalog = catalog.for_system_session();
@@ -9864,6 +9869,7 @@ mod tests {
     }
 
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_object_type() {
         let debug_stash_factory = DebugStashFactory::new().await;
         let stash = debug_stash_factory.open_debug().await;
@@ -9883,6 +9889,7 @@ mod tests {
     }
 
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_get_privileges() {
         let debug_stash_factory = DebugStashFactory::new().await;
         let stash = debug_stash_factory.open_debug().await;

src/adapter/src/catalog/builtin.rs

@@ -4206,6 +4206,7 @@ mod tests {
     // Connect to a running Postgres server and verify that our builtin
     // types and functions match it, in addition to some other things.
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_compare_builtins_postgres() {
         async fn inner(catalog: Catalog) {
             // Verify that all builtin functions:
@@ -4557,6 +4558,7 @@ mod tests {
 
     // Execute all builtin functions with all combinations of arguments from interesting datums.
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_smoketest_all_builtins() {
         fn inner(catalog: Catalog) {
             let conn_catalog = catalog.for_system_session();
@@ -4797,6 +4799,7 @@ mod tests {
 
     // Make sure pg views don't use types that only exist in Materialize.
     #[mz_ore::test(tokio::test)]
+    #[cfg_attr(miri, ignore)] //  unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
     async fn test_pg_views_forbidden_types() {
         Catalog::with_debug(SYSTEM_TIME.clone(), |catalog| async move {
             let conn_catalog = catalog.for_system_session();

src/adapter/src/catalog/storage.rs

@@ -2131,6 +2131,7 @@ mod test {
 
     proptest! {
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // slow
         fn proptest_database_key_roundtrip(key: DatabaseKey) {
             let proto = key.into_proto();
             let round = proto.into_rust().expect("to roundtrip");
@@ -2139,6 +2140,7 @@ mod test {
         }
 
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // slow
         fn proptest_database_value_roundtrip(value: DatabaseValue) {
             let proto = value.into_proto();
             let round = proto.into_rust().expect("to roundtrip");
@@ -2147,6 +2149,7 @@ mod test {
         }
 
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // slow
         fn proptest_schema_key_roundtrip(key: SchemaKey) {
             let proto = key.into_proto();
             let round = proto.into_rust().expect("to roundtrip");
@@ -2155,6 +2158,7 @@ mod test {
         }
 
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // slow
         fn proptest_schema_value_roundtrip(value: SchemaValue) {
             let proto = value.into_proto();
             let round = proto.into_rust().expect("to roundtrip");
@@ -2163,6 +2167,7 @@ mod test {
         }
 
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // slow
         fn proptest_item_key_roundtrip(key: ItemKey) {
             let proto = key.into_proto();
             let round = proto.into_rust().expect("to roundtrip");
@@ -2171,6 +2176,7 @@ mod test {
         }
 
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // slow
         fn proptest_item_value_roundtrip(value: ItemValue) {
             let proto = value.into_proto();
             let round = proto.into_rust().expect("to roundtrip");

src/adapter/src/config/params.rs

@@ -146,6 +146,7 @@ mod tests {
     use super::SynchronizedParameters;
 
     #[mz_ore::test]
+    #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
     fn test_github_18189() {
         let vars = SystemVars::default();
         let mut sync = SynchronizedParameters::new(vars);
@@ -156,6 +157,7 @@ mod tests {
     }
 
     #[mz_ore::test]
+    #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
     fn test_vars_are_synced() {
         let vars = SystemVars::default();
         let sync = SynchronizedParameters::new(vars);

src/adapter/src/coord/peek.rs

@@ -581,6 +581,7 @@ mod tests {
     use super::*;
 
     #[mz_ore::test]
+    #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
     fn test_fast_path_plan_as_text() {
         let typ = RelationType::new(vec![ColumnType {
             scalar_type: ScalarType::String,

src/adapter/tests/parameters.rs

@@ -80,6 +80,7 @@ use mz_repr::ScalarType;
 use mz_sql::plan::PlanContext;
 
 #[mz_ore::test(tokio::test)]
+#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
 async fn test_parameter_type_inference() {
     let test_cases = vec![
         (

src/adapter/tests/sql.rs

@@ -105,6 +105,7 @@ use tokio::sync::Mutex;
 // catalog.
 
 #[mz_ore::test(tokio::test)]
+#[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
 async fn datadriven() {
     datadriven::walk_async("tests/testdata/sql", |mut f| async {
         // The datadriven API takes an `FnMut` closure, and can't express to Rust that

src/adapter/tests/timestamp_selection.rs

@@ -261,6 +261,7 @@ fn parse_query_when(s: &str) -> QueryWhen {
 /// returns the chosen timestamp. Append `full` as an argument to it to see the entire
 /// TimestampDetermination.
 #[mz_ore::test]
+#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `decNumberFromInt32` on OS `linux`
 fn test_timestamp_selection() {
     datadriven::walk("tests/testdata/timestamp_selection", |tf| {
         let mut f = Frontiers {

src/avro/src/writer.rs

@@ -586,6 +586,7 @@ mod tests {
     }
 
     #[mz_ore::test]
+    #[cfg_attr(miri, ignore)] // slow
     fn test_writer_roundtrip() {
         let schema = Schema::from_str(SCHEMA).unwrap();
         let make_record = |a: i64, b| {

src/cluster-client/src/client.rs

@@ -208,6 +208,7 @@ mod tests {
         }
 
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // slow
         fn cluster_startup_epoch_protobuf_roundtrip(expect in any::<ClusterStartupEpoch>() ) {
             let actual = protobuf_roundtrip::<_, ProtoClusterStartupEpoch>(&expect);
             assert!(actual.is_ok());

src/compute-client/src/plan/join/mod.rs

@@ -421,6 +421,7 @@ mod tests {
         #![proptest_config(ProptestConfig::with_cases(32))]
 
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decContextDefault` on OS `linux`
         fn join_plan_protobuf_roundtrip(expect in any::<JoinPlan>() ) {
             let actual = protobuf_roundtrip::<_, ProtoJoinPlan>(&expect);
             assert!(actual.is_ok());

src/compute-client/src/plan/mod.rs

@@ -2197,6 +2197,7 @@ mod tests {
     proptest! {
         #![proptest_config(ProptestConfig::with_cases(10))]
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decContextDefault` on OS `linux`
         fn get_plan_protobuf_roundtrip(expect in any::<GetPlan>()) {
             let actual = protobuf_roundtrip::<_, ProtoGetPlan>(&expect);
             assert!(actual.is_ok());

src/compute-client/src/protocol/command.rs

@@ -522,6 +522,7 @@ mod tests {
         #![proptest_config(ProptestConfig::with_cases(32))]
 
         #[mz_ore::test]
+        #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decContextDefault` on OS `linux`
         fn peek_protobuf_roundtrip(expect in any::<Peek>() ) {
             let actual = protobuf_roundtrip::<_, ProtoPeek>(&expect);
             assert!(actual.is_ok());