Merge pull request #24091 from philip-stoev/zippy-persist-txn

zippy: Add a scenario that toggles persist-txn

ci/nightly/pipeline.template.yml

@@ -324,6 +324,17 @@ steps:
               composition: zippy
               args: [--scenario=UserTables, --actions=10000, --max-execution-time=30m]
 
+      - id: zippy-user-tables-persist-txn
+        label: "Zippy User Tables + toggle persist-txn"
+        timeout_in_minutes: 180
+        agents:
+          queue: linux-x86_64
+        artifact_paths: junit_*.xml
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: zippy
+              args: [--scenario=UserTablesTogglePersistTxn, --actions=10000, --max-execution-time=30m]
+
       - id: zippy-postgres-cdc
         label: "Zippy Postgres CDC"
         timeout_in_minutes: 120

misc/python/materialize/zippy/mz_actions.py

@@ -9,14 +9,40 @@
 
 
 from materialize.mzcompose.composition import Composition
+from materialize.mzcompose.services.materialized import Materialized
 from materialize.zippy.balancerd_capabilities import BalancerdIsRunning
 from materialize.zippy.crdb_capabilities import CockroachIsRunning
-from materialize.zippy.framework import Action, Capability
+from materialize.zippy.framework import Action, ActionFactory, Capabilities, Capability
 from materialize.zippy.minio_capabilities import MinioIsRunning
 from materialize.zippy.mz_capabilities import MzIsRunning
 from materialize.zippy.view_capabilities import ViewExists
 
 
+class MzStartParameterized(ActionFactory):
+    """Starts a Mz instance with custom paramters."""
+
+    @classmethod
+    def requires(cls) -> set[type[Capability]]:
+        return {CockroachIsRunning, MinioIsRunning}
+
+    @classmethod
+    def incompatible_with(cls) -> set[type[Capability]]:
+        return {MzIsRunning}
+
+    def __init__(
+        self, additional_system_parameter_defaults: dict[str, str] = {}
+    ) -> None:
+        self.additional_system_parameter_defaults = additional_system_parameter_defaults
+
+    def new(self, capabilities: Capabilities) -> list[Action]:
+        return [
+            MzStart(
+                capabilities=capabilities,
+                additional_system_parameter_defaults=self.additional_system_parameter_defaults,
+            )
+        ]
+
+
 class MzStart(Action):
     """Starts a Mz instance (all components are running in the same container)."""
 
@@ -28,8 +54,28 @@ def requires(cls) -> set[type[Capability]]:
     def incompatible_with(cls) -> set[type[Capability]]:
         return {MzIsRunning}
 
+    def __init__(
+        self,
+        capabilities: Capabilities,
+        additional_system_parameter_defaults: dict[str, str] = {},
+    ) -> None:
+        self.additional_system_parameter_defaults = additional_system_parameter_defaults
+        super().__init__(capabilities)
+
     def run(self, c: Composition) -> None:
-        c.up("materialized")
+        print(
+            f"Starting Mz with additional_system_parameter_defaults = {self.additional_system_parameter_defaults}"
+        )
+
+        with c.override(
+            Materialized(
+                external_minio=True,
+                external_cockroach=True,
+                sanity_restart=False,
+                additional_system_parameter_defaults=self.additional_system_parameter_defaults,
+            )
+        ):
+            c.up("materialized")
 
         for config_param in [
             "max_tables",

misc/python/materialize/zippy/scenarios.py

@@ -26,7 +26,13 @@
 )
 from materialize.zippy.kafka_capabilities import Envelope
 from materialize.zippy.minio_actions import MinioRestart, MinioStart
-from materialize.zippy.mz_actions import KillClusterd, MzRestart, MzStart, MzStop
+from materialize.zippy.mz_actions import (
+    KillClusterd,
+    MzRestart,
+    MzStart,
+    MzStartParameterized,
+    MzStop,
+)
 from materialize.zippy.peek_actions import PeekCancellation
 from materialize.zippy.pg_cdc_actions import CreatePostgresCdcTable
 from materialize.zippy.postgres_actions import (
@@ -142,6 +148,33 @@ def config(self) -> dict[ActionOrFactory, float]:
         }
 
 
+class UserTablesTogglePersistTxn(Scenario):
+    """A Zippy test using user tables with toggling persist_txn."""
+
+    def config(self) -> dict[ActionOrFactory, float]:
+        workload: dict[ActionOrFactory, float] = {
+            MzStop: 10,
+            CreateTableParameterized(): 10,
+            CreateViewParameterized(): 10,
+            ValidateTable: 10,
+            ValidateView: 10,
+            DML: 50,
+        }
+
+        starts: dict[ActionOrFactory, float] = {}
+        for persist_txn in ["off", "eager"]:
+            starts[
+                MzStartParameterized(
+                    additional_system_parameter_defaults={
+                        "persist_txn_tables": persist_txn
+                    }
+                )
+            ] = 1
+
+        workload.update(starts)
+        return workload
+
+
 class DebeziumPostgres(Scenario):
     """A Zippy test using Debezium Postgres exclusively."""
 

test/zippy/mzcompose.py

@@ -113,13 +113,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
         default=TransactionIsolation.STRICT_SERIALIZABLE,
     )
 
-    parser.add_argument(
-        "--size",
-        type=int,
-        default=None,
-        help="SIZE to use for sources, sinks, materialized views and clusters",
-    )
-
     parser.add_argument(
         "--cockroach-tag",
         type=str,
@@ -162,7 +155,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
             },
         ),
         Materialized(
-            default_size=args.size or Materialized.Size.DEFAULT_SIZE,
             external_minio=True,
             external_cockroach=True,
             sanity_restart=False,