diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index 209a1206c6549..3789b58313be0 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -324,6 +324,17 @@ steps: composition: zippy args: [--scenario=UserTables, --actions=10000, --max-execution-time=30m] + - id: zippy-user-tables-persist-txn + label: "Zippy User Tables + toggle persist-txn" + timeout_in_minutes: 180 + agents: + queue: linux-x86_64 + artifact_paths: junit_*.xml + plugins: + - ./ci/plugins/mzcompose: + composition: zippy + args: [--scenario=UserTablesTogglePersistTxn, --actions=10000, --max-execution-time=30m] + - id: zippy-postgres-cdc label: "Zippy Postgres CDC" timeout_in_minutes: 120 diff --git a/misc/python/materialize/zippy/mz_actions.py b/misc/python/materialize/zippy/mz_actions.py index 32a43bbc17f21..549faa06c9eee 100644 --- a/misc/python/materialize/zippy/mz_actions.py +++ b/misc/python/materialize/zippy/mz_actions.py @@ -9,14 +9,40 @@ from materialize.mzcompose.composition import Composition +from materialize.mzcompose.services.materialized import Materialized from materialize.zippy.balancerd_capabilities import BalancerdIsRunning from materialize.zippy.crdb_capabilities import CockroachIsRunning -from materialize.zippy.framework import Action, Capability +from materialize.zippy.framework import Action, ActionFactory, Capabilities, Capability from materialize.zippy.minio_capabilities import MinioIsRunning from materialize.zippy.mz_capabilities import MzIsRunning from materialize.zippy.view_capabilities import ViewExists +class MzStartParameterized(ActionFactory): + """Starts a Mz instance with custom paramters.""" + + @classmethod + def requires(cls) -> set[type[Capability]]: + return {CockroachIsRunning, MinioIsRunning} + + @classmethod + def incompatible_with(cls) -> set[type[Capability]]: + return {MzIsRunning} + + def __init__( + self, additional_system_parameter_defaults: dict[str, str] = {} + ) -> None: + self.additional_system_parameter_defaults = additional_system_parameter_defaults + + def new(self, capabilities: Capabilities) -> list[Action]: + return [ + MzStart( + capabilities=capabilities, + additional_system_parameter_defaults=self.additional_system_parameter_defaults, + ) + ] + + class MzStart(Action): """Starts a Mz instance (all components are running in the same container).""" @@ -28,8 +54,28 @@ def requires(cls) -> set[type[Capability]]: def incompatible_with(cls) -> set[type[Capability]]: return {MzIsRunning} + def __init__( + self, + capabilities: Capabilities, + additional_system_parameter_defaults: dict[str, str] = {}, + ) -> None: + self.additional_system_parameter_defaults = additional_system_parameter_defaults + super().__init__(capabilities) + def run(self, c: Composition) -> None: - c.up("materialized") + print( + f"Starting Mz with additional_system_parameter_defaults = {self.additional_system_parameter_defaults}" + ) + + with c.override( + Materialized( + external_minio=True, + external_cockroach=True, + sanity_restart=False, + additional_system_parameter_defaults=self.additional_system_parameter_defaults, + ) + ): + c.up("materialized") for config_param in [ "max_tables", diff --git a/misc/python/materialize/zippy/scenarios.py b/misc/python/materialize/zippy/scenarios.py index 14ddd11d85b78..0034be7f9315d 100644 --- a/misc/python/materialize/zippy/scenarios.py +++ b/misc/python/materialize/zippy/scenarios.py @@ -26,7 +26,13 @@ ) from materialize.zippy.kafka_capabilities import Envelope from materialize.zippy.minio_actions import MinioRestart, MinioStart -from materialize.zippy.mz_actions import KillClusterd, MzRestart, MzStart, MzStop +from materialize.zippy.mz_actions import ( + KillClusterd, + MzRestart, + MzStart, + MzStartParameterized, + MzStop, +) from materialize.zippy.peek_actions import PeekCancellation from materialize.zippy.pg_cdc_actions import CreatePostgresCdcTable from materialize.zippy.postgres_actions import ( @@ -142,6 +148,33 @@ def config(self) -> dict[ActionOrFactory, float]: } +class UserTablesTogglePersistTxn(Scenario): + """A Zippy test using user tables with toggling persist_txn.""" + + def config(self) -> dict[ActionOrFactory, float]: + workload: dict[ActionOrFactory, float] = { + MzStop: 10, + CreateTableParameterized(): 10, + CreateViewParameterized(): 10, + ValidateTable: 10, + ValidateView: 10, + DML: 50, + } + + starts: dict[ActionOrFactory, float] = {} + for persist_txn in ["off", "eager"]: + starts[ + MzStartParameterized( + additional_system_parameter_defaults={ + "persist_txn_tables": persist_txn + } + ) + ] = 1 + + workload.update(starts) + return workload + + class DebeziumPostgres(Scenario): """A Zippy test using Debezium Postgres exclusively.""" diff --git a/test/zippy/mzcompose.py b/test/zippy/mzcompose.py index 66cdcbf7c3093..3645cf3c5fa31 100644 --- a/test/zippy/mzcompose.py +++ b/test/zippy/mzcompose.py @@ -113,13 +113,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: default=TransactionIsolation.STRICT_SERIALIZABLE, ) - parser.add_argument( - "--size", - type=int, - default=None, - help="SIZE to use for sources, sinks, materialized views and clusters", - ) - parser.add_argument( "--cockroach-tag", type=str, @@ -162,7 +155,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: }, ), Materialized( - default_size=args.size or Materialized.Size.DEFAULT_SIZE, external_minio=True, external_cockroach=True, sanity_restart=False,