Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

improvement(upgrade_test): Move InfoEvent message into their respective functions #9991

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

improvement(upgrade_test): Move InfoEvent message into their respective functions #9991

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions performance_regression_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -809,9 +809,7 @@ def get_email_data(self): # pylint: disable=no-self-use

@latency_calculator_decorator(legend="Upgrade Node")
def upgrade_node(self, node): # pylint: disable=arguments-differ
InfoEvent(message='Upgrade Node %s begin' % node.name).publish()
self._upgrade_node(node)
InfoEvent(message='Upgrade Node %s ended' % node.name).publish()

def _stop_stress_when_finished(self): # pylint: disable=no-self-use
with EventsSeverityChangerFilter(new_severity=Severity.NORMAL, # killing stress creates Critical error
Expand Down
30 changes: 5 additions & 25 deletions upgrade_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ def upgrade_node(self, node, upgrade_sstables=True):
# https://github.com/scylladb/scylla/issues/10447#issuecomment-1194155163
def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_version=None): # noqa: PLR0915
# pylint: disable=too-many-branches,too-many-statements
InfoEvent(message=f"Upgrade Node {node.name} begin").publish()
new_scylla_repo = new_scylla_repo or self.params.get('new_scylla_repo')
new_version = new_version or self.params.get('new_version')
upgrade_node_packages = self.params.get('upgrade_node_packages')
Expand Down Expand Up @@ -373,6 +374,7 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v
InfoEvent(message='upgrade_node - ended to "upgradesstables_if_command_available"').publish()

self.db_cluster.wait_all_nodes_un()
InfoEvent(message=f"Upgrade Node {node.name} ended").publish()

def upgrade_os(self, nodes):
def upgrade(node):
Expand All @@ -393,7 +395,7 @@ def rollback_node(self, node, upgrade_sstables=True):
@decorate_with_context(ignore_abort_requested_errors)
def _rollback_node(self, node, upgrade_sstables=True):
# pylint: disable=too-many-branches,too-many-statements
InfoEvent(message='Rollbacking a Node').publish()
InfoEvent(message=f"Rollback Node {node.name} begin").publish()
result = node.remoter.run('scylla --version')
orig_ver = result.stdout.strip()
# flush all memtables to SSTables
Expand Down Expand Up @@ -453,6 +455,7 @@ def _rollback_node(self, node, upgrade_sstables=True):
self.upgradesstables_if_command_available(node)

self.db_cluster.wait_all_nodes_un()
InfoEvent(message=f"Rollback Node {node.name} ended").publish()

@staticmethod
def upgradesstables_if_command_available(node, queue=None): # pylint: disable=invalid-name
Expand Down Expand Up @@ -563,10 +566,8 @@ def test_upgrade_cql_queries(self):
# upgrade all the nodes in random order
for i in indexes:
self.db_cluster.node_to_upgrade = self.db_cluster.nodes[i]
InfoEvent(message='Upgrade Node %s begin' % self.db_cluster.node_to_upgrade.name).publish()
self.upgrade_node(self.db_cluster.node_to_upgrade)
time.sleep(300)
InfoEvent(message='Upgrade Node %s ended' % self.db_cluster.node_to_upgrade.name).publish()

InfoEvent(message='Run some Queries to verify data AFTER UPGRADE').publish()
self.verify_db_data()
Expand Down Expand Up @@ -685,9 +686,7 @@ def test_rolling_upgrade(self): # pylint: disable=too-many-locals,too-many-stat
InfoEvent(message=step).publish()
# upgrade first node
self.db_cluster.node_to_upgrade = self.db_cluster.nodes[indexes[0]]
InfoEvent(message='Upgrade Node %s begin' % self.db_cluster.node_to_upgrade.name).publish()
self.upgrade_node(self.db_cluster.node_to_upgrade)
InfoEvent(message='Upgrade Node %s ended' % self.db_cluster.node_to_upgrade.name).publish()
self.db_cluster.node_to_upgrade.check_node_health()

# wait for the prepare write workload to finish
Expand Down Expand Up @@ -719,9 +718,7 @@ def test_rolling_upgrade(self): # pylint: disable=too-many-locals,too-many-stat
InfoEvent(message=step).publish()
# upgrade second node
self.db_cluster.node_to_upgrade = self.db_cluster.nodes[indexes[1]]
InfoEvent(message='Upgrade Node %s begin' % self.db_cluster.node_to_upgrade.name).publish()
self.upgrade_node(self.db_cluster.node_to_upgrade)
InfoEvent(message='Upgrade Node %s ended' % self.db_cluster.node_to_upgrade.name).publish()
self.db_cluster.node_to_upgrade.check_node_health()

# wait for the 10m read workload to finish
Expand All @@ -739,9 +736,7 @@ def test_rolling_upgrade(self): # pylint: disable=too-many-locals,too-many-stat

InfoEvent(message='Step3 - Rollback Second Node ').publish()
# rollback second node
InfoEvent(message='Rollback Node %s begin' % self.db_cluster.nodes[indexes[1]].name).publish()
self.rollback_node(self.db_cluster.nodes[indexes[1]])
InfoEvent(message='Rollback Node %s ended' % self.db_cluster.nodes[indexes[1]].name).publish()
self.db_cluster.nodes[indexes[1]].check_node_health()

step = 'Step4 - Verify data during mixed cluster mode '
Expand All @@ -758,9 +753,7 @@ def test_rolling_upgrade(self): # pylint: disable=too-many-locals,too-many-stat
InfoEvent(message=step).publish()
for i in indexes[1:]:
self.db_cluster.node_to_upgrade = self.db_cluster.nodes[i]
InfoEvent(message='Upgrade Node %s begin' % self.db_cluster.node_to_upgrade.name).publish()
self.upgrade_node(self.db_cluster.node_to_upgrade)
InfoEvent(message='Upgrade Node %s ended' % self.db_cluster.node_to_upgrade.name).publish()
self.db_cluster.node_to_upgrade.check_node_health()
self.fill_and_verify_db_data('after upgraded %s' % self.db_cluster.node_to_upgrade.name)
self.search_for_idx_token_error_after_upgrade(node=self.db_cluster.node_to_upgrade,
Expand Down Expand Up @@ -899,21 +892,17 @@ def run_raft_topology_upgrade_procedure(self):
def _start_and_wait_for_node_upgrade(self, node: BaseNode, step: int) -> None:
InfoEvent(
message=f"Step {step} - Upgrade {node.name} from dc {node.dc_idx}").publish()
InfoEvent(message='Upgrade Node %s begins' % node.name).publish()
with ignore_ycsb_connection_refused(), ignore_topology_change_coordinator_errors():
self.upgrade_node(node, upgrade_sstables=self.params.get('upgrade_sstables'))
InfoEvent(message='Upgrade Node %s ended' % node.name).publish()
node.check_node_health()

def _start_and_wait_for_node_rollback(self, node: BaseNode, step: int) -> None:
InfoEvent(
message=f"Step {step} - "
f"Rollback {node.name} from dc {node.dc_idx}"
).publish()
InfoEvent(message='Rollback Node %s begin' % node).publish()
with ignore_ycsb_connection_refused():
self.rollback_node(node, upgrade_sstables=self.params.get('upgrade_sstables'))
InfoEvent(message='Rollback Node %s ended' % node).publish()
node.check_node_health()

def _run_stress_workload(self, workload_name: str, wait_for_finish: bool = False) -> [CassandraStressThread]:
Expand Down Expand Up @@ -1533,11 +1522,9 @@ def _custom_profile_rolling_upgrade(self, cs_user_profiles, new_scylla_repo=None
InfoEvent(message=step).publish()
# upgrade first node
self.db_cluster.node_to_upgrade = self.db_cluster.nodes[indexes[0]]
InfoEvent(message='Upgrade Node %s begin' % self.db_cluster.node_to_upgrade.name).publish()
# Call "_upgrade_node" to prevent running truncate test
self._upgrade_node(self.db_cluster.node_to_upgrade,
new_scylla_repo=new_scylla_repo, new_version=new_version)
InfoEvent(message='Upgrade Node %s ended' % self.db_cluster.node_to_upgrade.name).publish()
self.db_cluster.node_to_upgrade.check_node_health()

InfoEvent(message='after upgraded one node').publish()
Expand All @@ -1548,22 +1535,17 @@ def _custom_profile_rolling_upgrade(self, cs_user_profiles, new_scylla_repo=None
InfoEvent(message=step).publish()
# upgrade second node
self.db_cluster.node_to_upgrade = self.db_cluster.nodes[indexes[1]]
InfoEvent(message='Upgrade Node %s begin' % self.db_cluster.node_to_upgrade.name).publish()
# Call "_upgrade_node" to prevent running truncate test
self._upgrade_node(self.db_cluster.node_to_upgrade,
new_scylla_repo=new_scylla_repo, new_version=new_version)
InfoEvent(message='Upgrade Node %s ended' % self.db_cluster.node_to_upgrade.name).publish()
self.db_cluster.node_to_upgrade.check_node_health()

self.search_for_idx_token_error_after_upgrade(node=self.db_cluster.node_to_upgrade,
step=step+' - after upgraded two nodes')

InfoEvent(message='Step3 - Rollback Second Node ').publish()
# rollback second node
InfoEvent(message='Rollback Node %s begin' % self.db_cluster.nodes[indexes[1]].name).publish()
# Call "_rollback_node" to prevent running truncate test
# rollback second node, use "_rollback_node" to prevent running truncate test
self._rollback_node(self.db_cluster.nodes[indexes[1]])
InfoEvent(message='Rollback Node %s ended' % self.db_cluster.nodes[indexes[1]].name).publish()
self.db_cluster.nodes[indexes[1]].check_node_health()

step = 'Step4 - Verify data during mixed cluster mode '
Expand All @@ -1579,11 +1561,9 @@ def _custom_profile_rolling_upgrade(self, cs_user_profiles, new_scylla_repo=None
InfoEvent(message=step).publish()
for i in indexes[1:]:
self.db_cluster.node_to_upgrade = self.db_cluster.nodes[i]
InfoEvent(message='Upgrade Node %s begin' % self.db_cluster.node_to_upgrade.name).publish()
# Call "_upgrade_node" to prevent running truncate test
self._upgrade_node(self.db_cluster.node_to_upgrade,
new_scylla_repo=new_scylla_repo, new_version=new_version)
InfoEvent(message='Upgrade Node %s ended' % self.db_cluster.node_to_upgrade.name).publish()
self.db_cluster.node_to_upgrade.check_node_health()
self.search_for_idx_token_error_after_upgrade(node=self.db_cluster.node_to_upgrade,
step=step)
Expand Down