Skip to content

Commit

Permalink
fix: only generate stales for deployed services and moved it to dao
Browse files Browse the repository at this point in the history
  • Loading branch information
@SteBaum
SteBaum committed Jul 26, 2024
1 parent 7c2481c commit 45c8f83
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 105 deletions.
2 changes: 1 addition & 1 deletion tdp/cli/commands/status/generate_stales.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def generate_stales(
check_services_cleanliness(cluster_variables)

with Dao(db_engine) as dao:
stale_status_logs = dao.get_cluster_status().generate_stale_sch_logs(
stale_status_logs = dao.generate_stale_sch_logs(
cluster_variables=cluster_variables, collections=collections
)

Expand Down
2 changes: 1 addition & 1 deletion tdp/cli/commands/vars/edit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def edit(

# Generate stale component list and save it to the database
with Dao(db_engine) as dao:
stale_status_logs = dao.get_cluster_status().generate_stale_sch_logs(
stale_status_logs = dao.generate_stale_sch_logs(
cluster_variables=cluster_variables, collections=collections
)
dao.session.add_all(stale_status_logs)
Expand Down
103 changes: 1 addition & 102 deletions tdp/core/cluster_status.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,20 @@
from collections.abc import Iterable, MutableMapping
from typing import TYPE_CHECKING, Optional

from tdp.core.dag import Dag
from tdp.core.entities.hostable_entity_name import (
HostableEntityName,
create_hostable_entity_name,
)
from tdp.core.entities.hosted_entity import (
HostedEntity,
HostedServiceComponent,
create_hosted_entity,
)
from tdp.core.entities.hosted_entity_status import HostedEntityStatus
from tdp.core.models.sch_status_log_model import (
SCHStatusLogModel,
SCHStatusLogSourceEnum,
)

if TYPE_CHECKING:
from tdp.core.collections import Collections
from tdp.core.variables import ClusterVariables
pass

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -59,102 +54,6 @@ def __len__(self) -> int:
def __iter__(self):
return self._cluster_status.__iter__()

def generate_stale_sch_logs(
self,
*,
cluster_variables: ClusterVariables,
collections: Collections,
) -> set[SCHStatusLogModel]:
"""Generate logs for components that need to be configured or restarted.
This method identifies components that have undergone changes in their
versions and determines if they need to be configured, restarted, or both.
Note: If a component has neither config or restart operations, it is not
considered stale and is excluded from the results.
Args:
cluster_variables: Current configuration.
collections: Collections instance.
Returns:
Set of SCHStatusLog.
"""
logs: dict[HostedEntity, SCHStatusLogModel] = {}
source_reconfigure_operations: set[str] = set()

modified_entities = cluster_variables.get_modified_entities(self.values())

# Return early if no entity has modified configurations
if len(modified_entities) == 0:
return set()

# Create logs for the modified entities
for entity in modified_entities:
config_operation = collections.operations.get(f"{entity.name}_config")
start_operation = collections.operations.get(f"{entity.name}_start")
restart_operation = collections.operations.get(f"{entity.name}_restart")

# Add the config and start operations to the set to get their descendants
if config_operation:
source_reconfigure_operations.add(config_operation.name)
if start_operation:
source_reconfigure_operations.add(start_operation.name)

# Create a log to update the stale status of the entity if a config and/or
# restart operations are available
# Only source hosts affected by the modified configuration are considered as
# stale (while all hosts are considered as stale for the descendants)
if config_operation or restart_operation:
log = logs.setdefault(
entity,
SCHStatusLogModel(
service=entity.name.service,
component=(
entity.name.component
if isinstance(entity, HostedServiceComponent)
else None
),
host=entity.host,
source=SCHStatusLogSourceEnum.STALE,
),
)
if config_operation:
log.to_config = True
if restart_operation:
log.to_restart = True

# Create logs for the descendants of the modified entities
for operation in Dag(collections).get_operation_descendants(
nodes=list(source_reconfigure_operations), restart=True
):
# Only create a log when config or restart operation is available
if operation.action_name not in ["config", "restart"]:
continue

# Create a log for each host where the entity is deployed
for host in operation.host_names:
log = logs.setdefault(
create_hosted_entity(
create_hostable_entity_name(
operation.service_name, operation.component_name
),
host,
),
SCHStatusLogModel(
service=operation.service_name,
component=operation.component_name,
host=host,
source=SCHStatusLogSourceEnum.STALE,
),
)
if operation.action_name == "config":
log.to_config = True
elif operation.action_name == "restart":
log.to_restart = True

return set(logs.values())

def update_hosted_entity(
self,
entity: HostedEntity,
Expand Down
126 changes: 125 additions & 1 deletion tdp/dao.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,20 @@
from sqlalchemy.orm import sessionmaker

from tdp.core.cluster_status import ClusterStatus
from tdp.core.collections import Collections
from tdp.core.dag import Dag
from tdp.core.entities.hostable_entity_name import create_hostable_entity_name
from tdp.core.entities.hosted_entity import create_hosted_entity
from tdp.core.entities.hosted_entity import (
HostedEntity,
HostedServiceComponent,
create_hosted_entity,
)
from tdp.core.entities.hosted_entity_status import HostedEntityStatus
from tdp.core.models.deployment_model import DeploymentModel
from tdp.core.models.enums import SCHStatusLogSourceEnum
from tdp.core.models.operation_model import OperationModel
from tdp.core.models.sch_status_log_model import SCHStatusLogModel
from tdp.core.variables.cluster_variables import ClusterVariables


def _create_last_value_statement(column, non_null=False):
Expand Down Expand Up @@ -300,3 +308,119 @@ def get_deployments(
.limit(limit)
.offset(offset)
)

def generate_stale_sch_logs(
self,
cluster_variables: ClusterVariables,
collections: Collections,
) -> set[SCHStatusLogModel]:
"""Generate logs for components that need to be configured or restarted.
This method identifies components that have undergone changes in their
versions and determines if they need to be configured, restarted, or both.
Note: If a component has neither config or restart operations, it is not
considered stale and is excluded from the results.
Args:
cluster_variables: Current configuration.
collections: Collections instance.
Returns:
Set of SCHStatusLog.
"""
hosted_entity_status: list[HostedEntityStatus] = (
self.get_hosted_entity_statuses()
)
logs: dict[HostedEntity, SCHStatusLogModel] = {}
source_reconfigure_operations: set[str] = set()

modified_entities = cluster_variables.get_modified_entities(
ClusterStatus(hosted_entity_status).values()
)

# Return early if no entity has modified configurations
if len(modified_entities) == 0:
return set()

# Get the list of services of the current hosted_entities in the database
hosted_entities: list[HostedEntity] = [
entity_status.entity for entity_status in hosted_entity_status
]
hosted_entity_services: list[str] = [
entity.name.service for entity in hosted_entities
]

# Create logs for the modified entities
for entity in modified_entities:
config_operation = collections.operations.get(f"{entity.name}_config")
start_operation = collections.operations.get(f"{entity.name}_start")
restart_operation = collections.operations.get(f"{entity.name}_restart")

# Add the config and start operations to the set to get their descendants
if config_operation:
source_reconfigure_operations.add(config_operation.name)
if start_operation:
source_reconfigure_operations.add(start_operation.name)

# Create a log to update the stale status of the entity if a config and/or
# restart operations are available
# Only source hosts affected by the modified configuration are considered as
# stale (while all hosts are considered as stale for the descendants)
if (
config_operation
and entity in hosted_entities
or restart_operation
and entity in hosted_entities
):
log = logs.setdefault(
entity,
SCHStatusLogModel(
service=entity.name.service,
component=(
entity.name.component
if isinstance(entity, HostedServiceComponent)
else None
),
host=entity.host,
source=SCHStatusLogSourceEnum.STALE,
),
)
if config_operation:
log.to_config = True
if restart_operation:
log.to_restart = True

# Create logs for the descendants of the modified entities
for operation in Dag(collections).get_operation_descendants(
nodes=list(source_reconfigure_operations), restart=True
):
# Only create a log when config or restart operation is available
if (
operation.action_name not in ["config", "restart"]
or operation.service_name not in hosted_entity_services
):
continue

# Create a log for each host where the entity is deployed
for host in operation.host_names:
log = logs.setdefault(
create_hosted_entity(
create_hostable_entity_name(
operation.service_name, operation.component_name
),
host,
),
SCHStatusLogModel(
service=operation.service_name,
component=operation.component_name,
host=host,
source=SCHStatusLogSourceEnum.STALE,
),
)
if operation.action_name == "config":
log.to_config = True
elif operation.action_name == "restart":
log.to_restart = True

return set(logs.values())

0 comments on commit 45c8f83

Please sign in to comment.