Skip to content

Commit

Permalink
feat: enhance logging for on-call duty policy execution and improve u…
Browse files Browse the repository at this point in the history 
…ser notification handling
  • Loading branch information
@simlarsen
simlarsen committed Jan 20, 2025
1 parent 6c7f1b5 commit cb11a46
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 19 deletions.
28 changes: 14 additions & 14 deletions Common/Server/Services/OnCallDutyPolicyEscalationRuleService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import OnCallDutyPolicyEscalationRuleTeam from "Common/Models/DatabaseModels/OnC
import OnCallDutyPolicyEscalationRuleUser from "Common/Models/DatabaseModels/OnCallDutyPolicyEscalationRuleUser";
import OnCallDutyPolicyExecutionLogTimeline from "Common/Models/DatabaseModels/OnCallDutyPolicyExecutionLogTimeline";
import User from "Common/Models/DatabaseModels/User";
import logger from "../Utils/Logger";

export class Service extends DatabaseService<Model> {
public async startRuleExecution(
Expand All @@ -43,7 +44,7 @@ export class Service extends DatabaseService<Model> {
onCallPolicyId: ObjectID;
},
): Promise<void> {
// add log timeline.
logger.debug(`Starting rule execution for ruleId: ${ruleId.toString()}`);

const rule: Model | null = await this.findOneById({
id: ruleId,
Expand All @@ -63,6 +64,8 @@ export class Service extends DatabaseService<Model> {
);
}

logger.debug(`Found rule: ${JSON.stringify(rule)}`);

await OnCallDutyPolicyExecutionLogService.updateOneById({
id: options.onCallPolicyExecutionLogId,
data: {
Expand All @@ -76,6 +79,8 @@ export class Service extends DatabaseService<Model> {
},
});

logger.debug(`Updated execution log for ruleId: ${ruleId.toString()}`);

type GetNewLogFunction = () => OnCallDutyPolicyExecutionLogTimeline;

const getNewLog: GetNewLogFunction =
Expand Down Expand Up @@ -135,6 +140,8 @@ export class Service extends DatabaseService<Model> {
},
});

logger.debug(`Found users in rule: ${JSON.stringify(usersInRule)}`);

const teamsInRule: Array<OnCallDutyPolicyEscalationRuleTeam> =
await OnCallDutyPolicyEscalationRuleTeamService.findBy({
query: {
Expand All @@ -150,6 +157,8 @@ export class Service extends DatabaseService<Model> {
},
});

logger.debug(`Found teams in rule: ${JSON.stringify(teamsInRule)}`);

const schedulesInRule: Array<OnCallDutyPolicyEscalationRuleSchedule> =
await OnCallDutyPolicyEscalationRuleScheduleService.findBy({
query: {
Expand All @@ -165,7 +174,7 @@ export class Service extends DatabaseService<Model> {
},
});

// get unique users and notify all the users.
logger.debug(`Found schedules in rule: ${JSON.stringify(schedulesInRule)}`);

type StartUserNotificationRuleExecutionFunction = (
userId: ObjectID,
Expand All @@ -179,7 +188,7 @@ export class Service extends DatabaseService<Model> {
teamId: ObjectID | null,
scheduleId: ObjectID | null,
): Promise<void> => {
// no users in this rule. Skipping.
logger.debug(`Starting notification rule execution for userId: ${userId.toString()}`);
let log: OnCallDutyPolicyExecutionLogTimeline = getNewLog();
log.statusMessage = "Sending notification to user.";
log.status = OnCallDutyExecutionLogTimelineStatus.Executing;
Expand Down Expand Up @@ -236,7 +245,6 @@ export class Service extends DatabaseService<Model> {
null,
);
} else {
// no users in this rule. Skipping.
const log: OnCallDutyPolicyExecutionLogTimeline = getNewLog();
log.statusMessage =
"Skipped because notification sent to this user already.";
Expand All @@ -263,7 +271,6 @@ export class Service extends DatabaseService<Model> {
uniqueUserIds.push(userRule.userId!);
await startUserNotificationRuleExecution(userRule.userId!, null, null);
} else {
// no users in this rule. Skipping.
const log: OnCallDutyPolicyExecutionLogTimeline = getNewLog();
log.statusMessage =
"Skipped because notification sent to this user already.";
Expand All @@ -280,23 +287,16 @@ export class Service extends DatabaseService<Model> {
}

for (const scheduleRule of schedulesInRule) {
// get layers and users in this schedule and find a user to notify.

const userIdInSchedule: ObjectID | null =
await OnCallDutyPolicyScheduleService.getCurrentUserIdInSchedule(
scheduleRule.onCallDutyPolicyScheduleId!,
);

if (!userIdInSchedule) {
// no user active in this schedule. Skipping.

const log: OnCallDutyPolicyExecutionLogTimeline = getNewLog();

log.statusMessage =
"Skipped because no active users are found in this schedule.";

log.status = OnCallDutyExecutionLogTimelineStatus.Skipped;

log.onCallDutyScheduleId = scheduleRule.onCallDutyPolicyScheduleId!;

await OnCallDutyPolicyExecutionLogTimelineService.create({
Expand All @@ -321,7 +321,6 @@ export class Service extends DatabaseService<Model> {
scheduleRule.onCallDutyPolicyScheduleId!,
);
} else {
// no users in this rule. Skipping.
const log: OnCallDutyPolicyExecutionLogTimeline = getNewLog();
log.statusMessage =
"Skipped because notification sent to this user already.";
Expand All @@ -339,7 +338,6 @@ export class Service extends DatabaseService<Model> {
}

if (uniqueUserIds.length === 0) {
// no users in this rule. Skipping.
const log: OnCallDutyPolicyExecutionLogTimeline = getNewLog();
log.statusMessage = "Skipped because no users in this rule.";
log.status = OnCallDutyExecutionLogTimelineStatus.Skipped;
Expand All @@ -351,6 +349,8 @@ export class Service extends DatabaseService<Model> {
},
});
}

logger.debug(`Completed rule execution for ruleId: ${ruleId.toString()}`);
}

public constructor() {
Expand Down
18 changes: 17 additions & 1 deletion Common/Server/Services/OnCallDutyPolicyExecutionLogService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import ObjectID from "../../Types/ObjectID";
import Color from "../../Types/Color";
import AlertFeedService from "./AlertFeedService";
import { AlertFeedEventType } from "../../Models/DatabaseModels/AlertFeed";
import BadDataException from "../../Types/Exception/BadDataException";

export class Service extends DatabaseService<Model> {
public constructor() {
Expand Down Expand Up @@ -112,13 +113,28 @@ export class Service extends DatabaseService<Model> {
},
});


let userNotificationEventType: UserNotificationEventType | null = null;

if(createdItem.triggeredByIncidentId) {
userNotificationEventType = UserNotificationEventType.IncidentCreated;
}

if(createdItem.triggeredByAlertId) {
userNotificationEventType = UserNotificationEventType.AlertCreated;
}

if(!userNotificationEventType) {
throw new BadDataException("Invalid userNotificationEventType");
}

await OnCallDutyPolicyEscalationRuleService.startRuleExecution(
executionRule.id!,
{
projectId: createdItem.projectId!,
triggeredByIncidentId: createdItem.triggeredByIncidentId,
triggeredByAlertId: createdItem.triggeredByAlertId,
userNotificationEventType: UserNotificationEventType.IncidentCreated,
userNotificationEventType: userNotificationEventType,
onCallPolicyExecutionLogId: createdItem.id!,
onCallPolicyId: createdItem.onCallDutyPolicyId!,
},
Expand Down
63 changes: 59 additions & 4 deletions Worker/Jobs/OnCallDutyPolicyExecutionLog/ExecutePendingExecutions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,28 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (
executionLog: OnCallDutyPolicyExecutionLog,
): Promise<void> => {
try {


logger.debug(
`Executing on-call policy execution log: ${executionLog.id}`,
)

// get trigger by alert
if (executionLog.triggeredByAlertId) {

logger.debug(
`Triggered by alert: ${executionLog.triggeredByAlertId}`,
)

// check if this alert is ack.
const isAcknowledged: boolean = await AlertService.isAlertAcknowledged({
alertId: executionLog.triggeredByAlertId,
});

logger.debug(
`Alert is acknowledged: ${isAcknowledged}`,
)

if (isAcknowledged) {
// then mark this policy as executed.
await OnCallDutyPolicyExecutionLogService.updateOneById({
Expand All @@ -92,12 +107,21 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (

// get trigger by incident
if (executionLog.triggeredByIncidentId) {

logger.debug(
`Triggered by incident: ${executionLog.triggeredByIncidentId}`,
)

// check if this incident is ack.
const isAcknowledged: boolean =
await IncidentService.isIncidentAcknowledged({
incidentId: executionLog.triggeredByIncidentId,
});

logger.debug(
`Incident is acknowledged: ${isAcknowledged}`,
)

if (isAcknowledged) {
// then mark this policy as executed.
await OnCallDutyPolicyExecutionLogService.updateOneById({
Expand Down Expand Up @@ -126,10 +150,17 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (
currentDate,
);

logger.debug(
`Current date: ${currentDate}, Last executed at: ${lastExecutedAt}, Difference in minutes: ${getDifferenceInMinutes}`,
);

if (
getDifferenceInMinutes <
(executionLog.executeNextEscalationRuleInMinutes || 0)
) {
logger.debug(
`Execution not needed yet. Waiting for ${executionLog.executeNextEscalationRuleInMinutes} minutes.`,
);
return;
}

Expand All @@ -150,19 +181,26 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (
});

if (!nextEscalationRule) {
// check if we need to repeat this execution.
logger.debug(
`No next escalation rule found. Checking if we need to repeat this execution.`,
);

// check if we need to repeat this execution.
if (
executionLog.onCallPolicyExecutionRepeatCount &&
executionLog.onCallPolicyExecutionRepeatCount <
executionLog.onCallDutyPolicy!
.repeatPolicyIfNoOneAcknowledgesNoOfTimes!
executionLog.onCallDutyPolicy!
.repeatPolicyIfNoOneAcknowledgesNoOfTimes!
) {
// repeating execution

const newRepeatCount: number =
executionLog.onCallPolicyExecutionRepeatCount + 1;

logger.debug(
`Repeating execution. New repeat count: ${newRepeatCount}`,
);

await OnCallDutyPolicyExecutionLogService.updateOneById({
id: executionLog.id!,
data: {
Expand All @@ -174,7 +212,6 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (
});

// get first escalation rule.

const firstEscalationRule: OnCallDutyPolicyEscalationRule | null =
await OnCallDutyPolicyEscalationRuleService.findOneBy({
query: {
Expand All @@ -191,6 +228,10 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (
});

if (!firstEscalationRule) {
logger.debug(
`No first escalation rule found. Marking execution as complete.`,
);

// mark this as complete.
await OnCallDutyPolicyExecutionLogService.updateOneById({
id: executionLog.id!,
Expand All @@ -206,6 +247,10 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (
return;
}

logger.debug(
`Starting execution of the first escalation rule: ${firstEscalationRule.id}`,
);

// update the execution log.
await OnCallDutyPolicyEscalationRuleService.startRuleExecution(
firstEscalationRule.id!,
Expand All @@ -221,6 +266,11 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (

return;
}

logger.debug(
`No rules to execute and no repeats left. Marking execution as complete.`,
);

// mark this as complete as we have no rules to execute.
await OnCallDutyPolicyExecutionLogService.updateOneById({
id: executionLog.id!,
Expand All @@ -234,6 +284,11 @@ const executeOnCallPolicy: ExecuteOnCallPolicyFunction = async (
});
return;
}

logger.debug(
`Starting execution of the next escalation rule: ${nextEscalationRule.id}`,
);

await OnCallDutyPolicyEscalationRuleService.startRuleExecution(
nextEscalationRule!.id!,
{
Expand Down

0 comments on commit cb11a46

Please sign in to comment.