feat: implement mutex for incident creation and improve incident numb…

…er handling
OneUptime · Jan 12, 2025 · 982d051 · 982d051
1 parent b66b1db
commit 982d051
Show file tree

Hide file tree

Showing 5 changed files with 82 additions and 25 deletions.
diff --git a/Common/Server/Services/IncidentService.ts b/Common/Server/Services/IncidentService.ts
@@ -44,6 +44,9 @@ import OneUptimeDate from "../../Types/Date";
 import TelemetryUtil from "../Utils/Telemetry/Telemetry";
 import TelemetryType from "../../Types/Telemetry/TelemetryType";
 import logger from "../Utils/Logger";
+import Semaphore, {
+  SemaphoreMutex,
+} from "Common/Server/Infrastructure/Semaphore";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -189,11 +192,6 @@ export class Service extends DatabaseService<Model> {
     const projectId: ObjectID =
       createBy.props.tenantId || createBy.data.projectId!;
 
-    const incidentNumberForThisIncident: number =
-      (await this.getExistingIncidentNumberForProject({
-        projectId: projectId,
-      })) + 1;
-
     const incidentState: IncidentState | null =
       await IncidentStateService.findOneBy({
         query: {
@@ -214,6 +212,37 @@ export class Service extends DatabaseService<Model> {
       );
     }
 
+    let mutex: SemaphoreMutex | null = null;
+
+    try {
+      mutex = await Semaphore.lock({
+        key: projectId.toString(),
+        namespace: "IncidentService.incident-create",
+        lockTimeout: 15000,
+        acquireTimeout: 20000,
+      });
+
+      logger.debug(
+        "Mutex acquired - IncidentService.incident-create " +
+          projectId.toString() +
+          " at " +
+          OneUptimeDate.getCurrentDateAsFormattedString(),
+      );
+    } catch (err) {
+      logger.debug(
+        "Mutex acquire failed - IncidentService.incident-create " +
+          projectId.toString() +
+          " at " +
+          OneUptimeDate.getCurrentDateAsFormattedString(),
+      );
+      logger.error(err);
+    }
+
+    const incidentNumberForThisIncident: number =
+      (await this.getExistingIncidentNumberForProject({
+        projectId: projectId,
+      })) + 1;
+
     createBy.data.currentIncidentStateId = incidentState.id;
     createBy.data.incidentNumber = incidentNumberForThisIncident;
 
@@ -252,13 +281,19 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
-    return { createBy, carryForward: null };
+    return {
+      createBy,
+      carryForward: {
+        mutex: mutex,
+      },
+    };
   }
 
   protected override async onCreateSuccess(
     onCreate: OnCreate<Model>,
     createdItem: Model,
   ): Promise<Model> {
+    // these should never be null.
     if (!createdItem.projectId) {
       throw new BadDataException("projectId is required");
     }
@@ -267,6 +302,30 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("id is required");
     }
 
+    // release the mutex.
+    if (onCreate.carryForward && onCreate.carryForward.mutex) {
+      const mutex: SemaphoreMutex = onCreate.carryForward.mutex;
+      const projectId: ObjectID = createdItem.projectId!;
+
+      try {
+        await Semaphore.release(mutex);
+        logger.debug(
+          "Mutex released - IncidentService.incident-create " +
+            projectId.toString() +
+            " at " +
+            OneUptimeDate.getCurrentDateAsFormattedString(),
+        );
+      } catch (err) {
+        logger.debug(
+          "Mutex release failed -  IncidentService.incident-create " +
+            projectId.toString() +
+            " at " +
+            OneUptimeDate.getCurrentDateAsFormattedString(),
+        );
+        logger.error(err);
+      }
+    }
+
     if (!createdItem.currentIncidentStateId) {
       throw new BadDataException("currentIncidentStateId is required");
     }

diff --git a/Dashboard/src/Components/Incident/IncidentsTable.tsx b/Dashboard/src/Components/Incident/IncidentsTable.tsx
@@ -528,8 +528,7 @@ const IncidentsTable: FunctionComponent<ComponentProps> = (
             title: "Incident Number",
             type: FieldType.Text,
             getElement: (item: Incident): ReactElement => {
-
-              if(!item.incidentNumber) {
+              if (!item.incidentNumber) {
                 return <>-</>;
               }
 

diff --git a/Dashboard/src/Pages/Incidents/View/Index.tsx b/Dashboard/src/Pages/Incidents/View/Index.tsx
@@ -375,12 +375,12 @@ const IncidentView: FunctionComponent<
               title: "Incident Number",
               fieldType: FieldType.Element,
               getElement: (item: Incident): ReactElement => {
-                if(!item.incidentNumber) {
+                if (!item.incidentNumber) {
                   return <>-</>;
                 }
-  
+
                 return <># {item.incidentNumber}</>;
-              }
+              },
             },
             {
               field: {

diff --git a/Worker/DataMigrations/AddIncidentNumber.ts b/Worker/DataMigrations/AddIncidentNumber.ts
@@ -31,7 +31,7 @@ export default class AddIncidentNumber extends DataMigrationBase {
       // first fetch resolved state. Ended state order is -1 of resolved state.
 
       // get all incicents for this project
-      const incidents: Array<Incident> =  await IncidentService.findBy({
+      const incidents: Array<Incident> = await IncidentService.findBy({
         query: {
           projectId: project.id!,
         },
@@ -42,30 +42,29 @@ export default class AddIncidentNumber extends DataMigrationBase {
         skip: 0,
         limit: LIMIT_MAX,
         sort: {
-            createdAt: SortOrder.Descending
+          createdAt: SortOrder.Descending,
         },
         props: {
           isRoot: true,
         },
       });
 
-      const totalIncidentForProject = incidents.length;
-      let incidentCounter = totalIncidentForProject; // start from the last incident number
+      const totalIncidentForProject: number = incidents.length;
+      let incidentCounter: number = totalIncidentForProject; // start from the last incident number
 
-      for(const incident of incidents) {
+      for (const incident of incidents) {
         await IncidentService.updateOneById({
-            id: incident.id!,
-            data: {
-              incidentNumber: incidentCounter,
-            },
-            props: {
-              isRoot: true,
-            },
+          id: incident.id!,
+          data: {
+            incidentNumber: incidentCounter,
+          },
+          props: {
+            isRoot: true,
+          },
         });
 
         incidentCounter = incidentCounter - 1;
       }
-
     }
   }
 

diff --git a/Worker/DataMigrations/Index.ts b/Worker/DataMigrations/Index.ts
@@ -84,7 +84,7 @@ const DataMigrations: Array<DataMigrationBase> = [
   new RefreshDefaultUserNotificationSetting(),
   new AddServiceTypeColumnToMetricsTable(),
   new AddIsSubscriptionConfirmedToSubscribers(),
-  new AddIncidentNumber()
+  new AddIncidentNumber(),
 ];
 
 export default DataMigrations;