HOSTSD-246 Improve Data Service (#96)

Add DB migration 1.0.1

src/api/Areas/Services/Controllers/ServerItemController.cs

@@ -6,6 +6,7 @@
 using System.Net;
 using HSB.DAL.Services;
 using HSB.Keycloak;
+using Microsoft.AspNetCore.Http.Extensions;
 
 namespace HSB.API.Areas.Services.Controllers;
 
@@ -56,7 +57,11 @@ public ServerItemController(
     [SwaggerOperation(Tags = new[] { "Server Item" })]
     public IActionResult Find()
     {
-        var serverItems = _serverItemService.Find(o => true);
+        var uri = new Uri(this.Request.GetDisplayUrl());
+        var query = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(uri.Query);
+        var filter = new HSB.Models.Filters.ServerItemFilter(query);
+
+        var serverItems = _serverItemService.Find(filter);
         return new JsonResult(serverItems.Select(ci => new ServerItemModel(ci)));
     }
 
@@ -115,6 +120,23 @@ public IActionResult Update(ServerItemModel model)
         return new JsonResult(new ServerItemModel(entity));
     }
 
+    /// <summary>
+    ///
+    /// </summary>
+    /// <param name="model"></param>
+    /// <returns></returns>
+    [HttpDelete("{id}", Name = "DeleteServerItem-Services")]
+    [Produces(MediaTypeNames.Application.Json)]
+    [ProducesResponseType(typeof(ServerItemModel), (int)HttpStatusCode.OK)]
+    [ProducesResponseType(typeof(ErrorResponseModel), (int)HttpStatusCode.BadRequest)]
+    [SwaggerOperation(Tags = new[] { "Server Item" })]
+    public IActionResult Delete(ServerItemModel model)
+    {
+        _serverItemService.Remove(model.ToEntity());
+        _serverItemService.CommitTransaction();
+        return new JsonResult(model);
+    }
+
     /// <summary>
     ///
     /// </summary>

src/data-service/DataService.cs

@@ -107,6 +107,8 @@ public async Task RunAsync()
             }
         }
 
+        await ServerItemCleanupProcessAsync();
+
         this.Logger.LogInformation("Data Sync Service Completed");
     }
 
@@ -244,13 +246,35 @@ private async Task ProcessConfigurationItemsAsync(Models.DataSyncModel option)
         if (serverItemSN.Data == null) throw new ArgumentNullException(nameof(serverItemSN));
         if (configurationItemSN.Data == null) throw new ArgumentNullException(nameof(configurationItemSN));
 
+        var serviceNowKey = serverItemSN.Data.Id;
+
         if (serverItemSN.Data.InstallStatus != "1")
         {
             this.Logger.LogDebug("Server item install status: {status}", serverItemSN.Data.InstallStatus);
+
+            // Need to update with the latest status.
+            if (_serverItems.TryGetValue(serviceNowKey, out Hsb.ServerItemModel? serverItemHSB))
+            {
+                // Update the server item in HSB.
+                this.Logger.LogDebug("Update Server Item: '{id}'", configurationItemSN.Data?.Id);
+
+                serverItemHSB.RawData = serverItemSN.RawData;
+                serverItemHSB.RawDataCI = configurationItemSN.RawData;
+                serverItemHSB.InstallStatus = int.Parse(serverItemSN.Data.InstallStatus ?? "0");
+
+                serverItemHSB = await this.HsbApi.UpdateServerItemAsync(serverItemHSB);
+                if (serverItemHSB == null)
+                {
+                    this.Logger.LogError("Server Item was not returned from HSB: {id}", serviceNowKey);
+                    return null;
+                }
+                _serverItems[serverItemHSB.ServiceNowKey] = serverItemHSB;
+                return serverItemHSB;
+            }
+
             return null;
         }
 
-        var serviceNowKey = serverItemSN.Data.Id;
         var tenant = await ProcessTenantAsync(configurationItemSN.RawData, serverItemSN.RawData);
         var organization = await ProcessOrganizationAsync(configurationItemSN.RawData, serverItemSN.RawData);
         if (organization == null)
@@ -319,14 +343,14 @@ private async Task ProcessConfigurationItemsAsync(Models.DataSyncModel option)
         if (fileSystemItemSN.Data == null) throw new ArgumentNullException(nameof(fileSystemItemSN));
         if (configurationItemSN.Data == null) throw new ArgumentNullException(nameof(configurationItemSN));
 
+        var serviceNowKey = fileSystemItemSN.Data.Id;
+
         if (fileSystemItemSN.Data.InstallStatus != "1")
         {
             this.Logger.LogDebug("Server item install status: {status}", fileSystemItemSN.Data.InstallStatus);
             return null;
         }
 
-        var serviceNowKey = fileSystemItemSN.Data.Id;
-
         // Server does not currently exist, add it.
         if (!_serverItems.TryGetValue(serviceNowKey, out Hsb.ServerItemModel? serverItem))
         {
@@ -393,6 +417,7 @@ private async Task ProcessConfigurationItemsAsync(Models.DataSyncModel option)
 
             fileSystemItem.ClassName = fileSystemItemSN.Data.ClassName ?? "";
             fileSystemItem.Name = fileSystemItemSN.Data.Name ?? "";
+            fileSystemItem.InstallStatus = int.Parse(fileSystemItemSN.Data.InstallStatus ?? "0");
             fileSystemItem.Label = fileSystemItemSN.Data.Label ?? "";
             fileSystemItem.Category = fileSystemItemSN.Data.Category ?? "";
             fileSystemItem.Subcategory = fileSystemItemSN.Data.Subcategory ?? "";
@@ -689,6 +714,51 @@ private static string GenerateUniqueName(string name, IEnumerable<string> names,
         return operatingSystem;
     }
 
+    /// <summary>
+    /// Fetch all servers that were not updated in the prior run.
+    /// Make a request to ServiceNow to determine if these servers should be removed.
+    /// </summary>
+    /// <returns></returns>
+    private async Task ServerItemCleanupProcessAsync()
+    {
+        this.Logger.LogInformation("Starting servers cleanup process");
+        var filter = new Hsb.Filters.ServerItemFilter()
+        {
+            InstallStatus = 1, // Only fetch server items that are currently marked as installed.
+            UpdatedBeforeDate = DateTime.UtcNow.AddDays(-1), // Only fetch server items that haven't been updated in over a day.
+        };
+        var serverItems = await this.HsbApi.FetchServerItemsAsync(filter);
+        foreach (var serverItem in serverItems)
+        {
+            try
+            {
+                // For each server item make a request to ServiceNow to determine if it is still installed.
+                var serverItemSN = await this.ServiceNowApi.GetTableItemAsync<ServiceNow.BaseItemModel>(serverItem.ClassName, serverItem.ServiceNowKey);
+                if (serverItemSN?.Data == null)
+                {
+                    await this.HsbApi.DeleteServerItemAsync(serverItem);
+                }
+                else
+                {
+                    // Fetch the configuration item now.
+                    var configurationItemSN = await this.ServiceNowApi.GetTableItemAsync<ServiceNow.ConfigurationItemModel>(this.ServiceNowApi.Options.TableNames.ConfigurationItem, serverItem.ServiceNowKey);
+                    if (configurationItemSN != null)
+                    {
+                        // Update the server item with the latest information.
+                        await this.ProcessServerItemAsync(serverItemSN, configurationItemSN);
+                    }
+                    else
+                    {
+                        this.Logger.LogWarning("Configuration item could not be found: {key}", serverItem.ServiceNowKey);
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                this.Logger.LogError(ex, "Failed to fetch server item: {key}", serverItem.ServiceNowKey);
+            }
+        }
+    }
     #endregion
 }
 

src/data-service/Helpers/HsbApiService.cs

@@ -4,6 +4,7 @@
 using Microsoft.Extensions.Options;
 using HSB.Core.Http;
 using System.Net.Http.Json;
+using HSB.Core.Extensions;
 
 namespace HSB.DataService;
 
@@ -290,13 +291,15 @@ public async Task<IEnumerable<OrganizationModel>> FetchOrganizationsAsync()
     /// <summary>
     /// Fetch all server items from HSB.
     /// </summary>
+    /// <param name="filter"></param>
     /// <returns></returns>
-    public async Task<IEnumerable<ServerItemModel>> FetchServerItemsAsync()
+    public async Task<IEnumerable<ServerItemModel>> FetchServerItemsAsync(Models.Filters.ServerItemFilter? filter = null)
     {
         this.Logger.LogDebug("HSB - Fetching all server items");
         var builder = new UriBuilder($"{this.ApiClient.Client.BaseAddress}")
         {
-            Path = this.Options.Endpoints.ServerItems
+            Path = this.Options.Endpoints.ServerItems,
+            Query = filter?.GetQueryString() ?? "",
         };
         var results = await HsbSendAsync<IEnumerable<ServerItemModel>>(HttpMethod.Get, builder.Uri);
         return results ?? Array.Empty<ServerItemModel>();
@@ -333,6 +336,22 @@ public async Task<IEnumerable<ServerItemModel>> FetchServerItemsAsync()
         var results = await HsbSendAsync<ServerItemModel>(HttpMethod.Put, builder.Uri, JsonContent.Create(model));
         return results;
     }
+
+    /// <summary>
+    /// Delete server item in HSB.
+    /// </summary>
+    /// <param name="model"></param>
+    /// <returns></returns>
+    public async Task<ServerItemModel> DeleteServerItemAsync(ServerItemModel model)
+    {
+        this.Logger.LogDebug("HSB - Delete server item");
+        var builder = new UriBuilder($"{this.ApiClient.Client.BaseAddress}")
+        {
+            Path = $"{this.Options.Endpoints.ServerItems}/{model.ServiceNowKey}"
+        };
+        var results = await HsbSendAsync<ServerItemModel>(HttpMethod.Delete, builder.Uri, JsonContent.Create(model));
+        return results ?? model;
+    }
     #endregion
 
     #region File System Items

src/data-service/Helpers/IHsbApiService.cs

@@ -12,137 +12,145 @@ public interface IHsbApiService
     /// Fetch all data sync configuration items.
     /// </summary>
     /// <returns></returns>
-    public Task<IEnumerable<DataSyncModel>> FetchDataSyncAsync();
+    Task<IEnumerable<DataSyncModel>> FetchDataSyncAsync();
 
     /// <summary>
     /// Get the data sync for the specified name.
     /// </summary>
     /// <param name="name"></param>
     /// <returns></returns>
-    public Task<DataSyncModel?> GetDataSyncAsync(string name);
+    Task<DataSyncModel?> GetDataSyncAsync(string name);
 
     /// <summary>
     /// Update the data sync.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<DataSyncModel?> UpdateDataSyncAsync(DataSyncModel model);
+    Task<DataSyncModel?> UpdateDataSyncAsync(DataSyncModel model);
     #endregion
 
     #region Operating System Items
     /// <summary>
     /// Fetch all operating system items from HSB.
     /// </summary>
     /// <returns></returns>
-    public Task<IEnumerable<OperatingSystemItemModel>> FetchOperatingSystemItemsAsync();
+    Task<IEnumerable<OperatingSystemItemModel>> FetchOperatingSystemItemsAsync();
 
     /// <summary>
     /// Add operating system to HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<OperatingSystemItemModel?> AddOperatingSystemItemAsync(OperatingSystemItemModel model);
+    Task<OperatingSystemItemModel?> AddOperatingSystemItemAsync(OperatingSystemItemModel model);
 
     /// <summary>
     /// Update operating system in HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<OperatingSystemItemModel?> UpdateOperatingSystemItemAsync(OperatingSystemItemModel model);
+    Task<OperatingSystemItemModel?> UpdateOperatingSystemItemAsync(OperatingSystemItemModel model);
     #endregion
 
     #region Tenants
     /// <summary>
     /// Fetch all tenants from HSB.
     /// </summary>
     /// <returns></returns>
-    public Task<IEnumerable<TenantModel>> FetchTenantsAsync();
+    Task<IEnumerable<TenantModel>> FetchTenantsAsync();
 
     /// <summary>
     /// Add tenant to HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<TenantModel?> AddTenantAsync(TenantModel model);
+    Task<TenantModel?> AddTenantAsync(TenantModel model);
 
     /// <summary>
     /// Update tenant in HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<TenantModel?> UpdateTenantAsync(TenantModel model);
+    Task<TenantModel?> UpdateTenantAsync(TenantModel model);
     #endregion
 
     #region Organizations
     /// <summary>
     /// Fetch all organizations from HSB.
     /// </summary>
     /// <returns></returns>
-    public Task<IEnumerable<OrganizationModel>> FetchOrganizationsAsync();
+    Task<IEnumerable<OrganizationModel>> FetchOrganizationsAsync();
 
     /// <summary>
     /// Add organization to HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<OrganizationModel?> AddOrganizationAsync(OrganizationModel model);
+    Task<OrganizationModel?> AddOrganizationAsync(OrganizationModel model);
 
     /// <summary>
     /// Update organization in HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<OrganizationModel?> UpdateOrganizationAsync(OrganizationModel model);
+    Task<OrganizationModel?> UpdateOrganizationAsync(OrganizationModel model);
     #endregion
 
     #region Servers
     /// <summary>
     /// Fetch all server items from HSB.
     /// </summary>
+    /// <param name="filter"></param>
     /// <returns></returns>
-    public Task<IEnumerable<ServerItemModel>> FetchServerItemsAsync();
+    Task<IEnumerable<ServerItemModel>> FetchServerItemsAsync(Models.Filters.ServerItemFilter? filter = null);
 
     /// <summary>
     /// Add server item to HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<ServerItemModel?> AddServerItemAsync(ServerItemModel model);
+    Task<ServerItemModel?> AddServerItemAsync(ServerItemModel model);
 
     /// <summary>
     /// Update server item in HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<ServerItemModel?> UpdateServerItemAsync(ServerItemModel model);
+    Task<ServerItemModel?> UpdateServerItemAsync(ServerItemModel model);
+
+    /// <summary>
+    /// Delete server item in HSB.
+    /// </summary>
+    /// <param name="model"></param>
+    /// <returns></returns>
+    Task<ServerItemModel> DeleteServerItemAsync(ServerItemModel model);
     #endregion
 
     #region File System Items
     /// <summary>
     /// Fetch all file system items from HSB.
     /// </summary>
     /// <returns></returns>
-    public Task<IEnumerable<FileSystemItemModel>> FetchFileSystemItemsAsync();
+    Task<IEnumerable<FileSystemItemModel>> FetchFileSystemItemsAsync();
 
     /// <summary>
-    /// Get the file system item from HSB for the specified 'id'.
+    ///Get the file system item from HSB for the specified 'id'.
     /// </summary>
     /// <param name="id"></param>
     /// <returns></returns>
-    public Task<FileSystemItemModel?> GetFileSystemItemAsync(string id);
+    Task<FileSystemItemModel?> GetFileSystemItemAsync(string id);
 
     /// <summary>
     /// Add file system item to HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<FileSystemItemModel?> AddFileSystemItemAsync(FileSystemItemModel model);
+    Task<FileSystemItemModel?> AddFileSystemItemAsync(FileSystemItemModel model);
 
     /// <summary>
     /// Update file system item in HSB.
     /// </summary>
     /// <param name="model"></param>
     /// <returns></returns>
-    public Task<FileSystemItemModel?> UpdateFileSystemItemAsync(FileSystemItemModel model);
+    Task<FileSystemItemModel?> UpdateFileSystemItemAsync(FileSystemItemModel model);
     #endregion
 }