Refactor Logs (#539)

* HTTP error logging - Code cleanup with CodeMaid * gh-386 Refactor Workflow manager logs * Adjust time zone to UTC Signed-off-by: Victor Chang <[email protected]>
Project-MONAI · Oct 27, 2022 · f5e2f17 · f5e2f17
1 parent dcdf133
commit f5e2f17
Show file tree

Hide file tree

Showing 110 changed files with 1,062 additions and 616 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -21,7 +21,7 @@ on:
 env:
   BUILD_CONFIG: "Release"
   SOLUTION: "Monai.Deploy.WorkflowManager.sln"
-  
+
 jobs:
   build-and-deploy:
     runs-on: ubuntu-latest
@@ -36,7 +36,7 @@ jobs:
     outputs:
       semVer: ${{ steps.gitversion.outputs.semVer }}
       majorMinorPatch: ${{ steps.gitversion.outputs.majorMinorPatch }}
-    env: 
+    env:
       REGISTRY: ghcr.io
     permissions:
       packages: write
@@ -106,15 +106,15 @@ jobs:
 
       - name: Trivy Vulnerability Scanner
         uses: aquasecurity/trivy-action@master
-       if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.head_ref, 'release/') }}
+        if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.head_ref, 'release/') }}
         with:
           image-ref: ${{ fromJSON(steps.meta.outputs.json).tags[0] }}
           format: 'sarif'
           output: 'trivy-results.sarif'
 
       - name: Upload Trivy SARIF Report
         uses: github/codeql-action/upload-sarif@v2
-       if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.head_ref, 'release/') }}
+        if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.head_ref, 'release/') }}
         with:
           sarif_file: 'trivy-results.sarif'
 
@@ -184,7 +184,7 @@ jobs:
         run: |
           mkdir ~\release
           Compress-Archive -Path docs\_site\* -DestinationPath ~\release\mwm-docs-${{ env.SEMVER }}.zip
-          Get-ChildItem ~\release -Recurse   
+          Get-ChildItem ~\release -Recurse
 
       - name: Upload docs
         uses: actions/[email protected]

diff --git a/src/Monai.Deploy.WorkflowManager.sln b/src/Monai.Deploy.WorkflowManager.sln
@@ -80,7 +80,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.WorkflowManage
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.WorkflowManager.Shared", "Shared\Shared\Monai.Deploy.WorkflowManager.Shared.csproj", "{81E3F943-B992-4C81-AA09-A17C05081236}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monai.Deploy.WorkflowManager.Shared.Tests", "..\tests\UnitTests\Monai.Deploy.WorkflowManager.Shared.Tests\Monai.Deploy.WorkflowManager.Shared.Tests.csproj", "{C853A9E3-C53D-4B1A-BFDA-228689A8C94C}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.WorkflowManager.Shared.Tests", "..\tests\UnitTests\Monai.Deploy.WorkflowManager.Shared.Tests\Monai.Deploy.WorkflowManager.Shared.Tests.csproj", "{C853A9E3-C53D-4B1A-BFDA-228689A8C94C}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Monai.Deploy.WorkflowManager.Common.Tests", "..\tests\UnitTests\Common.Tests\Monai.Deploy.WorkflowManager.Common.Tests.csproj", "{75A4AEDA-0386-4B2D-9DBA-BC9AE733660E}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -228,6 +230,10 @@ Global
 		{C853A9E3-C53D-4B1A-BFDA-228689A8C94C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C853A9E3-C53D-4B1A-BFDA-228689A8C94C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C853A9E3-C53D-4B1A-BFDA-228689A8C94C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{75A4AEDA-0386-4B2D-9DBA-BC9AE733660E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{75A4AEDA-0386-4B2D-9DBA-BC9AE733660E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{75A4AEDA-0386-4B2D-9DBA-BC9AE733660E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{75A4AEDA-0386-4B2D-9DBA-BC9AE733660E}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -269,6 +275,7 @@ Global
 		{BF6569A1-1A5A-4358-9C02-1A6A5F0FBFD9} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7}
 		{81E3F943-B992-4C81-AA09-A17C05081236} = {37A19144-CEA5-47A2-9FFD-22C522E8B895}
 		{C853A9E3-C53D-4B1A-BFDA-228689A8C94C} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7}
+		{75A4AEDA-0386-4B2D-9DBA-BC9AE733660E} = {71DDEE7B-E213-4E39-A7F4-4646783A27F7}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {DC0D56C8-D8CB-45CE-B528-F3DCF86D63ED}

diff --git a/src/Shared/Shared/Monai.Deploy.WorkflowManager.Shared.csproj b/src/Shared/Shared/Monai.Deploy.WorkflowManager.Shared.csproj
@@ -41,7 +41,7 @@
     <EmbeddedResource Remove="Test\**" />
     <None Remove="Test\**" />
   </ItemGroup>
-
+  
   <ItemGroup>
     <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
   </ItemGroup>

diff --git a/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj b/src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj
@@ -1,4 +1,4 @@
-<!--
+<!--
   ~ Copyright 2022 MONAI Consortium
   ~
   ~ Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,11 +32,12 @@
   <ItemGroup>
     <Compile Include="..\..\AssemblyInfo.cs" Link="AssemblyInfo.cs" />
   </ItemGroup>
-
+  
   <ItemGroup>
     <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
   </ItemGroup>
 
+
   <ItemGroup>
     <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.8" />
     <PackageReference Include="MongoDB.Bson" Version="2.18.0" />

diff --git a/src/TaskManager/Database/Monai.Deploy.WorkflowManager.TaskManager.Database.csproj b/src/TaskManager/Database/Monai.Deploy.WorkflowManager.TaskManager.Database.csproj
@@ -1,4 +1,4 @@
-<!--
+<!--
   ~ Copyright 2022 MONAI Consortium
   ~
   ~ Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,12 +23,14 @@
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
   </PropertyGroup>
 
+
   <ItemGroup>
-    <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
+    <Compile Include="..\..\AssemblyInfo.cs" Link="AssemblyInfo.cs" />
   </ItemGroup>
 
+
   <ItemGroup>
-    <Compile Include="..\..\AssemblyInfo.cs" Link="AssemblyInfo.cs" />
+    <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/TaskManager/Plug-ins/Docker/IDockerClientFactory.cs b/src/TaskManager/Plug-ins/Docker/IDockerClientFactory.cs
@@ -27,7 +27,7 @@ public class DockerClientFactory : IDockerClientFactory
     {
         public IDockerClient CreateClient(Uri dockerEndpoint)
         {
-           return new DockerClientConfiguration(dockerEndpoint).CreateClient();
+            return new DockerClientConfiguration(dockerEndpoint).CreateClient();
         }
     }
 }
diff --git a/src/TaskManager/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj b/src/TaskManager/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj
@@ -55,6 +55,7 @@
     <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
   </ItemGroup>
 
+
   <ItemGroup>
     <PackageReference Include="AspNetCore.HealthChecks.MongoDb" Version="6.0.2" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.10" />

diff --git a/src/TaskManager/TaskManager/appsettings.json b/src/TaskManager/TaskManager/appsettings.json
@@ -95,5 +95,4 @@
     "LogHttpResponseBody": true
   },
   "AllowedHosts": "*",
-
 }
diff --git a/src/TaskManager/TaskManager/nlog.config b/src/TaskManager/TaskManager/nlog.config
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8" ?>
 
-<!-- 
+<!--
 Copyright 2022 MONAI Consortium
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,7 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License. 
+limitations under the License.
 -->
 
 <nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd"
@@ -24,8 +24,8 @@ limitations under the License.
 
   <!-- enable asp.net core layout renderers -->
   <extensions>
-    <add assembly="NLog.Web.AspNetCore"/>
-    <add assembly="NLog.Extensions.Logging"/>
+    <add assembly="NLog.Web.AspNetCore" />
+    <add assembly="NLog.Extensions.Logging" />
   </extensions>
 
   <variable name="logDir" value="${basedir}/logs/" />
@@ -34,10 +34,10 @@ limitations under the License.
   <time type="AccurateUTC" />
 
   <targets async="true">
-    <target xsi:type="File" name="allfile" fileName="${logDir}/MWM-${shortdate}.log" >
+    <target xsi:type="File" name="allfile" fileName="${logDir}/MWM-${shortdate}.log">
       <layout xsi:type="JsonLayout" includeEventProperties="true" includeScopeProperties="true">
         <attribute name="timestamp" layout="${date:format=o}" />
-        <attribute name="level" layout="${level:upperCase=true}"/>
+        <attribute name="level" layout="${level:upperCase=true}" />
         <attribute name="logger" layout="${logger}" />
         <attribute name="message" layout="${message:raw=true}" />
         <attribute name="properties" layout="${message:raw=true}" />
@@ -48,17 +48,17 @@ limitations under the License.
     <target xsi:type="ColoredConsole" name="lifetimeConsole" layout="${longdate}|${event-properties:item=EventId:whenEmpty=0}|${uppercase:${level}}|${logger}|${scopenested}|${message} ${exception:format=tostring}" />
 
     <target name="logstash" xsi:type="Network" address="${environment:LOGSTASH_URL}" newLine="true">
-      <layout xsi:type="JsonLayout" includeAllProperties="true" IncludeEventProperties="true" IncludeScopeProperties="true" >
+      <layout xsi:type="JsonLayout" includeAllProperties="true" IncludeEventProperties="true" IncludeScopeProperties="true">
         <attribute name="Tag" layout="${environment:ENVIRONMENT_NAME}" />
         <attribute name="ServiceName" layout="${servicename}" />
         <attribute name="ServiceVersion" layout="${serviceversion}" />
         <attribute name="MachineName" layout="${machinename}" />
-        <attribute name="CorrelationId" layout="${mdlc:item=correlationId}"/>
-        <attribute name="LoggerName" layout="${logger}"/>
-        <attribute name="Level" layout="${level:upperCase=true}"/>
-        <attribute name="Message" layout="${message}"/>
-        <attribute name="Exception" layout="${onexception:${literal:text=|}"/>
-        <attribute name="Exception2" layout="${exception:format=tostring}"/>
+        <attribute name="CorrelationId" layout="${mdlc:item=correlationId}" />
+        <attribute name="LoggerName" layout="${logger}" />
+        <attribute name="Level" layout="${level:upperCase=true}" />
+        <attribute name="Message" layout="${message}" />
+        <attribute name="Exception" layout="${onexception:${literal:text=|}" />
+        <attribute name="Exception2" layout="${exception:format=tostring}" />
       </layout>
     </target>
   </targets>
@@ -67,13 +67,12 @@ limitations under the License.
   <rules>
     <!--All logs, including from Microsoft-->
     <logger name="*" minlevel="Trace" writeTo="allfile" />
-    <logger name="*" minlevel="Trace" writeTo="logstash" >
+    <logger name="*" minlevel="Trace" writeTo="logstash">
       <filters FilterDefaultAction="Log">
         <when condition="length('${environment:LOGSTASH_URL}') &lt; 10" action="Ignore" />
       </filters>
     </logger>
 
-
     <!--Output hosting lifetime messages to console target for faster startup detection -->
     <logger name="Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware" minlevel="Info" writeTo="lifetimeConsole" final="true" />
     <logger name="Microsoft.Hosting.Lifetime" minlevel="Info" writeTo="lifetimeConsole" final="true" />
@@ -82,6 +81,5 @@ limitations under the License.
     <!--Skip non-critical Microsoft logs and so log only own logs (BlackHole) -->
     <logger name="Microsoft.*" maxlevel="Info" final="true" />
     <logger name="System.Net.Http.*" maxlevel="Info" final="true" />
-
   </rules>
 </nlog>
diff --git a/src/WorkflowManager/Authentication/Extensions/AuthKeys.cs b/src/WorkflowManager/Authentication/Extensions/AuthKeys.cs
@@ -25,6 +25,7 @@ public static class AuthKeys
 
         // Configuration Keys
         public const string ServerRealm = "ServerRealm";
+
         public const string ServerRealmKey = "ServerRealmKey";
         public const string Claims = "Claims";
         public const string OpenId = "OpenId";

diff --git a/src/WorkflowManager/Authentication/Extensions/ConfigurationExtension.cs b/src/WorkflowManager/Authentication/Extensions/ConfigurationExtension.cs
@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-using Microsoft.Extensions.Configuration;
 using Ardalis.GuardClauses;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Monai.Deploy.WorkflowManager.Logging;
 
 namespace Monai.Deploy.WorkflowManager.Authentication.Extensions
 {
@@ -32,7 +33,7 @@ public static bool BypassAuth(this IConfiguration configuration, ILogger logger)
 
             if (authenticationSettings[AuthKeys.OpenId] is null)
             {
-                logger.LogInformation("Bypass Authentication");
+                logger.BypassAuthentication();
                 return true;
             }
 
@@ -42,7 +43,7 @@ public static bool BypassAuth(this IConfiguration configuration, ILogger logger)
         /// <summary>
         /// Gets configuration values from the IConfiguration provider.
         /// under "WorkflowManagerAuthentication" key you are able to
-        /// have any label of configuration for example could have 
+        /// have any label of configuration for example could have
         /// </summary>
         /// <param name="configuration"></param>
         /// <param name="serverRealm"></param>

diff --git a/src/WorkflowManager/Authentication/Extensions/MonaiAuthenticationExtensions.cs b/src/WorkflowManager/Authentication/Extensions/MonaiAuthenticationExtensions.cs
@@ -14,16 +14,16 @@
  * limitations under the License.
  */
 
+using System.Text;
+using Microsoft.AspNetCore.Authentication;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
+using Microsoft.AspNetCore.Authorization;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using Microsoft.IdentityModel.Tokens;
-using Microsoft.AspNetCore.Authentication.JwtBearer;
-using Monai.Deploy.WorkflowManager.Common.Extensions;
-using System.Text;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Authentication;
 using Monai.Deploy.WorkflowManager.Authentication.Middleware;
-using Microsoft.Extensions.Logging;
+using Monai.Deploy.WorkflowManager.Common.Extensions;
 
 namespace Monai.Deploy.WorkflowManager.Authentication.Extensions
 {

diff --git a/src/WorkflowManager/Authentication/Middleware/BypassAuthenticationHandler.cs b/src/WorkflowManager/Authentication/Middleware/BypassAuthenticationHandler.cs
@@ -14,11 +14,12 @@
  * limitations under the License.
  */
 
-using Microsoft.AspNetCore.Authentication;
-using Microsoft.Extensions.Options;
 using System.Security.Claims;
 using System.Text.Encodings.Web;
+using Microsoft.AspNetCore.Authentication;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Monai.Deploy.WorkflowManager.Logging;
 
 namespace Monai.Deploy.WorkflowManager.Authentication.Middleware
 {
@@ -36,7 +37,7 @@ public BypassAuthenticationHandler(
         protected override Task<AuthenticateResult> HandleAuthenticateAsync()
         {
             var principal = new ClaimsPrincipal(new ClaimsIdentity(Array.Empty<Claim>(), Scheme.Name));
-            Logger.LogInformation("Bypassing request");
+            Logger.BypassAuthentication();
             return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name)));
         }
     }

diff --git a/src/WorkflowManager/Authentication/Middleware/EndpointAuthorizationMiddleware.cs b/src/WorkflowManager/Authentication/Middleware/EndpointAuthorizationMiddleware.cs
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-using Microsoft.Extensions.Configuration;
-using Microsoft.AspNetCore.Routing;
 using System.Net;
 using Microsoft.AspNetCore.Http;
-using Monai.Deploy.WorkflowManager.Authentication.Extensions;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Monai.Deploy.WorkflowManager.Authentication.Extensions;
 
 namespace Monai.Deploy.WorkflowManager.Authentication.Middleware
 {

diff --git a/src/WorkflowManager/Authentication/Monai.Deploy.WorkflowManager.Authentication.csproj b/src/WorkflowManager/Authentication/Monai.Deploy.WorkflowManager.Authentication.csproj
@@ -25,11 +25,12 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
+    <Compile Include="..\..\AssemblyInfo.cs" Link="AssemblyInfo.cs" />
   </ItemGroup>
 
+
   <ItemGroup>
-    <Compile Include="..\..\AssemblyInfo.cs" Link="AssemblyInfo.cs" />
+    <AdditionalFiles Include="..\..\.sonarlint\project-monai_monai-deploy-workflow-manager\CSharp\SonarLint.xml" Link="SonarLint.xml" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/WorkflowManager/Common/Extensions/FileExtensions.cs b/src/WorkflowManager/Common/Extensions/FileExtensions.cs
@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-
 using Monai.Deploy.Storage.API;
 
 namespace Monai.Deploy.WorkflowManager.Common.Extensions

diff --git a/src/WorkflowManager/Common/Interfaces/IWorkflowInstanceService.cs b/src/WorkflowManager/Common/Interfaces/IWorkflowInstanceService.cs
@@ -51,7 +51,6 @@ public interface IWorkflowInstanceService : IPaginatedApi<WorkflowInstance>
         /// <returns></returns>
         public Task<long> FilteredCountAsync(Status? status = null, string? payloadId = null);
 
-
         /// <summary>
         /// Get all failed workflow instance's within a time span.
         /// </summary>
-Original file line number
+Diff line change
@@ Expand Up / @@ -95,5 +95,4 @@ @@
         "LogHttpResponseBody": true
       },
       "AllowedHosts": "*",
     }