splunk log and first dashboard (#245)

test setting update source type add splunk-config secrets add ticket-number to log context temp initial setup for splunk Co-authored-by: peggy-gov-bc-ca <[email protected]>
bcgov · Sep 27, 2021 · 1986075 · 1986075
1 parent 5130cc0
commit 1986075
Show file tree

Hide file tree

Showing 14 changed files with 323 additions and 73 deletions.
diff --git a/.github/workflows/imageUploadTesting.yml b/.github/workflows/imageUploadTesting.yml
@@ -4,8 +4,82 @@
 name: Main-Build .NET API Image and Push to Openshift Registry
 on:
   push:
-    branches: [feature/workerDeploy]
+    branches: [feature/splunk]
 jobs:
+  build-citizen-api:
+    runs-on: ubuntu-20.04
+    environment: Development
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v2
+
+      - name: Get git commit short sha
+        id: sha
+        run: |
+          shortSha=$(echo $(git rev-parse --short HEAD) | cut -c1-6)
+          echo "::set-output name=sha_short::$shortSha"
+
+      - name: Display short sha
+        run: echo ${{ steps.sha.outputs.sha_short }}
+
+      - name: Login to OpenShift
+        uses: docker/login-action@v1
+        with:
+          registry: ${{secrets.OPENSHIFT_EXTERNAL_REPOSITORY}}
+          username: ${{secrets.OPENSHIFT_SA_USERNAME}}
+          password: ${{secrets.OPENSHIFT_SA_PASSWORD}}
+
+      - name: Build and push
+        env:
+          COMPOSE_DOCKER_CLI_BUILD: 1
+          DOCKER_BUILDKIT: 1
+          CONTEXT: ./src/backend/TrafficCourtsApi
+          DOCKER_FILE: ./src/backend/TrafficCourtsApi/Dockerfile.citizen-api
+          IMAGE: ${{secrets.OPENSHIFT_EXTERNAL_REPOSITORY}}/${{secrets.OPENSHIFT_TOOLS_NAMESPACE}}/citizen-api:${{secrets.OC_APP}}
+        run: |
+          docker build \
+            --build-arg ASPNETCORE_ENVIRONMENT=Development \
+            --file ${DOCKER_FILE} \
+            --tag ${IMAGE} \
+            ${CONTEXT}
+            docker push ${IMAGE}
+
+  build-ticket-search:
+    runs-on: ubuntu-20.04
+    environment: Development
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v2
+
+      - name: Get git commit short sha
+        id: sha
+        run: |
+          shortSha=$(echo $(git rev-parse --short HEAD) | cut -c1-6)
+          echo "::set-output name=sha_short::$shortSha"
+
+      - name: Display short sha
+        run: echo ${{ steps.sha.outputs.sha_short }}
+
+      - name: Login to OpenShift
+        uses: docker/login-action@v1
+        with:
+          registry: ${{secrets.OPENSHIFT_EXTERNAL_REPOSITORY}}
+          username: ${{secrets.OPENSHIFT_SA_USERNAME}}
+          password: ${{secrets.OPENSHIFT_SA_PASSWORD}}
+
+      - name: Build and push
+        env:
+          COMPOSE_DOCKER_CLI_BUILD: 1
+          DOCKER_BUILDKIT: 1
+          CONTEXT: ./src/ticket-search
+          IMAGE: ${{secrets.OPENSHIFT_EXTERNAL_REPOSITORY}}/${{secrets.OPENSHIFT_TOOLS_NAMESPACE}}/ticket-search:${{secrets.OC_APP}}
+        run: |
+          docker build \
+            --build-arg ASPNETCORE_ENVIRONMENT=Development \
+            --tag ${IMAGE} \
+            ${CONTEXT}
+            docker push ${IMAGE}
+
   build-ticket-worker:
     runs-on: ubuntu-20.04
     environment: Development
@@ -33,4 +107,4 @@ jobs:
             --tag ${IMAGE} \
             --file ${DOCKER_FILE} \
             ${CONTEXT}
-          docker push ${IMAGE}
+            docker push ${IMAGE}
diff --git a/gitops/components/apps/citizen-api/base/citizen-api-deployment-config.yaml b/gitops/components/apps/citizen-api/base/citizen-api-deployment-config.yaml
@@ -42,13 +42,17 @@ spec:
               value: "http://paybc-mock-0198bb-dev.apps.silver.devops.gov.bc.ca/"
             - name: PaybcApi__CallbackBaseUrl
               value: "https://dev.violationtickets.gov.bc.ca/paymentComplete?"
+            - name: SOURCE_TYPE
+              value: "citizen-api"
           envFrom:
             - secretRef:
                 name: rabbitmq-configuration
+            - secretRef:
+                name: splunk-config
           resources:
             limits:
-              cpu: 100m
-              memory: 256Mi
+              cpu: 250m
+              memory: 512Mi
             requests:
               cpu: 100m
               memory: 256Mi

diff --git a/gitops/components/apps/ticket-search/base/ticket-search-deployment-config.yaml b/gitops/components/apps/ticket-search/base/ticket-search-deployment-config.yaml
@@ -36,12 +36,16 @@ spec:
               value: Debug
             - name: ASPNETCORE_ENVIRONMENT
               value: Development
+            - name: SOURCE_TYPE
+              value: "ticket-search"
           envFrom:
             - secretRef:
                 name: ticket-search-rsi-config
+            - secretRef:
+                name: splunk-config
           resources:
             limits:
-              cpu: 150m
+              cpu: 250m
               memory: 512Mi
             requests:
               cpu: 100m

diff --git a/gitops/components/apps/ticket-worker/base/ticket-worker-deployment-config.yaml b/gitops/components/apps/ticket-worker/base/ticket-worker-deployment-config.yaml
@@ -36,13 +36,17 @@ spec:
           env:
             - name: Serilog__MinimumLevel
               value: Debug
+            - name: SOURCE_TYPE
+              value: "ticket-worker"
           envFrom:
             - secretRef:
                 name: rabbitmq-configuration
+            - secretRef:
+                name: splunk-config
           resources:
             limits:
-              cpu: 100m
-              memory: 256Mi
+              cpu: 250m
+              memory: 512Mi
             requests:
               cpu: 100m
               memory: 256Mi

diff --git a/gitops/components/secrets/splunk-config.yaml b/gitops/components/secrets/splunk-config.yaml
@@ -0,0 +1,9 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: splunk-config
+data:
+  SPLUNK__TOKEN: base64 encoded splunk token
+  SPLUNK__URL: >-
+    base64 encoded splunk url
+type: Opaque
diff --git a/splunk-dash-board/operational-support.html b/splunk-dash-board/operational-support.html
@@ -0,0 +1,137 @@
+<dashboard theme="dark">
+  <label>TCO - Operational Support</label>
+  <description>Support Dashboard for Traffic Court Online</description>
+  <fieldset submitButton="false">
+    <input type="time" token="reporttime">
+      <label>Time Range</label>
+      <default>
+        <earliest>-24h@h</earliest>
+        <latest>now</latest>
+      </default>
+    </input>
+    <input type="dropdown" token="index">
+      <label>Environment</label>
+      <choice value="prod_traffic_court">Production</choice>
+      <choice value="test_traffic_court">Test</choice>
+      <choice value="dev_traffic_court">Development</choice>
+      <default>dev_traffic_court</default>
+      <initialValue>dev_traffic_court</initialValue>
+    </input>
+    <input type="dropdown" token="sourcetype">
+      <label>Service</label>
+      <choice value="*">All</choice>
+      <choice value="citizen-api">citizen-api</choice>
+      <choice value="ticket-search">ticket-search</choice>
+      <choice value="ticket-worker">ticket-worker</choice>
+      <default>*</default>
+      <initialValue>*</initialValue>
+    </input>
+  </fieldset>
+  <row>
+     <panel>
+      <title>Ticket Search Statistics </title>
+      <single>
+        <search>
+          <query>index="$index$" Level="Information" RenderedMessage="get ticket search query*" | stats count(*)</query>
+          <earliest>$reporttime.earliest$</earliest>
+          <latest>$reporttime.latest$</latest>
+          <sampleRatio>1</sampleRatio>
+        </search>
+        <option name="colorMode">block</option>
+        <option name="drilldown">none</option>
+        <option name="rangeColors">["0xdc4e41","0x53a051"]</option>
+        <option name="rangeValues">[0]</option>
+        <option name="refresh.display">progressbar</option>
+        <option name="useColors">1</option>
+      </single>
+    </panel>  
+    <panel>
+      <title>Shell ticket Created Statistics </title>
+      <single>
+        <search>
+          <query>index="$index$" Level="Information" RenderedMessage="Shell ticket is created.*" | stats count(*)</query>
+          <earliest>$reporttime.earliest$</earliest>
+          <latest>$reporttime.latest$</latest>
+          <sampleRatio>1</sampleRatio>
+        </search>
+        <option name="colorMode">block</option>
+        <option name="drilldown">none</option>
+        <option name="rangeColors">["0xdc4e41","0x53a051",]</option>
+        <option name="rangeValues">[0]</option>
+        <option name="refresh.display">progressbar</option>
+        <option name="useColors">1</option>
+      </single>
+    </panel> 
+    <panel>
+        <title>Ticket Paid Statistics </title>
+        <single>
+          <search>
+            <query>index="$index$" Level="Information" RenderedMessage="update ticket payment" | stats count(*)</query>
+            <earliest>$reporttime.earliest$</earliest>
+            <latest>$reporttime.latest$</latest>
+            <sampleRatio>1</sampleRatio>
+          </search>
+          <option name="colorMode">block</option>
+          <option name="drilldown">none</option>
+          <option name="rangeColors">["0xdc4e41","0x53a051"]</option>
+          <option name="rangeValues">[0]</option>
+          <option name="refresh.display">progressbar</option>
+          <option name="useColors">1</option>
+        </single>
+    </panel>
+    <panel>
+        <title>Ticket Dispute Statistics </title>
+        <single>
+          <search>
+            <query>index="$index$" Level="Information" RenderedMessage="Dispute created.*" | stats count(*)</query>
+            <earliest>$reporttime.earliest$</earliest>
+            <latest>$reporttime.latest$</latest>
+            <sampleRatio>1</sampleRatio>
+          </search>
+          <option name="colorMode">block</option>
+          <option name="drilldown">none</option>
+          <option name="rangeColors">["0xdc4e41","0x53a051"]</option>
+          <option name="rangeValues">[0]</option>
+          <option name="refresh.display">progressbar</option>
+          <option name="useColors">1</option>
+        </single>
+    </panel>  
+  </row>
+  <row>
+    <panel>
+      <title>Error Statistics</title>
+      <single>
+        <search>
+          <query>index="$index$" Level="Error" |  stats count(*)</query>
+          <earliest>$reporttime.earliest$</earliest>
+          <latest>$reporttime.latest$</latest>
+          <sampleRatio>1</sampleRatio>
+        </search>
+        <option name="colorMode">block</option>
+        <option name="drilldown">none</option>
+        <option name="rangeColors">["0x53a051","0xdc4e41"]</option>
+        <option name="rangeValues">[0]</option>
+        <option name="refresh.display">progressbar</option>
+        <option name="useColors">1</option>
+      </single>
+    </panel>
+    <panel>
+      <title>Error Statistics</title>
+      <chart>
+        <search>
+          <query>index="$index$" Level="Error" sourcetype="$sourcetype$" | stats count by sourcetype  | sort -count</query>
+          <earliest>$reporttime.earliest$</earliest>
+          <latest>$reporttime.latest$</latest>
+          <sampleRatio>1</sampleRatio>
+        </search>
+        <option name="charting.axisY.scale">linear</option>
+        <option name="charting.chart">bar</option>
+        <option name="charting.chart.showDataLabels">all</option>
+        <option name="charting.chart.stackMode">default</option>
+        <option name="charting.drilldown">none</option>
+        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
+        <option name="refresh.display">progressbar</option>
+      </chart>
+    </panel>
+  </row>
+</dashboard>
diff --git a/src/backend/TrafficCourtsApi/TrafficCourts.Common/Configuration/SplunkEventCollector.cs b/src/backend/TrafficCourtsApi/TrafficCourts.Common/Configuration/SplunkEventCollector.cs
@@ -30,6 +30,7 @@ public static void Configure(HostBuilderContext hostingContext, LoggerConfigurat
             // otherwise, the event collector could be configured using the configuration system
             var splunkUrl = configuration["Splunk:Url"];
             var splunkToken = configuration["Splunk:Token"];
+            var sourceType = configuration["SOURCE_TYPE"];
 
             if (string.IsNullOrWhiteSpace(splunkToken) || string.IsNullOrWhiteSpace(splunkUrl))
             {
@@ -41,6 +42,7 @@ public static void Configure(HostBuilderContext hostingContext, LoggerConfigurat
                     .WriteTo.EventCollector(
                         splunkHost: splunkUrl,
                         eventCollectorToken: splunkToken,
+                        sourceType: sourceType,
 #pragma warning disable CA2000 // Dispose objects before losing scope
                         messageHandler: new HttpClientHandler
                         {