Reporting: Variable change to deprecate step Splunk Hook; logging in …

…telemetry (SAP#3539)

* Adds fatalError variable with helper setter and getter helper functions
* Adds logging function to telemetry package (always executed)

pkg/log/errors.go

@@ -16,6 +16,7 @@ const (
 )
 
 var errorCategory ErrorCategory = ErrorUndefined
+var fatalError []byte
 
 func (e ErrorCategory) String() string {
 	return [...]string{
@@ -63,3 +64,13 @@ func SetErrorCategory(category ErrorCategory) {
 func GetErrorCategory() ErrorCategory {
 	return errorCategory
 }
+
+// SetFatalErrorDetail sets the fatal error to be stored
+func SetFatalErrorDetail(error []byte) {
+	fatalError = error
+}
+
+// GetFatalErrorDetail retrieves the error which is currently known to the execution of a step
+func GetFatalErrorDetail() []byte {
+	return fatalError
+}

pkg/log/errors_test.go

@@ -1,10 +1,11 @@
 package log
 
 import (
+	"encoding/json"
 	"fmt"
-	"testing"
-
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"testing"
 )
 
 func TestSetErrorCategory(t *testing.T) {
@@ -17,3 +18,73 @@ func TestGetErrorCategory(t *testing.T) {
 	errorCategory = ErrorCompliance
 	assert.Equal(t, GetErrorCategory(), errorCategory)
 }
+
+func TestSetFatalErrorDetail(t *testing.T) {
+	sampleError := logrus.Fields{"Message": "Error happened"}
+	errDetails, _ := json.Marshal(&sampleError)
+
+	tests := []struct {
+		name       string
+		error      []byte
+		want       []byte
+		errPresent bool
+	}{
+		{
+			name:       "set fatal error",
+			error:      errDetails,
+			want:       errDetails,
+			errPresent: false,
+		},
+		{
+			name:       "set fatal error - override",
+			error:      errDetails,
+			want:       errDetails,
+			errPresent: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.errPresent {
+				SetFatalErrorDetail(tt.error)
+			}
+			SetFatalErrorDetail(tt.error)
+			assert.Equalf(t, tt.want, fatalError, "GetFatalErrorDetail()")
+			fatalError = nil // reset error
+		})
+	}
+}
+
+func TestGetFatalErrorDetail(t *testing.T) {
+	sampleError := logrus.Fields{"Message": "Error happened"}
+	errDetails, _ := json.Marshal(&sampleError)
+
+	tests := []struct {
+		name       string
+		errDetails []byte
+		want       string
+	}{
+		{
+			name:       "returns fatal error",
+			errDetails: errDetails,
+			want:       "{\"Message\":\"Error happened\"}",
+		},
+		{
+			name:       "no fatal error set - returns empty",
+			errDetails: nil,
+			want:       "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.errDetails != nil {
+				SetFatalErrorDetail(tt.errDetails)
+			}
+			str := ""
+			data := []byte(str)
+			fmt.Println(data)
+			fatalErrorDetail := string(GetFatalErrorDetail())
+			assert.Equalf(t, tt.want, fatalErrorDetail, "GetFatalErrorDetail()")
+			fatalError = nil // resets fatal error
+		})
+	}
+}

pkg/log/fatalHook.go

@@ -34,6 +34,7 @@ func (f *FatalHook) Fire(entry *logrus.Entry) error {
 	details["category"] = GetErrorCategory().String()
 	details["result"] = "failure"
 	details["correlationId"] = f.CorrelationID
+	details["time"] = entry.Time
 
 	fileName := "errorDetails.json"
 	if details["stepName"] != nil {
@@ -44,10 +45,12 @@ func (f *FatalHook) Fire(entry *logrus.Entry) error {
 	errDetails, _ := json.Marshal(&details)
 	// Logging information needed for error reporting -  do not modify.
 	Entry().Infof("fatal error: errorDetails%v", string(errDetails))
+	// Sets the fatal error details in the logging framework to be consumed in the stepTelemetryData
+	SetFatalErrorDetail(errDetails)
 	_, err := ioutil.ReadFile(filePath)
 	if err != nil {
 		// do not overwrite file in case it already exists
-		// this helps to report the first error which occured - instead of the last one
+		// this helps to report the first error which occurred - instead of the last one
 		ioutil.WriteFile(filePath, errDetails, 0666)
 	}
 

pkg/telemetry/data.go

@@ -69,6 +69,23 @@ type CustomData struct {
 	Custom5         string `json:"e_30,omitempty"`
 }
 
+// StepTelemetryData definition for telemetry reporting and monitoring
+type StepTelemetryData struct {
+	PipelineURLHash string                 `json:"PipelineURLHash"`
+	BuildURLHash    string                 `json:"BuildURLHash"`
+	StageName       string                 `json:"StageName"`
+	StepName        string                 `json:"StepName"`
+	ErrorCode       string                 `json:"ErrorCode"`
+	Duration        string                 `json:"Duration"`
+	ErrorCategory   string                 `json:"ErrorCategory"`
+	CorrelationID   string                 `json:"CorrelationID"`
+	CommitHash      string                 `json:"CommitHash"`
+	Branch          string                 `json:"Branch"`
+	GitOwner        string                 `json:"GitOwner"`
+	GitRepository   string                 `json:"GitRepository"`
+	ErrorDetail     map[string]interface{} `json:"ErrorDetail"`
+}
+
 // Data object definition containing all telemetry data
 type Data struct {
 	BaseData

pkg/telemetry/telemetry.go

@@ -2,6 +2,7 @@ package telemetry
 
 import (
 	"crypto/sha1"
+	"encoding/json"
 	"fmt"
 	"github.com/SAP/jenkins-library/pkg/orchestrator"
 	"time"
@@ -118,6 +119,9 @@ func (t *Telemetry) GetData() Data {
 
 // Send telemetry information to SWA
 func (t *Telemetry) Send() {
+	// always log step telemetry data to logfile used for internal use-case
+	t.logStepTelemetryData()
+
 	// skip if telemetry is disabled
 	if t.disabled {
 		return
@@ -129,3 +133,39 @@ func (t *Telemetry) Send() {
 	log.Entry().WithField("request", request.String()).Debug("Sending telemetry data")
 	t.client.SendRequest(http.MethodGet, request.String(), nil, nil, nil)
 }
+
+func (t *Telemetry) logStepTelemetryData() {
+
+	var fatalError map[string]interface{}
+	if t.data.ErrorCode != "0" {
+		// retrieve the error information from the logCollector
+		err := json.Unmarshal(log.GetFatalErrorDetail(), &fatalError)
+		if err != nil {
+			log.Entry().WithError(err).Error("could not unmarshal fatal error struct")
+		}
+	}
+
+	stepTelemetryData := StepTelemetryData{
+		PipelineURLHash: t.data.PipelineURLHash,
+		BuildURLHash:    t.data.BuildURLHash,
+		StageName:       t.data.StageName,
+		StepName:        t.data.BaseData.StepName,
+		ErrorCode:       t.data.CustomData.ErrorCode,
+		Duration:        t.data.CustomData.Duration,
+		ErrorCategory:   t.data.CustomData.ErrorCategory,
+		ErrorDetail:     fatalError,
+		CorrelationID:   t.provider.GetBuildUrl(),
+		CommitHash:      t.provider.GetCommit(),
+		Branch:          t.provider.GetBranch(),
+		GitOwner:        t.provider.GetRepoUrl(), // TODO not correct
+		GitRepository:   t.provider.GetRepoUrl(), // TODO not correct
+	}
+	stepTelemetryJSON, err := json.Marshal(stepTelemetryData)
+	if err != nil {
+		log.Entry().Error("could not marshal step telemetry data")
+		log.Entry().Infof("Step telemetry data: {n/a}")
+	} else {
+		// log step monitoring data, changes here need to change the regex in the internal piper lib
+		log.Entry().Infof("Step telemetry data:%v", string(stepTelemetryJSON))
+	}
+}