refactor and add new tags, add readme (#15)

* refactor and add new tags, add readme * update readme
Canva · May 14, 2023 · 68485c3 · 68485c3
1 parent 1dcdec4
commit 68485c3
Show file tree

Hide file tree

Showing 7 changed files with 230 additions and 170 deletions.
diff --git a/enricher/ecs/ecs.go b/enricher/ecs/ecs.go
@@ -55,13 +55,13 @@ func NewEnricher() *Enricher {
 // EnrichRecord modifies existing record.
 func (enr *Enricher) EnrichRecord(r map[interface{}]interface{}, t time.Time) map[interface{}]interface{} {
 	resource := map[interface{}]interface{}{
-		mappings.RESOURCE_CLOUD_ACCOUNT_ID: enr.canvaAWSAccount,
-		"service.name":                     enr.canvaAppName,
-		"cloud.platform":                   "aws_ecs",
-		"aws.ecs.launchtype":               "EC2",
-		"aws.ecs.task.family":              enr.ecsTaskFamily,
-		"aws.ecs.task.revision":            enr.ecsTaskRevision,
-		"aws.log.group.names":              enr.logGroup,
+		mappings.RESOURCE_ACCOUNT_ID: enr.canvaAWSAccount,
+		"service.name":               enr.canvaAppName,
+		"cloud.platform":             "aws_ecs",
+		"aws.ecs.launchtype":         "EC2",
+		"aws.ecs.task.family":        enr.ecsTaskFamily,
+		"aws.ecs.task.revision":      enr.ecsTaskRevision,
+		"aws.log.group.names":        enr.logGroup,
 	}
 	body := make(map[interface{}]interface{})
 
@@ -94,9 +94,9 @@ func (enr *Enricher) EnrichRecord(r map[interface{}]interface{}, t time.Time) ma
 		}
 	}
 	return map[interface{}]interface{}{
-		"resource":                  resource,
-		"body":                      body,
-		"timestamp":                 timestamp,
-		mappings.OBSERVED_TIMESTAMP: t.UnixMilli(),
+		mappings.RESOURCE_FIELD_NAME: resource,
+		"body":                       body,
+		"timestamp":                  timestamp,
+		mappings.OBSERVED_TIMESTAMP:  t.UnixMilli(),
 	}
 }
diff --git a/enricher/ecs/ecs_test.go b/enricher/ecs/ecs_test.go
@@ -45,19 +45,19 @@ func TestEnrichRecords(t *testing.T) {
 				time.Date(2009, time.November, 10, 23, 7, 5, 432000000, time.UTC),
 			},
 			want: map[interface{}]interface{}{
-				"resource": map[interface{}]interface{}{
-					mappings.RESOURCE_CLOUD_ACCOUNT_ID: "canva_aws_account_val",
-					"service.name":                     "canva_app_name_val",
-					"cloud.platform":                   "aws_ecs",
-					"aws.ecs.launchtype":               "EC2",
-					"aws.ecs.task.family":              "ecs_task_family_val",
-					"aws.ecs.task.revision":            10001,
-					"aws.log.group.names":              "log_group_val",
-					"host.id":                          "ec2_instance_id_val",
-					"aws.ecs.cluster.name":             "ecs_cluster_val",
-					"aws.ecs.task.arn":                 "ecs_task_arn_val",
-					"container.id":                     "container_id_val",
-					"container.name":                   "container_name_val",
+				mappings.RESOURCE_FIELD_NAME: map[interface{}]interface{}{
+					mappings.RESOURCE_ACCOUNT_ID: "canva_aws_account_val",
+					"service.name":               "canva_app_name_val",
+					"cloud.platform":             "aws_ecs",
+					"aws.ecs.launchtype":         "EC2",
+					"aws.ecs.task.family":        "ecs_task_family_val",
+					"aws.ecs.task.revision":      10001,
+					"aws.log.group.names":        "log_group_val",
+					"host.id":                    "ec2_instance_id_val",
+					"aws.ecs.cluster.name":       "ecs_cluster_val",
+					"aws.ecs.task.arn":           "ecs_task_arn_val",
+					"container.id":               "container_id_val",
+					"container.name":             "container_name_val",
 				},
 				"body": map[interface{}]interface{}{
 					"other_key_1": "other_value_1",

diff --git a/enricher/eks/README.md b/enricher/eks/README.md
@@ -0,0 +1,28 @@
+# EKS enricher
+This EKS enricher follows the Standardize Tagging [documentation](https://canvadev.atlassian.net/wiki/spaces/OB/pages/2869725127/Standardized+Telemetry+Tagging).
+
+##  Static Tags
+Static tags are tags that do not change during runtime. These values are parsed as environment variables in `mappings.go`.
+
+These are the following tags that are being enriched: 
+
+### Resource 
+- `cloud.partition`
+- `cloud.account.id`
+- `cloud.account.name`
+- `cloud.region`
+- `cloud.account.function`
+- `organization`
+- `cloud.provider`
+- `cloud.platform`
+
+### Kubernetes
+- `cluster.name`
+
+## Dropping logs
+EKS enricher is dropping logs if `log` field is empty.
+
+## Missing Kubernetes Metadata
+Refer to this [investigation](https://docs.google.com/document/d/1vRCUKMeo6ypnAq34iwQN7LtDsXxmlj0aYEfRofwV7A4/edit?usp=sharing).
+
+EKS enricher is currently mitigating this issue by adding `container_name` with `_missing_metadata`.
diff --git a/enricher/eks/eks.go b/enricher/eks/eks.go
@@ -9,12 +9,19 @@ import (
 )
 
 type Enricher struct {
-	// AWS Account ID
-	AccountId string `env:"CANVA_AWS_ACCOUNT,required"`
-	// Canva Account Group Function
-	CanvaAccountFunction string `env:"CANVA_ACCOUNT_FUNCTION,required"`
+	CloudAccountId            string `env:"CLOUD_ACCOUNT_ID,required"`
+	CloudAccountName          string `env:"CLOUD_ACCOUNT_NAME,required"`
+	CloudRegion               string `env:"CLOUD_REGION,required"`
+	K8sClusterName            string `env:"K8S_CLUSTER_NAME,required"`
+	CloudPartition            string `env:"CLOUD_PARTITION,required"`
+	CloudAccountGroupFunction string `env:"CLOUD_ACCOUNT_GROUP_FUNCTION,required"`
+	Organization              string `env:"ORGANIZATION,required"`
+	CloudProvider             string `env:"CLOUD_PROVIDER,required"`
+	CloudPlatform             string `env:"CLOUD_PLATFORM,required"`
 }
 
+// NewEnricher returns a enricher with env vars being parsed.
+// These env vars are derived from mappings.go.
 func NewEnricher() (*Enricher, error) {
 	enricher := Enricher{}
 	if err := env.Parse(&enricher); err != nil {
@@ -28,14 +35,20 @@ var _ enricher.IEnricher = (*Enricher)(nil)
 
 func (e Enricher) EnrichRecord(r map[interface{}]interface{}, t time.Time) map[interface{}]interface{} {
 	// Drop log if "log" field is empty
-	if r["log"] == nil {
+	if r[mappings.LOG_FIELD_NAME] == nil {
 		return nil
 	}
 
 	// add resource attributes
-	r["resource"] = map[interface{}]interface{}{
-		mappings.RESOURCE_CLOUD_ACCOUNT_ID: e.AccountId,
-		mappings.RESOURCE_ACCOUNT_GROUP:    e.CanvaAccountFunction,
+	r[mappings.RESOURCE_FIELD_NAME] = map[interface{}]interface{}{
+		mappings.RESOURCE_ACCOUNT_ID:             e.CloudAccountId,
+		mappings.RESOURCE_ACCOUNT_NAME:           e.CloudAccountName,
+		mappings.RESOURCE_ACCOUNT_GROUP_FUNCTION: e.CloudAccountGroupFunction,
+		mappings.RESOURCE_PARTITION:              e.CloudPartition,
+		mappings.RESOURCE_REGION:                 e.CloudRegion,
+		mappings.RESOURCE_ORGANIZATION:           e.Organization,
+		mappings.RESOURCE_PLATFORM:               e.CloudPlatform,
+		mappings.RESOURCE_PROVIDER:               e.CloudProvider,
 	}
 
 	r[mappings.OBSERVED_TIMESTAMP] = t.UnixMilli()
@@ -48,5 +61,7 @@ func (e Enricher) EnrichRecord(r map[interface{}]interface{}, t time.Time) map[i
 		}
 	}
 
+	r[mappings.KUBERNETES_RESOURCE_FIELD_NAME].(map[interface{}]interface{})[mappings.KUBERNETES_RESOURCE_CLUSTER_NAME] = e.K8sClusterName
+
 	return r
 }