Merge pull request #788 from guidonguido/gng/custom-metrics-server

.github/workflows/build-matrix.json

@@ -27,6 +27,13 @@
         "build-args": "COMPONENT=examagent",
         "harbor-project": "crownlabs-core"
     },
+    {
+      "component": "instmetrics",
+      "context": "./operators",
+      "dockerfile": "./operators/build/golang-common/Dockerfile",
+      "build-args": "COMPONENT=instmetrics",
+      "harbor-project": "crownlabs-core"
+    },
     {
         "component": "crownlabs-image-list",
         "context": "./operators",

deploy/crownlabs/Chart.lock

@@ -23,5 +23,8 @@ dependencies:
 - name: exam-agent
   repository: file://../../operators/deploy/exam-agent
   version: 0.1.0
+- name: instmetrics
+  repository: file://../../operators/deploy/instmetrics
+  version: 0.1.0
 digest: sha256:5e61e140806766e46aa2014b018ba66c40653756b213e3ff042782b0d662a6a8
 generated: "2022-04-02T18:37:42.140831116+02:00"

deploy/crownlabs/Chart.yaml

@@ -59,3 +59,8 @@ dependencies:
   version: "0.1.0"
   repository: file://../../operators/deploy/exam-agent
   condition: exam-agent.enabled
+
+- name: instmetrics
+  version: "0.1.0"
+  repository: file://../../operators/deploy/instmetrics
+  condition: instmetrics.enabled

deploy/crownlabs/values.yaml

@@ -150,3 +150,14 @@ exam-agent:
   image:
     repository: crownlabs/exam-agent
     pullPolicy: IfNotPresent
+
+instmetrics:
+  image:
+    repository: crownlabs/instmetrics
+  configurations:
+    runtimeEndpoint: unix:///run/dockershim.sock
+    containerRuntime: /run/dockershim.sock
+    dockerSocket: /var/run/docker.sock
+    connectionTimeout:  10s
+    updatePeriod: 4s
+    grpcPort: 9090

operators/cmd/instmetrics/main.go

@@ -0,0 +1,83 @@
+// Copyright 2020-2022 Politecnico di Torino
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/docker/docker/client"
+	"k8s.io/klog/v2"
+	"k8s.io/klog/v2/klogr"
+
+	clctx "github.com/netgroup-polito/CrownLabs/operators/pkg/context"
+	"github.com/netgroup-polito/CrownLabs/operators/pkg/instmetrics"
+)
+
+func main() {
+	grpcPort := flag.Int("grpc-port", 9090, "Grpc server listening port")
+	runtimeEndpoint := flag.String("runtime-endpoint", "unix:///run/containerd/containerd.sock", "Container runtime endpoint for CRI-API")
+	connectionTimeout := flag.Duration("connection-timeout", 5*time.Second, "Timeout of connection to the CRI-API")
+	updatePeriod := flag.Duration("update-period", 1*time.Second, "Metrics update period and timeout in seconds for requests to CRI-API")
+
+	klog.InitFlags(nil)
+	flag.Parse()
+
+	log := klogr.NewWithOptions().WithName("instmetrics")
+	ctx := clctx.LoggerIntoContext(context.Background(), log)
+
+	remoteRuntimeClient, err := instmetrics.GetRuntimeService(ctx, *connectionTimeout, *runtimeEndpoint)
+	if err != nil {
+		log.Error(err, "Error creating remoteRuntimeServiceClient")
+		os.Exit(1)
+	}
+
+	var statsScraper instmetrics.StatsScraper
+
+	switch *runtimeEndpoint {
+	case "unix:///run/dockershim.sock":
+		dockerCli, err := client.NewClientWithOpts(client.WithVersion("v1.41"))
+		if err != nil {
+			log.Error(err, "Unable to initialize docker API client")
+			os.Exit(1)
+		}
+		statsScraper = instmetrics.DockerMetricsScraper{DockerClient: dockerCli, ContStatsListMutex: &sync.RWMutex{}}
+	default:
+		statsScraper = instmetrics.CRIMetricsScraper{RuntimeClient: remoteRuntimeClient}
+	}
+
+	go func() {
+		http.Handle("/ready", &instmetrics.ReadinessProbeHandler{RuntimeClient: remoteRuntimeClient, Log: log.WithName("probeHandler"), Ready: false})
+		if err = http.ListenAndServe(":8081", nil); err != nil {
+			log.Error(err, "Error serving readiness probe")
+		}
+	}()
+
+	err = instmetrics.Server{
+		MetricsScraperPeriod: *updatePeriod,
+		Log:                  log.WithName("gRPCServer"),
+		Port:                 *grpcPort,
+		RuntimeClient:        remoteRuntimeClient,
+		StatsScraper:         &statsScraper,
+	}.Start(ctx)
+	if err != nil {
+		log.Error(err, "Unable to initialize gRPC server")
+		os.Exit(1)
+	}
+}

operators/deploy/instmetrics/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

operators/deploy/instmetrics/Chart.yaml

@@ -0,0 +1,20 @@
+apiVersion: v2
+name: instmetrics
+description: The CrownLabs CustomMetricsServer
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+icon: https://crownlabs.polito.it/images/logo.svg

operators/deploy/instmetrics/templates/_helpers.tpl

@@ -0,0 +1,60 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "instmetrics.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "instmetrics.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+The version of the application to be deployed
+*/}}
+{{- define "instmetrics.version" -}}
+{{- if .Values.global }}
+{{- .Values.image.tag | default .Values.global.version | default .Chart.AppVersion }}
+{{- else }}
+{{- .Values.image.tag | default .Chart.AppVersion }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "instmetrics.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "instmetrics.labels" -}}
+helm.sh/chart: {{ include "instmetrics.chart" . }}
+{{ include "instmetrics.selectorLabels" . }}
+app.kubernetes.io/version: {{ include "instmetrics.version" . | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "instmetrics.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "instmetrics.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

operators/deploy/instmetrics/templates/daemonset.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "instmetrics.fullname" . }}
+  labels:
+    {{- include "instmetrics.labels" . | nindent 4 }}
+{{- with .Values.daemonsetAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+{{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "instmetrics.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+    {{- with .Values.podAnnotations }}
+      annotations:
+        {{ toYaml . | indent 8 }}
+    {{- end }}
+      labels:
+        {{- include "instmetrics.selectorLabels" . | nindent 8 }}
+    spec:
+      tolerations:
+        {{ toYaml .Values.tolerations | indent 8 }}
+      automountServiceAccountToken: {{ .Values.automountServiceAccountToken }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.containerSecurityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ include "instmetrics.version" . }}"
+          imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+          args:
+          - "--runtime-endpoint={{ .Values.configurations.runtimeEndpoint }}"
+          - "--connection-timeout={{ .Values.configurations.connectionTimeout }}"
+          - "--update-period={{ .Values.configurations.updatePeriod }}"
+          - "--grpc-port={{ .Values.configurations.grpcPort }}"
+          ports:
+            - name: grpc
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+            - name: probes
+              containerPort: 8081
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: probes
+            initialDelaySeconds: 3
+            periodSeconds: 3
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+          - name: container-runtime
+            mountPath: "{{ .Values.configurations.containerRuntime }}"
+            readOnly: true
+          - name: docker-socket
+            mountPath: "{{ .Values.configurations.dockerSocket }}"
+            readOnly: true
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      volumes:
+        - name: container-runtime
+          hostPath:
+            path: "{{ .Values.configurations.containerRuntime }}"
+        - name: docker-socket
+          hostPath:
+            path: "{{ .Values.configurations.dockerSocket }}"

operators/deploy/instmetrics/templates/service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "instmetrics.fullname" . }}
+  labels:
+    {{- include "instmetrics.labels" . | nindent 4 }}
+spec:
+  ports:
+  - port: {{ .Values.service.port }}
+    targetPort: grpc
+    protocol: TCP
+    name: grpc
+  selector:
+    {{- include "instmetrics.selectorLabels" . | nindent 4 }}
+  # Traffic is restricted to be routed only to endpoinds on the same node.
+  # This is because metrics of a specific container are colleted only by 
+  # instmetrics server(s) running on the same Node of the container.
+  internalTrafficPolicy: Local
+  type: {{ .Values.service.type }}

operators/deploy/instmetrics/values.yaml

@@ -0,0 +1,61 @@
+# Default values for instmetrics
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+image:
+  repository: crownlabs/instmetrics
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart version.
+  tag: ""
+
+configurations:
+  runtimeEndpoint: unix:///run/dockershim.sock
+  containerRuntime: /run/dockershim.sock
+  dockerSocket: /var/run/docker.sock
+  connectionTimeout:  10s
+  updatePeriod: 4s
+  grpcPort: 9090
+
+automountServiceAccountToken: false
+
+nameOverride: ""
+fullnameOverride: ""
+
+daemonsetAnnotations: {}
+podAnnotations: {}
+
+podSecurityContext: 
+  runAsNonRoot: false
+  runAsUser: 0
+  runAsGroup: 0
+
+containerSecurityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+    - ALL
+  privileged: false
+
+service:
+  type: ClusterIP
+  port: 9090
+
+
+resources:
+  limits:
+    cpu: 1000m
+    memory: 500Mi
+  requests:
+    cpu: 200m
+    memory: 150Mi
+
+tolerations: []
+  # these tolerations are to have the daemonset runnable on control plane nodes
+  # uncomment them to enable the daemonset pods to be also scheduled on control plane nodes
+  # - key: node-role.kubernetes.io/control-plane
+  #   operator: Exists
+  #   effect: NoSchedule
+  # - key: node-role.kubernetes.io/master
+  #   operator: Exists
+  #   effect: NoSchedule
+