diff --git a/README.md b/README.md index 314d822..74cd37b 100644 --- a/README.md +++ b/README.md @@ -304,12 +304,34 @@ node driver to patch pods. To deploy this, simply uncomment the This will add the flag and the required permissions when building the kustomization. +#### Status Labels + +To reflect the container scaling status in the k8s API, the manager can set +status labels on a pod. This requires the flag `-status-labels=true`, which is +set by default in the production deployment. + +The resulting labels have the following structure: + +```yaml +status.zeropod.ctrox.dev/: +``` + +So if our pod has two containers, one of them running and one in scaled-down +state, the labels would be set like this: + +```yaml +labels: + status.zeropod.ctrox.dev/container1: RUNNING + status.zeropod.ctrox.dev/container2: SCALED_DOWN +``` + #### Flags ``` -metrics-addr=":8080" sets the address of the metrics server -debug enables debug logging -in-place-scaling=false enable in-place resource scaling, requires InPlacePodVerticalScaling feature flag +-status-labels=false update pod labels to reflect container status ``` ## Metrics diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 3193848..c64a10b 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -19,6 +19,7 @@ var ( debug = flag.Bool("debug", false, "enable debug logs") inPlaceScaling = flag.Bool("in-place-scaling", false, "enable in-place resource scaling, requires InPlacePodVerticalScaling feature flag") + statusLabels = flag.Bool("status-labels", false, "update pod labels to reflect container status") ) func main() { @@ -44,6 +45,9 @@ func main() { } podHandlers := []manager.PodHandler{} + if *statusLabels { + podHandlers = append(podHandlers, manager.NewPodLabeller()) + } if *inPlaceScaling { podHandlers = append(podHandlers, manager.NewPodScaler()) } diff --git a/config/in-place-scaling/kustomization.yaml b/config/in-place-scaling/kustomization.yaml index 70abbf1..cf4363c 100644 --- a/config/in-place-scaling/kustomization.yaml +++ b/config/in-place-scaling/kustomization.yaml @@ -1,7 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component -resources: - - rbac.yaml patches: - patch: |- - op: add diff --git a/config/kind/kustomization.yaml b/config/kind/kustomization.yaml index 87ce003..b28fdf7 100644 --- a/config/kind/kustomization.yaml +++ b/config/kind/kustomization.yaml @@ -2,6 +2,8 @@ resources: - ../base components: - ../in-place-scaling + - ../pod-updater + - ../status-labels images: - name: manager newName: ghcr.io/ctrox/zeropod-manager diff --git a/config/pod-updater/kustomization.yaml b/config/pod-updater/kustomization.yaml new file mode 100644 index 0000000..b8f594c --- /dev/null +++ b/config/pod-updater/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - rbac.yaml diff --git a/config/in-place-scaling/rbac.yaml b/config/pod-updater/rbac.yaml similarity index 100% rename from config/in-place-scaling/rbac.yaml rename to config/pod-updater/rbac.yaml diff --git a/config/production/kustomization.yaml b/config/production/kustomization.yaml index b78f029..6a28e6e 100644 --- a/config/production/kustomization.yaml +++ b/config/production/kustomization.yaml @@ -1,8 +1,11 @@ resources: - ../base -# uncommment to enable in-place scaling -# components: -# - ../in-place-scaling +# pod-updater is required if status-labels or in-place-scaling is enabled +components: +- ../pod-updater +- ../status-labels +# uncommment to enable in-place-scaling +# - ../in-place-scaling images: - name: installer newName: ghcr.io/ctrox/zeropod-installer diff --git a/config/status-labels/kustomization.yaml b/config/status-labels/kustomization.yaml new file mode 100644 index 0000000..c330504 --- /dev/null +++ b/config/status-labels/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: -status-labels=true + target: + kind: DaemonSet diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index b4603cc..cb3417b 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -5,10 +5,12 @@ import ( "fmt" "net/http" "runtime" + "strings" "sync" "testing" "time" + v1 "github.com/ctrox/zeropod/api/shim/v1" "github.com/ctrox/zeropod/manager" "github.com/ctrox/zeropod/zeropod" "github.com/prometheus/client_golang/prometheus" @@ -240,6 +242,29 @@ func TestE2E(t *testing.T) { }, time.Minute, time.Second) }) + t.Run("status labels", func(t *testing.T) { + pod := testPod(scaleDownAfter(0), agnContainer("agn", 8080), agnContainer("agn2", 8081)) + + cleanupPod := createPodAndWait(t, ctx, client, pod) + defer cleanupPod() + require.Eventually(t, func() bool { + if err := client.Get(ctx, objectName(pod), pod); err != nil { + return false + } + labelCount := 0 + expectedLabels := 2 + for k, v := range pod.GetLabels() { + if strings.HasPrefix(k, manager.StatusLabelKeyPrefix) { + if v == v1.ContainerPhase_SCALED_DOWN.String() { + labelCount++ + } + } + } + + return labelCount == expectedLabels + }, time.Minute, time.Second) + }) + t.Run("metrics", func(t *testing.T) { // create two pods to test metric merging runningPod := testPod(scaleDownAfter(time.Hour)) diff --git a/manager/pod_labeller.go b/manager/pod_labeller.go new file mode 100644 index 0000000..72d6e6f --- /dev/null +++ b/manager/pod_labeller.go @@ -0,0 +1,40 @@ +package manager + +import ( + "context" + "log/slog" + "path" + + v1 "github.com/ctrox/zeropod/api/shim/v1" + corev1 "k8s.io/api/core/v1" +) + +const ( + StatusLabelKeyPrefix = "status.zeropod.ctrox.dev" +) + +type PodLabeller struct { + log *slog.Logger +} + +func NewPodLabeller() *PodLabeller { + log := slog.With("component", "podupdater") + log.Info("init") + return &PodLabeller{log: log} +} + +func (pl *PodLabeller) Handle(ctx context.Context, status *v1.ContainerStatus, pod *corev1.Pod) error { + clog := pl.log.With("container", status.Name, "pod", status.PodName, + "namespace", status.PodNamespace, "phase", status.Phase) + clog.Info("status event") + + pl.setLabel(pod, status) + return nil +} + +func (pu *PodLabeller) setLabel(pod *corev1.Pod, status *v1.ContainerStatus) { + if pod.Labels == nil { + pod.Labels = map[string]string{} + } + pod.Labels[path.Join(StatusLabelKeyPrefix, status.Name)] = status.Phase.String() +} diff --git a/manager/pod_labeller_test.go b/manager/pod_labeller_test.go new file mode 100644 index 0000000..8a400b2 --- /dev/null +++ b/manager/pod_labeller_test.go @@ -0,0 +1,74 @@ +package manager + +import ( + "context" + "log/slog" + "testing" + + v1 "github.com/ctrox/zeropod/api/shim/v1" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestPodLabeller(t *testing.T) { + slog.SetLogLoggerLevel(slog.LevelDebug) + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cases := map[string]struct { + statusEventPhase v1.ContainerPhase + beforeEvent map[string]string + expected map[string]string + }{ + "no labels set": { + statusEventPhase: v1.ContainerPhase_RUNNING, + beforeEvent: nil, + expected: map[string]string{ + "status.zeropod.ctrox.dev/first-container": v1.ContainerPhase_RUNNING.String(), + }, + }, + "existing labels are kept": { + statusEventPhase: v1.ContainerPhase_RUNNING, + beforeEvent: map[string]string{"existing": "label"}, + expected: map[string]string{ + "existing": "label", + "status.zeropod.ctrox.dev/first-container": v1.ContainerPhase_RUNNING.String(), + }, + }, + "status label is updated": { + statusEventPhase: v1.ContainerPhase_SCALED_DOWN, + beforeEvent: map[string]string{ + "status.zeropod.ctrox.dev/first-container": v1.ContainerPhase_RUNNING.String(), + }, + expected: map[string]string{ + "status.zeropod.ctrox.dev/first-container": v1.ContainerPhase_SCALED_DOWN.String(), + }, + }, + } + + for name, tc := range cases { + tc := tc + t.Run(name, func(t *testing.T) { + pod := newPod(nil) + pod.SetLabels(tc.beforeEvent) + + if err := NewPodLabeller().Handle( + context.Background(), + &v1.ContainerStatus{ + Name: pod.Spec.Containers[0].Name, + PodName: pod.Name, + PodNamespace: pod.Namespace, + Phase: tc.statusEventPhase, + }, + pod, + ); err != nil { + t.Fatal(err) + } + + assert.Equal(t, pod.GetLabels(), tc.expected) + }) + } +} diff --git a/manager/pod_scaler_test.go b/manager/pod_scaler_test.go index d5ccc55..64b8854 100644 --- a/manager/pod_scaler_test.go +++ b/manager/pod_scaler_test.go @@ -108,12 +108,20 @@ func newPod(req corev1.ResourceList) *corev1.Pod { Namespace: "default", }, Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "first-container", - Resources: corev1.ResourceRequirements{ - Requests: req, + Containers: []corev1.Container{ + { + Name: "first-container", + Resources: corev1.ResourceRequirements{ + Requests: req, + }, }, - }}, + { + Name: "second-container", + Resources: corev1.ResourceRequirements{ + Requests: req, + }, + }, + }, }, } } diff --git a/manager/status.go b/manager/status.go index a03ac4a..ea6514e 100644 --- a/manager/status.go +++ b/manager/status.go @@ -1,3 +1,6 @@ +// Package manager contains most of the implementation of the zeropod-manager +// node daemon. It takes care of loading eBPF programs, providing metrics and +// monitors the shims for status updates. package manager import (