Skip to content

Commit

Permalink
Add resources for etcd
Browse files Browse the repository at this point in the history
Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com>
  • Loading branch information
thibaultmg committed Jun 3, 2024
1 parent 375b31f commit ffd6fab
Showing 3 changed files with 443 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -32,6 +32,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"

"github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/hypershift"
"github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/microshift"
"github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/openshift"
"github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/rendering"
"github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/util"
@@ -247,6 +248,21 @@ func (r *ObservabilityAddonReconciler) Reconcile(ctx context.Context, req ctrl.R
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to render prometheus templates: %w", err)
}

if !isHubMetricsCollector {
microshiftVersion, err := microshift.IsMicroshiftCluster(ctx, r.Client)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to check if the cluster is microshift: %w", err)
}

if len(microshiftVersion) > 0 {
mcs := microshift.NewMicroshift(namespace)
if err := mcs.Render(ctx, toDeploy); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to render microshift templates: %w", err)
}
}
}

deployer := deploying.NewDeployer(r.Client)
for _, res := range toDeploy {
if res.GetNamespace() != namespace {
417 changes: 417 additions & 0 deletions operators/endpointmetrics/pkg/microshift/microshift.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,417 @@
// Copyright (c) Red Hat, Inc.
// Copyright Contributors to the Open Cluster Management project
// Licensed under the Apache License 2.0

package microshift

import (
"context"
"fmt"
"os"

promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
etcdClientCertSecretName = "etcd-client-cert" //nolint:gosec
)

type Microshift struct {
// client client.Client
addonNamespace string
}

func NewMicroshift(addonNs string) *Microshift {
return &Microshift{
// client: client,
addonNamespace: addonNs,
}
}

// Render renders the resources for the microshift cluster
// If the cluster is not a microshift cluster, it modifies the resources
// to adapt to the microshift cluster
func (m *Microshift) Render(ctx context.Context, resources []*unstructured.Unstructured) error {
jobRes, err := m.renderCronJobExposingMicroshiftSecrets()
if err != nil {
return fmt.Errorf("failed to render cronjob for secrets: %w", err)
}
resources = append(resources, jobRes...)

etcdRes, err := m.renderEtcdResources()
if err != nil {
return fmt.Errorf("failed to render etcd resources: %w", err)
}
resources = append(resources, etcdRes...)

if err := m.renderPrometheus(resources); err != nil {
return fmt.Errorf("failed to render prometheus: %w", err)
}

return nil
}

// renderPrometheus modifies the prometheus resource to adapt to the microshift cluster
// It adds the etcd client key and certificate secret to the prometheus pod
func (m *Microshift) renderPrometheus(res []*unstructured.Unstructured) error {
promRes, err := getResource(res, "Prometheus", "prometheus")
if err != nil {
return fmt.Errorf("failed to get prometheus resource: %w", err)
}

prom := &promv1.Prometheus{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(promRes.Object, prom); err != nil {
return fmt.Errorf("failed to convert unstructured object to prometheus object: %w", err)
}

prom.Spec.Secrets = append(prom.Spec.Secrets, etcdClientCertSecretName)

return nil

}

// renderCronJobExposingMicroshiftSecrets creates a cronjob to expose Microshift's host secrets needed in Microshift itself.
// For example, Microshift clusters run etcd directly on the host. It exposes its metrics via a secured port.
// The job ensures that etcd client key and certificate are exposed as a secret in the addon namespace.
func (m *Microshift) renderCronJobExposingMicroshiftSecrets() ([]*unstructured.Unstructured, error) {
ret := []*unstructured.Unstructured{}
jobName := "microshift-secrets-exposer"

// Create a cronjob to update the etcd client key and certificate secret
// every hour
cronJob := &batchv1.CronJob{
ObjectMeta: metav1.ObjectMeta{
Name: jobName,
Namespace: m.addonNamespace,
},
Spec: batchv1.CronJobSpec{
Schedule: "0 * * * *",
JobTemplate: batchv1.JobTemplateSpec{
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "microshift-certs-updater",
Image: "registry.access.redhat.com/ubi9/ubi-minimal@sha256:ef6fb6b3b38ef6c85daebeabebc7ff3151b9dd1500056e6abc9c3295e4b78a51",
Command: []string{"/bin/sh", "-c"},
Args: []string{fmt.Sprintf(
`
kubectl create secret generic %s --from-file=key=/tmp/etcd-certs/ca.key --from-file=cert=/tmp/etcd-certs/ca.crt --dry-run=client -o yaml | kubectl apply -f -
`, etcdClientCertSecretName),
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "etcd-certs",
MountPath: "/tmp/etcd-certs",
ReadOnly: true,
},
},
SecurityContext: &corev1.SecurityContext{
RunAsUser: new(int64), // 0 for root user
AllowPrivilegeEscalation: new(bool),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"ALL"},
},
},
},
},
RestartPolicy: corev1.RestartPolicyOnFailure,
ServiceAccountName: jobName,
Volumes: []corev1.Volume{
{
Name: "etcd-certs",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/var/lib/microshift/certs/etcd-signer/",
Type: newHostPathType(corev1.HostPathDirectory),
},
},
},
},
},
},
},
},
},
}

unstructuredCronJob, err := convertToUnstructured(cronJob)
if err != nil {
return nil, fmt.Errorf("failed to convert cronjob to unstructured: %w", err)
}
ret = append(ret, unstructuredCronJob)

// Add service account to the cronjob
sa := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: jobName,
Namespace: m.addonNamespace,
},
}

unstructuredSA, err := convertToUnstructured(sa)
if err != nil {
return nil, fmt.Errorf("failed to convert service account to unstructured: %w", err)
}
ret = append(ret, unstructuredSA)

// Add permissions to the service account to update the secret and run as root
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: jobName,
Namespace: m.addonNamespace,
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"secrets"},
Verbs: []string{"create", "update"},
},
},
}

unstructuredRole, err := convertToUnstructured(role)
if err != nil {
return nil, fmt.Errorf("failed to convert role to unstructured: %w", err)
}
ret = append(ret, unstructuredRole)

roleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: jobName,
Namespace: m.addonNamespace,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: jobName,
Namespace: m.addonNamespace,
},
},
RoleRef: rbacv1.RoleRef{
Kind: "Role",
Name: jobName,
APIGroup: "rbac.authorization.k8s.io",
},
}

unstructuredRoleBinding, err := convertToUnstructured(roleBinding)
if err != nil {
return nil, fmt.Errorf("failed to convert role binding to unstructured: %w", err)
}
ret = append(ret, unstructuredRoleBinding)

// Add cluster role for hostmount and anyuid permissions- apiGroups:
clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: jobName,
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{"security.openshift.io"},
Resources: []string{"securitycontextconstraints"},
ResourceNames: []string{"hostmount-anyuid"},
Verbs: []string{"use"},
},
},
}

unstructuredClusterRole, err := convertToUnstructured(clusterRole)
if err != nil {
return nil, fmt.Errorf("failed to convert cluster role to unstructured: %w", err)
}
ret = append(ret, unstructuredClusterRole)

clusterRoleBinding := &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: jobName,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: jobName,
Namespace: m.addonNamespace,
},
},
RoleRef: rbacv1.RoleRef{
Kind: "ClusterRole",
Name: jobName,
APIGroup: "rbac.authorization.k8s.io",
},
}

unstructuredClusterRoleBinding, err := convertToUnstructured(clusterRoleBinding)
if err != nil {
return nil, fmt.Errorf("failed to convert cluster role binding to unstructured: %w", err)
}
ret = append(ret, unstructuredClusterRoleBinding)

return ret, nil
}

// renderServiceMonitors modifies or creates the service monitors to adapt to the microshift cluster
func (m *Microshift) renderEtcdResources() ([]*unstructured.Unstructured, error) {
ret := []*unstructured.Unstructured{}

hostIP := os.Getenv("HOST_IP")
if hostIP == "" {
return nil, fmt.Errorf("HOST_IP env var is not set")
}

// Expose etcd endpoint in the addon namespace
endpoint := &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "etcd",
Namespace: m.addonNamespace,
Labels: map[string]string{
"app": "etcd",
},
},
Subsets: []corev1.EndpointSubset{
{
Addresses: []corev1.EndpointAddress{
{
IP: hostIP,
},
},
Ports: []corev1.EndpointPort{
{
Name: "metrics",
Port: 2381,
Protocol: corev1.ProtocolTCP,
},
},
},
},
}

unstructuredEndpoint, err := convertToUnstructured(endpoint)
if err != nil {
return nil, fmt.Errorf("failed to convert endpoint to unstructured: %w", err)
}
ret = append(ret, unstructuredEndpoint)

service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "etcd",
Namespace: m.addonNamespace,
Labels: map[string]string{
"app": "etcd",
},
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "metrics",
Port: 2381,
TargetPort: intstr.FromInt(2381),
},
},
Selector: map[string]string{
"app": "etcd",
},
},
}

unstructuredService, err := convertToUnstructured(service)
if err != nil {
return nil, fmt.Errorf("failed to convert service to unstructured: %w", err)
}
ret = append(ret, unstructuredService)

// render service monitor for etcd
smon := &promv1.ServiceMonitor{
ObjectMeta: metav1.ObjectMeta{
Name: "etcd",
Namespace: m.addonNamespace,
},
Spec: promv1.ServiceMonitorSpec{
Endpoints: []promv1.Endpoint{
{
Scheme: "https",
Path: "/metrics",
Interval: "15s",
// Use secret etcd-cert to scrape etcd metrics
TLSConfig: &promv1.TLSConfig{
CertFile: "/etc/prometheus/secrets/etcd-cert/ca.crt",
KeyFile: "/etc/prometheus/secrets/etcd-cert/ca.key",
CAFile: "/etc/prometheus/secrets/etcd-cert/ca.crt",
},
},
},
Selector: metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "etcd",
},
},
NamespaceSelector: promv1.NamespaceSelector{
MatchNames: []string{m.addonNamespace},
},
},
}

unstructuredSMon, err := convertToUnstructured(smon)
if err != nil {
return nil, fmt.Errorf("failed to convert service monitor to unstructured: %w", err)
}
ret = append(ret, unstructuredSMon)

return ret, nil
}

// IsMicroshiftCluster checks if the cluster is a microshift cluster.
// It verifies the existence of the configmap microshift-version in namespace kube-public.
// If the configmap exists, it returns the version of the microshift cluster.
// If the configmap does not exist, it returns an empty string.
func IsMicroshiftCluster(ctx context.Context, client client.Client) (string, error) {
res := &corev1.ConfigMap{}
err := client.Get(ctx, types.NamespacedName{
Name: "microshift-version",
Namespace: "kube-public",
}, res)
if err != nil {
if errors.IsNotFound(err) {
return "", nil
}
return "", err
}

return res.Data["version"], nil
}

func getResource(res []*unstructured.Unstructured, kind, name string) (*unstructured.Unstructured, error) {
for _, r := range res {
if r.GetKind() == kind && r.GetName() == name {
return r, nil
}
}
return nil, errors.NewNotFound(schema.GroupResource{
Group: "",
Resource: kind,
}, name)
}

func convertToUnstructured(obj interface{}) (*unstructured.Unstructured, error) {
unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
if err != nil {
return nil, err
}

return &unstructured.Unstructured{Object: unstructuredObj}, nil
}

func newHostPathType(pathType corev1.HostPathType) *corev1.HostPathType {
return &pathType
}
Original file line number Diff line number Diff line change
@@ -409,6 +409,16 @@ func createManifestWorks(
Value: "true",
})

// Add host ip env for endpoint operator. It is needed for microshift to scrape host processes metrics
spec.Containers[0].Env = append(spec.Containers[0].Env, corev1.EnvVar{
Name: "HOST_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
},
})

dep.ObjectMeta.Name = config.HubEndpointOperatorName
}

0 comments on commit ffd6fab

Please sign in to comment.