Skip to content

Commit

Permalink
Monitor nodepool-builder PV for disk usage, simplify PodMonitor
Browse files Browse the repository at this point in the history 
Change-Id: Ide819a0480f4e57fb30071a0df4be763c198aeae
  • Loading branch information
@mhuin
mhuin committed Dec 18, 2023
1 parent 0b1bb42 commit 3288f7e
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 67 deletions.
94 changes: 30 additions & 64 deletions controllers/nodepool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (

appsv1 "k8s.io/api/apps/v1"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/yaml"
)
Expand Down Expand Up @@ -57,17 +56,17 @@ var timestampOutputCallback string

const (
nodepoolIdent = "nodepool"
launcherIdent = nodepoolIdent + "-launcher"
LauncherIdent = nodepoolIdent + "-launcher"
shortIdent = "np"
launcherPortName = "nlwebapp"
launcherPort = 8006
buildLogsHttpdPort = 8080
BuildLogsHttpdPortName = "buildlogs-http"
NodepoolProvidersSecretsName = "nodepool-providers-secrets"
builderIdent = nodepoolIdent + "-builder"
BuilderIdent = nodepoolIdent + "-builder"
)

var nodepoolStatsdExporterPortName = monitoring.GetStatsdExporterPort(shortIdent)
var NodepoolStatsdExporterPortName = monitoring.GetStatsdExporterPort(shortIdent)

var configScriptVolumeMount = apiv1.VolumeMount{
Name: "nodepool-tooling-vol",
Expand Down Expand Up @@ -100,7 +99,7 @@ func createImageBuildLogForwarderSidecar(r *SFController, annotations map[string

volumeMounts := []apiv1.VolumeMount{
{
Name: builderIdent,
Name: BuilderIdent,
SubPath: "builds",
MountPath: "/watch/",
},
Expand Down Expand Up @@ -203,43 +202,6 @@ func mkStatsdMappingConfig(cloudsYaml map[string]interface{}) (string, error) {
return statsdMappingConfig, err
}

func (r *SFController) EnsureNodepoolPodMonitor() bool {
selector := metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "run",
Operator: metav1.LabelSelectorOpIn,
Values: []string{launcherIdent, builderIdent},
},
{
Key: "app",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"sf"},
},
},
}
desiredNodepoolMonitor := monitoring.MkPodMonitor("nodepool-monitor", r.ns, []string{nodepoolStatsdExporterPortName}, selector)
// add annotations so we can handle lifecycle
annotations := map[string]string{
"version": "1",
}
desiredNodepoolMonitor.ObjectMeta.Annotations = annotations
currentNPM := monitoringv1.PodMonitor{}
if !r.GetM(desiredNodepoolMonitor.Name, &currentNPM) {
r.CreateR(&desiredNodepoolMonitor)
return false
} else {
if !utils.MapEquals(&currentNPM.ObjectMeta.Annotations, &annotations) {
r.log.V(1).Info("Nodepool PodMonitor configuration changed, updating...")
currentNPM.Spec = desiredNodepoolMonitor.Spec
currentNPM.ObjectMeta.Annotations = annotations
r.UpdateR(&currentNPM)
return false
}
}
return true
}

// create default alerts
func (r *SFController) ensureNodepoolPromRule(cloudsYaml map[string]interface{}) bool {
var extraOpenStackMappings []monitoring.StatsdMetricMapping
Expand Down Expand Up @@ -455,6 +417,13 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
statsdExporterVolume,
}

nodeExporterVolumeMount := []apiv1.VolumeMount{
{
Name: BuilderIdent,
MountPath: "/var/lib/nodepool",
},
}

volumeMounts := append(initialVolumeMounts, []apiv1.VolumeMount{
{
Name: "zookeeper-client-tls",
Expand All @@ -465,10 +434,6 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
Name: "nodepool-config",
MountPath: "/etc/nodepool",
},
{
Name: builderIdent,
MountPath: "/var/lib/nodepool",
},
configScriptVolumeMount,
{
Name: "nodepool-tooling-vol",
Expand Down Expand Up @@ -512,6 +477,8 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
},
}...)

volumeMounts = append(volumeMounts, nodeExporterVolumeMount...)

annotations := map[string]string{
"nodepool.yaml": utils.Checksum([]byte(generateConfigScript)),
"nodepool-logging.yaml": utils.Checksum([]byte(loggingConfig)),
Expand All @@ -521,7 +488,7 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
"statsd_mapping": utils.Checksum([]byte(nodepoolStatsdMappingConfig)),
"image": base.NodepoolBuilderImage,
"nodepool-providers-secrets": getProvidersSecretsVersion(providersSecrets, providerSecretsExists),
"serial": "11",
"serial": "12",
}

initContainer := base.MkContainer("nodepool-builder-init", base.BusyboxImage)
Expand All @@ -535,15 +502,12 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
Name: "nodepool-config",
MountPath: "/etc/nodepool/",
},
{
Name: builderIdent,
MountPath: "/var/lib/nodepool",
},
configScriptVolumeMount,
}
initContainer.VolumeMounts = append(initContainer.VolumeMounts, nodeExporterVolumeMount...)

nb := r.mkStatefulSet(
builderIdent, base.NodepoolBuilderImage, r.getStorageConfOrDefault(r.cr.Spec.Nodepool.Builder.Storage),
BuilderIdent, base.NodepoolBuilderImage, r.getStorageConfOrDefault(r.cr.Spec.Nodepool.Builder.Storage),
apiv1.ReadWriteOnce)

nb.Spec.Template.Spec.InitContainers = []apiv1.Container{initContainer}
Expand All @@ -568,11 +532,14 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
monitoring.MkStatsdExporterSideCarContainer(shortIdent, "statsd-config", relayAddress),
)

diskUsageExporter := monitoring.MkNodeExporterSideCarContainer(BuilderIdent, nodeExporterVolumeMount)
nb.Spec.Template.Spec.Containers = append(nb.Spec.Template.Spec.Containers, diskUsageExporter)

// Append image build logs HTTPD sidecar
buildLogsContainer := base.MkContainer("build-logs-httpd", HTTPDImage)
buildLogsContainer.VolumeMounts = []apiv1.VolumeMount{
{
Name: builderIdent,
Name: BuilderIdent,
SubPath: "builds",
MountPath: "/var/www/html/builds",
},
Expand All @@ -593,24 +560,24 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
)

svc := base.MkServicePod(
builderIdent, r.ns, builderIdent+"-0", []int32{buildLogsHttpdPort}, builderIdent)
BuilderIdent, r.ns, BuilderIdent+"-0", []int32{buildLogsHttpdPort}, BuilderIdent)
r.EnsureService(&svc)

current, changed := r.ensureStatefulset(nb)
if changed {
return false
}

pvcReadiness := r.reconcileExpandPVC(builderIdent+"-"+builderIdent+"-0", r.cr.Spec.Nodepool.Builder.Storage)
pvcReadiness := r.reconcileExpandPVC(BuilderIdent+"-"+BuilderIdent+"-0", r.cr.Spec.Nodepool.Builder.Storage)

routeReady := r.ensureHTTPSRoute(r.cr.Name+"-nodepool-builder", r.cr.Spec.FQDN, builderIdent, "/nodepool/builds",
routeReady := r.ensureHTTPSRoute(r.cr.Name+"-nodepool-builder", r.cr.Spec.FQDN, BuilderIdent, "/nodepool/builds",
buildLogsHttpdPort, map[string]string{
"haproxy.router.openshift.io/rewrite-target": "/builds/",
}, r.cr.Spec.LetsEncrypt)

var isReady = r.IsStatefulSetReady(current) && routeReady && pvcReadiness

conds.UpdateConditions(&r.cr.Status.Conditions, builderIdent, isReady)
conds.UpdateConditions(&r.cr.Status.Conditions, BuilderIdent, isReady)

return isReady
}
Expand Down Expand Up @@ -720,7 +687,7 @@ func (r *SFController) DeployNodepoolLauncher(statsdExporterVolume apiv1.Volume,
}

current := appsv1.Deployment{}
if r.GetM(launcherIdent, &current) {
if r.GetM(LauncherIdent, &current) {
if !utils.MapEquals(&current.Spec.Template.ObjectMeta.Annotations, &annotations) {
r.log.V(1).Info("Nodepool-launcher configuration changed, rollout pods ...")
current.Spec = nl.DeepCopy().Spec
Expand All @@ -732,16 +699,16 @@ func (r *SFController) DeployNodepoolLauncher(statsdExporterVolume apiv1.Volume,
r.CreateR(&current)
}

srv := base.MkService(launcherIdent, r.ns, launcherIdent, []int32{launcherPort}, launcherIdent)
srv := base.MkService(LauncherIdent, r.ns, LauncherIdent, []int32{launcherPort}, LauncherIdent)
r.GetOrCreate(&srv)

routeReady := r.ensureHTTPSRoute(r.cr.Name+"-nodepool-launcher", r.cr.Spec.FQDN, launcherIdent, "/nodepool/api",
routeReady := r.ensureHTTPSRoute(r.cr.Name+"-nodepool-launcher", r.cr.Spec.FQDN, LauncherIdent, "/nodepool/api",
launcherPort, map[string]string{
"haproxy.router.openshift.io/rewrite-target": "/",
}, r.cr.Spec.LetsEncrypt)

isDeploymentReady := r.IsDeploymentReady(&current)
conds.UpdateConditions(&r.cr.Status.Conditions, launcherIdent, isDeploymentReady)
conds.UpdateConditions(&r.cr.Status.Conditions, LauncherIdent, isDeploymentReady)

return isDeploymentReady && routeReady
}
Expand All @@ -767,12 +734,11 @@ func (r *SFController) DeployNodepool() map[string]bool {
statsdVolume := base.MkVolumeCM("statsd-config", "np-statsd-config-map")

// Ensure monitoring - TODO add to condition
r.EnsureNodepoolPodMonitor()
r.ensureNodepoolPromRule(cloudsYaml)

deployments[launcherIdent] = r.DeployNodepoolLauncher(
deployments[LauncherIdent] = r.DeployNodepoolLauncher(
statsdVolume, nodepoolStatsdMappingConfig, volumeMounts, nodepoolProvidersSecrets, providerSecretsResourceExists)
deployments[builderIdent] = r.DeployNodepoolBuilder(statsdVolume, nodepoolStatsdMappingConfig,
deployments[BuilderIdent] = r.DeployNodepoolBuilder(statsdVolume, nodepoolStatsdMappingConfig,
volumeMounts, nodepoolProvidersSecrets, providerSecretsResourceExists)
return deployments
}
17 changes: 14 additions & 3 deletions controllers/softwarefactory_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,10 @@ func (r *SFController) cleanup() {
if r.GetM("zuul-monitor", &currentZPM) {
r.DeleteR(&currentZPM)
}

currentNPM := monitoringv1.PodMonitor{}
if r.GetM("nodepool-monitor", &currentNPM) {
r.DeleteR(&currentNPM)
}
// remove a legacy Route definition for Zuul
r.DeleteR(&apiroutev1.Route{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -209,8 +212,16 @@ func (r *SFController) Step() sfv1.SoftwareFactoryStatus {

if services["Zookeeper"] {
nodepool := r.DeployNodepool()
services["NodePoolLauncher"] = nodepool[launcherIdent]
services["NodePoolBuilder"] = nodepool[builderIdent]
services["NodePoolLauncher"] = nodepool[LauncherIdent]
services["NodePoolBuilder"] = nodepool[BuilderIdent]
if services["NodePoolLauncher"] && services["NodePoolBuilder"] {
monitoredPorts = append(
monitoredPorts,
sfmonitoring.GetTruncatedPortName(BuilderIdent, sfmonitoring.NodeExporterPortNameSuffix),
NodepoolStatsdExporterPortName,
)
selectorRunList = append(selectorRunList, LauncherIdent, BuilderIdent)
}
}

if services["Zuul"] {
Expand Down

0 comments on commit 3288f7e

Please sign in to comment.