Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Emiting reconcile_elapsed_ms metrics #6534

Merged
merged 4 commits into from
Feb 4, 2025
Merged

Emiting reconcile_elapsed_ms metrics #6534

Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6575b4d
Emiting reconcile_time_ms metrics
NamanMahor Jan 30, 2025
9c220ae
fix linting error
NamanMahor Jan 30, 2025
a5218d3
removing reconcile log event as metrics were added
NamanMahor Jan 31, 2025
1c64d3d
review comments
NamanMahor Feb 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 31 additions & 3 deletions runtime/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,6 @@ func (c *Controller) Run(ctx context.Context) error {
if len(hanging) != 0 {
loopErr = fmt.Errorf("reconciles for resources %v have hung for more than %s after cancelation", hanging, reconcileCancelationTimeout.String())
stop = true
break
}
case <-c.catalog.hasEventsCh: // The catalog has events to process
// Need a write lock to call resetEvents.
Expand All @@ -257,7 +256,6 @@ func (c *Controller) Run(ctx context.Context) error {
c.mu.RUnlock()
case <-ctx.Done(): // We've been asked to stop
stop = true
break
}
}

Expand All @@ -275,7 +273,7 @@ func (c *Controller) Run(ctx context.Context) error {
}
c.mu.RUnlock()

// Allow 10 seconds for closing invocations and reconcilers
// Allow 30 seconds for closing invocations and reconcilers
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()

Expand Down Expand Up @@ -1334,6 +1332,36 @@ func (c *Controller) processCompletedInvocation(inv *invocation) error {
// Cleanup - must remove it from c.invocations before any error conditions can occur (otherwise, closing the event loop can hang)
delete(c.invocations, nameStr(inv.name))

// Emit metrics
reconcileTimeMs := time.Since(inv.startedOn).Milliseconds()
commonDims := []attribute.KeyValue{
attribute.String("resource_id", inv.name.Name),
attribute.String("resource_type", inv.name.Kind),
}
if inv.isRename && inv.isDelete {
// not sure about when this will occur
commonDims = append(commonDims, attribute.String("reconcile_operation", "Delete/Rename"))
} else if inv.isRename {
commonDims = append(commonDims, attribute.String("reconcile_operation", "Rename"))
} else if inv.isDelete {
commonDims = append(commonDims, attribute.String("reconcile_operation", "Delete"))
} else {
commonDims = append(commonDims, attribute.String("reconcile_operation", "Normal"))
}

if !inv.cancelledOn.IsZero() {
commonDims = append(commonDims, attribute.String("reconcile_status", "Cancelled"))
} else if inv.result.Err != nil {
commonDims = append(commonDims, attribute.String("reconcile_status", "Error"), attribute.String("reconcile_error", inv.result.Err.Error()))
} else {
commonDims = append(commonDims, attribute.String("reconcile_status", "Succeeded"))
}

if !inv.result.Retrigger.IsZero() {
commonDims = append(commonDims, attribute.String("retrigger_time", inv.result.Retrigger.Format(time.RFC3339)))
}
c.Activity.RecordMetric(context.Background(), "reconcile_time_ms", float64(reconcileTimeMs), commonDims...)

// Log result
logArgs := []zap.Field{
zap.String("name", inv.name.Name),
Expand Down
Loading