Skip to content

Commit

Permalink
revert self monitor degraded on remote es error
Browse files Browse the repository at this point in the history
  • Loading branch information
juliaElastic committed Nov 21, 2023
1 parent 4c99949 commit 8182bab
Show file tree
Hide file tree
Showing 4 changed files with 8 additions and 589 deletions.
48 changes: 8 additions & 40 deletions internal/pkg/policy/self.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,14 @@ LOOP:
break LOOP
case <-cT.C:
state, err := m.process(ctx)
m.log.Trace().Str("state", state.String()).Msg("self monitor state")
if err != nil {
return err
}
cT.Reset(m.checkTime)
if state == client.UnitStateHealthy {
// running; can stop
break LOOP
}
case hits := <-s.Output():
policies := make([]model.Policy, len(hits))
for i, hit := range hits {
Expand All @@ -119,10 +122,13 @@ LOOP:
}
}
state, err := m.processPolicies(ctx, policies)
m.log.Trace().Str("state", state.String()).Msg("self monitor state")
if err != nil {
return err
}
if state == client.UnitStateHealthy {
// running; can stop
break LOOP
}
}
}

Expand Down Expand Up @@ -212,44 +218,6 @@ func (m *selfMonitorT) updateState(ctx context.Context) (client.UnitState, error
return client.UnitStateStarting, nil
}

remoteOutputErrorMap := m.bulker.GetRemoteOutputErrorMap()
hasError := false
remoteESPayload := make(map[string]interface{})
for key, value := range remoteOutputErrorMap {
if value != "" {
hasError = true
remoteESPayload[key] = value
break
}
}
if hasError {
m.state = client.UnitStateDegraded
m.reporter.UpdateState(client.UnitStateDegraded, "Could not connect to remote ES output", remoteESPayload) //nolint:errcheck // not clear what to do in failure cases
return m.state, nil
} else {
bulkerMap := m.bulker.GetBulkerMap()
for outputName, outputBulker := range bulkerMap {
res, err := outputBulker.Client().Ping(outputBulker.Client().Ping.WithContext(ctx))
if err != nil {
m.log.Error().Err(err).Msg("error calling remote es ping")
m.state = client.UnitStateDegraded
message := fmt.Sprintf("Could not ping remote ES: %s, error: %s", outputName, err.Error())
m.reporter.UpdateState(m.state, message, nil) //nolint:errcheck // not clear what to do in failure cases
hasError = true
break
} else if res.StatusCode != 200 {
m.state = client.UnitStateDegraded
message := fmt.Sprintf("Could not connect to remote ES output: %s, status code: %d", outputName, res.StatusCode)
m.reporter.UpdateState(m.state, message, nil) //nolint:errcheck // not clear what to do in failure cases
hasError = true
break
}
}
if hasError {
return m.state, nil
}
}

state := client.UnitStateHealthy
extendMsg := ""
var payload map[string]interface{}
Expand Down
Loading

0 comments on commit 8182bab

Please sign in to comment.