Fix: Flapping addon staus due to 409 response on remote write to the hub

collectors/metrics/pkg/forwarder/forwarder.go

@@ -504,7 +504,13 @@ func (w *Worker) forward(ctx context.Context) error {
 
 	req := &http.Request{Method: "POST", URL: w.to}
 	if err := w.toClient.RemoteWrite(ctx, req, families, w.interval); err != nil {
-		updateStatus(statuslib.ForwardFailed, "Failed to send metrics")
+		var httpError *metricsclient.HTTPError
+		// Avoid degrading the status on 409
+		if errors.As(err, &httpError) && httpError.StatusCode == http.StatusConflict {
+			updateStatus(statuslib.ForwardSuccessful, "Cluster metrics sent successfully")
+		} else {
+			updateStatus(statuslib.ForwardFailed, "Failed to send metrics")
+		}
 		return err
 	}
 

collectors/metrics/pkg/metricsclient/metricsclient.go

@@ -44,6 +44,15 @@ const (
 	maxSeriesLength = 10000
 )
 
+type HTTPError struct {
+	StatusCode int
+	Message    string
+}
+
+func (e *HTTPError) Error() string {
+	return fmt.Sprintf("HTTP %d: %s", e.StatusCode, e.Message)
+}
+
 type Client struct {
 	client      *http.Client
 	maxBytes    int64
@@ -555,7 +564,10 @@ func (c *Client) sendRequest(ctx context.Context, serverURL string, body []byte)
 			logger.Log(c.logger, logger.Warn, err)
 		}
 
-		retErr := fmt.Errorf("response status code is %s, response body is %s", resp.Status, string(bodyBytes))
+		retErr := &HTTPError{
+			StatusCode: resp.StatusCode,
+			Message:    string(bodyBytes),
+		}
 
 		if isTransientResponseError(resp) {
 			return retErr

collectors/metrics/pkg/metricsclient/metricsclient_test.go

@@ -328,6 +328,10 @@ func TestClient_RemoteWrite(t *testing.T) {
 			expect: func(t *testing.T, err error, retryCount int) {
 				assert.Error(t, err)
 				assert.Equal(t, 1, retryCount)
+				// Ensure the http error is wrapped
+				var httpError *HTTPError
+				assert.ErrorAs(t, err, &httpError)
+				assert.Equal(t, http.StatusConflict, httpError.StatusCode)
 			},
 		},
 	}