From 09f3bb2dba38b5f84a6dcb92cfc9c29adec05ffd Mon Sep 17 00:00:00 2001
From: Thibault Mange <22740367+thibaultmg@users.noreply.github.com>
Date: Tue, 14 Jan 2025 11:07:41 +0100
Subject: [PATCH] report available status on 409

Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com>
---
 collectors/metrics/pkg/forwarder/forwarder.go      |  8 +++++++-
 .../metrics/pkg/metricsclient/metricsclient.go     | 14 +++++++++++++-
 .../pkg/metricsclient/metricsclient_test.go        |  4 ++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/collectors/metrics/pkg/forwarder/forwarder.go b/collectors/metrics/pkg/forwarder/forwarder.go
index 1df1760c3..a8689875e 100644
--- a/collectors/metrics/pkg/forwarder/forwarder.go
+++ b/collectors/metrics/pkg/forwarder/forwarder.go
@@ -504,7 +504,13 @@ func (w *Worker) forward(ctx context.Context) error {
 
 	req := &http.Request{Method: "POST", URL: w.to}
 	if err := w.toClient.RemoteWrite(ctx, req, families, w.interval); err != nil {
-		updateStatus(statuslib.ForwardFailed, "Failed to send metrics")
+		var httpError *metricsclient.HTTPError
+		// Avoid degrading the status on 409
+		if errors.As(err, &httpError) && httpError.StatusCode == http.StatusConflict {
+			updateStatus(statuslib.ForwardSuccessful, "Cluster metrics sent successfully")
+		} else {
+			updateStatus(statuslib.ForwardFailed, "Failed to send metrics")
+		}
 		return err
 	}
 
diff --git a/collectors/metrics/pkg/metricsclient/metricsclient.go b/collectors/metrics/pkg/metricsclient/metricsclient.go
index 4f28c065e..45d595d6c 100644
--- a/collectors/metrics/pkg/metricsclient/metricsclient.go
+++ b/collectors/metrics/pkg/metricsclient/metricsclient.go
@@ -44,6 +44,15 @@ const (
 	maxSeriesLength = 10000
 )
 
+type HTTPError struct {
+	StatusCode int
+	Message    string
+}
+
+func (e *HTTPError) Error() string {
+	return fmt.Sprintf("HTTP %d: %s", e.StatusCode, e.Message)
+}
+
 type Client struct {
 	client      *http.Client
 	maxBytes    int64
@@ -555,7 +564,10 @@ func (c *Client) sendRequest(ctx context.Context, serverURL string, body []byte)
 			logger.Log(c.logger, logger.Warn, err)
 		}
 
-		retErr := fmt.Errorf("response status code is %s, response body is %s", resp.Status, string(bodyBytes))
+		retErr := &HTTPError{
+			StatusCode: resp.StatusCode,
+			Message:    string(bodyBytes),
+		}
 
 		if isTransientResponseError(resp) {
 			return retErr
diff --git a/collectors/metrics/pkg/metricsclient/metricsclient_test.go b/collectors/metrics/pkg/metricsclient/metricsclient_test.go
index 05402bac4..eedec3ef4 100644
--- a/collectors/metrics/pkg/metricsclient/metricsclient_test.go
+++ b/collectors/metrics/pkg/metricsclient/metricsclient_test.go
@@ -328,6 +328,10 @@ func TestClient_RemoteWrite(t *testing.T) {
 			expect: func(t *testing.T, err error, retryCount int) {
 				assert.Error(t, err)
 				assert.Equal(t, 1, retryCount)
+				// Ensure the http error is wrapped
+				var httpError *HTTPError
+				assert.ErrorAs(t, err, &httpError)
+				assert.Equal(t, http.StatusConflict, httpError.StatusCode)
 			},
 		},
 	}