Added metrics for lock acquiring attempts

pkg/server/prometheus_exporter.go

@@ -1,41 +1,55 @@
 package server
 
 import (
-	"github.com/prometheus/client_golang/prometheus"
 	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
+	buildInfoGaugeVec = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "sidecache_admission_build_info",
+			Help: "Build info for sidecache admission webhook",
+		}, []string{"version"})
+
 	cacheHitCounter = prometheus.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: "sidecache_" + ProjectName,
 			Name:      "cache_hit_counter",
 			Help:      "Cache hit count",
 		})
 
+	lockAcquiringAttemptsHistogram = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: "sidecache_" + ProjectName,
+			Name:      "lock_acquiring_attempts_histogram",
+			Help:      "Lock acquiring attempts histogram",
+			Buckets:   []float64{0.999, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100, 200, 300, 500, 1000},
+		})
+
 	totalRequestCounter = prometheus.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: "sidecache_" + ProjectName,
 			Name:      "all_request_hit_counter",
 			Help:      "All request hit counter",
 		})
-
-	buildInfoGaugeVec = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "sidecache_admission_build_info",
-			Help: "Build info for sidecache admission webhook",
-		}, []string{"version"})
 )
 
 type Prometheus struct {
-	CacheHitCounter     prometheus.Counter
-	TotalRequestCounter prometheus.Counter
+	CacheHitCounter                prometheus.Counter
+	LockAcquiringAttemptsHistogram prometheus.Histogram
+	TotalRequestCounter            prometheus.Counter
 }
 
 func NewPrometheusClient() *Prometheus {
-	prometheus.MustRegister(cacheHitCounter, totalRequestCounter, buildInfoGaugeVec)
+	prometheus.MustRegister(buildInfoGaugeVec, cacheHitCounter, lockAcquiringAttemptsHistogram, totalRequestCounter)
 
-	return &Prometheus{TotalRequestCounter: totalRequestCounter, CacheHitCounter: cacheHitCounter}
+	return &Prometheus{
+		CacheHitCounter:                cacheHitCounter,
+		LockAcquiringAttemptsHistogram: lockAcquiringAttemptsHistogram,
+		TotalRequestCounter:            totalRequestCounter,
+	}
 }
 
 func BuildInfo(admission string) {

pkg/server/server.go

@@ -7,7 +7,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
-	"github.com/zeriontech/sidecache/pkg/lock"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -19,6 +18,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/zeriontech/sidecache/pkg/cache"
+	"github.com/zeriontech/sidecache/pkg/lock"
 	"go.uber.org/zap"
 )
 
@@ -144,14 +144,14 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) {
 	}()
 
 	path := strings.Split(r.URL.Path, "/")
-	key := "lock:" + path[1]
+	key := "lock:" + path[1] + "/" + path[2]
 	resultKey := server.HashURL(server.ReorderQueryString(r.URL))
 
 	if UseLock {
-		attempt := 0
+		attempt := 1
 		for {
 			// check the cache
-			server.Logger.Info("checking the cache", zap.String("resultKey", resultKey), zap.Int("attempt", attempt+1))
+			server.Logger.Info("checking the cache", zap.String("resultKey", resultKey), zap.Int("attempt", attempt))
 			if cachedDataBytes := server.CheckCache(resultKey); cachedDataBytes != nil {
 				serveFromCache(cachedDataBytes, server, w, r)
 				return
@@ -160,7 +160,8 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) {
 			// try to acquire the lock
 			server.Logger.Info("acquiring the lock", zap.String("key", key))
 			if err := server.LockMgr.Acquire(key, LockTtl); err == nil {
-				server.Logger.Info("lock acquired", zap.String("key", key))
+				server.Prometheus.LockAcquiringAttemptsHistogram.Observe(float64(attempt))
+				server.Logger.Info("lock acquired", zap.String("key", key), zap.Int("attempt", attempt))
 				defer func() {
 					// release the lock
 					if err := server.LockMgr.Release(key); err != nil {
@@ -251,9 +252,8 @@ func (server CacheServer) ReorderQueryString(url *url.URL) string {
 }
 
 func (server CacheServer) GetBackoff(attempt int) time.Duration {
-	if attempt < 10 {
+	if attempt <= 10 {
 		return 100 * time.Millisecond
-	} else {
-		return 500 * time.Millisecond
 	}
+	return 500 * time.Millisecond
 }