diff --git a/pkg/server/prometheus_exporter.go b/pkg/server/prometheus_exporter.go index 70521fe..f46f13d 100644 --- a/pkg/server/prometheus_exporter.go +++ b/pkg/server/prometheus_exporter.go @@ -1,11 +1,18 @@ package server import ( - "github.com/prometheus/client_golang/prometheus" "strings" + + "github.com/prometheus/client_golang/prometheus" ) var ( + buildInfoGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "sidecache_admission_build_info", + Help: "Build info for sidecache admission webhook", + }, []string{"version"}) + cacheHitCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "sidecache_" + ProjectName, @@ -13,29 +20,36 @@ var ( Help: "Cache hit count", }) + lockAcquiringAttemptsHistogram = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "sidecache_" + ProjectName, + Name: "lock_acquiring_attempts_histogram", + Help: "Lock acquiring attempts histogram", + Buckets: []float64{0.999, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100, 200, 300, 500, 1000}, + }) + totalRequestCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "sidecache_" + ProjectName, Name: "all_request_hit_counter", Help: "All request hit counter", }) - - buildInfoGaugeVec = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "sidecache_admission_build_info", - Help: "Build info for sidecache admission webhook", - }, []string{"version"}) ) type Prometheus struct { - CacheHitCounter prometheus.Counter - TotalRequestCounter prometheus.Counter + CacheHitCounter prometheus.Counter + LockAcquiringAttemptsHistogram prometheus.Histogram + TotalRequestCounter prometheus.Counter } func NewPrometheusClient() *Prometheus { - prometheus.MustRegister(cacheHitCounter, totalRequestCounter, buildInfoGaugeVec) + prometheus.MustRegister(buildInfoGaugeVec, cacheHitCounter, lockAcquiringAttemptsHistogram, totalRequestCounter) - return &Prometheus{TotalRequestCounter: totalRequestCounter, CacheHitCounter: cacheHitCounter} + return &Prometheus{ + CacheHitCounter: cacheHitCounter, + LockAcquiringAttemptsHistogram: lockAcquiringAttemptsHistogram, + TotalRequestCounter: totalRequestCounter, + } } func BuildInfo(admission string) { diff --git a/pkg/server/server.go b/pkg/server/server.go index 19d0560..61914e1 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -7,7 +7,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "github.com/zeriontech/sidecache/pkg/lock" "io" "io/ioutil" "net/http" @@ -19,6 +18,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/zeriontech/sidecache/pkg/cache" + "github.com/zeriontech/sidecache/pkg/lock" "go.uber.org/zap" ) @@ -144,14 +144,14 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) { }() path := strings.Split(r.URL.Path, "/") - key := "lock:" + path[1] + key := "lock:" + path[1] + "/" + path[2] resultKey := server.HashURL(server.ReorderQueryString(r.URL)) if UseLock { - attempt := 0 + attempt := 1 for { // check the cache - server.Logger.Info("checking the cache", zap.String("resultKey", resultKey), zap.Int("attempt", attempt+1)) + server.Logger.Info("checking the cache", zap.String("resultKey", resultKey), zap.Int("attempt", attempt)) if cachedDataBytes := server.CheckCache(resultKey); cachedDataBytes != nil { serveFromCache(cachedDataBytes, server, w, r) return @@ -160,7 +160,8 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) { // try to acquire the lock server.Logger.Info("acquiring the lock", zap.String("key", key)) if err := server.LockMgr.Acquire(key, LockTtl); err == nil { - server.Logger.Info("lock acquired", zap.String("key", key)) + server.Prometheus.LockAcquiringAttemptsHistogram.Observe(float64(attempt)) + server.Logger.Info("lock acquired", zap.String("key", key), zap.Int("attempt", attempt)) defer func() { // release the lock if err := server.LockMgr.Release(key); err != nil { @@ -251,9 +252,8 @@ func (server CacheServer) ReorderQueryString(url *url.URL) string { } func (server CacheServer) GetBackoff(attempt int) time.Duration { - if attempt < 10 { + if attempt <= 10 { return 100 * time.Millisecond - } else { - return 500 * time.Millisecond } + return 500 * time.Millisecond }