Merge pull request #248 from porters-xyz/prod-tweaks

gateway/common/config.go

@@ -20,7 +20,7 @@ const (
     REDIS_ADDR            = "REDIS_ADDR"
     REDIS_USER            = "REDIS_USER"
     REDIS_PASSWORD        = "REDIS_PASSWORD"
-
+    INSTRUMENT_ENABLED    = "ENABLE_INSTRUMENT"
 )
 
 // This may evolve to include config outside env, or use .env file for
@@ -42,6 +42,7 @@ func setupConfig() *Config {
         config.defaults[NUM_WORKERS] = "10"
         config.defaults[HOST] = "localhost"
         config.defaults[PORT] = "9000"
+        config.defaults[INSTRUMENT_ENABLED] = "false"
     })
     return config
 }
@@ -71,3 +72,12 @@ func GetConfigInt(key string) int {
     }
     return intval
 }
+
+func Enabled(key string) bool {
+    configval := GetConfig(key)
+    boolval, err := strconv.ParseBool(configval)
+    if err != nil {
+        boolval = false
+    }
+    return boolval
+}

gateway/common/context.go

@@ -2,12 +2,21 @@ package common
 
 import (
     "context"
+    "time"
+)
+
+const (
+    INSTRUMENT string = "INSTRUMENT_START"
 )
 
 type Contextable interface {
     ContextKey() string
 }
 
+type Instrument struct {
+    Timestamp time.Time
+}
+
 func UpdateContext(ctx context.Context, entity Contextable) context.Context {
     return context.WithValue(ctx, entity.ContextKey(), entity)
 }
@@ -20,3 +29,13 @@ func FromContext(ctx context.Context, contextkey string) (any, bool) {
         return nil, false
     }
 }
+
+func StartInstrument() *Instrument {
+    return &Instrument{
+        Timestamp: time.Now(),
+    }
+}
+
+func (i *Instrument) ContextKey() string {
+    return INSTRUMENT
+}

gateway/common/prometheus.go

@@ -1,6 +1,7 @@
 package common
 
 import (
+    "time"
 
     "github.com/prometheus/client_golang/prometheus"
     "github.com/prometheus/client_golang/prometheus/promauto"
@@ -12,6 +13,7 @@ const (
     STATUS     = "status"
     TENANT     = "tenant"
     QUEUE      = "queue"
+    STAGE      = "stage"
 )
 
 var (
@@ -23,4 +25,9 @@ var (
         Name: "gateway_job_queue",
         Help: "If this grows too big it may effect performance, should scale up",
     }, []string{QUEUE})
+    LatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+        Name: "gateway_added_latency",
+        Help: "Shows how much the proxy process is adding to request",
+        Buckets: prometheus.ExponentialBucketsRange(float64(10 * time.Millisecond), float64(20 * time.Second), 10),
+    }, []string{STAGE})
 )

gateway/proxy/proxy.go

@@ -113,10 +113,33 @@ func setupProxy(remote *url.URL) *httputil.ReverseProxy {
             log.Debug("lifecycle incomplete", "mask", lifecycle)
             cancel(err)
         }
+
+        if common.Enabled(common.INSTRUMENT_ENABLED) {
+            ctx := req.Context()
+            instr, ok := common.FromContext(ctx, common.INSTRUMENT)
+            if ok {
+                start := instr.(*common.Instrument).Timestamp
+                elapsed := time.Now().Sub(start)
+                common.LatencyHistogram.WithLabelValues("setup").Observe(float64(elapsed))
+
+                ctx = common.UpdateContext(ctx, common.StartInstrument())
+                *req = *req.WithContext(ctx)
+            }
+        }
     }
 
     revProxy.ModifyResponse = func(resp *http.Response) error {
         ctx := resp.Request.Context()
+
+        if common.Enabled(common.INSTRUMENT_ENABLED) {
+            instr, ok := common.FromContext(ctx, common.INSTRUMENT)
+            if ok {
+                start := instr.(*common.Instrument).Timestamp
+                elapsed := time.Now().Sub(start)
+                common.LatencyHistogram.WithLabelValues("serve").Observe(float64(elapsed))
+            }
+        }
+
         var err error
         for _, p := range (*reg).plugins {
             h, ok := p.(PostHandler)
@@ -161,8 +184,11 @@ func setupProxy(remote *url.URL) *httputil.ReverseProxy {
 func setupContext(req *http.Request) {
     // TODO read ctx from request and make any modifications
     ctx := req.Context()
-    lifecyclectx := common.UpdateContext(ctx, &Lifecycle{})
-    *req = *req.WithContext(lifecyclectx)
+    ctx = common.UpdateContext(ctx, &Lifecycle{})
+    if common.Enabled(common.INSTRUMENT_ENABLED) {
+        ctx = common.UpdateContext(ctx, common.StartInstrument())
+    }
+    *req = *req.WithContext(ctx)
 }
 
 func lookupPoktId(req *http.Request) string {

services/gatewaykit/fly.prod.toml

@@ -29,7 +29,7 @@ primary_region = 'sea'
     port = 8080
     force_https = false
 [[vm]]
-  memory = '1gb'
+  memory = '2gb'
   cpu_kind = 'shared'
   cpus = 1