From 4c97e429f83ac7060f299e7d0cebea36d3d73891 Mon Sep 17 00:00:00 2001 From: Samuel Behan Date: Wed, 17 Mar 2021 17:16:02 +0100 Subject: [PATCH 1/3] feat(exporter) add per_consumer status metric (kong_http_consumer_status) --- kong/plugins/prometheus/exporter.lua | 15 ++++++++++++++- kong/plugins/prometheus/handler.lua | 10 ++++++++-- kong/plugins/prometheus/schema.lua | 4 +++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/kong/plugins/prometheus/exporter.lua b/kong/plugins/prometheus/exporter.lua index 0194077..a9e3b3a 100644 --- a/kong/plugins/prometheus/exporter.lua +++ b/kong/plugins/prometheus/exporter.lua @@ -79,6 +79,10 @@ local function init() "Total bandwidth in bytes " .. "consumed per service/route in Kong", {"service", "route", "type"}) + metrics.consumer_status = prometheus:counter("http_consumer_status", + "HTTP status codes for customer per service/route in Kong", + {"service", "route", "code", "consumer"}) + if enterprise then enterprise.init(prometheus) end @@ -92,6 +96,7 @@ end -- Since in the prometheus library we create a new table for each diverged label -- so putting the "more dynamic" label at the end will save us some memory local labels_table = {0, 0, 0} +local labels_table4 = {0, 0, 0, 0} local upstream_target_addr_health_table = { { value = 0, labels = { 0, 0, 0, "healthchecks_off" } }, { value = 0, labels = { 0, 0, 0, "healthy" } }, @@ -113,7 +118,7 @@ end local log if ngx.config.subsystem == "http" then - function log(message) + function log(message, serialized) if not metrics then kong.log.err("prometheus: can not log metrics because of an initialization " .. "error, please make sure that you've declared " @@ -168,6 +173,14 @@ if ngx.config.subsystem == "http" then labels_table[3] = "kong" metrics.latency:observe(kong_proxy_latency, labels_table) end + + if serialized.consumer ~= nil then + labels_table4[1] = labels_table[1] + labels_table4[2] = labels_table[2] + labels_table4[3] = message.response.status + labels_table4[4] = serialized.consumer + metrics.consumer_status:inc(1, labels_table4) + end end else diff --git a/kong/plugins/prometheus/handler.lua b/kong/plugins/prometheus/handler.lua index 92c2c59..9b31f0d 100644 --- a/kong/plugins/prometheus/handler.lua +++ b/kong/plugins/prometheus/handler.lua @@ -15,9 +15,15 @@ function PrometheusHandler.init_worker() end -function PrometheusHandler.log() +function PrometheusHandler.log(self, conf) local message = kong.log.serialize() - prometheus.log(message) + + local serialized = {} + if conf.per_consumer and message.consumer ~= nil then + serialized.consumer = message.consumer.username + end + + prometheus.log(message, serialized) end diff --git a/kong/plugins/prometheus/schema.lua b/kong/plugins/prometheus/schema.lua index 2327500..d749077 100644 --- a/kong/plugins/prometheus/schema.lua +++ b/kong/plugins/prometheus/schema.lua @@ -12,7 +12,9 @@ return { fields = { { config = { type = "record", - fields = {}, + fields = { + { per_consumer = { type = "boolean", default = false }, }, + }, custom_validator = validate_shared_dict, }, }, }, From 96e4a0b614c54e9d4d341ec869f453dd3137a8ac Mon Sep 17 00:00:00 2001 From: Samuel Behan Date: Wed, 17 Mar 2021 18:32:28 +0100 Subject: [PATCH 2/3] docs(README) add kong_http_consumer_status sample output --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 7002ffc..6077b24 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,9 @@ kong_bandwidth{type="ingress",service="google"} 254 # HELP kong_datastore_reachable Datastore reachable from Kong, 0 is unreachable # TYPE kong_datastore_reachable gauge kong_datastore_reachable 1 +# HELP kong_http_consumer_status HTTP status codes for customer per service/route in Kong +# TYPE kong_http_consumer_status counter +kong_http_consumer_status{service="upstream",route="default",code="200",consumer="consumer1"} 5185 # HELP kong_http_status HTTP status codes per service in Kong # TYPE kong_http_status counter kong_http_status{code="301",service="google"} 2 From 29d8446498d7543b96cbffc9db11248e191e39f1 Mon Sep 17 00:00:00 2001 From: Samuel Behan Date: Thu, 18 Mar 2021 09:31:51 +0100 Subject: [PATCH 3/3] feat(exporter) add url location and GET param counters, with regex extract --- README.md | 6 ++ kong/plugins/prometheus/exporter.lua | 83 ++++++++++++++++++++++++++++ kong/plugins/prometheus/handler.lua | 7 ++- kong/plugins/prometheus/schema.lua | 4 ++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6077b24..c16f330 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,12 @@ kong_http_consumer_status{service="upstream",route="default",code="200",consumer # HELP kong_http_status HTTP status codes per service in Kong # TYPE kong_http_status counter kong_http_status{code="301",service="google"} 2 +# HELP kong_http_url_location_consumer_total HTTP status codes for specific URL location in Kong +# TYPE kong_http_url_location_consumer_total counter +kong_http_url_location_consumer_total{service="upstream",route="default",location="hello",consumer="consumer1"} 5 +# HELP kong_http_url_param_consumer_total HTTP status codes for specific GET param in Kong +# TYPE kong_http_url_param_consumer_total counter +kong_http_url_param_consumer_total{service="upstream",route="default",param="123456",consumer="consumer1"} 5 # HELP kong_latency Latency added by Kong, total request time and upstream latency for each service in Kong # TYPE kong_latency histogram kong_latency_bucket{type="kong",service="google",le="00001.0"} 1 diff --git a/kong/plugins/prometheus/exporter.lua b/kong/plugins/prometheus/exporter.lua index a9e3b3a..46774bf 100644 --- a/kong/plugins/prometheus/exporter.lua +++ b/kong/plugins/prometheus/exporter.lua @@ -83,6 +83,23 @@ local function init() "HTTP status codes for customer per service/route in Kong", {"service", "route", "code", "consumer"}) + -- per location / url param + metrics.param_total = prometheus:counter("http_url_param_total", + "HTTP status codes for specific GET param in Kong", + {"service", "route", "param"}) + + metrics.param_consumer_total = prometheus:counter("http_url_param_consumer_total", + "HTTP status codes for specific GET param in Kong", + {"service", "route", "param", "consumer"}) + + metrics.location_total = prometheus:counter("http_url_location_total", + "HTTP status codes for specific URL location in Kong", + {"service", "route", "location"}) + + metrics.location_consumer_total = prometheus:counter("http_url_location_consumer_total", + "HTTP status codes for specific URL location in Kong", + {"service", "route", "location", "consumer"}) + if enterprise then enterprise.init(prometheus) end @@ -181,6 +198,72 @@ if ngx.config.subsystem == "http" then labels_table4[4] = serialized.consumer metrics.consumer_status:inc(1, labels_table4) end + + if serialized.param_list then + local value + local args, err = ngx.req.get_uri_args() + for _, param in ipairs(serialized.param_list) do + if args[param] ~= nil and type(args[param]) ~= 'table' then + value = args[param] + break + end + end + + if value ~= nil then + if serialized.param_extract ~= nil then + local match, err = ngx.re.match(value, serialized.param_extract, 'aio') + if err then + kong.log.err("prometheus: failed to extract param value becase of a regex error - " .. err) + value = nil + elseif match == nil or (not match[1] and not match['param']) then + value = nil + elseif match['param'] then + value = match['param'] + else + value = match[1] + end + end + + if value ~= nil then + if serialized.consumer ~= nil then + labels_table4[3] = value + labels_table4[4] = serialized.consumer + metrics.param_consumer_total:inc(1, labels_table4) + else + labels_table[3] = value + metrics.param_total:inc(1, labels_table) + end + end + end + end + + if serialized.location then + local value = ngx.var.uri + if serialized.location_extract ~= nil then + local match, err = ngx.re.match(value, serialized.location_extract, 'aio') + if err then + kong.log.err("prometheus: failed to extract location portion becase of a regex error - " .. err) + value = nil + elseif match == nil or (not match[1] and not match['location']) then + value = nil + elseif match['location'] then + value = match['location'] + else + value = match[1] + end + end + + if value ~= nil then + if serialized.consumer ~= nil then + labels_table4[3] = value + labels_table4[4] = serialized.consumer + metrics.location_consumer_total:inc(1, labels_table4) + else + labels_table[3] = value + metrics.location_total:inc(1, labels_table) + end + end + end end else diff --git a/kong/plugins/prometheus/handler.lua b/kong/plugins/prometheus/handler.lua index 9b31f0d..98c82ac 100644 --- a/kong/plugins/prometheus/handler.lua +++ b/kong/plugins/prometheus/handler.lua @@ -18,7 +18,12 @@ end function PrometheusHandler.log(self, conf) local message = kong.log.serialize() - local serialized = {} + local serialized = { + param_list = conf.param_collect_list, + param_extract = conf.param_value_extract, + location = conf.location_collect, + location_extract = conf.location_extract, + } if conf.per_consumer and message.consumer ~= nil then serialized.consumer = message.consumer.username end diff --git a/kong/plugins/prometheus/schema.lua b/kong/plugins/prometheus/schema.lua index d749077..a8ccc34 100644 --- a/kong/plugins/prometheus/schema.lua +++ b/kong/plugins/prometheus/schema.lua @@ -13,6 +13,10 @@ return { { config = { type = "record", fields = { + { param_collect_list = { type = "array", elements = { type = "string", match = "^[a-z_]+$" }, }, }, + { param_value_extract = { type = "string" }, }, -- regex + { location_collect = { type = "boolean", default = false }, }, + { location_extract = { type = "string" }, }, -- regex { per_consumer = { type = "boolean", default = false }, }, }, custom_validator = validate_shared_dict,