Merge pull request #918 from 3scale/upstream-metrics

Add Prometheus metrics for the upstream

CHANGELOG.md

@@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
-- Prometheus metrics for the 3scale batching policy [PR #902](https://github.com/3scale/apicast/pull/902)
+- Prometheus metrics for: the 3scale batching policy and the upstream API [PR #902](https://github.com/3scale/apicast/pull/902), [PR #918](https://github.com/3scale/apicast/pull/918)
 - Support for path in the upstream URL [PR #905](https://github.com/3scale/apicast/pull/905)
 - OIDC Authentication policy (only useable directly by the configuration file) [PR #904](https://github.com/3scale/apicast/pull/904)
 

gateway/src/apicast/metrics/upstream.lua

@@ -0,0 +1,39 @@
+local tonumber = tonumber
+
+local prometheus = require('apicast.prometheus')
+
+local _M = {}
+
+local upstream_status_codes = prometheus(
+  'counter',
+  'upstream_status',
+  'HTTP status from upstream servers',
+  { 'status' }
+)
+
+local upstream_resp_times = prometheus(
+  'histogram',
+  'upstream_resp_times',
+  'Response times from upstream servers'
+)
+
+local function inc_status_codes_counter(status)
+  if tonumber(status) and upstream_status_codes then
+    upstream_status_codes:inc(1, { status })
+  end
+end
+
+local function add_resp_time(response_time)
+  local time = tonumber(response_time)
+
+  if time and upstream_resp_times then
+    upstream_resp_times:observe(time)
+  end
+end
+
+function _M.report(status, response_time)
+  inc_status_codes_counter(status)
+  add_resp_time(response_time)
+end
+
+return _M

gateway/src/apicast/policy/nginx_metrics/nginx_metrics.lua

@@ -9,6 +9,8 @@ local select = select
 local find = string.find
 local pairs = pairs
 
+local upstream_metrics = require('apicast.metrics.upstream')
+
 local new = _M.new
 
 local log_levels_list = {
@@ -110,4 +112,8 @@ function _M:metrics()
   end
 end
 
+function _M.log()
+  upstream_metrics.report(ngx.var.upstream_status, ngx.var.upstream_response_time)
+end
+
 return _M

spec/metrics/upstream_spec.lua

@@ -0,0 +1,59 @@
+describe('upstream metrics', function()
+  describe('report', function()
+    local upstream_metrics
+    local test_counter = { inc = function() end }
+    local test_histogram = { observe = function() end }
+
+    before_each(function()
+      -- Make Prometheus return stubbed counters and histograms
+      stub(test_counter, 'inc')
+      stub(test_histogram, 'observe')
+
+      local Prometheus = require('apicast.prometheus')
+      getmetatable(Prometheus).__call = function(_, type)
+        if type == 'counter' then
+          return test_counter
+        elseif type == 'histogram' then
+          return test_histogram
+        end
+      end
+
+      package.loaded['apicast.metrics.upstream'] = nil
+      upstream_metrics = require('apicast.metrics.upstream')
+    end)
+
+    after_each(function()
+      package.loaded['apicast.prometheus'] = nil
+      require('apicast.prometheus')
+
+      package.loaded['apicast.metrics.upstream'] = nil
+      require('apicast.metrics.upstream')
+    end)
+
+    it('increases the counter of status codes', function()
+      upstream_metrics.report(200, 0.1)
+      assert.stub(test_counter.inc).was_called_with(test_counter, 1, { 200 })
+    end)
+
+    it('adds the latency to the histogram', function()
+      upstream_metrics.report(200, 0.1)
+      assert.stub(test_histogram.observe).was_called_with(test_histogram, 0.1)
+    end)
+
+    describe('when the status is nil or empty', function()
+      it('does not increase the counter of status codes', function()
+        upstream_metrics.report(nil, 0.1)
+        upstream_metrics.report('', 0.1)
+        assert.stub(test_counter.inc).was_not_called()
+      end)
+    end)
+
+    describe('when the latency is nil or empty', function()
+      it('does not add the latency to the histogram', function()
+        upstream_metrics.report(200, nil)
+        upstream_metrics.report(200, '')
+        assert.stub(test_histogram.observe).was_not_called()
+      end)
+    end)
+  end)
+end)

t/prometheus-metrics.t

@@ -190,3 +190,48 @@ METRICS_OUTPUT
 ]
 --- no_error_log
 [error]
+
+=== TEST 4: the metrics endpoint shows metrics about the upstream
+In particular, it shows the status codes and the response times
+--- configuration
+{
+  "services": [
+    {
+      "id": 42,
+      "proxy": {
+        "policy_chain": [
+          {
+            "name": "apicast.policy.upstream",
+            "configuration": {
+              "rules": [
+                {
+                  "regex": "/",
+                  "url": "http://test:$TEST_NGINX_SERVER_PORT"
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  ]
+}
+--- upstream
+  location / {
+     content_by_lua_block {
+       ngx.exit(200);
+     }
+  }
+--- request eval
+["GET /", "GET /metrics"]
+--- more_headers eval
+["", "Host: metrics"]
+--- error_code eval
+[ 200, 200 ]
+--- response_body_like eval
+[
+"",
+qr/upstream_resp_times(.|\n)*upstream_status\{status="200"\} 1/
+]
+--- no_error_log
+[error]