Instrument backend handler response count

Toby Lorne · Toby Lorne · commit b443d3dd9cf7 · 2020-03-02T09:49:53.000Z
Every time the backend handler responds with a status code, we should
count that response

Signed-off-by: Toby Lorne &lt;toby.lornewelch-richards@digital.cabinet-office.gov.uk&gt;
diff --git a/handlers/backend_handler.go b/handlers/backend_handler.go
@@ -144,6 +144,11 @@ func (bt *backendTransport) RoundTrip(req *http.Request) (resp *http.Response, e
 			"backend_id":    bt.backendID,
 			"response_code": fmt.Sprintf("%d", responseCode),
 		}).Add(durationSeconds)
+
+		BackendHandlerResponseCountMetric.With(prometheus.Labels{
+			"backend_id":    bt.backendID,
+			"response_code": fmt.Sprintf("%d", responseCode),
+		}).Inc()
 	}()
 
 	resp, err = bt.wrapped.RoundTrip(req)
diff --git a/handlers/backend_handler_test.go b/handlers/backend_handler_test.go
@@ -141,6 +141,7 @@ var _ = Describe("Backend handler", func() {
 	Context("metrics", func() {
 		var (
 			beforeRequestCountMetric            float64
+			beforeResponseCountMetric           float64
 			beforeResponseDurationSecondsMetric float64
 		)
 
@@ -152,6 +153,15 @@ var _ = Describe("Backend handler", func() {
 			)
 		}
 
+		var measureResponseCount = func(responseCode string) float64 {
+			return promtest.ToFloat64(
+				handlers.BackendHandlerResponseCountMetric.With(prometheus.Labels{
+					"backend_id":    "backend-metrics",
+					"response_code": responseCode,
+				}),
+			)
+		}
+
 		var measureResponseDurationSeconds = func(responseCode string) float64 {
 			return promtest.ToFloat64(
 				handlers.BackendHandlerResponseDurationSecondsMetric.With(prometheus.Labels{
@@ -179,6 +189,7 @@ var _ = Describe("Backend handler", func() {
 					rw.WriteHeader(http.StatusOK)
 				})
 
+				beforeResponseCountMetric = measureResponseCount("200")
 				beforeResponseDurationSecondsMetric = measureResponseDurationSeconds("200")
 
 				router.ServeHTTP(
@@ -193,6 +204,12 @@ var _ = Describe("Backend handler", func() {
 				).To(Equal(float64(1)))
 			})
 
+			It("should count the number of proxied responses", func() {
+				Expect(
+					measureResponseCount("200") - beforeResponseCountMetric,
+				).To(Equal(float64(1)))
+			})
+
 			It("should record the duration of proxied responses", func() {
 				Expect(
 					measureResponseDurationSeconds("200") - beforeResponseDurationSecondsMetric,
@@ -207,6 +224,7 @@ var _ = Describe("Backend handler", func() {
 					rw.WriteHeader(http.StatusOK)
 				})
 
+				beforeResponseCountMetric = measureResponseCount("504")
 				beforeResponseDurationSecondsMetric = measureResponseDurationSeconds("504")
 
 				router.ServeHTTP(
@@ -221,6 +239,12 @@ var _ = Describe("Backend handler", func() {
 				).To(Equal(float64(1)))
 			})
 
+			It("should count the number of proxied responses", func() {
+				Expect(
+					measureResponseCount("504") - beforeResponseCountMetric,
+				).To(Equal(float64(1)))
+			})
+
 			It("should record the duration of proxied responses", func() {
 				Expect(
 					measureResponseDurationSeconds("504") - beforeResponseDurationSecondsMetric,
diff --git a/handlers/metrics.go b/handlers/metrics.go
@@ -27,6 +27,17 @@ var (
 		},
 	)
 
+	BackendHandlerResponseCountMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "router_backend_handler_response_count",
+			Help: "Number of responses returned by router backend handlers",
+		},
+		[]string{
+			"backend_id",
+			"response_code",
+		},
+	)
+
 	BackendHandlerResponseDurationSecondsMetric = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "router_backend_handler_response_duration_seconds",
@@ -43,5 +54,6 @@ func initMetrics() {
 	prometheus.MustRegister(RedirectHandlerRedirectCountMetric)
 
 	prometheus.MustRegister(BackendHandlerRequestCountMetric)
+	prometheus.MustRegister(BackendHandlerResponseCountMetric)
 	prometheus.MustRegister(BackendHandlerResponseDurationSecondsMetric)
 }
