Add setting to set prefix on Prometheus metrics (#3483)

* Add setting to set prefix on Prometheus metrics

* Add test to cover empty prefix branch

Author: leplatrem

docs/configuration/settings.rst

@@ -503,6 +503,8 @@ Prometheus metrics can be enabled with (disabled by default):
 
     kinto.includes = kinto.plugins.prometheus
 
+    # kinto.prometheus_prefix = kinto-prod
+
 Metrics can then be crawled from the ``/__metrics__`` endpoint.
 
 

kinto/plugins/prometheus.py

@@ -28,7 +28,7 @@ def get_registry():
 
 
 def _fix_metric_name(s):
-    return s.replace("-", "_").replace(".", "_")
+    return s.replace("-", "_").replace(".", "_").replace(" ", "_")
 
 
 class Timer:
@@ -68,8 +68,20 @@ def stop(self):
 
 @implementer(metrics.IMetricsService)
 class PrometheusService:
+    def __init__(self, prefix=""):
+        prefix_clean = ""
+        if prefix:
+            # In GCP Console, the metrics are grouped by the first
+            # word before the first underscore. Here we make sure the specified
+            # prefix is not mixed up with metrics names.
+            # (eg. `remote-settings` -> `remotesettings_`, `kinto_` -> `kinto_`)
+            prefix_clean = _fix_metric_name(prefix).replace("_", "") + "_"
+        self.prefix = prefix_clean.lower()
+
     def timer(self, key):
         global _METRICS
+        key = self.prefix + key
+
         if key not in _METRICS:
             _METRICS[key] = prometheus_module.Summary(
                 _fix_metric_name(key), f"Summary of {key}", registry=get_registry()
@@ -84,6 +96,7 @@ def timer(self, key):
 
     def observe(self, key, value, labels=[]):
         global _METRICS
+        key = self.prefix + key
 
         if key not in _METRICS:
             _METRICS[key] = prometheus_module.Summary(
@@ -106,6 +119,7 @@ def observe(self, key, value, labels=[]):
 
     def count(self, key, count=1, unique=None):
         global _METRICS
+        key = self.prefix + key
 
         labels = []
 
@@ -183,4 +197,7 @@ def includeme(config):
             pass
     _METRICS.clear()
 
-    config.registry.registerUtility(PrometheusService(), metrics.IMetricsService)
+    settings = config.get_settings()
+    prefix = settings.get("prometheus_prefix", settings["project_name"])
+
+    config.registry.registerUtility(PrometheusService(prefix=prefix), metrics.IMetricsService)

tests/plugins/test_prometheus.py

@@ -31,6 +31,7 @@ class PrometheusWebTest(support.BaseWebTest, unittest.TestCase):
     def get_app_settings(cls, extras=None):
         settings = super().get_app_settings(extras)
         settings["includes"] = "kinto.plugins.prometheus"
+        settings["project_name"] = "kinto PROD"
         return settings
 
 
@@ -59,15 +60,15 @@ def test_timer_can_be_used_as_context_manager(self):
             self.assertEqual(my_func(1, 1), 2)
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("TYPE func_latency_context summary", resp.text)
+        self.assertIn("TYPE kintoprod_func_latency_context summary", resp.text)
 
     def test_timer_can_be_used_as_decorator(self):
         decorated = self.app.app.registry.metrics.timer("func.latency.decorator")(my_func)
 
         self.assertEqual(decorated(1, 1), 2)
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("TYPE func_latency_decorator summary", resp.text)
+        self.assertIn("TYPE kintoprod_func_latency_decorator summary", resp.text)
 
     def test_timer_can_be_used_as_decorator_on_partial_function(self):
         partial = functools.partial(my_func, 3)
@@ -76,39 +77,39 @@ def test_timer_can_be_used_as_decorator_on_partial_function(self):
         self.assertEqual(decorated(3), 6)
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("TYPE func_latency_partial summary", resp.text)
+        self.assertIn("TYPE kintoprod_func_latency_partial summary", resp.text)
 
     def test_observe_a_single_value(self):
         self.app.app.registry.metrics.observe("price", 111)
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("price_sum 111", resp.text)
+        self.assertIn("kintoprod_price_sum 111", resp.text)
 
     def test_observe_a_single_value_with_labels(self):
         self.app.app.registry.metrics.observe("size", 3.14, labels=[("endpoint", "/buckets")])
 
         resp = self.app.get("/__metrics__")
-        self.assertIn('size_sum{endpoint="/buckets"} 3.14', resp.text)
+        self.assertIn('kintoprod_size_sum{endpoint="/buckets"} 3.14', resp.text)
 
     def test_count_by_key(self):
         self.app.app.registry.metrics.count("key")
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("key_total 1.0", resp.text)
+        self.assertIn("kintoprod_key_total 1.0", resp.text)
 
     def test_count_by_key_value(self):
         self.app.app.registry.metrics.count("bigstep", count=2)
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("bigstep_total 2.0", resp.text)
+        self.assertIn("kintoprod_bigstep_total 2.0", resp.text)
 
     def test_count_by_key_grouped(self):
         self.app.app.registry.metrics.count("http", unique=[("status", "500")])
         self.app.app.registry.metrics.count("http", unique=[("status", "200")])
 
         resp = self.app.get("/__metrics__")
-        self.assertIn('http_total{status="500"} 1.0', resp.text)
-        self.assertIn('http_total{status="200"} 1.0', resp.text)
+        self.assertIn('kintoprod_http_total{status="500"} 1.0', resp.text)
+        self.assertIn('kintoprod_http_total{status="200"} 1.0', resp.text)
 
     def test_metrics_cant_be_mixed(self):
         self.app.app.registry.metrics.count("counter")
@@ -127,16 +128,32 @@ def test_metrics_names_and_labels_are_transformed(self):
         self.app.app.registry.metrics.count("http.home.status", unique=[("code.get", "200")])
 
         resp = self.app.get("/__metrics__")
-        self.assertIn('http_home_status_total{code_get="200"} 1.0', resp.text)
+        self.assertIn('kintoprod_http_home_status_total{code_get="200"} 1.0', resp.text)
 
     def test_count_with_legacy_string_generic_group(self):
         self.app.app.registry.metrics.count("champignons", unique="boletus")
 
         resp = self.app.get("/__metrics__")
-        self.assertIn('champignons_total{group="boletus"} 1.0', resp.text)
+        self.assertIn('kintoprod_champignons_total{group="boletus"} 1.0', resp.text)
 
     def test_count_with_legacy_string_basic_group(self):
         self.app.app.registry.metrics.count("mushrooms", unique="species.boletus")
 
         resp = self.app.get("/__metrics__")
-        self.assertIn('mushrooms_total{species="boletus"} 1.0', resp.text)
+        self.assertIn('kintoprod_mushrooms_total{species="boletus"} 1.0', resp.text)
+
+
+@skip_if_no_prometheus
+class PrometheusNoPrefixTest(PrometheusWebTest):
+    @classmethod
+    def get_app_settings(cls, extras=None):
+        settings = super().get_app_settings(extras)
+        settings["project_name"] = "Some Project"
+        settings["prometheus_prefix"] = ""
+        return settings
+
+    def test_metrics_have_no_prefix(self):
+        self.app.app.registry.metrics.observe("price", 111)
+
+        resp = self.app.get("/__metrics__")
+        self.assertIn("TYPE price summary", resp.text)