MM-55630: Generate a new dashboard with enabled coordinator metrics

config/coordinator.sample.json

@@ -14,55 +14,63 @@
     "Queries": [
       {
         "Description": "Percentage of HTTP 5xx server errors",
+        "Legend": "Percentage of requests responded from the server with a 5xx status code",
         "Query": "(sum(rate(mattermost_api_time_count{status_code=~\"5..\"}[1m]))/sum(rate(mattermost_api_time_count[1m])))*100",
         "Threshold": 0.025,
         "MinIntervalSec": 60,
         "Alert": true
       },
       {
         "Description": "Average client request duration",
+        "Legend": "Average duration of all requests as measured from the client side",
         "Query": "sum(rate(loadtest_http_request_time_sum[1m]))/sum(rate(loadtest_http_request_time_count[1m]))",
         "Threshold": 0.1,
         "MinIntervalSec": 60,
         "Alert": true
       },
       {
         "Description": "99th percentile of client request duration",
+        "Legend": "99th percentile duration of all requests as measured from the client side",
         "Query": "histogram_quantile(0.99, sum(rate(loadtest_http_request_time_bucket[1m])) by (le))",
         "Threshold": 2.0,
         "MinIntervalSec": 60,
         "Alert": true
       },
       {
         "Description": "Percentage of HTTP 5xx client errors",
+        "Legend": "Percentage of requests responded with a 5xx status code, as seen from the client",
         "Query": "(sum(rate(loadtest_http_errors_total{status_code=~\"5..\"}[1m]))/sum(rate(loadtest_http_request_time_count[1m])))*100",
         "Threshold": 0.025,
         "MinIntervalSec": 60,
         "Alert": true
       },
       {
         "Description": "Percentage of client timeouts",
+        "Legend": "Percentage of timeouts over all requests done",
         "Query": "(sum(rate(loadtest_http_timeouts_total[1m]))/sum(rate(loadtest_http_request_time_count[1m]))) * 100",
         "Threshold": 0.025,
         "MinIntervalSec": 60,
         "Alert": true
       },
       {
         "Description": "CPU utilization",
+        "Legend": "Percentage of CPU used, as an average over all app nodes",
         "Query": "100 - 100 * (avg(irate(node_cpu_seconds_total{instance=~\"app.*\",mode=\"idle\"}[5m])))",
         "Threshold": 85,
         "MinIntervalSec": 60,
         "Alert": true
       },
       {
         "Description": "Memory utilization",
+        "Legend": "Percentage of memory used, as an average over all app nodes",
         "Query": "100 - 100 * avg(node_memory_MemAvailable_bytes{instance=~\"app.*\"} / node_memory_MemTotal_bytes{instance=~\"app.*\"})",
         "Threshold": 85,
         "MinIntervalSec": 60,
         "Alert": true
       },
       {
         "Description": "Percentage of TCP timeouts in the proxy node",
+        "Legend": "Percentage of timeouts over all TCP connections established in the proxy",
         "Query": "100 * rate(node_netstat_TcpExt_TCPTimeouts{instance=~\"proxy:9100\"}[1m]) / (rate(node_netstat_TcpExt_TCPTimeouts{instance=~\"proxy:9100\"}[1m]) + node_netstat_Tcp_CurrEstab{instance=~\"proxy:9100\"})",
         "Threshold": 0.2,
         "MinIntervalSec": 60,
@@ -71,6 +79,7 @@
 
       {
         "Description": "Percentage of TCP timeouts in the app nodes",
+        "Legend": "Percentage of timeouts over all TCP connections established by the app nodes, as an average over all app nodes",
         "Query": "100 * avg(rate(node_netstat_TcpExt_TCPTimeouts{instance=~\"app.*\"}[1m])) / (avg(rate(node_netstat_TcpExt_TCPTimeouts{instance=~\"app.*\"}[1m])) + avg(node_netstat_Tcp_CurrEstab{instance=~\"app.*\"}))",
         "Threshold": 3.0,
         "MinIntervalSec": 60,

coordinator/performance/prometheus/config.go

@@ -13,6 +13,9 @@ type Configuration struct {
 type Query struct {
 	// The description for the query.
 	Description string `validate:"notempty"`
+	// An optional string for populating the legend of this query's panel
+	// in the Grafana dashboard containing all coordinator metrics
+	Legend string
 	// The PromQL query to be run.
 	Query string `validate:"notempty"`
 	// The value over which the performance monitor will fire an alert

deployment/terraform/assets/coordinator_dashboard_tmpl.json

@@ -0,0 +1,143 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 5,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {{range $idx, $panel := .Panels}}
+    {{if $idx}},{{end}}{{/* Prepend each panel with a comma except for the first one */}}
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P207FEC8D12DA302B"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "dashed"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": {{$panel.Threshold}}
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": {{$panel.Height}},
+        "w": {{$panel.Width}},
+        "x": {{$panel.PosX}},
+        "y": {{$panel.PosY}}
+      },
+      "id": {{$panel.Id}},
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P207FEC8D12DA302B"
+          },
+          "editorMode": "code",
+          "expr": "{{$panel.Query}}",
+          "instant": false,
+          "legendFormat": "{{$panel.Legend}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "{{$panel.Title}}",
+      "type": "timeseries"
+    }
+    {{end}}
+  ],
+  "refresh": "10s",
+  "schemaVersion": 39,
+  "tags": [
+    "mattermost",
+    "mattermost-perf"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Mattermost Load-Test Coordinator Metrics",
+  "uid": "ad8e66e9-4558-4543-92d3-7fd2c0a037f7",
+  "version": 1,
+  "weekStart": ""
+}

deployment/terraform/create.go

@@ -684,6 +684,7 @@ func (t *Terraform) init() error {
 	assets.RestoreAssets(t.config.TerraformStateDir, "datasource.yaml")
 	assets.RestoreAssets(t.config.TerraformStateDir, "dashboard.yaml")
 	assets.RestoreAssets(t.config.TerraformStateDir, "dashboard_data.json")
+	assets.RestoreAssets(t.config.TerraformStateDir, "coordinator_dashboard_tmpl.json")
 	assets.RestoreAssets(t.config.TerraformStateDir, "es_dashboard_data.json")
 
 	// We lock to make this call safe for concurrent use

deployment/terraform/metrics.go

@@ -13,8 +13,10 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"text/template"
 	"time"
 
+	"github.com/mattermost/mattermost-load-test-ng/coordinator"
 	"github.com/mattermost/mattermost-load-test-ng/deployment/terraform/ssh"
 	"gopkg.in/yaml.v3"
 
@@ -80,6 +82,27 @@ func (t *Terraform) UploadDashboard(dashboard string) (string, error) {
 	return url, nil
 }
 
+type PanelData struct {
+	Id        int
+	Title     string
+	Legend    string
+	Height    int
+	Width     int
+	PosX      int
+	PosY      int
+	Query     string
+	Threshold float64
+}
+
+const (
+	panelWidth  = 12
+	panelHeight = 9
+)
+
+type DashboardData struct {
+	Panels []PanelData
+}
+
 func (t *Terraform) setupMetrics(extAgent *ssh.ExtAgent) error {
 	// Updating Prometheus config
 	sshc, err := extAgent.NewClient(t.output.MetricsServer.PublicIP)
@@ -225,6 +248,43 @@ func (t *Terraform) setupMetrics(extAgent *ssh.ExtAgent) error {
 		return fmt.Errorf("error while uploading dashboard_json: output: %s, error: %w", out, err)
 	}
 
+	// Upload coordinator metrics dashboard
+	coordConfig, err := coordinator.ReadConfig("")
+	if err != nil {
+		return fmt.Errorf("error while reading coordinator's config: %w", err)
+	}
+	panels := []PanelData{}
+	i := 0
+	for _, query := range coordConfig.MonitorConfig.Queries {
+		if query.Alert {
+			panels = append(panels, PanelData{
+				Id:        i,
+				Title:     query.Description,
+				Legend:    query.Legend,
+				Height:    panelHeight,
+				Width:     panelWidth,
+				PosX:      panelWidth * (i % 2),
+				PosY:      panelHeight * (i / 2),
+				Query:     strings.ReplaceAll(query.Query, "\"", "\\\""),
+				Threshold: query.Threshold,
+			})
+			i++
+		}
+	}
+	// Create the coordinator dashboard only if there is at least one panel
+	if len(panels) > 0 {
+		dashboard := DashboardData{panels}
+		var b bytes.Buffer
+		tmpl, err := template.ParseFiles(t.getAsset("coordinator_dashboard_tmpl.json"))
+		if err != nil {
+			return err
+		}
+		tmpl.Execute(&b, dashboard)
+		if out, err := sshc.Upload(&b, "/var/lib/grafana/dashboards/coordinator_dashboard.json", true); err != nil {
+			return fmt.Errorf("error while uploading coordinator_dashboard.json: output: %s, error: %w", out, err)
+		}
+	}
+
 	if t.output.HasElasticSearch() {
 		buf, err = os.ReadFile(t.getAsset("es_dashboard_data.json"))
 		if err != nil {