Add split brain metric (#100)

* Implement metric structure

* Add documentation in wip and some pseudo implementation

* Implement splitbrain metric

* update doc

* Improve error handling inc case some postifix miss

* Refactor and implement configurable path splitbrain

* change default drbdsetup-path config value

* update doc and metric with refacto

* add fake collector

* Implement test for metric

change location to var/tmp/drbd

Author: MalloZup

doc/metric_spec.md

@@ -206,8 +206,8 @@ The DRBD subsystems collect devices stats by parsing its configuration the JSON
 0. [Sample](../test/drbd.metrics)
 1. [`ha_cluster_drbd_resources`](#ha_cluster_drbd_resources)
 2. [`ha_cluster_drbd_connections`](#ha_cluster_drbd_connections)
-3. [`ha_cluster_drbd_connections_sync`](#ha_cluster_drbd_connections_sync`)
-
+3. [`ha_cluster_drbd_connections_sync`](#ha_cluster_drbd_connections_sync)
+4. [`ha_cluster_drbd_split_brain`](#ha_cluster_drbd_split_brain)
 
 ### `ha_cluster_drbd_connections`
 
@@ -249,3 +249,36 @@ Either the value is `1`, or the line is absent altogether.
 - `disk_state`: one of `attaching|failed|negotiating|inconsistent|outdated|dunknown|consistent|uptodate`
 
 The total number of lines for this metric will be the cardinality of `name` times the cardinality of `volume`.
+
+### `ha_cluster_drbd_split_brain`
+
+#### Description
+
+This metric signal if there is a split brain occuring per resource and volume.
+Either the value is `1`, or the line is absent altogether.
+
+This metric is a special metric comparing to others, because in order to make this metric working you will need to set a drbd customer split-brain handler. Look at the end 
+
+#### Labels
+
+- `resource`: the name of the resource.
+- `volume`: the volume number
+
+#### Setting up the DRBD split-brain hook
+
+In order to get the `split_brain` metric working:
+
+1) copy hook into all drbd nodes:
+
+get the hook from:
+https://github.com/SUSE/ha-sap-terraform-deployments/blob/72c9d3ecf6c3f6dd18ccb7bcbde4b40722d5c641/salt/drbd_node/files/notify-split-brain-haclusterexporter-suse-metric.sh
+
+2) on the drbd configuration enable the hook:
+
+```split_brain: "/usr/lib/drbd/notify-split-brain-haclusterexporter-suse-metric.sh"`
+
+Refer to upstream doc: https://docs.linbit.com/docs/users-guide-8.4/#s-configure-split-brain-behavior
+
+It is important for the exporter that he hook should create the files in that location and naming. 
+
+Remember to remove the files manually after the split brain is solved

drbd_metrics.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"encoding/json"
+	"io/ioutil"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -37,10 +39,11 @@ var (
 		"resources":        NewMetricDesc("drbd", "resources", "The DRBD resources; 1 line per name, per volume", []string{"resource", "role", "volume", "disk_state"}),
 		"connections":      NewMetricDesc("drbd", "connections", "The DRBD resource connections; 1 line per per resource, per peer_node_id", []string{"resource", "peer_node_id", "peer_role", "volume", "peer_disk_state"}),
 		"connections_sync": NewMetricDesc("drbd", "connections_sync", "The in sync percentage value for DRBD resource connections", []string{"resource", "peer_node_id", "volume"}),
+		"split_brain":      NewMetricDesc("drbd", "split_brain", "Whether a split brain has been detected; 1 line per resource, per volume.", []string{"resource", "volume"}),
 	}
 )
 
-func NewDrbdCollector(drbdSetupPath string) (*drbdCollector, error) {
+func NewDrbdCollector(drbdSetupPath string, drbdSplitBrainPath string) (*drbdCollector, error) {
 	err := CheckExecutables(drbdSetupPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not initialize DRBD collector")
@@ -51,12 +54,14 @@ func NewDrbdCollector(drbdSetupPath string) (*drbdCollector, error) {
 			metrics: drbdMetrics,
 		},
 		drbdSetupPath,
+		drbdSplitBrainPath,
 	}, nil
 }
 
 type drbdCollector struct {
 	DefaultCollector
-	drbdsetupPath string
+	drbdsetupPath      string
+	drbdSplitBrainPath string
 }
 
 func (c *drbdCollector) Collect(ch chan<- prometheus.Metric) {
@@ -65,6 +70,9 @@ func (c *drbdCollector) Collect(ch chan<- prometheus.Metric) {
 
 	log.Infoln("Collecting DRBD metrics...")
 
+	// set split brain metric
+	c.setDrbdSplitBrainMetric(ch)
+
 	drbdStatusRaw, err := exec.Command(c.drbdsetupPath, "status", "--json").Output()
 	if err != nil {
 		log.Warnf("Error while retrieving drbd infos %s", err)
@@ -101,6 +109,7 @@ func (c *drbdCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 		}
 	}
+
 }
 
 func parseDrbdStatus(statusRaw []byte) ([]drbdStatus, error) {
@@ -111,3 +120,32 @@ func parseDrbdStatus(statusRaw []byte) ([]drbdStatus, error) {
 	}
 	return drbdDevs, nil
 }
+
+func (c *drbdCollector) setDrbdSplitBrainMetric(ch chan<- prometheus.Metric) {
+
+	// set split brain metric
+	// by default if the custom hook is not set, the exporter will not be able to detect it
+	files, err := ioutil.ReadDir(c.drbdSplitBrainPath)
+	if err != nil {
+		log.Warnf("Error while reading directory %s: %s", c.drbdSplitBrainPath, err)
+	}
+
+	for _, f := range files {
+		// check if in directory there are file of syntax we expect (nil is when there is not any)
+		match, _ := filepath.Glob(c.drbdSplitBrainPath + "/drbd-split-brain-detected-*")
+		if match == nil {
+			continue
+		}
+		resAndVolume := strings.Split(f.Name(), "drbd-split-brain-detected-")[1]
+
+		// avoid to have index out range panic error (in case the there is not resource-volume syntax)
+		if len(strings.Split(resAndVolume, "-")) != 2 {
+			continue
+		}
+		//Resource (0) volume (1) place in slice
+		resourceAndVolume := strings.Split(resAndVolume, "-")
+
+		ch <- c.makeGaugeMetric("split_brain", float64(1), resourceAndVolume[0], resourceAndVolume[1])
+
+	}
+}

drbd_metrics_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"os"
 	"testing"
 )
 
@@ -152,14 +153,14 @@ func TestDrbdParsing(t *testing.T) {
 }
 
 func TestNewDrbdCollector(t *testing.T) {
-	_, err := NewDrbdCollector("test/fake_drbdsetup.sh")
+	_, err := NewDrbdCollector("test/fake_drbdsetup.sh", "splitbrainpath")
 	if err != nil {
 		t.Errorf("Unexpected error, got: %v", err)
 	}
 }
 
 func TestNewDrbdCollectorChecksDrbdsetupExistence(t *testing.T) {
-	_, err := NewDrbdCollector("test/nonexistent")
+	_, err := NewDrbdCollector("test/nonexistent", "splitbrainfake")
 	if err == nil {
 		t.Fatal("a non nil error was expected")
 	}
@@ -169,7 +170,7 @@ func TestNewDrbdCollectorChecksDrbdsetupExistence(t *testing.T) {
 }
 
 func TestNewDrbdCollectorChecksDrbdsetupExecutableBits(t *testing.T) {
-	_, err := NewDrbdCollector("test/dummy")
+	_, err := NewDrbdCollector("test/dummy", "splibrainfake")
 	if err == nil {
 		t.Fatalf("a non nil error was expected")
 	}
@@ -180,7 +181,26 @@ func TestNewDrbdCollectorChecksDrbdsetupExecutableBits(t *testing.T) {
 
 func TestDRBDCollector(t *testing.T) {
 	clock = StoppedClock{}
+	splitBrainDir := "/var/tmp/drbd/splitbrain"
+	testFiles := [3]string{
+		"drbd-split-brain-detected-resource01-vol01",
+		"drbd-split-brain-detected-resource02-vol02",
+		"drbd-split-brain-detected-missingthingsWrongSkippedMetricS",
+	}
+	// create dir for putting temp file if not existings
+	if _, err := os.Stat(splitBrainDir); os.IsNotExist(err) {
+		err := os.MkdirAll(splitBrainDir, os.ModePerm)
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+	}
 
-	collector, _ := NewDrbdCollector("test/fake_drbdsetup.sh")
+	for _, testFile := range testFiles {
+		os.Create(splitBrainDir + "/" + testFile)
+	}
+	defer os.RemoveAll(splitBrainDir)
+
+	collector, _ := NewDrbdCollector("test/fake_drbdsetup.sh", splitBrainDir)
 	expectMetrics(t, collector, "drbd.metrics")
+
 }

ha_cluster_exporter.go

@@ -126,6 +126,7 @@ func init() {
 	flag.String("sbd-path", "/usr/sbin/sbd", "path to sbd executable")
 	flag.String("sbd-config-path", "/etc/sysconfig/sbd", "path to sbd configuration")
 	flag.String("drbdsetup-path", "/sbin/drbdsetup", "path to drbdsetup executable")
+	flag.String("drbdsplitbrain-path", "/var/run/drbd/splitbrain", "path to drbd splitbrain hooks temporary files")
 
 	err := config.BindPFlags(flag.CommandLine)
 	if err != nil {
@@ -181,7 +182,7 @@ func main() {
 		log.Info("SBD collector registered")
 	}
 
-	drbdCollector, err := NewDrbdCollector(config.GetString("drbdsetup-path"))
+	drbdCollector, err := NewDrbdCollector(config.GetString("drbdsetup-path"), config.GetString("drbdsplitbrain-path"))
 	if err != nil {
 		log.Warn(err)
 	} else {

test/drbd.metrics

@@ -10,3 +10,7 @@ ha_cluster_drbd_connections_sync{peer_node_id="1",resource="1-single-1",volume="
 # TYPE ha_cluster_drbd_resources gauge
 ha_cluster_drbd_resources{disk_state="uptodate",resource="1-single-0",role="Secondary",volume="0"} 1 1234
 ha_cluster_drbd_resources{disk_state="uptodate",resource="1-single-1",role="Secondary",volume="0"} 1 1234
+# HELP ha_cluster_drbd_split_brain Whether a split brain has been detected; 1 line per resource, per volume.
+# TYPE ha_cluster_drbd_split_brain gauge
+ha_cluster_drbd_split_brain{resource="resource01",volume="vol01"} 1 1234
+ha_cluster_drbd_split_brain{resource="resource02",volume="vol02"} 1 1234