Add TLS certs expiration metric

Author: sylr

cmd/traefik/traefik.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"crypto/x509"
 	"encoding/json"
 	stdlog "log"
 	"net/http"
@@ -14,6 +15,7 @@ import (
 	"github.com/coreos/go-systemd/daemon"
 	assetfs "github.com/elazarl/go-bindata-assetfs"
 	"github.com/go-acme/lego/v4/challenge"
+	gokitmetrics "github.com/go-kit/kit/metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/traefik/paerser/cli"
 	"github.com/traefik/traefik/v2/autogen/genstatic"
@@ -260,6 +262,11 @@ func setupServer(staticConfiguration *static.Configuration) (*server.Server, err
 	watcher.AddListener(func(conf dynamic.Configuration) {
 		ctx := context.Background()
 		tlsManager.UpdateConfigs(ctx, conf.TLS.Stores, conf.TLS.Options, conf.TLS.Certificates)
+
+		gauge := metricsRegistry.TLSCertsNotAfterTimestampGauge()
+		for _, certificate := range tlsManager.GetCertificates() {
+			appendCertMetric(gauge, certificate)
+		}
 	})
 
 	// Metrics
@@ -432,6 +439,20 @@ func registerMetricClients(metricsConfig *types.Metrics) []metrics.Registry {
 	return registries
 }
 
+func appendCertMetric(gauge gokitmetrics.Gauge, certificate *x509.Certificate) {
+	sort.Strings(certificate.DNSNames)
+
+	labels := []string{
+		"cn", certificate.Subject.CommonName,
+		"serial", certificate.SerialNumber.String(),
+		"sans", strings.Join(certificate.DNSNames, ","),
+	}
+
+	notAfter := float64(certificate.NotAfter.Unix())
+
+	gauge.With(labels...).Set(notAfter)
+}
+
 func setupAccessLog(conf *types.AccessLog) *accesslog.Handler {
 	if conf == nil {
 		return nil

cmd/traefik/traefik_test.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"strings"
+	"testing"
+
+	"github.com/go-kit/kit/metrics"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// FooCert is a PEM-encoded TLS cert.
+// generated from src/crypto/tls:
+// go run generate_cert.go  --rsa-bits 1024 --host foo.org,foo.com  --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h
+const fooCert = `-----BEGIN CERTIFICATE-----
+MIICHzCCAYigAwIBAgIQXQFLeYRwc5X21t457t2xADANBgkqhkiG9w0BAQsFADAS
+MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw
+MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
+iQKBgQDCjn67GSs/khuGC4GNN+tVo1S+/eSHwr/hWzhfMqO7nYiXkFzmxi+u14CU
+Pda6WOeps7T2/oQEFMxKKg7zYOqkLSbjbE0ZfosopaTvEsZm/AZHAAvoOrAsIJOn
+SEiwy8h0tLA4z1SNR6rmIVQWyqBZEPAhBTQM1z7tFp48FakCFwIDAQABo3QwcjAO
+BgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUw
+AwEB/zAdBgNVHQ4EFgQUDHG3ASzeUezElup9zbPpBn/vjogwGwYDVR0RBBQwEoIH
+Zm9vLm9yZ4IHZm9vLmNvbTANBgkqhkiG9w0BAQsFAAOBgQBT+VLMbB9u27tBX8Aw
+ZrGY3rbNdBGhXVTksrjiF+6ZtDpD3iI56GH9zLxnqvXkgn3u0+Ard5TqF/xmdwVw
+NY0V/aWYfcL2G2auBCQrPvM03ozRnVUwVfP23eUzX2ORNHCYhd2ObQx4krrhs7cJ
+SWxtKwFlstoXY3K2g9oRD9UxdQ==
+-----END CERTIFICATE-----`
+
+// BarCert is a PEM-encoded TLS cert.
+// generated from src/crypto/tls:
+// go run generate_cert.go  --rsa-bits 1024 --host bar.org,bar.com  --ca --start-date "Jan 1 00:00:00 1970" --duration=10000h
+const barCert = `-----BEGIN CERTIFICATE-----
+MIICHTCCAYagAwIBAgIQcuIcNEXzBHPoxna5S6wG4jANBgkqhkiG9w0BAQsFADAS
+MRAwDgYDVQQKEwdBY21lIENvMB4XDTcwMDEwMTAwMDAwMFoXDTcxMDIyMTE2MDAw
+MFowEjEQMA4GA1UEChMHQWNtZSBDbzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
+gYEAqtcrP+KA7D6NjyztGNIPMup9KiBMJ8QL+preog/YHR7SQLO3kGFhpS3WKMab
+SzMypC3ZX1PZjBP5ZzwaV3PFbuwlCkPlyxR2lOWmullgI7mjY0TBeYLDIclIzGRp
+mpSDDSpkW1ay2iJDSpXjlhmwZr84hrCU7BRTQJo91fdsRTsCAwEAAaN0MHIwDgYD
+VR0PAQH/BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMB
+Af8wHQYDVR0OBBYEFK8jnzFQvBAgWtfzOyXY4VSkwrTXMBsGA1UdEQQUMBKCB2Jh
+ci5vcmeCB2Jhci5jb20wDQYJKoZIhvcNAQELBQADgYEAJz0ifAExisC/ZSRhWuHz
+7qs1i6Nd4+YgEVR8dR71MChP+AMxucY1/ajVjb9xlLys3GPE90TWSdVppabEVjZY
+Oq11nPKc50ItTt8dMku6t0JHBmzoGdkN0V4zJCBqdQJxhop8JpYJ0S9CW0eT93h3
+ipYQSsmIINGtMXJ8VkP/MlM=
+-----END CERTIFICATE-----`
+
+type gaugeMock struct {
+	metrics map[string]float64
+	labels  string
+}
+
+func (g gaugeMock) With(labelValues ...string) metrics.Gauge {
+	g.labels = strings.Join(labelValues, ",")
+	return g
+}
+
+func (g gaugeMock) Set(value float64) {
+	g.metrics[g.labels] = value
+}
+
+func (g gaugeMock) Add(delta float64) {
+	panic("implement me")
+}
+
+func TestAppendCertMetric(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		certs    []string
+		expected map[string]float64
+	}{
+		{
+			desc:     "No certs",
+			certs:    []string{},
+			expected: map[string]float64{},
+		},
+		{
+			desc:  "One cert",
+			certs: []string{fooCert},
+			expected: map[string]float64{
+				"cn,,serial,123624926713171615935660664614975025408,sans,foo.com,foo.org": 3.6e+09,
+			},
+		},
+		{
+			desc:  "Two certs",
+			certs: []string{fooCert, barCert},
+			expected: map[string]float64{
+				"cn,,serial,123624926713171615935660664614975025408,sans,foo.com,foo.org": 3.6e+09,
+				"cn,,serial,152706022658490889223053211416725817058,sans,bar.com,bar.org": 3.6e+07,
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			gauge := &gaugeMock{
+				metrics: map[string]float64{},
+			}
+
+			for _, cert := range test.certs {
+				block, _ := pem.Decode([]byte(cert))
+				parsedCert, err := x509.ParseCertificate(block.Bytes)
+				require.NoError(t, err)
+
+				appendCertMetric(gauge, parsedCert)
+			}
+
+			assert.Equal(t, test.expected, gauge.metrics)
+		})
+	}
+}

docs/content/providers/docker.md

@@ -98,8 +98,8 @@ See the list of labels in the dedicated [routing](../routing/providers/docker.md
 By default, Traefik watches for [container level labels](https://docs.docker.com/config/labels-custom-metadata/) on a standalone Docker Engine.
 
 When using Docker Compose, labels are specified by the directive
-[`labels`](https://docs.docker.com/compose/compose-file/#labels) from the
-["services" objects](https://docs.docker.com/compose/compose-file/#service-configuration-reference).
+[`labels`](https://docs.docker.com/compose/compose-file/compose-file-v3/#labels) from the
+["services" objects](https://docs.docker.com/compose/compose-file/compose-file-v3/#service-configuration-reference).
 
 !!! tip "Not Only Docker"
     Please note that any tool like Nomad, Terraform, Ansible, etc.
@@ -186,7 +186,7 @@ set the [`swarmMode`](#swarmmode) directive to `true`.
 While in Swarm Mode, Traefik uses labels found on services, not on individual containers.
 
 Therefore, if you use a compose file with Swarm Mode, labels should be defined in the
-[`deploy`](https://docs.docker.com/compose/compose-file/#labels-1) part of your service.
+[`deploy`](https://docs.docker.com/compose/compose-file/compose-file-v3/#labels-1) part of your service.
 
 This behavior is only enabled for docker-compose version 3+ ([Compose file reference](https://docs.docker.com/compose/compose-file)).
 

docs/content/routing/providers/docker.md

@@ -124,7 +124,7 @@ Attach labels to your containers and let Traefik do the rest!
     !!! important "Labels in Docker Swarm Mode"
         While in Swarm Mode, Traefik uses labels found on services, not on individual containers.
         Therefore, if you use a compose file with Swarm Mode, labels should be defined in the `deploy` part of your service.
-        This behavior is only enabled for docker-compose version 3+ ([Compose file reference](https://docs.docker.com/compose/compose-file/#labels-1)).
+        This behavior is only enabled for docker-compose version 3+ ([Compose file reference](https://docs.docker.com/compose/compose-file/compose-file-v3/#labels-1)).
 
 ## Routing Configuration
 

pkg/metrics/datadog.go

@@ -20,18 +20,19 @@ var datadogTicker *time.Ticker
 
 // Metric names consistent with https://github.com/DataDog/integrations-extras/pull/64
 const (
-	ddMetricsServiceReqsName      = "service.request.total"
-	ddMetricsServiceLatencyName   = "service.request.duration"
-	ddRetriesTotalName            = "service.retries.total"
-	ddConfigReloadsName           = "config.reload.total"
-	ddConfigReloadsFailureTagName = "failure"
-	ddLastConfigReloadSuccessName = "config.reload.lastSuccessTimestamp"
-	ddLastConfigReloadFailureName = "config.reload.lastFailureTimestamp"
-	ddEntryPointReqsName          = "entrypoint.request.total"
-	ddEntryPointReqDurationName   = "entrypoint.request.duration"
-	ddEntryPointOpenConnsName     = "entrypoint.connections.open"
-	ddOpenConnsName               = "service.connections.open"
-	ddServerUpName                = "service.server.up"
+	ddMetricsServiceReqsName        = "service.request.total"
+	ddMetricsServiceLatencyName     = "service.request.duration"
+	ddRetriesTotalName              = "service.retries.total"
+	ddConfigReloadsName             = "config.reload.total"
+	ddConfigReloadsFailureTagName   = "failure"
+	ddLastConfigReloadSuccessName   = "config.reload.lastSuccessTimestamp"
+	ddLastConfigReloadFailureName   = "config.reload.lastFailureTimestamp"
+	ddEntryPointReqsName            = "entrypoint.request.total"
+	ddEntryPointReqDurationName     = "entrypoint.request.duration"
+	ddEntryPointOpenConnsName       = "entrypoint.connections.open"
+	ddOpenConnsName                 = "service.connections.open"
+	ddServerUpName                  = "service.server.up"
+	ddTLSCertsNotAfterTimestampName = "tls.certs.notAfterTimestamp"
 )
 
 // RegisterDatadog registers the metrics pusher if this didn't happen yet and creates a datadog Registry instance.
@@ -41,10 +42,11 @@ func RegisterDatadog(ctx context.Context, config *types.Datadog) Registry {
 	}
 
 	registry := &standardRegistry{
-		configReloadsCounter:         datadogClient.NewCounter(ddConfigReloadsName, 1.0),
-		configReloadsFailureCounter:  datadogClient.NewCounter(ddConfigReloadsName, 1.0).With(ddConfigReloadsFailureTagName, "true"),
-		lastConfigReloadSuccessGauge: datadogClient.NewGauge(ddLastConfigReloadSuccessName),
-		lastConfigReloadFailureGauge: datadogClient.NewGauge(ddLastConfigReloadFailureName),
+		configReloadsCounter:           datadogClient.NewCounter(ddConfigReloadsName, 1.0),
+		configReloadsFailureCounter:    datadogClient.NewCounter(ddConfigReloadsName, 1.0).With(ddConfigReloadsFailureTagName, "true"),
+		lastConfigReloadSuccessGauge:   datadogClient.NewGauge(ddLastConfigReloadSuccessName),
+		lastConfigReloadFailureGauge:   datadogClient.NewGauge(ddLastConfigReloadFailureName),
+		tlsCertsNotAfterTimestampGauge: datadogClient.NewGauge(ddTLSCertsNotAfterTimestampName),
 	}
 
 	if config.AddEntryPointsLabels {

pkg/metrics/datadog_test.go

@@ -36,6 +36,7 @@ func TestDatadog(t *testing.T) {
 		"traefik.entrypoint.request.duration:10000.000000|h|#entrypoint:test\n",
 		"traefik.entrypoint.connections.open:1.000000|g|#entrypoint:test\n",
 		"traefik.service.server.up:1.000000|g|#service:test,url:http://127.0.0.1,one:two\n",
+		"traefik.tls.certs.notAfterTimestamp:1.000000|g|#key:value\n",
 	}
 
 	udp.ShouldReceiveAll(t, expected, func() {
@@ -50,5 +51,6 @@ func TestDatadog(t *testing.T) {
 		datadogRegistry.EntryPointReqDurationHistogram().With("entrypoint", "test").Observe(10000)
 		datadogRegistry.EntryPointOpenConnsGauge().With("entrypoint", "test").Set(1)
 		datadogRegistry.ServiceServerUpGauge().With("service", "test", "url", "http://127.0.0.1", "one", "two").Set(1)
+		datadogRegistry.TLSCertsNotAfterTimestampGauge().With("key", "value").Set(1)
 	})
 }

pkg/metrics/influxdb.go

@@ -26,18 +26,19 @@ type influxDBWriter struct {
 var influxDBTicker *time.Ticker
 
 const (
-	influxDBMetricsServiceReqsName      = "traefik.service.requests.total"
-	influxDBMetricsServiceLatencyName   = "traefik.service.request.duration"
-	influxDBRetriesTotalName            = "traefik.service.retries.total"
-	influxDBConfigReloadsName           = "traefik.config.reload.total"
-	influxDBConfigReloadsFailureName    = influxDBConfigReloadsName + ".failure"
-	influxDBLastConfigReloadSuccessName = "traefik.config.reload.lastSuccessTimestamp"
-	influxDBLastConfigReloadFailureName = "traefik.config.reload.lastFailureTimestamp"
-	influxDBEntryPointReqsName          = "traefik.entrypoint.requests.total"
-	influxDBEntryPointReqDurationName   = "traefik.entrypoint.request.duration"
-	influxDBEntryPointOpenConnsName     = "traefik.entrypoint.connections.open"
-	influxDBOpenConnsName               = "traefik.service.connections.open"
-	influxDBServerUpName                = "traefik.service.server.up"
+	influxDBMetricsServiceReqsName        = "traefik.service.requests.total"
+	influxDBMetricsServiceLatencyName     = "traefik.service.request.duration"
+	influxDBRetriesTotalName              = "traefik.service.retries.total"
+	influxDBConfigReloadsName             = "traefik.config.reload.total"
+	influxDBConfigReloadsFailureName      = influxDBConfigReloadsName + ".failure"
+	influxDBLastConfigReloadSuccessName   = "traefik.config.reload.lastSuccessTimestamp"
+	influxDBLastConfigReloadFailureName   = "traefik.config.reload.lastFailureTimestamp"
+	influxDBEntryPointReqsName            = "traefik.entrypoint.requests.total"
+	influxDBEntryPointReqDurationName     = "traefik.entrypoint.request.duration"
+	influxDBEntryPointOpenConnsName       = "traefik.entrypoint.connections.open"
+	influxDBOpenConnsName                 = "traefik.service.connections.open"
+	influxDBServerUpName                  = "traefik.service.server.up"
+	influxDBTLSCertsNotAfterTimestampName = "traefik.tls.certs.notAfterTimestamp"
 )
 
 const (
@@ -55,10 +56,11 @@ func RegisterInfluxDB(ctx context.Context, config *types.InfluxDB) Registry {
 	}
 
 	registry := &standardRegistry{
-		configReloadsCounter:         influxDBClient.NewCounter(influxDBConfigReloadsName),
-		configReloadsFailureCounter:  influxDBClient.NewCounter(influxDBConfigReloadsFailureName),
-		lastConfigReloadSuccessGauge: influxDBClient.NewGauge(influxDBLastConfigReloadSuccessName),
-		lastConfigReloadFailureGauge: influxDBClient.NewGauge(influxDBLastConfigReloadFailureName),
+		configReloadsCounter:           influxDBClient.NewCounter(influxDBConfigReloadsName),
+		configReloadsFailureCounter:    influxDBClient.NewCounter(influxDBConfigReloadsFailureName),
+		lastConfigReloadSuccessGauge:   influxDBClient.NewGauge(influxDBLastConfigReloadSuccessName),
+		lastConfigReloadFailureGauge:   influxDBClient.NewGauge(influxDBLastConfigReloadFailureName),
+		tlsCertsNotAfterTimestampGauge: influxDBClient.NewGauge(influxDBTLSCertsNotAfterTimestampName),
 	}
 
 	if config.AddEntryPointsLabels {

pkg/metrics/influxdb_test.go

@@ -64,6 +64,16 @@ func TestInfluxDB(t *testing.T) {
 	})
 
 	assertMessage(t, msgEntrypoint, expectedEntrypoint)
+
+	expectedTLS := []string{
+		`(traefik\.tls\.certs\.notAfterTimestamp,key=value value=1) [\d]{19}`,
+	}
+
+	msgTLS := udp.ReceiveString(t, func() {
+		influxDBRegistry.TLSCertsNotAfterTimestampGauge().With("key", "value").Set(1)
+	})
+
+	assertMessage(t, msgTLS, expectedTLS)
 }
 
 func TestInfluxDBHTTP(t *testing.T) {
@@ -121,6 +131,15 @@ func TestInfluxDBHTTP(t *testing.T) {
 	msgEntrypoint := <-c
 
 	assertMessage(t, *msgEntrypoint, expectedEntrypoint)
+
+	expectedTLS := []string{
+		`(traefik\.tls\.certs\.notAfterTimestamp,key=value value=1) [\d]{19}`,
+	}
+
+	influxDBRegistry.TLSCertsNotAfterTimestampGauge().With("key", "value").Set(1)
+	msgTLS := <-c
+
+	assertMessage(t, *msgTLS, expectedTLS)
 }
 
 func assertMessage(t *testing.T, msg string, patterns []string) {

pkg/metrics/metrics.go

@@ -21,6 +21,9 @@ type Registry interface {
 	LastConfigReloadSuccessGauge() metrics.Gauge
 	LastConfigReloadFailureGauge() metrics.Gauge
 
+	// TLS
+	TLSCertsNotAfterTimestampGauge() metrics.Gauge
+
 	// entry point metrics
 	EntryPointReqsCounter() metrics.Counter
 	EntryPointReqsTLSCounter() metrics.Counter
@@ -50,6 +53,7 @@ func NewMultiRegistry(registries []Registry) Registry {
 	var configReloadsFailureCounter []metrics.Counter
 	var lastConfigReloadSuccessGauge []metrics.Gauge
 	var lastConfigReloadFailureGauge []metrics.Gauge
+	var tlsCertsNotAfterTimestampGauge []metrics.Gauge
 	var entryPointReqsCounter []metrics.Counter
 	var entryPointReqsTLSCounter []metrics.Counter
 	var entryPointReqDurationHistogram []ScalableHistogram
@@ -74,6 +78,9 @@ func NewMultiRegistry(registries []Registry) Registry {
 		if r.LastConfigReloadFailureGauge() != nil {
 			lastConfigReloadFailureGauge = append(lastConfigReloadFailureGauge, r.LastConfigReloadFailureGauge())
 		}
+		if r.TLSCertsNotAfterTimestampGauge() != nil {
+			tlsCertsNotAfterTimestampGauge = append(tlsCertsNotAfterTimestampGauge, r.TLSCertsNotAfterTimestampGauge())
+		}
 		if r.EntryPointReqsCounter() != nil {
 			entryPointReqsCounter = append(entryPointReqsCounter, r.EntryPointReqsCounter())
 		}
@@ -113,6 +120,7 @@ func NewMultiRegistry(registries []Registry) Registry {
 		configReloadsFailureCounter:    multi.NewCounter(configReloadsFailureCounter...),
 		lastConfigReloadSuccessGauge:   multi.NewGauge(lastConfigReloadSuccessGauge...),
 		lastConfigReloadFailureGauge:   multi.NewGauge(lastConfigReloadFailureGauge...),
+		tlsCertsNotAfterTimestampGauge: multi.NewGauge(tlsCertsNotAfterTimestampGauge...),
 		entryPointReqsCounter:          multi.NewCounter(entryPointReqsCounter...),
 		entryPointReqsTLSCounter:       multi.NewCounter(entryPointReqsTLSCounter...),
 		entryPointReqDurationHistogram: NewMultiHistogram(entryPointReqDurationHistogram...),
@@ -133,6 +141,7 @@ type standardRegistry struct {
 	configReloadsFailureCounter    metrics.Counter
 	lastConfigReloadSuccessGauge   metrics.Gauge
 	lastConfigReloadFailureGauge   metrics.Gauge
+	tlsCertsNotAfterTimestampGauge metrics.Gauge
 	entryPointReqsCounter          metrics.Counter
 	entryPointReqsTLSCounter       metrics.Counter
 	entryPointReqDurationHistogram ScalableHistogram
@@ -169,6 +178,10 @@ func (r *standardRegistry) LastConfigReloadFailureGauge() metrics.Gauge {
 	return r.lastConfigReloadFailureGauge
 }
 
+func (r *standardRegistry) TLSCertsNotAfterTimestampGauge() metrics.Gauge {
+	return r.tlsCertsNotAfterTimestampGauge
+}
+
 func (r *standardRegistry) EntryPointReqsCounter() metrics.Counter {
 	return r.entryPointReqsCounter
 }