Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add create conn counter
Browse files Browse the repository at this point in the history
@djshow832
djshow832 committed Dec 19, 2023
1 parent 4cbcac4 commit 5187ab3
Showing 6 changed files with 196 additions and 44 deletions.
170 changes: 128 additions & 42 deletions pkg/metrics/grafana/tiproxy_summary.json
View file
Original file line number Diff line number Diff line change
@@ -223,7 +223,7 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "TiProxy current connection counts.",
"description": "TiProxy uptime since the last restart.",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -258,6 +258,92 @@
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tiproxy\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Uptime",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "TiProxy current connection counts.",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 0
},
"id": 6,
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sideWidth": null,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"repeat": null,
"seriesOverrides": [ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
@@ -316,16 +402,16 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "TiProxy disconnection count per minute.",
"description": "TiProxy create connection count per minute.",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"x": 0,
"y": 0
},
"id": 6,
"id": 7,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -353,17 +439,17 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(tiproxy_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)",
"expr": "sum(increase(tiproxy_server_create_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{type}}",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Disconnection OPM",
"title": "Create Connection OPM",
"tooltip": {
"shared": true,
"sort": 0,
@@ -402,16 +488,16 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "TiProxy current goroutine counts.",
"description": "TiProxy disconnection count per minute.",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"x": 12,
"y": 0
},
"id": 7,
"id": 8,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -439,24 +525,17 @@
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tiproxy\"}",
"expr": "sum(increase(tiproxy_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"legendFormat": "{{type}}",
"refId": "A"
},
{
"expr": "sum(go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiproxy\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "total",
"refId": "B"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Goroutine Count",
"title": "Disconnection OPM",
"tooltip": {
"shared": true,
"sort": 0,
@@ -495,16 +574,16 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "TiProxy uptime since the last restart.",
"description": "TiProxy current goroutine counts.",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"x": 0,
"y": 0
},
"id": 8,
"id": 9,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -532,17 +611,24 @@
"steppedLine": false,
"targets": [
{
"expr": "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tiproxy\"}",
"expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tiproxy\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"refId": "A"
},
{
"expr": "sum(go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiproxy\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "total",
"refId": "B"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Uptime",
"title": "Goroutine Count",
"tooltip": {
"shared": true,
"sort": 0,
@@ -558,15 +644,15 @@
},
"yaxes": [
{
"format": "s",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "s",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
@@ -593,7 +679,7 @@
"x": 0,
"y": 0
},
"id": 9,
"id": 10,
"panels": [
{
"aliasColors": { },
@@ -610,7 +696,7 @@
"x": 0,
"y": 0
},
"id": 10,
"id": 11,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -710,7 +796,7 @@
"x": 12,
"y": 0
},
"id": 11,
"id": 12,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -796,7 +882,7 @@
"x": 0,
"y": 0
},
"id": 12,
"id": 13,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -882,7 +968,7 @@
"x": 12,
"y": 0
},
"id": 13,
"id": 14,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -968,7 +1054,7 @@
"x": 0,
"y": 0
},
"id": 14,
"id": 15,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -1054,7 +1140,7 @@
"x": 12,
"y": 0
},
"id": 15,
"id": 16,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -1143,7 +1229,7 @@
"x": 0,
"y": 0
},
"id": 16,
"id": 17,
"panels": [
{
"aliasColors": { },
@@ -1160,7 +1246,7 @@
"x": 0,
"y": 0
},
"id": 17,
"id": 18,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -1246,7 +1332,7 @@
"x": 12,
"y": 0
},
"id": 18,
"id": 19,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -1332,7 +1418,7 @@
"x": 0,
"y": 0
},
"id": 19,
"id": 20,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -1435,7 +1521,7 @@
"x": 0,
"y": 0
},
"id": 20,
"id": 21,
"panels": [
{
"aliasColors": { },
@@ -1452,7 +1538,7 @@
"x": 0,
"y": 0
},
"id": 21,
"id": 22,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -1552,7 +1638,7 @@
"x": 12,
"y": 0
},
"id": 22,
"id": 23,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -1638,7 +1724,7 @@
"x": 0,
"y": 0
},
"id": 23,
"id": 24,
"legend": {
"alignAsTable": false,
"avg": false,
19 changes: 17 additions & 2 deletions pkg/metrics/grafana/tiproxy_summary.jsonnet
View file
Original file line number Diff line number Diff line change
@@ -106,6 +106,20 @@ local connectionP = graphPanel.new(
)
);

local createConnP = graphPanel.new(
title='Create Connection OPM',
datasource=myDS,
legend_rightSide=true,
description='TiProxy create connection count per minute.',
format='short',
)
.addTarget(
prometheus.target(
'sum(increase(tiproxy_server_create_connection_total{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
legendFormat='{{instance}}',
)
);

local disconnP = graphPanel.new(
title='Disconnection OPM',
datasource=myDS,
@@ -420,10 +434,11 @@ newDash
serverRow
.addPanel(cpuP, gridPos=leftPanelPos)
.addPanel(memP, gridPos=rightPanelPos)
.addPanel(connectionP, gridPos=leftPanelPos)
.addPanel(uptimeP, gridPos=leftPanelPos)
.addPanel(connectionP, gridPos=rightPanelPos)
.addPanel(createConnP, gridPos=leftPanelPos)
.addPanel(disconnP, gridPos=rightPanelPos)
.addPanel(goroutineP, gridPos=leftPanelPos)
.addPanel(uptimeP, gridPos=rightPanelPos)
,
gridPos=rowPos
)
1 change: 1 addition & 0 deletions pkg/metrics/metrics.go
View file
Original file line number Diff line number Diff line change
@@ -154,6 +154,7 @@ func registerProxyMetrics() {
prometheus.MustRegister(collectors.NewGoCollector(collectors.WithGoCollections(collectors.GoRuntimeMetricsCollection | collectors.GoRuntimeMemStatsCollection)))

prometheus.MustRegister(ConnGauge)
prometheus.MustRegister(CreateConnCounter)
prometheus.MustRegister(DisConnCounter)
prometheus.MustRegister(MaxProcsGauge)
prometheus.MustRegister(ServerEventCounter)
8 changes: 8 additions & 0 deletions pkg/metrics/server.go
View file
Original file line number Diff line number Diff line change
@@ -26,6 +26,14 @@ var (
Help: "Number of connections.",
})

CreateConnCounter = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: ModuleProxy,
Subsystem: LabelServer,
Name: "create_connection_total",
Help: "Number of create connections.",
})

DisConnCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: ModuleProxy,
1 change: 1 addition & 0 deletions pkg/proxy/proxy.go
View file
Original file line number Diff line number Diff line change
@@ -180,6 +180,7 @@ func (s *SQLServer) onConn(ctx context.Context, conn net.Conn, addr string) {
s.mu.Unlock()

metrics.ConnGauge.Inc()
metrics.CreateConnCounter.Inc()

defer func() {
s.mu.Lock()
41 changes: 41 additions & 0 deletions pkg/proxy/proxy_test.go
View file
Original file line number Diff line number Diff line change
@@ -19,12 +19,53 @@ import (
"github.com/pingcap/tiproxy/lib/util/waitgroup"
"github.com/pingcap/tiproxy/pkg/manager/cert"
"github.com/pingcap/tiproxy/pkg/manager/router"
"github.com/pingcap/tiproxy/pkg/metrics"
"github.com/pingcap/tiproxy/pkg/proxy/backend"
"github.com/pingcap/tiproxy/pkg/proxy/client"
pnet "github.com/pingcap/tiproxy/pkg/proxy/net"
"github.com/stretchr/testify/require"
)

func TestCreateConn(t *testing.T) {
lg, _ := logger.CreateLoggerForTest(t)
certManager := cert.NewCertManager()
require.NoError(t, certManager.Init(&config.Config{}, lg, nil))
server, err := NewSQLServer(lg, config.ProxyServer{}, certManager, &panicHsHandler{})
require.NoError(t, err)
server.Run(context.Background(), nil)
defer func() {
require.NoError(t, server.Close())
}()

createConn := func() net.Conn {
conn, err := net.Dial("tcp", server.listeners[0].Addr().String())
require.NoError(t, err)
return conn
}
checkMetrics := func(totalConns, createConns int) {
require.Eventually(t, func() bool {
connGauge, err := metrics.ReadGauge(metrics.ConnGauge)
require.NoError(t, err)
if totalConns != int(connGauge) {
return false
}
connCounter, err := metrics.ReadCounter(metrics.CreateConnCounter)
require.NoError(t, err)
return createConns == connCounter
}, time.Second, 10*time.Millisecond)
}

checkMetrics(0, 0)
conn1 := createConn()
checkMetrics(1, 1)
conn2 := createConn()
checkMetrics(2, 2)
require.NoError(t, conn1.Close())
checkMetrics(1, 2)
require.NoError(t, conn2.Close())
checkMetrics(0, 2)
}

func TestGracefulCloseConn(t *testing.T) {
// Graceful shutdown finishes immediately if there's no connection.
lg, _ := logger.CreateLoggerForTest(t)

0 comments on commit 5187ab3

Please sign in to comment.