From cf42cb15fe63602630492bb788e8f6f1b691b172 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 18 Aug 2023 15:13:30 +0200 Subject: [PATCH] Add documentation for metrics endpoint (#816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: André Colomb --- .gitignore | 1 + _script/find-metrics/find-metrics.go | 203 +++++++++++++++++++++++++++ _script/go.mod | 8 +- _script/go.sum | 9 ++ conf.py | 2 +- includes/metrics-list.rst | 115 +++++++++++++++ refresh-metrics.sh | 8 ++ users/index.rst | 1 + users/metrics.rst | 14 ++ 9 files changed, 359 insertions(+), 2 deletions(-) create mode 100644 _script/find-metrics/find-metrics.go create mode 100644 includes/metrics-list.rst create mode 100755 refresh-metrics.sh create mode 100644 users/metrics.rst diff --git a/.gitignore b/.gitignore index 86e8c4aa9..b95c06847 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ _build/ _deployed _deployed.old +_syncthing diff --git a/_script/find-metrics/find-metrics.go b/_script/find-metrics/find-metrics.go new file mode 100644 index 000000000..404ef47d9 --- /dev/null +++ b/_script/find-metrics/find-metrics.go @@ -0,0 +1,203 @@ +// Copyright (C) 2023 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// Usage: go run script/find-metrics.go > metrics.md +// +// This script finds all of the metrics in the Syncthing codebase and prints +// them in Markdown format. It's used to generate the metrics documentation +// for the Syncthing docs. +package main + +import ( + "flag" + "fmt" + "go/ast" + "go/token" + "log" + "os" + "strconv" + "strings" + + "golang.org/x/exp/slices" + "golang.org/x/tools/go/packages" +) + +type metric struct { + subsystem string + name string + help string + kind string +} + +func main() { + flag.Parse() + if flag.NArg() != 1 { + fmt.Println("Usage: find-metrics ") + os.Exit(1) + } + + opts := &packages.Config{ + Dir: flag.Arg(0), + Mode: packages.NeedSyntax | packages.NeedName | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports | packages.NeedDeps, + } + + pkgs, err := packages.Load(opts, "github.com/syncthing/syncthing/...") + if err != nil { + log.Fatalln(err) + } + + var coll metricCollector + for _, pkg := range pkgs { + for _, file := range pkg.Syntax { + ast.Inspect(file, coll.Visit) + } + } + coll.print() +} + +type metricCollector struct { + metrics []metric +} + +func (c *metricCollector) Visit(n ast.Node) bool { + if gen, ok := n.(*ast.GenDecl); ok { + // We're only interested in var declarations (var metricWhatever = + // promauto.NewCounter(...) etc). + if gen.Tok != token.VAR { + return false + } + + for _, spec := range gen.Specs { + // We want to look at the value given to a var (the NewCounter() + // etc call). + if vsp, ok := spec.(*ast.ValueSpec); ok { + // There should be only one value. + if len(vsp.Values) != 1 { + continue + } + + // The value should be a function call. + call, ok := vsp.Values[0].(*ast.CallExpr) + if !ok { + continue + } + + // The call should be a selector expression + // (package.Identifer). + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + continue + } + + // The package selector should be `promauto`. + selID, ok := sel.X.(*ast.Ident) + if !ok || selID.Name != "promauto" { + continue + } + + // The function should be one of the New* functions. + var kind string + switch sel.Sel.Name { + case "NewCounter": + kind = "counter" + case "NewGauge": + kind = "gauge" + case "NewCounterVec": + kind = "counter vector" + case "NewGaugeVec": + kind = "gauge vector" + default: + continue + } + + // The arguments to the function should be a single + // composite (struct literal). Grab all of the fields in the + // declaration into a map so we can easily access them. + args := make(map[string]string) + for _, el := range call.Args[0].(*ast.CompositeLit).Elts { + kv := el.(*ast.KeyValueExpr) + key := kv.Key.(*ast.Ident).Name // e.g., "Name" + val := kv.Value.(*ast.BasicLit).Value // e.g., `"foo"` + args[key], _ = strconv.Unquote(val) + } + + // Build the full name of the metric from the namespace + + // subsystem + name, like Prometheus does. + var parts []string + if v := args["Namespace"]; v != "" { + parts = append(parts, v) + } + if v := args["Subsystem"]; v != "" { + parts = append(parts, v) + } + if v := args["Name"]; v != "" { + parts = append(parts, v) + } + fullName := strings.Join(parts, "_") + + // Add the metric to the list. + c.metrics = append(c.metrics, metric{ + subsystem: args["Subsystem"], + name: fullName, + help: args["Help"], + kind: kind, + }) + } + } + } + return true +} + +func (c *metricCollector) print() { + slices.SortFunc(c.metrics, func(a, b metric) int { + if a.subsystem != b.subsystem { + return strings.Compare(a.subsystem, b.subsystem) + } + return strings.Compare(a.name, b.name) + }) + + var prevSubsystem string + for _, m := range c.metrics { + if m.subsystem != prevSubsystem { + fmt.Println(header(fmt.Sprintf("Package *%s*", m.subsystem), "~")) + prevSubsystem = m.subsystem + } + fmt.Println(header(fmt.Sprintf("Metric *%v* (%s)", m.name, m.kind), "^")) + fmt.Println(wordwrap(sentenceize(m.help), 72)) + fmt.Println() + } +} + +func header(header, underline string) string { + under := strings.Repeat(underline, len(header)) + return fmt.Sprintf("%s\n%s\n", header, under) +} + +func sentenceize(s string) string { + if s == "" { + return "" + } + if !strings.HasSuffix(s, ".") { + return s + "." + } + return s +} + +func wordwrap(s string, width int) string { + var lines []string + for _, line := range strings.Split(s, "\n") { + for len(line) > width { + i := strings.LastIndex(line[:width], " ") + if i == -1 { + i = width + } + lines = append(lines, line[:i]) + line = line[i+1:] + } + lines = append(lines, line) + } + return strings.Join(lines, "\n") +} diff --git a/_script/go.mod b/_script/go.mod index caf9a78c7..a16e98ad1 100644 --- a/_script/go.mod +++ b/_script/go.mod @@ -2,9 +2,15 @@ module syncthing.net/docs go 1.20 -require github.com/google/go-github/v49 v49.1.0 +require ( + github.com/google/go-github/v49 v49.1.0 + golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb + golang.org/x/tools v0.12.0 +) require ( github.com/google/go-querystring v1.1.0 // indirect golang.org/x/crypto v0.6.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/sys v0.11.0 // indirect ) diff --git a/_script/go.sum b/_script/go.sum index 844163625..e4268036c 100644 --- a/_script/go.sum +++ b/_script/go.sum @@ -6,4 +6,13 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/conf.py b/conf.py index b78584c17..6a09ba5e4 100644 --- a/conf.py +++ b/conf.py @@ -83,7 +83,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', 'draft', 'README.rst', 'users/faq-parts'] +exclude_patterns = ['_build', '_syncthing', 'draft', 'README.rst', 'users/faq-parts'] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/includes/metrics-list.rst b/includes/metrics-list.rst new file mode 100644 index 000000000..dc12187fd --- /dev/null +++ b/includes/metrics-list.rst @@ -0,0 +1,115 @@ +Package *events* +~~~~~~~~~~~~~~~~ + +Metric *syncthing_events_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total number of created/forwarded/dropped events. + +Package *fs* +~~~~~~~~~~~~ + +Metric *syncthing_fs_operation_bytes_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total number of filesystem bytes transferred, per filesystem root and +operation. + +Metric *syncthing_fs_operation_seconds_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total time spent in filesystem operations, per filesystem root and +operation. + +Metric *syncthing_fs_operations_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total number of filesystem operations, per filesystem root and +operation. + +Package *model* +~~~~~~~~~~~~~~~ + +Metric *syncthing_model_folder_processed_bytes_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total amount of data processed during folder syncing, per folder ID and +data source (network/local_origin/local_other/local_shifted/skipped). + +Metric *syncthing_model_folder_pull_seconds_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total time spent in folder pull iterations, per folder ID. + +Metric *syncthing_model_folder_pulls_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total number of folder pull iterations, per folder ID. + +Metric *syncthing_model_folder_scan_seconds_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total time spent in folder scan iterations, per folder ID. + +Metric *syncthing_model_folder_scans_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total number of folder scan iterations, per folder ID. + +Metric *syncthing_model_folder_state* (gauge vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Current folder state. + +Metric *syncthing_model_folder_summary* (gauge vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Current folder summary data (counts for global/local/need +files/directories/symlinks/deleted/bytes). + +Package *protocol* +~~~~~~~~~~~~~~~~~~ + +Metric *syncthing_protocol_recv_bytes_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total amount of data received, per device. + +Metric *syncthing_protocol_recv_decompressed_bytes_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total amount of data received, after decompression, per device. + +Metric *syncthing_protocol_recv_messages_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total number of messages received, per device. + +Metric *syncthing_protocol_sent_bytes_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total amount of data sent, per device. + +Metric *syncthing_protocol_sent_messages_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total number of messages sent, per device. + +Metric *syncthing_protocol_sent_uncompressed_bytes_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total amount of data sent, before compression, per device. + +Package *scanner* +~~~~~~~~~~~~~~~~~ + +Metric *syncthing_scanner_hashed_bytes_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total amount of data hashed, per folder. + +Metric *syncthing_scanner_scanned_items_total* (counter vector) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Total number of items (files/directories) inspected, per folder. + diff --git a/refresh-metrics.sh b/refresh-metrics.sh new file mode 100755 index 000000000..b7cbc70b2 --- /dev/null +++ b/refresh-metrics.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -euo pipefail + +rm -rf _syncthing +git clone --depth 1 https://github.com/syncthing/syncthing.git _syncthing +pushd _script +go run ./find-metrics ../_syncthing > ../includes/metrics-list.rst +popd diff --git a/users/index.rst b/users/index.rst index 3d052b75f..4e7f0814f 100644 --- a/users/index.rst +++ b/users/index.rst @@ -15,6 +15,7 @@ Usage guilisten ldap tuning + metrics syncing untrusted diff --git a/users/metrics.rst b/users/metrics.rst new file mode 100644 index 000000000..947087d9c --- /dev/null +++ b/users/metrics.rst @@ -0,0 +1,14 @@ +Prometheus-Style Metrics +======================== + +Syncthing provides an endpoint for Prometheus-style metrics. Metrics are +served on the ``/metrics`` path on the GUI / API address. The metrics endpoint +requires authentication when the GUI / API is configured to require +authentication; see :doc:`/dev/rest` for details. + +Metrics +------- + +The following metrics are available. + +.. include:: ../includes/metrics-list.rst