Add cluster scanner metrics in metrics-v3

endpoint: /minio/metrics/v3/cluster/scanner
metrics:
 - bucket_scans_finished (counter)
 - bucket_scans_started (counter)
 - directories_scanned (counter)
 - last_activity_nano_seconds (gauge)
 - objects_scanned (counter)
 - versions_scanned (counter)

cmd/metrics-v3-cluster-scanner.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+ "context"
+ "time"
+)
+
+const (
+ scannerBucketScansFinished = "bucket_scans_finished"
+ scannerBucketScansStarted = "bucket_scans_started"
+ scannerDirectoriesScanned = "directories_scanned"
+ scannerObjectsScanned = "objects_scanned"
+ scannerVersionsScanned = "versions_scanned"
+ scannerLastActivityNanoSeconds = "last_activity_nano_seconds"
+)
+
+var (
+ scannerBucketScansFinishedMD = NewCounterMD(scannerBucketScansFinished,
+ "Total number of bucket scans finished since server start")
+ scannerBucketScansStartedMD = NewCounterMD(scannerBucketScansStarted,
+ "Total number of bucket scans started since server start")
+ scannerDirectoriesScannedMD = NewCounterMD(scannerDirectoriesScanned,
+ "Total number of directories scanned since server start")
+ scannerObjectsScannedMD = NewCounterMD(scannerObjectsScanned,
+ "Total number of unique objects scanned since server start")
+ scannerVersionsScannedMD = NewCounterMD(scannerVersionsScanned,
+ "Total number of object versions scanned since server start")
+ scannerLastActivityNanoSecondsMD = NewGaugeMD(scannerLastActivityNanoSeconds,
+ "Time elapsed (in nano seconds) since last scan activity.")
+)
+
+// loadClusterScannerMetrics - `MetricsLoaderFn` for cluster webhook
+// such as failed objects and directories scanned.
+func loadClusterScannerMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
+ m.Set(scannerBucketScansFinished, float64(globalScannerMetrics.lifetime(scannerMetricScanBucketDrive)))
+ m.Set(scannerBucketScansStarted, float64(globalScannerMetrics.lifetime(scannerMetricScanBucketDrive)+uint64(globalScannerMetrics.activeDrives())))
+ m.Set(scannerDirectoriesScanned, float64(globalScannerMetrics.lifetime(scannerMetricScanFolder)))
+ m.Set(scannerObjectsScanned, float64(globalScannerMetrics.lifetime(scannerMetricScanObject)))
+ m.Set(scannerVersionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricApplyVersion)))
+
+ dui, err := c.dataUsageInfo.Get()
+ if err != nil {
+ metricsLogIf(ctx, err)
+ } else {
+ m.Set(scannerLastActivityNanoSeconds, float64(time.Since(dui.LastUpdate)))
+ }
+
+ return nil
+}

cmd/metrics-v3.go

@@ -42,6 +42,7 @@ const (
  clusterUsageObjectsCollectorPath collectorPath = "/cluster/usage/objects"
  clusterUsageBucketsCollectorPath collectorPath = "/cluster/usage/buckets"
  clusterErasureSetCollectorPath collectorPath = "/cluster/erasure-set"
+ clusterScannerCollectorPath collectorPath = "/cluster/scanner"
 )
 
 const (
@@ -203,6 +204,18 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
  loadClusterErasureSetMetrics,
  )
 
+ clusterScannerMG := NewMetricsGroup(clusterScannerCollectorPath,
+ []MetricDescriptor{
+ scannerBucketScansFinishedMD,
+ scannerBucketScansStartedMD,
+ scannerDirectoriesScannedMD,
+ scannerObjectsScannedMD,
+ scannerVersionsScannedMD,
+ scannerLastActivityNanoSecondsMD,
+ },
+ loadClusterScannerMetrics,
+ )
+
  allMetricGroups := []*MetricsGroup{
  apiRequestsMG,
  apiBucketMG,
@@ -214,6 +227,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
  clusterUsageObjectsMG,
  clusterUsageBucketsMG,
  clusterErasureSetMG,
+ clusterScannerMG,
  }
 
  // Bucket metrics are special, they always include the bucket label. These

docs/metrics/v3.md

@@ -149,6 +149,16 @@ The standard metrics groups for ProcessCollector and GoCollector are not shown b
 | `minio_cluster_health_capacity_usable_total_bytes` | `gauge` | Total cluster usable storage capacity in bytes | |
 | `minio_cluster_health_capacity_usable_free_bytes` | `gauge` | Total cluster usable storage free in bytes | |
 
+### `/cluster/scanner`
+
+| Name | Type | Help | Labels |
+|----------------------------------------------------|-----------|------------------------------------------------------------|--------|
+| `minio_cluster_scanner_bucket_scans_finished` | `counter` | Total number of bucket scans finished since server start | |
+| `minio_cluster_scanner_bucket_scans_started` | `counter` | Total number of bucket scans started since server start | |
+| `minio_cluster_scanner_directories_scanned` | `counter` | Total number of directories scanned since server start | |
+| `minio_cluster_scanner_last_activity_nano_seconds` | `gauge` | Time elapsed (in nano seconds) since last scan activity | |
+| `minio_cluster_scanner_objects_scanned` | `counter` | Total number of unique objects scanned since server start | |
+| `minio_cluster_scanner_versions_scanned` | `counter` | Total number of object versions scanned since server start | |
 
 ### `/cluster/usage/objects`