Error handling while running the Gluster commands

.github/workflows/on-pr-submit.yml

@@ -26,7 +26,7 @@ jobs:
  run: make lint
  - name: Install Binnacle
  run: |
- curl -L https://github.com/kadalu/binnacle/releases/latest/download/binnacle -o binnacle
+ curl -L https://github.com/kadalu/binnacle/releases/download/0.6.1/binnacle -o binnacle
  chmod +x ./binnacle
  sudo mv ./binnacle /usr/local/bin/binnacle
  binnacle --version

Makefile

@@ -5,7 +5,7 @@ fmt-check:
 
 lint:
  cd lint && shards install
- ./lint/bin/ameba src
+ ./lint/bin/ameba --except Documentation/DocumentationAdmonition src
 
 fmt:
  crystal tool format src

lint/shard.lock

@@ -2,5 +2,5 @@ version: 2.0
 shards:
  ameba:
  git: https://github.com/crystal-ameba/ameba.git
- version: 1.0.0
+ version: 1.6.1
 

lint/shard.yml

@@ -12,4 +12,3 @@ license: GPLv3
 development_dependencies:
  ameba:
  github: crystal-ameba/ameba
- version: 1.0.0

shard.lock

@@ -2,23 +2,23 @@ version: 2.0
 shards:
  backtracer:
  git: https://github.com/sija/backtracer.cr.git
- version: 1.2.1
+ version: 1.2.2
 
  crometheus:
  git: https://github.com/darwinnn/crometheus.git
- version: 0.3.0+git.commit.8394850abd90b976aa205f58de49a886c91aa802
+ version: 0.3.0+git.commit.c71a13174d02e3767b62faa1771132d4245e1ce5
 
  exception_page:
  git: https://github.com/crystal-loot/exception_page.git
- version: 0.2.2
+ version: 0.4.1
 
  glustercli:
  git: https://github.com/aravindavk/glustercli-crystal.git
- version: 0.2.0+git.commit.0881f270471be31d4f6d06f45d896e17adace0db
+ version: 0.2.0+git.commit.fb2f36881d79523990ca8b5f23212d4181718c8f
 
  kemal:
  git: https://github.com/kemalcr/kemal.git
- version: 1.2.0
+ version: 1.5.0
 
  radix:
  git: https://github.com/luislavena/radix.git

src/args.cr

@@ -27,15 +27,15 @@ module GlusterMetricsExporter
  @@config.gluster_host = name
  end
 
- parser.on("-v", "--verbose", "Enable verbose output (default: #{@@config.verbose})") do
+ parser.on("-v", "--verbose", "Enable verbose output (default: #{@@config.verbose?})") do
  @@config.verbose = true
  end
 
- parser.on("--disable-all", "Disable all Metrics (default: #{@@config.disable_all_metrics})") do
+ parser.on("--disable-all", "Disable all Metrics (default: #{@@config.disable_all_metrics?})") do
  @@config.disable_all_metrics = true
  end
 
- parser.on("--disable-volumes-all", "Disable all Volumes (default: #{@@config.disable_volumes_all})") do
+ parser.on("--disable-volumes-all", "Disable all Volumes (default: #{@@config.disable_volumes_all?})") do
  @@config.disable_volumes_all = true
  end
 

src/config.cr

@@ -6,23 +6,24 @@ module GlusterMetricsExporter
 
  property metrics_path = "/metrics",
  port = 9713,
- disable_volumes_all = false,
  gluster_executable_path = "/usr/sbin/gluster",
- disable_all_metrics = false,
  log_level = "info",
  log_dir = "/var/log/gluster-metrics-exporter",
  log_file = "exporter.log",
  gluster_log_dir = "/var/log/glusterfs",
  glusterd_dir = "/var/lib/glusterd",
  gluster_cli_socket_path = "",
- verbose = false,
  gluster_host = "",
  disabled_volumes = [] of String,
  enabled_volumes = [] of String,
  disabled_metrics = [] of String,
  enabled_metrics = [] of String,
  all_metrics = [] of String
 
+ property? verbose = false,
+ disable_volumes_all = false,
+ disable_all_metrics = false
+
  def initialize
  end
 
@@ -35,7 +36,7 @@ module GlusterMetricsExporter
 
  def self.set_enabled_metrics
  metrics = [] of String
- if @@config.disable_all_metrics
+ if @@config.disable_all_metrics?
  # If --disable-all is passed then enabled list
  # will only have whatever passed as --enable
  # For example --disable-all --enable "volume_status"

src/metrics/errors.cr

@@ -0,0 +1,19 @@
+module GlusterMetricsExporter
+ Crometheus.alias ErrorGauge = Crometheus::Gauge[:name]
+
+ @@error_count = ErrorGauge.new(:error_count, "Metrics collection errors")
+
+ def self.clear_error_metrics
+ @@error_count.clear
+ end
+
+ handle_metrics(["error"]) do |metrics_data|
+ # Reset all Metrics to avoid stale data. Careful if
+ # counter type is used
+ clear_error_metrics
+
+ metrics_data.errors.each do |err|
+ @@error_count[name: err.name].set(1)
+ end
+ end
+end

src/metrics/helpers.cr

@@ -1,13 +1,23 @@
 require "glustercli"
 
 module GlusterMetricsExporter
+ struct MetricError
+ include JSON::Serializable
+
+ property name = ""
+
+ def initialize(@name)
+ end
+ end
+
  class MetricsData
  include JSON::Serializable
 
  property volumes = [] of GlusterCLI::VolumeInfo,
  peers = [] of GlusterCLI::NodeInfo,
  local_metrics = Hash(String, GlusterCLI::LocalMetrics).new,
- exporter_health = Hash(String, Int32).new
+ exporter_health = Hash(String, Int32).new,
+ errors = [] of MetricError
 
  def self.collect
  data = MetricsData.new
@@ -24,11 +34,21 @@ module GlusterMetricsExporter
  status_collect = true
  end
 
- data.volumes = cli.list_volumes(status: status_collect)
+ begin
+ data.volumes = cli.list_volumes(status: status_collect)
+ rescue ex : GlusterCLI::CommandException
+ data.errors << MetricError.new("list_volumes")
+ Log.error &.emit("Error while collecting the Volumes metrics", error: ex.message)
+ end
  end
 
  if GlusterMetricsExporter.config.enabled?("peer")
- data.peers = cli.list_peers
+ begin
+ data.peers = cli.list_peers
+ rescue ex : GlusterCLI::CommandException
+ data.errors << MetricError.new("list_peers")
+ Log.error &.emit("Error while collecting the Peers metrics", error: ex.message)
+ end
  end
 
  # TODO: API calls concurrently

src/metrics/peer.cr

@@ -20,7 +20,7 @@ module GlusterMetricsExporter
 
  metrics_data.peers.each do |peer|
  # Peer State 1 => Connected, 0 => Disconnected/Unknown
- state = peer.connected ? 1 : 0
+ state = peer.connected? ? 1 : 0
  @@peer_state[hostname: peer.hostname].set(state)
  end
  end

tests/install.t

@@ -4,7 +4,7 @@ USE_REMOTE_PLUGIN "docker"
 nodes = ["gserver1", "gserver2", "gserver3"]
 
 # Static build Kadalu Storage Manager
-TEST "docker run --rm -i -v $PWD:/workspace -w /workspace crystallang/crystal:1.2.0-alpine /bin/sh -c \"apk add --update --no-cache --force-overwrite sqlite-dev sqlite-static && shards install && shards build --static\""
+TEST "docker run --rm -i -v $PWD:/workspace -w /workspace crystallang/crystal:1.12-alpine /bin/sh -c \"shards install && shards build --static\""
 
 # Install the Static binary to all containers/nodes
 # and copy the service files