From 415b1b2d97578b690848476d7d2b2a4c010ac2e9 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 24 Jun 2024 21:09:50 -0700 Subject: [PATCH 1/4] Add homedirectory reporting grafana dashboard --- dashboards/homedirs.jsonnet | 143 ++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100755 dashboards/homedirs.jsonnet diff --git a/dashboards/homedirs.jsonnet b/dashboards/homedirs.jsonnet new file mode 100755 index 0000000..801a3f0 --- /dev/null +++ b/dashboards/homedirs.jsonnet @@ -0,0 +1,143 @@ +#!/usr/bin/env -S jsonnet -J ../vendor +local grafonnet = import 'grafonnet/main.libsonnet'; +local dashboard = grafonnet.dashboard; +local prometheus = grafonnet.query.prometheus; +local table = grafonnet.panel.table; + +local common = import './common.libsonnet'; + +local homedirUsage = + table.new('Home directory usage') + + table.queryOptions.withTargets([ + prometheus.new( + '$PROMETHEUS_DS', + ||| + min(dirsize_latest_mtime{namespace=~"$hub"}) by (directory) * 1000 + ||| + ) + + prometheus.withLegendFormat('Last Modified') + + prometheus.withInstant(true) + + prometheus.withFormat('table') + , + prometheus.new( + '$PROMETHEUS_DS', + ||| + max(dirsize_total_size_bytes{namespace=~"$hub"}) by (directory) + ||| + ) + + prometheus.withLegendFormat('Total Size') + + prometheus.withInstant(true) + + prometheus.withFormat('table'), + prometheus.new( + '$PROMETHEUS_DS', + ||| + max(dirsize_total_size_bytes{namespace=~"$hub"}) by (directory) + / + ignoring (directory) group_left sum(dirsize_total_size_bytes{namespace=~"$hub"}) + ||| + ) + + prometheus.withLegendFormat('% of total space used') + + prometheus.withInstant(true) + + prometheus.withFormat('table'), + prometheus.new( + '$PROMETHEUS_DS', + ||| + max(dirsize_entries_count{namespace=~"$hub"}) by (directory) + ||| + ) + + prometheus.withLegendFormat('Number of entries') + + prometheus.withInstant(true) + + prometheus.withFormat('table'), + ]) + + table.queryOptions.withTransformations([ + table.queryOptions.transformation.withId('joinByField') + + table.queryOptions.transformation.withOptions({ + byField: 'directory', + mode: 'outer', + }), + + table.queryOptions.transformation.withId('organize') + + table.queryOptions.transformation.withOptions({ + // Grafana adds an individual 'Time #N' column for each timeseries we get. + // They all display the same time. We don't care about time *at all*, since + // all these are instant data query targets. So we hide all the time values. + excludeByName: { + 'Time 1': true, + 'Time 2': true, + 'Time 3': true, + 'Time 4': true, + }, + // Explicitly rename the column headers, so they do not display 'Value #N' + renameByName: { + 'Value #A': 'Last Modified', + 'Value #B': 'Size', + 'Value #C': '% of total space usage', + 'Value #D': 'Number of Entries', + }, + }), + ]) + + { + fieldConfig: { + overrides: [ + { + matcher: { + id: 'byName', + options: 'Size', + }, + properties: [ + { + id: 'unit', + value: 'bytes', + }, + ], + }, + { + matcher: { + id: 'byName', + options: 'Last Modified', + }, + properties: [ + { + id: 'unit', + value: 'dateTimeFromNow', + }, + ], + }, + { + matcher: { + id: 'byName', + options: 'Number of Entries', + }, + properties: [ + { + id: 'unit', + value: 'short', + }, + ], + }, + { + matcher: { + id: 'byName', + options: '% of total space usage', + }, + properties: [ + { + id: 'unit', + value: 'percentunit', + }, + ], + }, + ], + }, + }; + +dashboard.new('Home Directory Usage Dashboard') ++ dashboard.withTags(['jupyterhub']) ++ dashboard.withEditable(true) ++ dashboard.withVariables([ + common.variables.prometheus, + common.variables.hub, +]) ++ dashboard.withPanels( + homedirUsage +) From 729f48c04ef922ad50da6cf6a8be94555f7a1039 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 19 Jul 2024 09:25:41 -0700 Subject: [PATCH 2/4] Make panel full width --- dashboards/homedirs.jsonnet | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dashboards/homedirs.jsonnet b/dashboards/homedirs.jsonnet index 801a3f0..1757f93 100755 --- a/dashboards/homedirs.jsonnet +++ b/dashboards/homedirs.jsonnet @@ -139,5 +139,7 @@ dashboard.new('Home Directory Usage Dashboard') common.variables.hub, ]) + dashboard.withPanels( - homedirUsage + grafonnet.util.grid.makeGrid([ + homedirUsage, + ], panelWidth=24, panelHeight=24) ) From fa134c0efc421c4d3d8e04070929bb78cf10c314 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 19 Jul 2024 09:51:35 -0700 Subject: [PATCH 3/4] Add inline documentation --- dashboards/homedirs.jsonnet | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/dashboards/homedirs.jsonnet b/dashboards/homedirs.jsonnet index 1757f93..be43335 100755 --- a/dashboards/homedirs.jsonnet +++ b/dashboards/homedirs.jsonnet @@ -8,26 +8,35 @@ local common = import './common.libsonnet'; local homedirUsage = table.new('Home directory usage') + + table.panelOptions.withDescription( + ||| + Home directory usage by various users on the hub. + + Requires an installation of https://github.com/yuvipanda/prometheus-dirsize-exporter to work. + If this table is empty, your infrastructure administrator needs to deploy that exporter correctly. + ||| + ) + table.queryOptions.withTargets([ + // Last Modified prometheus.new( '$PROMETHEUS_DS', ||| min(dirsize_latest_mtime{namespace=~"$hub"}) by (directory) * 1000 ||| ) - + prometheus.withLegendFormat('Last Modified') - + prometheus.withInstant(true) + + prometheus.withInstant(true) // Only fetch latest value + prometheus.withFormat('table') , + // Total Size prometheus.new( '$PROMETHEUS_DS', ||| max(dirsize_total_size_bytes{namespace=~"$hub"}) by (directory) ||| ) - + prometheus.withLegendFormat('Total Size') + prometheus.withInstant(true) + prometheus.withFormat('table'), + // % of total usage prometheus.new( '$PROMETHEUS_DS', ||| @@ -36,19 +45,19 @@ local homedirUsage = ignoring (directory) group_left sum(dirsize_total_size_bytes{namespace=~"$hub"}) ||| ) - + prometheus.withLegendFormat('% of total space used') + prometheus.withInstant(true) + prometheus.withFormat('table'), + // Total number of files prometheus.new( '$PROMETHEUS_DS', ||| max(dirsize_entries_count{namespace=~"$hub"}) by (directory) ||| ) - + prometheus.withLegendFormat('Number of entries') + prometheus.withInstant(true) + prometheus.withFormat('table'), ]) + // Transform table from multiple series with same key to one unified table with shared key 'directory' + table.queryOptions.withTransformations([ table.queryOptions.transformation.withId('joinByField') + table.queryOptions.transformation.withOptions({ @@ -60,14 +69,17 @@ local homedirUsage = + table.queryOptions.transformation.withOptions({ // Grafana adds an individual 'Time #N' column for each timeseries we get. // They all display the same time. We don't care about time *at all*, since - // all these are instant data query targets. So we hide all the time values. + // all these are instant data query targets that only display latest + // values. So we hide all the time values. excludeByName: { 'Time 1': true, 'Time 2': true, 'Time 3': true, 'Time 4': true, }, - // Explicitly rename the column headers, so they do not display 'Value #N' + // Tables do not use the legend keys, and show Value #N for each Time #N. We + // explicitly rename these here. This depends on the ordering of these series + // above, so if the ordering changes, so must this. renameByName: { 'Value #A': 'Last Modified', 'Value #B': 'Size', @@ -78,10 +90,12 @@ local homedirUsage = ]) + { fieldConfig: { + // Set units for all the columns. These can not be set elsewhere for tables overrides: [ { matcher: { id: 'byName', + // This is name provided by the `renameByName` transform options: 'Size', }, properties: [ @@ -94,6 +108,7 @@ local homedirUsage = { matcher: { id: 'byName', + // This is name provided by the `renameByName` transform options: 'Last Modified', }, properties: [ @@ -106,6 +121,7 @@ local homedirUsage = { matcher: { id: 'byName', + // This is name provided by the `renameByName` transform options: 'Number of Entries', }, properties: [ @@ -118,6 +134,7 @@ local homedirUsage = { matcher: { id: 'byName', + // This is name provided by the `renameByName` transform options: '% of total space usage', }, properties: [ From af69423621093e4f3027b062c79f339eb03328d3 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 19 Jul 2024 09:52:53 -0700 Subject: [PATCH 4/4] Add note about home directory usage dashboard --- docs/tutorials/deploy.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorials/deploy.md b/docs/tutorials/deploy.md index b3502c8..53bdd04 100644 --- a/docs/tutorials/deploy.md +++ b/docs/tutorials/deploy.md @@ -221,6 +221,8 @@ spec: You will likely only need to adjust the `claimName` above to use this example. +This is required for the "Home directory usage" dashboard. + ## Deploy the dashbaords There's a helper `deploy.py` script that can deploy the dashboards to any grafana installation.