Merge pull request #280 from grafana/fix-node-disk-panels

Fixed "Disk Writes" and "Disk Reads" panels

Author: pracucci

CHANGELOG.md

@@ -13,6 +13,7 @@
 * [ENHANCEMENT] Sort legend descending in the CPU/memory panels. #271
 * [ENHANCEMENT] Add config option to enable streaming of chunks in block-based ingesters. #276
 * [BUGFIX] Fixed `CortexQuerierHighRefetchRate` alert. #268
+* [BUGFIX] Fixed "Disk Writes" and "Disk Reads" panels. #280
 
 ## 1.7.0 / 2021-02-24
 

cortex-mixin/config.libsonnet

@@ -45,7 +45,10 @@
     // Whether resources dashboards are enabled (based on cAdvisor metrics).
     resources_dashboards_enabled: false,
 
-    // Used on panels that show metrics per instance.  i.e. 'pod' in a kubernetes install
+    // The label used to differentiate between different application instances (i.e. 'pod' in a kubernetes install).
     per_instance_label: 'pod',
+
+    // The label used to differentiate between different nodes (i.e. servers).
+    per_node_label: 'instance',
   },
 }

cortex-mixin/dashboards/alertmanager-resources.libsonnet

@@ -60,7 +60,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.panel('Writes') +
         $.queryPanel(
-          'sum by(%s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('alertmanager')],
+          'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('alertmanager')],
           '{{%s}} - {{device}}' % $._config.per_instance_label
         ) +
         $.stack +
@@ -69,7 +69,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.panel('Reads') +
         $.queryPanel(
-          'sum by(%s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('alertmanager')],
+          'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('alertmanager')],
           '{{%s}} - {{device}}' % $._config.per_instance_label
         ) +
         $.stack +

cortex-mixin/dashboards/compactor-resources.libsonnet

@@ -36,7 +36,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.panel('Disk Writes') +
         $.queryPanel(
-          'sum by(%s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('compactor')],
+          'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('compactor')],
           '{{%s}} - {{device}}' % $._config.per_instance_label
         ) +
         $.stack +
@@ -45,7 +45,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.panel('Disk Reads') +
         $.queryPanel(
-          'sum by(%s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('compactor')],
+          'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('compactor')],
           '{{%s}} - {{device}}' % $._config.per_instance_label
         ) +
         $.stack +

cortex-mixin/dashboards/dashboard-utils.libsonnet

@@ -257,9 +257,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
       { yaxes: $.yaxes('percentunit') },
     ),
 
-
   filterNodeDiskContainer(containerName)::
     |||
-      ignoring(%s) group_right() (label_replace(count by(%s, device) (container_fs_writes_bytes_total{%s,container="%s",device!~".*sda.*"}), "device", "$1", "device", "/dev/(.*)") * 0)
-    ||| % [$._config.per_instance_label, $._config.per_instance_label, $.namespaceMatcher(), containerName],
+      ignoring(%s) group_right() (label_replace(count by(%s, %s, device) (container_fs_writes_bytes_total{%s,container="%s",device!~".*sda.*"}), "device", "$1", "device", "/dev/(.*)") * 0)
+    ||| % [$._config.per_instance_label, $._config.per_node_label, $._config.per_instance_label, $.namespaceMatcher(), containerName],
 }

cortex-mixin/dashboards/reads-resources.libsonnet

@@ -105,7 +105,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.panel('Disk Writes') +
         $.queryPanel(
-          'sum by(%s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('store-gateway')],
+          'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('store-gateway')],
           '{{%s}} - {{device}}' % $._config.per_instance_label
         ) +
         $.stack +
@@ -114,7 +114,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.panel('Disk Reads') +
         $.queryPanel(
-          'sum by(%s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('store-gateway')],
+          'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('store-gateway')],
           '{{%s}} - {{device}}' % $._config.per_instance_label
         ) +
         $.stack +

cortex-mixin/dashboards/writes-resources.libsonnet

@@ -58,7 +58,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.panel('Disk Writes') +
         $.queryPanel(
-          'sum by(%s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('ingester')],
+          'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('ingester')],
           '{{%s}} - {{device}}' % $._config.per_instance_label
         ) +
         $.stack +
@@ -67,7 +67,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.panel('Disk Reads') +
         $.queryPanel(
-          'sum by(%s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('ingester')],
+          'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('ingester')],
           '{{%s}} - {{device}}' % $._config.per_instance_label
         ) +
         $.stack +