estorozhenko/monitoring
1
0
Fork 0
mirror of https://gitverse.ru/kpa39l/monitoring.git synced 2026-09-29 09:55:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Этап 6: починка NO DATA в Grafana (3 причины)+relabel instance→hostname; исправлены имена garage-метрик в панелях и алертах; опыт в EXPERIENCE.md

Browse Source
This commit is contained in:
kpa39l
2026-09-03 08:39:53 +00:00
parent 7437864b50
commit 66ff07c52a
8 changed files with 927 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
grafana/dashboards/garage-cluster.json
+598 -45
View File
@@ -9,136 +9,689 @@
"links": [],
"panels": [
{
"datasource": { "type": "prometheus", "uid": "Prometheus" },
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"color": {
"mode": "thresholds"
},
"mappings": [
{ "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" }
{
"options": {
"0": {
"color": "red",
"text": "DOWN"
},
"1": {
"color": "green",
"text": "UP"
}
},
"type": "value"
}
],
"thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 0 },
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.1.0",
"targets": [
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "up{job=\"garage\"}", "legendFormat": "{{ instance }}", "refId": "A" }
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"expr": "up{job=\"garage\"}",
"legendFormat": "{{ instance }}",
"refId": "A"
}
],
"title": "Garage nodes up",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "Prometheus" },
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } },
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 6, "y": 0 },
"gridPos": {
"h": 8,
"w": 6,
"x": 6,
"y": 0
},
"id": 2,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs\"} / 1024 / 1024 / 1024", "legendFormat": "{{ instance }}", "refId": "A" }
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"expr": "node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs\"} / 1024 / 1024 / 1024",
"legendFormat": "{{ instance }}",
"refId": "A"
}
],
"title": "Node disk free (GB)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "Prometheus" },
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } },
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "garage_block_count", "legendFormat": "blocks {{ instance }}", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "garage_block_resync_queue_length", "legendFormat": "resync queue {{ instance }}", "refId": "B" }
{
"expr": "block_resync_queue_length",
"legendFormat": "resync queue {{ instance }}",
"refId": "A"
},
{
"expr": "block_resync_errored_blocks",
"legendFormat": "errored {{ instance }}",
"refId": "B"
}
],
"title": "Garage blocks",
"title": "Garage blocks (resync)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "Prometheus" },
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 4,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "garage_rpc_node_health_is_up", "legendFormat": "{{ instance }}", "refId": "A" }
{
"expr": "cluster_layout_node_connected",
"legendFormat": "{{ role_zone }} {{ instance }}",
"refId": "A"
}
],
"title": "Garage RPC node health",
"title": "Garage node health (layout)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "Prometheus" },
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } },
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 5,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "rate(garage_api_s3_request_counter[5m])", "legendFormat": "{{ instance }} {{ api_endpoint }}", "refId": "A" }
{
"expr": "rate(api_s3_request_counter[5m])",
"legendFormat": "{{ instance }} {{ api_endpoint }}",
"refId": "A"
}
],
"title": "S3 requests /sec",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 16
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"title": "tproxy live sessions/streams",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"expr": "tproxy_sessions_live",
"legendFormat": "sessions {{ instance }}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"expr": "tproxy_streams_live",
"legendFormat": "streams {{ instance }}",
"refId": "B"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 6,
"y": 16
},
"id": 7,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"title": "tproxy traffic /sec",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"expr": "rate(tproxy_bytes_up_total[5m])",
"legendFormat": "up {{ instance }}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"expr": "rate(tproxy_bytes_down_total[5m])",
"legendFormat": "down {{ instance }}",
"refId": "B"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"title": "tproxy backend errors",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"expr": "tproxy_backend_dial_failures_total",
"legendFormat": "dial failures {{ instance }}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"expr": "tproxy_streams_rejected_total",
"legendFormat": "rejected {{ instance }}",
"refId": "B"
}
]
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [ "garage" ],
"templating": { "list": [] },
"time": { "from": "now-6h", "to": "now" },
"tags": [
"garage"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Garage Cluster",
"uid": "garage-cluster",
"version": 1,
"version": 2,
"weekStart": ""
}
grafana/provisioning/datasources/datasources.yml
+2 -1
View File
@@ -4,7 +4,8 @@ datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
uid: Prometheus
url: http://172.28.0.1:9090
isDefault: true
editable: true