Этап 6: починка NO DATA в Grafana (3 причины)+relabel instance→hostname; исправлены имена garage-метрик в панелях и алертах; опыт в EXPERIENCE.md

2026-09-29 09:55:09 +00:00 · 2026-09-03 08:39:53 +00:00
parent 7437864b50
commit 66ff07c52a
8 changed files with 927 additions and 189 deletions
@@ -1,34 +1,47 @@
 groups:
-  - name: garage
-    rules:
-      - alert: GarageNodeDown
-        expr: up{job="garage"} == 0
-        for: 2m
-        labels:
-          severity: critical
-        annotations:
-          summary: "Garage node {{ $labels.instance }} is down"
-
-      - alert: GarageNoQuorum
-        expr: (count(up{job="garage"} == 1) < 2)
-        for: 2m
-        labels:
-          severity: critical
-        annotations:
-          summary: "Garage cluster lost quorum (<2 nodes up)"
-
-      - alert: GarageResyncErrors
-        expr: garage_block_resync_error_count > 0
-        for: 10m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Garage resync errors on {{ $labels.instance }}"
-
-      - alert: GarageNodeUnstable
-        expr: garage_rpc_node_health_is_up == 0
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Garage RPC health of some node is down"
+- name: garage
+  rules:
+  - alert: GarageNodeDown
+    expr: up{job="garage"} == 0
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: Garage node {{ $labels.instance }} is down
+  - alert: GarageNoQuorum
+    expr: (count(up{job="garage"} == 1) < 2)
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: Garage cluster lost quorum (<2 nodes up)
+  - alert: GarageResyncErrors
+    expr: block_resync_errored_blocks > 0
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      summary: Garage resync errors on {{ $labels.instance }}
+  - alert: GarageNodeUnstable
+    expr: cluster_layout_node_connected == 0
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Garage RPC health of some node is down
+- name: tproxy
+  rules:
+  - alert: TProxyDown
+    expr: up{job="tproxy"} == 0
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: tproxy-server (vps03) is down or unreachable
+  - alert: TProxyBackendErrors
+    expr: increase(tproxy_backend_dial_failures_total[5m]) > 0
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      summary: tproxy-server backend dial failures (MTProxy unreachable)