Этап 6: починка NO DATA в Grafana (3 причины)+relabel instance→hostname; исправлены имена garage-метрик в панелях и алертах; опыт в EXPERIENCE.md

2026-09-29 09:55:09 +00:00 · 2026-09-03 08:39:53 +00:00
parent 7437864b50
commit 66ff07c52a
8 changed files with 927 additions and 189 deletions
@@ -2,61 +2,109 @@ global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
-    monitor: 'garage-cluster'
-
+    monitor: garage-cluster
 rule_files:
-  - /etc/prometheus/alerts.yml
-
+- /etc/prometheus/alerts.yml
 scrape_configs:
-  # Garage node metrics (admin API on WG) — Bearer auth
-  - job_name: 'garage'
-    metrics_path: /metrics
-    scheme: http
-    authorization:
-      type: Bearer
-      credentials: c213debe864061cefe501f7791d557b6dba701710b41791b4a4a0fb036690d1d
-    static_configs:
-      - targets: ['10.8.0.1:3903', '10.8.0.2:3903', '10.8.0.4:3903']
-        labels:
-          cluster: garage
-
-  # HTTP health probe via blackbox-exporter (returns probe_success)
-  - job_name: 'garage_health'
-    metrics_path: /probe
-    params:
-      module: [http_2xx]
-    static_configs:
-      - targets: ['http://10.8.0.1:3903/health']
-        labels:
-          instance: vps01
-      - targets: ['http://10.8.0.2:3903/health']
-        labels:
-          instance: bigbox
-      - targets: ['http://10.8.0.4:3903/health']
-        labels:
-          instance: vps02
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: target
-      - target_label: __address__
-        replacement: 127.0.0.1:9115
-
-  # Node exporter on each host (system metrics via WG)
-  - job_name: 'node'
-    static_configs:
-      - targets: ['10.8.0.2:9100']
-        labels:
-          host: bigbox
-      - targets: ['10.8.0.1:9100']
-        labels:
-          host: vps01
-      - targets: ['10.8.0.4:9100']
-        labels:
-          host: vps02
-
-  # Prometheus self
-  - job_name: 'prometheus'
-    static_configs:
-      - targets: ['localhost:9090']
+- job_name: garage
+  metrics_path: /metrics
+  scheme: http
+  authorization:
+    type: Bearer
+    credentials: c213debe864061cefe501f7791d557b6dba701710b41791b4a4a0fb036690d1d
+  static_configs:
+  - targets:
+    - 10.8.0.1:3903
+    - 10.8.0.2:3903
+    - 10.8.0.4:3903
+    labels:
+      cluster: garage
+  relabel_configs:
+  - source_labels:
+    - __address__
+    regex: 10.8.0.1:3903
+    target_label: instance
+    replacement: vps01:3903
+  - source_labels:
+    - __address__
+    regex: 10.8.0.2:3903
+    target_label: instance
+    replacement: bigbox:3903
+  - source_labels:
+    - __address__
+    regex: 10.8.0.4:3903
+    target_label: instance
+    replacement: vps02:3903
+- job_name: garage_health
+  metrics_path: /probe
+  params:
+    module:
+    - http_2xx
+  static_configs:
+  - targets:
+    - http://10.8.0.1:3903/health
+    labels:
+      instance: vps01
+  - targets:
+    - http://10.8.0.2:3903/health
+    labels:
+      instance: bigbox
+  - targets:
+    - http://10.8.0.4:3903/health
+    labels:
+      instance: vps02
+  relabel_configs:
+  - source_labels:
+    - __address__
+    target_label: __param_target
+  - source_labels:
+    - __param_target
+    target_label: target
+  - target_label: __address__
+    replacement: 127.0.0.1:9115
+- job_name: node
+  static_configs:
+  - targets:
+    - 10.8.0.2:9100
+    labels:
+      host: bigbox
+  - targets:
+    - 10.8.0.1:9100
+    labels:
+      host: vps01
+  - targets:
+    - 10.8.0.4:9100
+    labels:
+      host: vps02
+  relabel_configs:
+  - source_labels:
+    - __address__
+    regex: 10.8.0.2:9100
+    target_label: instance
+    replacement: bigbox:9100
+  - source_labels:
+    - __address__
+    regex: 10.8.0.1:9100
+    target_label: instance
+    replacement: vps01:9100
+  - source_labels:
+    - __address__
+    regex: 10.8.0.4:9100
+    target_label: instance
+    replacement: vps02:9100
+- job_name: prometheus
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: tproxy
+  static_configs:
+  - targets:
+    - 127.0.0.1:18081
+    labels:
+      host: vps03
+      service: tproxy
+  relabel_configs:
+  - target_label: instance
+    replacement: vps03:8081
+  - target_label: host
+    replacement: vps03