add blackbox-exporter, loki/promtail configs; fix prometheus: host network for WG access

prometheus.yml

@@ -8,7 +8,7 @@ rule_files:
   - /etc/prometheus/alerts.yml
 
 scrape_configs:
-  # Garage node metrics (admin API on WG)
+  # Garage node metrics (admin API on WG) — Bearer auth
   - job_name: 'garage'
     metrics_path: /metrics
     scheme: http
@@ -20,16 +20,30 @@ scrape_configs:
         labels:
           cluster: garage
 
-  # Health check for each node (returns 200 if quorum OK)
+  # HTTP health probe via blackbox-exporter (returns probe_success)
   - job_name: 'garage_health'
-    metrics_path: /health
-    scheme: http
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
     static_configs:
-      - targets: ['10.8.0.1:3903', '10.8.0.2:3903', '10.8.0.4:3903']
+      - targets: ['http://10.8.0.1:3903/health']
         labels:
-          cluster: garage
+          instance: vps01
+      - targets: ['http://10.8.0.2:3903/health']
+        labels:
+          instance: bigbox
+      - targets: ['http://10.8.0.4:3903/health']
+        labels:
+          instance: vps02
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: target
+      - target_label: __address__
+        replacement: 127.0.0.1:9115
 
-  # Node exporter on each host (system metrics)
+  # Node exporter on each host (system metrics via WG)
   - job_name: 'node'
     static_configs:
       - targets: ['10.8.0.2:9100']