feat: VESTI dashboard — textfile-коллектор vesti_* метрик + дашборд (доступность компонентов)

2026-09-29 01:50:07 +00:00 · 2026-09-13 15:06:10 +00:00
parent 3c5e9e5a71
commit 847fcfad9c
4 changed files with 1389 additions and 4 deletions
@@ -1,15 +1,36 @@
 # Мониторинг кластера — Статус

-Обновлено: 2026-09-08 (сессия @session:default/20260908_180858_730d1b)
+Обновлено: 2026-09-13 (добавлен дашборд VESTI)

 ## Текущее состояние
 Стек мониторинга на bigbox (/opt/monitoring): Prometheus 2.53 + Grafana 11.1 +
 Loki + promtail. Собирает: Garage-кластер (vps01/bigbox/vps02 :3903), системы
-всех 4 нод (node-exporter :9100 по WireGuard), tproxy (vps03, loopback-туннель).
+всех 4 нод (node-exporter :9100 по WireGuard), tproxy (vps03, loopback-туннель),
+**VESTI (компоненты/доступность)**.
 Дашборды: **4 per-node** (Node vps01/vps02/vps03/bigbox) + Garage Cluster +
-Vinograd WAN + tproxy-панели. vps03 подключён к WG (10.8.0.3), порты наружу
+Vinograd WAN + tproxy-панели + **VESTI**. vps03 подключён к WG (10.8.0.3), порты наружу
 не светятся. Все 4 node-таргета up. Всё в git (gitverse + gitea mirror).

+## VESTI (добавлено 2026-09-13)
+- Метрики: textfile-коллектор node-exporter на bigbox, скрипт
+  `/opt/vesti/scripts/vesti-metrics.sh` (запуск: `/etc/cron.d/vesti-metrics`, каждую минуту, root).
+  Пишет `/var/lib/node_exporter/textfile_collector/vesti.prom`.
+- Метрики `vesti_*` (label `component="vesti"`), собираются job'ом `node` (10.8.0.2:9100):
+  `vesti_web_up`, `vesti_publisher_health`, `vesti_publisher_bot`, `vesti_publisher_proxy`,
+  `vesti_publisher_channels`, `vesti_telegram_tunnel`, `vesti_publisher_docker`,
+  `vesti_web_systemd`, `vesti_db_ok`, `vesti_db_posts_total`, `vesti_db_new_total`,
+  `vesti_db_published_total`, `vesti_db_last_run_ts`.
+- node-exporter на bigbox: `ARGS="--web.listen-address=10.8.0.2:9100 --collector.textfile.directory=/var/lib/node_exporter/textfile_collector"`
+  (в /etc/default/prometheus-node-exporter).
+- Дашборд: `grafana/dashboards/vesti/vesti.json` (uid `vesti`, папка **vesti**), 18 панелей:
+  stat-панели доступности (web/publisher/Docker/tunnel/DB/systemd) + детали publisher
+  (bot/proxy/channels) + БД (posts/new/published + время с последнего рan) + история доступности.
+- Провайдер `vesti-dashboards` добавлен в `grafana/provisioning/dashboards/dashboards.yml`
+  (path: /var/lib/grafana/dashboards/vesti).
+- ⚠️ Известное: systemd1 на bigbox снова завис (systemctl висит) — таймер vesti-metrics.timer
+  не заведён, вместо него cron.d (работает). При следующем ребуте systemd1 оживёт;
+  тогда можно перевести на systemd-timer (unit-файлы уже написаны: /etc/systemd/system/vesti-metrics.{service,timer}).
+
 ## Сделано (за сессию 2026-09-08)
 - vps03 подключён к WireGuard: 10.8.0.3/24, hub vps01 (5.129.217.146:51820),
  пиры добавлены на vps01/bigbox/vps02 (AllowedIPs пира vps01 расширен до
@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+# Генератор дашборда VESTI для Grafana (provisioning).
+# Соглашения те же, что в node-<host>.json: datasource uid=Prometheus,
+# stat-панели с mapping 0/1, timeseries.
+import json
+
+DS = {"type": "prometheus", "uid": "Prometheus"}
+
+# --- утилиты ---
+def stat_panel(title, expr, mapping, idx, w=3, h=3, x=0, y=0, legend=""):
+    return {
+        "datasource": DS,
+        "fieldConfig": {
+            "defaults": {
+                "color": {"mode": "thresholds"},
+                "mappings": ([{"options": mapping, "type": "value"}] if mapping else []),
+                "thresholds": {
+                    "mode": "absolute",
+                    "steps": [{"color": "red", "value": None}, {"color": "green", "value": 1}],
+                },
+                "unit": "short",
+            },
+            "overrides": [],
+        },
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "id": idx,
+        "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
+            "textMode": "auto",
+        },
+        "pluginVersion": "11.1.0",
+        "targets": [{"datasource": DS, "expr": expr, "legendFormat": legend, "refId": "A"}],
+        "title": title,
+        "type": "stat",
+    }
+
+def timeseries_panel(title, exprs, idx, w=12, h=8, x=0, y=0, unit="short"):
+    targets = []
+    for e, l in exprs:
+        targets.append({"datasource": DS, "expr": e, "legendFormat": l, "refId": chr(65 + len(targets))})
+    return {
+        "datasource": DS,
+        "fieldConfig": {
+            "defaults": {
+                "color": {"mode": "palette-classic"},
+                "custom": {
+                    "axisCenteredZero": False, "axisColorMode": "text", "axisLabel": "",
+                    "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10,
+                    "gradientMode": "none", "hideFrom": {"legend": False, "tooltip": False, "viz": False},
+                    "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5,
+                    "scaleDistribution": {"type": "linear"}, "showPoints": "never",
+                    "spanNulls": False, "stacking": {"group": "A", "mode": "none"},
+                    "thresholdsStyle": {"mode": "off"},
+                },
+                "mappings": [],
+                "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}]},
+                "unit": unit,
+            },
+            "overrides": [],
+        },
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "id": idx,
+        "options": {
+            "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": True},
+            "tooltip": {"mode": "multi", "sort": "none"},
+        },
+        "targets": targets,
+        "title": title,
+        "type": "timeseries",
+    }
+
+panels = []
+y = 0
+idx = 1
+
+# Row 1: Доступность (stat, 3x4 = 12 панелей в 2 ряда по 6)
+row_title = {
+    "collapsed": False,
+    "datasource": DS,
+    "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+    "id": idx, "panels": [], "title": "Доступность компонентов", "type": "row",
+}
+idx += 1
+y += 1
+panels.append(row_title)
+
+stats = [
+    ("Web :8400 (systemd)", "vesti_web_up{component=\"vesti\"}", {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}),
+    ("Publisher :8410 (healthz)", "vesti_publisher_health{component=\"vesti\"}", {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}),
+    ("Publisher (Docker)", "vesti_publisher_docker{component=\"vesti\"}", {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}),
+    ("Telegram tunnel :1080", "vesti_telegram_tunnel{component=\"vesti\"}", {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}),
+    ("DB vesti.db", "vesti_db_ok{component=\"vesti\"}", {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}),
+    ("Web systemd unit", "vesti_web_systemd{component=\"vesti\"}", {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}),
+]
+x = 0
+for i, (title, expr, mp) in enumerate(stats):
+    panels.append(stat_panel(title, expr, mp, idx, w=4, h=3, x=(i % 6) * 4, y=y + (i // 6) * 3))
+    idx += 1
+y += 6  # 2 ряда по 3 = 6 строк
+
+# Row 2: Publisher healthz детали (bot/proxy/channels)
+row2 = {
+    "collapsed": False, "datasource": DS,
+    "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+    "id": idx, "panels": [], "title": "Publisher — детали", "type": "row",
+}
+idx += 1
+y += 1
+panels.append(row2)
+for i, (title, expr) in enumerate([
+    ("Bot identity", 'vesti_publisher_bot{component="vesti"}'),
+    ("Proxy SOCKS5", 'vesti_publisher_proxy{component="vesti"}'),
+    ("Channels", 'vesti_publisher_channels{component="vesti"}'),
+]):
+    panels.append(stat_panel(title, expr, None, idx, w=4, h=3, x=i * 4, y=y))
+    idx += 1
+y += 3
+
+# Row 3: БД
+row3 = {
+    "collapsed": False, "datasource": DS,
+    "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+    "id": idx, "panels": [], "title": "База данных VESTI", "type": "row",
+}
+idx += 1
+y += 1
+panels.append(row3)
+
+# stat: постов в БД, новых, опубликовано, последний рan
+panels.append(stat_panel("Постов в БД", 'vesti_db_posts_total{component="vesti"}', None, idx, w=4, h=3, x=0, y=y))
+idx += 1
+panels.append(stat_panel("Новых", 'vesti_db_new_total{component="vesti"}', None, idx, w=4, h=3, x=4, y=y))
+idx += 1
+panels.append(stat_panel("Опубликовано", 'vesti_db_published_total{component="vesti"}', None, idx, w=4, h=3, x=8, y=y))
+idx += 1
+# временной ряд последнего рan (ts → время)
+panels.append(timeseries_panel("Последний рan краулера", [('time() - vesti_db_last_run_ts{component="vesti"}', "сек назад")], idx, w=12, h=4, x=12, y=y, unit="s"))
+idx += 1
+y += 4
+
+# Row 4: Временные ряды доступности (возврат по вре)
+row4 = {
+    "collapsed": False, "datasource": DS,
+    "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+    "id": idx, "panels": [], "title": "История доступности (1/0)", "type": "row",
+}
+idx += 1
+y += 1
+panels.append(row4)
+panels.append(timeseries_panel(
+    "Доступность компонентов",
+    [(f'{m}{{component="vesti"}}', t) for m, t in [
+        ("vesti_web_up", "web"),
+        ("vesti_publisher_health", "publisher"),
+        ("vesti_publisher_docker", "docker"),
+        ("vesti_telegram_tunnel", "tunnel"),
+        ("vesti_db_ok", "db"),
+        ("vesti_web_systemd", "web_sys"),
+    ]],
+    idx, w=24, h=8, x=0, y=y, unit="short"))
+idx += 1
+y += 8
+
+dashboard = {
+    "annotations": {"list": []},
+    "editable": True,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 1,
+    "id": None,
+    "links": [],
+    "panels": panels,
+    "refresh": "15s",
+    "schemaVersion": 1,
+    "tags": ["vesti", "bigbox"],
+    "time": {"from": "now-6h", "to": "now"},
+    "timepicker": [{"refresh": "15s"}],
+    "title": "VESTI",
+    "uid": "vesti",
+    "version": 1,
+}
+
+with open("/opt/monitoring/grafana/dashboards/vesti/vesti.json", "w") as f:
+    json.dump(dashboard, f, indent=2, ensure_ascii=False)
+print("written /opt/monitoring/grafana/dashboards/vesti/vesti.json", len(panels), "pabels")
@@ -26,4 +26,13 @@ providers:
    disableDeletion: false
    updateIntervalSeconds: 30
    options:
-      path: /var/lib/grafana/dashboards/vinogorod
+      path: /var/lib/grafana/dashboards/vinogorod
+
+  - name: 'vesti-dashboards'
+    orgId: 1
+    folder: 'vesti'
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    options:
+      path: /var/lib/grafana/dashboards/vesti