Initial import: vesti.nixg.ru — новостной апрув-проект (web, crawler, classifier, publisher, openspec)

2026-09-29 18:05:03 +00:00 · 2026-09-13 15:58:32 +00:00
commit c3f59f7b7a
113 changed files with 7065 additions and 0 deletions
@@ -0,0 +1,140 @@
+#!/bin/bash
+# VESTI — textfile-коллектор для node-exporter (bigbox).
+# Пишет /var/lib/node_exporter/textfile_collector/vesti.prom (Prometheus-формат).
+# Метрики:
+#   vesti_web_up           1 = веб :8400 отвечает 200
+#   vesti_publisher_health 1 = publisher :8410 /healthz ок, 0 = нет
+#   vesti_publisher_bot    1 = publisher отвечает, объект bot задан
+#   vesti_publisher_proxy  1 = publisher отвечает, proxy задан
+#   vesti_publisher_channels N = число каналов из healthz
+#   vesti_telegram_tunnel  1 = SOCKS5-туннель :1080 слушает
+#   vesti_publisher_docker 1 = контейнер vesti-publisher running
+#   vesti_web_systemd      1 = юнит vesti-web.active
+#   vesti_db_ok            1 = БД vesti.db открывается
+#   vesti_db_posts_total   N = постов в БД
+#   vesti_db_new_total     N = постов со статусом 'new'
+#   vesti_db_published_total N = постов 'published'
+#   vesti_db_last_run_ts   unix-ts последнего run (0, если нет)
+#
+# Помечено label'ом component="vesti". Скрипт тихий: всегда успех, файл пишется
+# атомарно (tmp+mv). Периодичность — systemd timer (vesti-metrics.timer) каждые 30с.
+set -u
+
+OUT=/var/lib/node_exporter/textfile_collector/vesti.prom
+TMP="${OUT}.tmp"
+
+# --- безопасные подстановки: число или 0, строка без кавычек/переводов ---
+clean() { tr -cd '0-9' <<<"$1"; }
+cleanstr() { tr -d '\n\r"' <<<"$1"; }
+
+web_up=0; pub_health=0; pub_bot=0; pub_proxy=0; pub_channels=0
+tunnel=0; pub_docker=0; web_sys=0; db_ok=0
+posts=0; new=0; published=0; last_run=0
+
+# 1) веб :8400 — достаточно login-страницы (200)
+if curl -sf --max-time 5 -o /dev/null http://127.0.0.1:8400/login; then
+  web_up=1
+fi
+
+# 2) publisher :8410 /healthz — JSON
+HEALTH=$(curl -sf --max-time 5 http://127.0.0.1:8410/healthz 2>/dev/null) && pub_health=1
+if [ -n "$HEALTH" ]; then
+  # {"status":"ok","bot":"...","token_set":true,"proxy":"...","channels":[...]}
+  case "$HEALTH" in
+    *'"bot"'*'"'*) pub_bot=1 ;;
+  esac
+  case "$HEALTH" in
+    *'"proxy"'*'"'*) pub_proxy=1 ;;
+  esac
+  # число каналов
+  CH=$(grep -o '\[[^]]*\]' <<<"$HEALTH" | head -1 | tr -cd ',"@0-9a-zA-Z_.-' | grep -o '@' | wc -l)
+  # если нет массива — нет каналов
+  pub_channels=$(clean "$CH")
+fi
+
+# 3) SOCKS5-туннель :1080 (tcp connect)
+if timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/1080' 2>/dev/null; then
+  tunnel=1
+fi
+
+# 4) контейнер vesti-publisher
+if docker ps --filter name=^vesti-publisher$ --filter status=running --format '{{.Names}}' 2>/dev/null | grep -q vesti-publisher; then
+  pub_docker=1
+fi
+
+# 5) systemd vesti-web
+if systemctl is-active --quiet vesti-web; then
+  web_sys=1
+fi
+
+# 6) БД vesti.db
+if DB=$(/opt/vesti/.venv/bin/python - <<'PY' 2>/dev/null
+import sqlite3
+c = sqlite3.connect('/opt/vesti/db/vesti.db', timeout=5)
+def one(q):
+    try:
+        return c.execute(q).fetchone()[0]
+    except Exception:
+        return None
+posts  = one('SELECT count(*) FROM posts') or 0
+new    = one("SELECT count(*) FROM posts WHERE status='new'") or 0
+pub    = one("SELECT count(*) FROM posts WHERE status='published'") or 0
+run    = one('SELECT max(started_at) FROM runs')
+import datetime as dt
+ts = 0
+if run:
+    try:
+        ts = int(dt.datetime.fromisoformat(run).timestamp())
+    except Exception:
+        ts = 0
+print(posts, new, pub, ts)
+PY
+); then
+  db_ok=1
+  set -- $DB
+  posts=$(clean "${1:-0}"); new=$(clean "${2:-0}"); published=$(clean "${3:-0}"); last_run=$(clean "${4:-0}")
+fi
+
+# --- запись (атомарно) ---
+{
+  echo '# HELP vesti_web_up VESTI web :8400 responds'
+  echo '# TYPE vesti_web_up gauge'
+  echo "vesti_web_up{component=\"vesti\"} $web_up"
+  echo '# HELP vesti_publisher_health VESTI publisher :8410 /healthz ok'
+  echo '# TYPE vesti_publisher_health gauge'
+  echo "vesti_publisher_health{component=\"vesti\"} $pub_health"
+  echo '# HELP vesti_publisher_bot publisher reports bot identity'
+  echo '# TYPE vesti_publisher_bot gauge'
+  echo "vesti_publisher_bot{component=\"vesti\"} $pub_bot"
+  echo '# HELP vesti_publisher_proxy publisher reports SOCKS5 proxy'
+  echo '# TYPE vesti_publisher_proxy gauge'
+  echo "vesti_publisher_proxy{component=\"vesti\"} $pub_proxy"
+  echo '# HELP vesti_publisher_channels number of channels in publisher healthz'
+  echo '# TYPE vesti_publisher_channels gauge'
+  echo "vesti_publisher_channels{component=\"vesti\"} $pub_channels"
+  echo '# HELP vesti_telegram_tunnel SOCKS5 tunnel :1080 listening'
+  echo '# TYPE vesti_telegram_tunnel gauge'
+  echo "vesti_telegram_tunnel{component=\"vesti\"} $tunnel"
+  echo '# HELP vesti_publisher_docker vesti-publisher container running'
+  echo '# TYPE vesti_publisher_docker gauge'
+  echo "vesti_publisher_docker{component=\"vesti\"} $pub_docker"
+  echo '# HELP vesti_web_systemd vesti-web systemd unit active'
+  echo '# TYPE vesti_web_systemd gauge'
+  echo "vesti_web_systemd{component=\"vesti\"} $web_sys"
+  echo '# HELP vesti_db_ok vesti.db accessible'
+  echo '# TYPE vesti_db_ok gauge'
+  echo "vesti_db_ok{component=\"vesti\"} $db_ok"
+  echo '# HELP vesti_db_posts_total posts in DB'
+  echo '# TYPE vesti_db_posts_total gauge'
+  echo "vesti_db_posts_total{component=\"vesti\"} $posts"
+  echo '# HELP vesti_db_new_total posts with status=new'
+  echo '# TYPE vesti_db_new_total gauge'
+  echo "vesti_db_new_total{component=\"vesti\"} $new"
+  echo '# HELP vesti_db_published_total posts with status=published'
+  echo '# TYPE vesti_db_published_total gauge'
+  echo "vesti_db_published_total{component=\"vesti\"} $published"
+  echo '# HELP vesti_db_last_run_ts unix ts of last crawler run'
+  echo '# TYPE vesti_db_last_run_ts gauge'
+  echo "vesti_db_last_run_ts{component=\"vesti\"} $last_run"
+} > "$TMP" && chown prometheus:prometheus "$TMP" && chmod 644 "$TMP" && mv -f "$TMP" "$OUT"
+exit 0