Initial commit: Email Assistant project

scripts/mail_index.py

@@ -0,0 +1,317 @@
+#!/usr/bin/env python3
+"""
+mail_index.py — SQLite-индекс всех email.md для быстрого поиска и трекинга.
+
+Создаёт /opt/hermes/email/mail_index.db с таблицами:
+  - emails: path, uid, folder, date, from_addr, to_addrs, subject,
+            contacts_extracted, contacts_skipped, first_seen, last_scanned
+  - email_fts (FTS5): полнотекстовый поиск по subject + from + to
+
+Использование:
+  python3 mail_index.py                — полная переиндексация
+  python3 mail_index.py --incremental  — только новые файлы (по mtime last_index)
+  python3 mail_index.py --search "запрос"  — поиск по индексу
+"""
+
+import os
+import re
+import sqlite3
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+EMAIL_ROOT = Path("/opt/hermes/email")
+DB_PATH = EMAIL_ROOT / "mail_index.db"
+CONTACTS_DIR = EMAIL_ROOT / "contacts"
+LAST_SCAN_FILE = CONTACTS_DIR / "last_scan.json"
+STATE_DIR = EMAIL_ROOT / "state"
+
+# Папки, которые не индексируем
+EXCLUDE_DIRS = {"contacts", "state"}
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS emails (
+    path TEXT PRIMARY KEY,              -- относительный путь от EMAIL_ROOT
+    uid INTEGER,                        -- числовой UID из пути
+    folder TEXT,                        -- INBOX, INBOX/!Scan, Sent...
+    date TEXT,                          -- дата из frontmatter (ISO)
+    from_addr TEXT,                     -- отправитель
+    to_addrs TEXT,                      -- получатели
+    subject TEXT,                       -- тема
+    body_preview TEXT,                  -- первые 500 символов тела (без HTML)
+    contacts_extracted INTEGER DEFAULT 0,  -- 0/1
+    contacts_skipped INTEGER DEFAULT 0,    -- 0/1 (нет подписи / LLM error)
+    first_seen TEXT,                    -- когда проиндексировано
+    last_scanned TEXT,                  -- последняя проверка contacts
+    file_mtime REAL                     -- mtime файла для инкрементальной проверки
+);
+
+CREATE INDEX IF NOT EXISTS idx_emails_folder ON emails(folder);
+CREATE INDEX IF NOT EXISTS idx_emails_uid ON emails(uid);
+CREATE INDEX IF NOT EXISTS idx_emails_contacts ON emails(contacts_extracted);
+CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS email_fts USING fts5(
+    subject, from_addr, to_addrs, body_preview,
+    content='emails',
+    content_rowid='rowid',
+    tokenize='unicode61'
+);
+
+-- Триггеры для синхронизации FTS
+CREATE TRIGGER IF NOT EXISTS emails_ai AFTER INSERT ON emails BEGIN
+    INSERT INTO email_fts(rowid, subject, from_addr, to_addrs, body_preview)
+    VALUES (new.rowid, new.subject, new.from_addr, new.to_addrs, new.body_preview);
+END;
+
+CREATE TRIGGER IF NOT EXISTS emails_ad AFTER DELETE ON emails BEGIN
+    INSERT INTO email_fts(email_fts, rowid, subject, from_addr, to_addrs, body_preview)
+    VALUES ('delete', old.rowid, old.subject, old.from_addr, old.to_addrs, old.body_preview);
+END;
+
+CREATE TRIGGER IF NOT EXISTS emails_au AFTER UPDATE ON emails BEGIN
+    INSERT INTO email_fts(email_fts, rowid, subject, from_addr, to_addrs, body_preview)
+    VALUES ('delete', old.rowid, old.subject, old.from_addr, old.to_addrs, old.body_preview);
+    INSERT INTO email_fts(rowid, subject, from_addr, to_addrs, body_preview)
+    VALUES (new.rowid, new.subject, new.from_addr, new.to_addrs, new.body_preview);
+END;
+"""
+
+
+def get_db():
+    """Открыть/создать БД."""
+    conn = sqlite3.connect(str(DB_PATH))
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.executescript(SCHEMA)
+    return conn
+
+
+def parse_frontmatter(content):
+    """Парсит YAML-frontmatter из email.md."""
+    meta = {}
+    m = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
+    if not m:
+        return meta
+    yaml_block = m.group(1)
+    for line in yaml_block.split('\n'):
+        line = line.strip()
+        if ':' in line:
+            key, _, val = line.partition(':')
+            key = key.strip().lower()
+            val = val.strip().strip('"').strip("'")
+            # Обрабатываем списки
+            if val.startswith('[') and val.endswith(']'):
+                val = val[1:-1].replace('"', '').replace("'", '').strip()
+            meta[key] = val
+    return meta
+
+
+def clean_html(text):
+    """Удалить HTML-теги и мусор."""
+    text = re.sub(r'<#part[^>]*>', '', text)
+    text = re.sub(r'<#/part>', '', text)
+    text = re.sub(r'<[^>]+>', '', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'&[a-z]+;', ' ', text)
+    text = re.sub(r'https?://\S+', '', text)
+    return text.strip()[:500]
+
+
+def extract_uid_from_path(path):
+    """Извлекает UID из пути: .../YYYY/MM/UID/email.md"""
+    m = re.search(r'/(\d+)/email\.md$', str(path))
+    if m:
+        return int(m.group(1))
+    return None
+
+
+def find_email_md_files(root_path):
+    """Найти все email.md, исключая contacts/ и state/."""
+    files = []
+    root_path = Path(root_path)
+    for f in sorted(root_path.rglob("email.md")):
+        rel = f.relative_to(root_path)
+        parts = rel.parts
+        # Пропускаем служебные папки
+        if any(p in EXCLUDE_DIRS for p in parts):
+            continue
+        files.append(f)
+    return files
+
+
+def index_emails(incremental=False):
+    """Проиндексировать все email.md в SQLite."""
+    conn = get_db()
+    cursor = conn.cursor()
+
+    if incremental:
+        # Получаем последний mtime из БД
+        cursor.execute("SELECT COALESCE(MAX(file_mtime), 0) FROM emails")
+        last_mtime = cursor.fetchone()[0] or 0
+        print(f"📇 Инкрементальная индексация (mtime > {last_mtime})", flush=True)
+    else:
+        last_mtime = 0
+        print("📇 Полная индексация", flush=True)
+
+    email_files = find_email_md_files(EMAIL_ROOT)
+
+    new_count = 0
+    updated_count = 0
+    total = len(email_files)
+
+    for i, f in enumerate(email_files):
+        rel_path = str(f.relative_to(EMAIL_ROOT))
+        mtime = os.path.getmtime(f)
+
+        if incremental and mtime <= last_mtime:
+            continue
+
+        content = f.read_text(encoding='utf-8', errors='replace')
+        meta = parse_frontmatter(content)
+        uid = extract_uid_from_path(f)
+        folder = meta.get('folder', '') or str(f.parent.parent.parent.relative_to(EMAIL_ROOT))
+        subject = meta.get('subject', '')
+        from_addr = meta.get('from', '')
+        to_addrs = meta.get('to', '')
+        date_val = meta.get('date', '')
+
+        # Тело
+        body_match = re.search(r'^---\s*\n(.*?)\n---\s*\n(.*)', content, re.DOTALL)
+        body_preview = ''
+        if body_match:
+            body_preview = clean_html(body_match.group(2))
+        if not body_preview:
+            body_preview = clean_html(content)
+
+        cursor.execute("""
+            INSERT INTO emails (path, uid, folder, date, from_addr, to_addrs, subject,
+                                body_preview, first_seen, file_mtime)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(path) DO UPDATE SET
+                uid=excluded.uid,
+                folder=excluded.folder,
+                date=excluded.date,
+                from_addr=excluded.from_addr,
+                to_addrs=excluded.to_addrs,
+                subject=excluded.subject,
+                body_preview=excluded.body_preview,
+                file_mtime=excluded.file_mtime
+        """, (
+            rel_path, uid, folder, date_val, from_addr, to_addrs, subject,
+            body_preview, datetime.now(timezone.utc).isoformat(), mtime
+        ))
+
+        if cursor.rowcount == 1 and cursor.lastrowid:
+            new_count += 1
+        else:
+            updated_count += 1
+
+        if (i + 1) % 200 == 0:
+            conn.commit()
+            print(f"  {i+1}/{total} ({new_count} new, {updated_count} updated)", flush=True)
+
+    conn.commit()
+
+    # Обновляем FTS5 (перестроить для согласованности)
+    cursor.execute("INSERT INTO email_fts(email_fts) VALUES('rebuild')")
+    conn.commit()
+
+    print(f"\n✅ Готово: {total} файлов, {new_count} новых, {updated_count} обновлено")
+    print(f"  База: {DB_PATH}")
+
+    cursor.execute("SELECT COUNT(*) FROM emails")
+    total_in_db = cursor.fetchone()[0]
+    cursor.execute("SELECT COUNT(*) FROM emails WHERE contacts_extracted=1")
+    extracted = cursor.fetchone()[0]
+    print(f"  Всего в БД: {total_in_db}, контакты извлечены: {extracted}")
+
+    conn.close()
+
+
+def search(query, limit=10):
+    """Поиск по FTS5."""
+    conn = get_db()
+    cursor = conn.cursor()
+
+    try:
+        cursor.execute("""
+            SELECT e.path, e.folder, e.date, e.from_addr, e.subject, e.contacts_extracted,
+                   e.contacts_skipped, e.body_preview
+            FROM email_fts f
+            JOIN emails e ON e.rowid = f.rowid
+            WHERE email_fts MATCH ?
+            ORDER BY rank
+            LIMIT ?
+        """, (query, limit))
+
+        results = cursor.fetchall()
+        if not results:
+            print(f"  Ничего не найдено по запросу: {query}")
+            conn.close()
+            return
+
+        print(f"  Найдено: {len(results)} писем\n")
+        for r in results:
+            extracted = "✓" if r["contacts_extracted"] else "—"
+            skipped = "✗" if r["contacts_skipped"] else " "
+            print(f"  [{extracted}{skipped}] {r['folder']}")
+            print(f"    От: {r['from_addr']}")
+            print(f"    Тема: {r['subject']}")
+            print(f"    Дата: {r['date']}")
+            print(f"    Путь: {r['path']}")
+            print()
+    except sqlite3.OperationalError as e:
+        print(f"  Ошибка поиска: {e}")
+        print(f"  Используй: python3 mail_index.py --search '\"точная фраза\"'")
+        print(f"  Или: python3 mail_index.py --search 'тема OR отправитель'")
+
+    conn.close()
+
+
+def stats():
+    """Статистика индекса."""
+    conn = get_db()
+    cursor = conn.cursor()
+
+    cursor.execute("SELECT COUNT(*) FROM emails")
+    total = cursor.fetchone()[0]
+    cursor.execute("SELECT COUNT(*) FROM emails WHERE contacts_extracted=1")
+    extracted = cursor.fetchone()[0]
+    cursor.execute("SELECT COUNT(*) FROM emails WHERE contacts_skipped=1")
+    skipped = cursor.fetchone()[0]
+    cursor.execute("SELECT folder, COUNT(*) FROM emails GROUP BY folder ORDER BY COUNT(*) DESC")
+    by_folder = cursor.fetchall()
+
+    print(f"📊 Статистика индекса писем")
+    print(f"  Всего: {total}")
+    print(f"  Контакты извлечены: {extracted}")
+    print(f"  Пропущено (нет подписи): {skipped}")
+    print(f"  Осталось: {total - extracted - skipped}")
+    print(f"\n  По папкам:")
+    for r in by_folder:
+        print(f"    {r['folder']}: {r['COUNT(*)']}")
+
+    conn.close()
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Индекс писем в SQLite")
+    parser.add_argument("--incremental", action="store_true", help="Только новые письма")
+    parser.add_argument("--search", type=str, help="Поиск по индексу")
+    parser.add_argument("--stats", action="store_true", help="Статистика")
+    args = parser.parse_args()
+
+    if args.search:
+        search(args.search)
+    elif args.stats:
+        stats()
+    else:
+        index_emails(incremental=args.incremental)
+
+
+if __name__ == "__main__":
+    main()