317 lines
12 KiB
Python
317 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
mail_index.py — SQLite-индекс всех email.md для быстрого поиска и трекинга.
|
||
|
||
Создаёт /opt/hermes/email/mail_index.db с таблицами:
|
||
- emails: path, uid, folder, date, from_addr, to_addrs, subject,
|
||
contacts_extracted, contacts_skipped, first_seen, last_scanned
|
||
- email_fts (FTS5): полнотекстовый поиск по subject + from + to
|
||
|
||
Использование:
|
||
python3 mail_index.py — полная переиндексация
|
||
python3 mail_index.py --incremental — только новые файлы (по mtime last_index)
|
||
python3 mail_index.py --search "запрос" — поиск по индексу
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import sys
|
||
import time
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
EMAIL_ROOT = Path("/opt/hermes/email")
|
||
DB_PATH = EMAIL_ROOT / "mail_index.db"
|
||
CONTACTS_DIR = EMAIL_ROOT / "contacts"
|
||
LAST_SCAN_FILE = CONTACTS_DIR / "last_scan.json"
|
||
STATE_DIR = EMAIL_ROOT / "state"
|
||
|
||
# Папки, которые не индексируем
|
||
EXCLUDE_DIRS = {"contacts", "state"}
|
||
|
||
SCHEMA = """
|
||
CREATE TABLE IF NOT EXISTS emails (
|
||
path TEXT PRIMARY KEY, -- относительный путь от EMAIL_ROOT
|
||
uid INTEGER, -- числовой UID из пути
|
||
folder TEXT, -- INBOX, INBOX/!Scan, Sent...
|
||
date TEXT, -- дата из frontmatter (ISO)
|
||
from_addr TEXT, -- отправитель
|
||
to_addrs TEXT, -- получатели
|
||
subject TEXT, -- тема
|
||
body_preview TEXT, -- первые 500 символов тела (без HTML)
|
||
contacts_extracted INTEGER DEFAULT 0, -- 0/1
|
||
contacts_skipped INTEGER DEFAULT 0, -- 0/1 (нет подписи / LLM error)
|
||
first_seen TEXT, -- когда проиндексировано
|
||
last_scanned TEXT, -- последняя проверка contacts
|
||
file_mtime REAL -- mtime файла для инкрементальной проверки
|
||
);
|
||
|
||
CREATE INDEX IF NOT EXISTS idx_emails_folder ON emails(folder);
|
||
CREATE INDEX IF NOT EXISTS idx_emails_uid ON emails(uid);
|
||
CREATE INDEX IF NOT EXISTS idx_emails_contacts ON emails(contacts_extracted);
|
||
CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date);
|
||
|
||
CREATE VIRTUAL TABLE IF NOT EXISTS email_fts USING fts5(
|
||
subject, from_addr, to_addrs, body_preview,
|
||
content='emails',
|
||
content_rowid='rowid',
|
||
tokenize='unicode61'
|
||
);
|
||
|
||
-- Триггеры для синхронизации FTS
|
||
CREATE TRIGGER IF NOT EXISTS emails_ai AFTER INSERT ON emails BEGIN
|
||
INSERT INTO email_fts(rowid, subject, from_addr, to_addrs, body_preview)
|
||
VALUES (new.rowid, new.subject, new.from_addr, new.to_addrs, new.body_preview);
|
||
END;
|
||
|
||
CREATE TRIGGER IF NOT EXISTS emails_ad AFTER DELETE ON emails BEGIN
|
||
INSERT INTO email_fts(email_fts, rowid, subject, from_addr, to_addrs, body_preview)
|
||
VALUES ('delete', old.rowid, old.subject, old.from_addr, old.to_addrs, old.body_preview);
|
||
END;
|
||
|
||
CREATE TRIGGER IF NOT EXISTS emails_au AFTER UPDATE ON emails BEGIN
|
||
INSERT INTO email_fts(email_fts, rowid, subject, from_addr, to_addrs, body_preview)
|
||
VALUES ('delete', old.rowid, old.subject, old.from_addr, old.to_addrs, old.body_preview);
|
||
INSERT INTO email_fts(rowid, subject, from_addr, to_addrs, body_preview)
|
||
VALUES (new.rowid, new.subject, new.from_addr, new.to_addrs, new.body_preview);
|
||
END;
|
||
"""
|
||
|
||
|
||
def get_db():
|
||
"""Открыть/создать БД."""
|
||
conn = sqlite3.connect(str(DB_PATH))
|
||
conn.row_factory = sqlite3.Row
|
||
conn.execute("PRAGMA journal_mode=WAL")
|
||
conn.execute("PRAGMA synchronous=NORMAL")
|
||
conn.executescript(SCHEMA)
|
||
return conn
|
||
|
||
|
||
def parse_frontmatter(content):
|
||
"""Парсит YAML-frontmatter из email.md."""
|
||
meta = {}
|
||
m = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
|
||
if not m:
|
||
return meta
|
||
yaml_block = m.group(1)
|
||
for line in yaml_block.split('\n'):
|
||
line = line.strip()
|
||
if ':' in line:
|
||
key, _, val = line.partition(':')
|
||
key = key.strip().lower()
|
||
val = val.strip().strip('"').strip("'")
|
||
# Обрабатываем списки
|
||
if val.startswith('[') and val.endswith(']'):
|
||
val = val[1:-1].replace('"', '').replace("'", '').strip()
|
||
meta[key] = val
|
||
return meta
|
||
|
||
|
||
def clean_html(text):
|
||
"""Удалить HTML-теги и мусор."""
|
||
text = re.sub(r'<#part[^>]*>', '', text)
|
||
text = re.sub(r'<#/part>', '', text)
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
text = re.sub(r'\s+', ' ', text)
|
||
text = re.sub(r'&[a-z]+;', ' ', text)
|
||
text = re.sub(r'https?://\S+', '', text)
|
||
return text.strip()[:500]
|
||
|
||
|
||
def extract_uid_from_path(path):
|
||
"""Извлекает UID из пути: .../YYYY/MM/UID/email.md"""
|
||
m = re.search(r'/(\d+)/email\.md$', str(path))
|
||
if m:
|
||
return int(m.group(1))
|
||
return None
|
||
|
||
|
||
def find_email_md_files(root_path):
|
||
"""Найти все email.md, исключая contacts/ и state/."""
|
||
files = []
|
||
root_path = Path(root_path)
|
||
for f in sorted(root_path.rglob("email.md")):
|
||
rel = f.relative_to(root_path)
|
||
parts = rel.parts
|
||
# Пропускаем служебные папки
|
||
if any(p in EXCLUDE_DIRS for p in parts):
|
||
continue
|
||
files.append(f)
|
||
return files
|
||
|
||
|
||
def index_emails(incremental=False):
|
||
"""Проиндексировать все email.md в SQLite."""
|
||
conn = get_db()
|
||
cursor = conn.cursor()
|
||
|
||
if incremental:
|
||
# Получаем последний mtime из БД
|
||
cursor.execute("SELECT COALESCE(MAX(file_mtime), 0) FROM emails")
|
||
last_mtime = cursor.fetchone()[0] or 0
|
||
print(f"📇 Инкрементальная индексация (mtime > {last_mtime})", flush=True)
|
||
else:
|
||
last_mtime = 0
|
||
print("📇 Полная индексация", flush=True)
|
||
|
||
email_files = find_email_md_files(EMAIL_ROOT)
|
||
|
||
new_count = 0
|
||
updated_count = 0
|
||
total = len(email_files)
|
||
|
||
for i, f in enumerate(email_files):
|
||
rel_path = str(f.relative_to(EMAIL_ROOT))
|
||
mtime = os.path.getmtime(f)
|
||
|
||
if incremental and mtime <= last_mtime:
|
||
continue
|
||
|
||
content = f.read_text(encoding='utf-8', errors='replace')
|
||
meta = parse_frontmatter(content)
|
||
uid = extract_uid_from_path(f)
|
||
folder = meta.get('folder', '') or str(f.parent.parent.parent.relative_to(EMAIL_ROOT))
|
||
subject = meta.get('subject', '')
|
||
from_addr = meta.get('from', '')
|
||
to_addrs = meta.get('to', '')
|
||
date_val = meta.get('date', '')
|
||
|
||
# Тело
|
||
body_match = re.search(r'^---\s*\n(.*?)\n---\s*\n(.*)', content, re.DOTALL)
|
||
body_preview = ''
|
||
if body_match:
|
||
body_preview = clean_html(body_match.group(2))
|
||
if not body_preview:
|
||
body_preview = clean_html(content)
|
||
|
||
cursor.execute("""
|
||
INSERT INTO emails (path, uid, folder, date, from_addr, to_addrs, subject,
|
||
body_preview, first_seen, file_mtime)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(path) DO UPDATE SET
|
||
uid=excluded.uid,
|
||
folder=excluded.folder,
|
||
date=excluded.date,
|
||
from_addr=excluded.from_addr,
|
||
to_addrs=excluded.to_addrs,
|
||
subject=excluded.subject,
|
||
body_preview=excluded.body_preview,
|
||
file_mtime=excluded.file_mtime
|
||
""", (
|
||
rel_path, uid, folder, date_val, from_addr, to_addrs, subject,
|
||
body_preview, datetime.now(timezone.utc).isoformat(), mtime
|
||
))
|
||
|
||
if cursor.rowcount == 1 and cursor.lastrowid:
|
||
new_count += 1
|
||
else:
|
||
updated_count += 1
|
||
|
||
if (i + 1) % 200 == 0:
|
||
conn.commit()
|
||
print(f" {i+1}/{total} ({new_count} new, {updated_count} updated)", flush=True)
|
||
|
||
conn.commit()
|
||
|
||
# Обновляем FTS5 (перестроить для согласованности)
|
||
cursor.execute("INSERT INTO email_fts(email_fts) VALUES('rebuild')")
|
||
conn.commit()
|
||
|
||
print(f"\n✅ Готово: {total} файлов, {new_count} новых, {updated_count} обновлено")
|
||
print(f" База: {DB_PATH}")
|
||
|
||
cursor.execute("SELECT COUNT(*) FROM emails")
|
||
total_in_db = cursor.fetchone()[0]
|
||
cursor.execute("SELECT COUNT(*) FROM emails WHERE contacts_extracted=1")
|
||
extracted = cursor.fetchone()[0]
|
||
print(f" Всего в БД: {total_in_db}, контакты извлечены: {extracted}")
|
||
|
||
conn.close()
|
||
|
||
|
||
def search(query, limit=10):
|
||
"""Поиск по FTS5."""
|
||
conn = get_db()
|
||
cursor = conn.cursor()
|
||
|
||
try:
|
||
cursor.execute("""
|
||
SELECT e.path, e.folder, e.date, e.from_addr, e.subject, e.contacts_extracted,
|
||
e.contacts_skipped, e.body_preview
|
||
FROM email_fts f
|
||
JOIN emails e ON e.rowid = f.rowid
|
||
WHERE email_fts MATCH ?
|
||
ORDER BY rank
|
||
LIMIT ?
|
||
""", (query, limit))
|
||
|
||
results = cursor.fetchall()
|
||
if not results:
|
||
print(f" Ничего не найдено по запросу: {query}")
|
||
conn.close()
|
||
return
|
||
|
||
print(f" Найдено: {len(results)} писем\n")
|
||
for r in results:
|
||
extracted = "✓" if r["contacts_extracted"] else "—"
|
||
skipped = "✗" if r["contacts_skipped"] else " "
|
||
print(f" [{extracted}{skipped}] {r['folder']}")
|
||
print(f" От: {r['from_addr']}")
|
||
print(f" Тема: {r['subject']}")
|
||
print(f" Дата: {r['date']}")
|
||
print(f" Путь: {r['path']}")
|
||
print()
|
||
except sqlite3.OperationalError as e:
|
||
print(f" Ошибка поиска: {e}")
|
||
print(f" Используй: python3 mail_index.py --search '\"точная фраза\"'")
|
||
print(f" Или: python3 mail_index.py --search 'тема OR отправитель'")
|
||
|
||
conn.close()
|
||
|
||
|
||
def stats():
|
||
"""Статистика индекса."""
|
||
conn = get_db()
|
||
cursor = conn.cursor()
|
||
|
||
cursor.execute("SELECT COUNT(*) FROM emails")
|
||
total = cursor.fetchone()[0]
|
||
cursor.execute("SELECT COUNT(*) FROM emails WHERE contacts_extracted=1")
|
||
extracted = cursor.fetchone()[0]
|
||
cursor.execute("SELECT COUNT(*) FROM emails WHERE contacts_skipped=1")
|
||
skipped = cursor.fetchone()[0]
|
||
cursor.execute("SELECT folder, COUNT(*) FROM emails GROUP BY folder ORDER BY COUNT(*) DESC")
|
||
by_folder = cursor.fetchall()
|
||
|
||
print(f"📊 Статистика индекса писем")
|
||
print(f" Всего: {total}")
|
||
print(f" Контакты извлечены: {extracted}")
|
||
print(f" Пропущено (нет подписи): {skipped}")
|
||
print(f" Осталось: {total - extracted - skipped}")
|
||
print(f"\n По папкам:")
|
||
for r in by_folder:
|
||
print(f" {r['folder']}: {r['COUNT(*)']}")
|
||
|
||
conn.close()
|
||
|
||
|
||
def main():
|
||
import argparse
|
||
parser = argparse.ArgumentParser(description="Индекс писем в SQLite")
|
||
parser.add_argument("--incremental", action="store_true", help="Только новые письма")
|
||
parser.add_argument("--search", type=str, help="Поиск по индексу")
|
||
parser.add_argument("--stats", action="store_true", help="Статистика")
|
||
args = parser.parse_args()
|
||
|
||
if args.search:
|
||
search(args.search)
|
||
elif args.stats:
|
||
stats()
|
||
else:
|
||
index_emails(incremental=args.incremental)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |