feat(sources): форма добавления источника — одно поле Address, авто-название из TG/RSS (address-field-enhancement)

- parse_address(address, crawler) в sources/sources.py: telegram (t.me/name, @name,
  https://t.me/s/name) → slug/channel/url; rss → feed_url/url/slug из домена.
- GET /sources/preview: telegram — название через Telethon get_entity (title),
  rss — через feedparser (feed.title); slug и name возвращаются в JSON.
- _form_source: единое поле address вместо url/channel/feed_url; slug из адреса.
- /sources: direction datalist из DIRECTIONS + уникальные значения из БД,
  свободный ввод остаётся.
- Форма: поле Address + кнопка «Найти» (JS fetch → заполнение slug/name),
  direction datalist динамический.

Также в sources.py входит незакоммиченная ранее веб-CRUD-логика (upsert_source_yaml,
delete_source_yaml, sync_yaml_to_db_and_back и др.) — работала на проде, теперь в git.
This commit is contained in:
kpa39l
2026-09-16 17:03:23 +00:00
parent 87134ed839
commit 48c25f13d4
5 changed files with 329 additions and 48 deletions
+77 -9
View File
@@ -25,7 +25,7 @@ except ImportError:
# CRUD источников: работа с sources.yaml (источник истины) + БД синк
from sources.sources import (
db_source_to_yaml,
add_or_update_source, delete_source, set_source_enabled,
add_or_update_source, delete_source, set_source_enabled, parse_address,
)
BASE_DIR = Path(__file__).resolve().parent.parent
@@ -857,32 +857,100 @@ def sources_page(request: Request):
FROM sources s LEFT JOIN rss_state rs ON rs.slug=s.slug
ORDER BY s.enabled DESC, s.priority, s.slug"""
).fetchall()
# direction: хардкод-список + реальные значения из БД (для datalist)
dir_rows = conn.execute(
"SELECT DISTINCT direction FROM sources WHERE direction IS NOT NULL AND direction != '' ORDER BY direction"
).fetchall()
conn.close()
directions = sorted(set(DIRECTIONS) | {r[0] for r in dir_rows})
html = tpl.get_template("sources.html").render(
request=request, sources=sources,
request=request, sources=sources, directions=directions,
did=request.query_params.get("did", ""),
)
return HTMLResponse(html)
@app.get("/sources/preview")
def sources_preview(request: Request):
"""GET /sources/preview?address=...&crawler=... → JSON {slug, name, ...}.
telegram: название канала через Telethon get_entity (title).
rss: название фида через feedparser (feed.title).
Ошибки — {"error": "..."} со статусом 200 (чтобы JS читал).
Note: sync def (не async) — внутри asyncio.run() для Telethon,
из async-цикла FastAPI его вызвать нельзя.
"""
_require_auth(request)
address = (request.query_params.get("address") or "").strip()
crawler = (request.query_params.get("crawler") or "telegram").strip()
if not address:
return {"error": "адрес пуст"}
parsed = parse_address(address, crawler)
if not parsed:
return {"error": f"не могу разобрать адрес для crawler={crawler}"}
name = None
try:
if crawler == "telegram":
from crawler.telegram_crawler import make_client
from config import TG_SESSION_DIR
import asyncio
async def _get_title():
client = make_client(TG_SESSION_DIR)
await client.start()
try:
ent = await client.get_entity(parsed["channel"])
return getattr(ent, "title", None)
finally:
await client.disconnect()
name = asyncio.run(_get_title())
else:
import feedparser
try:
d = feedparser.parse(parsed["feed_url"])
if d.bozo and not d.feed.get("title"):
return {"error": f"фид не читается: {getattr(d, 'bozo_exception', '')}", **parsed}
name = d.feed.get("title") or None
except Exception as e:
return {"error": f"RSS: {e}", **parsed}
except Exception as e:
return {"error": f"не удалось получить название: {e}", **parsed}
out = {"slug": parsed["slug"], "name": name}
out.update(parsed)
return out
def _form_source(form) -> dict:
"""Собирает источник из формы (str поля, None для пустых)."""
"""Собирает источник из формы (str поля, None для пустых).
Поле `address` — единое: парсится по типу crawler в url/channel/feed_url.
slug: если не заполнен (или ручной) — берётся из адреса.
"""
def s(k, default=None):
v = form.get(k)
return (v or "").strip() if v else default
return {
"slug": s("slug"),
crawler = s("crawler", "telegram")
parsed = parse_address(s("address") or "", crawler)
src = {
"slug": s("slug") or parsed.get("slug"),
"name": s("name"),
"url": s("url"),
"channel": s("channel"),
"crawler": s("crawler", "telegram"),
"feed_url": s("feed_url"),
"crawler": crawler,
"direction": s("direction"),
"lang": s("lang", "ru"),
"priority": s("priority", "P1"),
"enabled": bool(form.get("enabled")),
"own": bool(form.get("own")),
}
if crawler == "telegram":
src["channel"] = parsed.get("channel") or s("channel")
src["url"] = parsed.get("url") or s("url")
src["feed_url"] = s("feed_url")
else:
src["feed_url"] = parsed.get("feed_url") or s("feed_url")
src["url"] = parsed.get("url") or s("url")
src["channel"] = s("channel")
return src
@app.post("/sources/add")
+30 -17
View File
@@ -23,13 +23,13 @@
<input type="text" name="name" class="form-control form-control-sm" placeholder="My Channel">
</div>
<div class="col-auto">
<label class="form-label mb-0 small">URL</label>
<input type="text" name="url" class="form-control form-control-sm" placeholder="https://t.me/... или RSS">
<label class="form-label mb-0 small">Address</label>
<input type="text" name="address" class="form-control form-control-sm"
placeholder="@handle, t.me/name, https://t.me/name или https://.../rss" id="add_address">
</div>
<div class="col-auto">
<label class="form-label mb-0 small">channel (telegram)</label>
<input type="text" name="channel" class="form-control form-control-sm"
placeholder="@handle или t.me/..." id="add_channel">
<button type="button" class="btn btn-sm btn-outline-secondary" id="btn_fetch"
onclick="fetchSourcePreview()">🔎 Найти</button>
</div>
<div class="col-auto">
<label class="form-label mb-0 small">crawler</label>
@@ -38,16 +38,11 @@
<option value="rss">rss</option>
</select>
</div>
<div class="col-auto">
<label class="form-label mb-0 small">feed_url (rss)</label>
<input type="text" name="feed_url" class="form-control form-control-sm" placeholder="https://.../rss">
</div>
<div class="col-auto">
<label class="form-label mb-0 small">direction</label>
<input type="text" name="direction" class="form-control form-control-sm" placeholder="linux" list="dirs">
<datalist id="dirs">
<option value="linux"><option value="tech"><option value="politics">
<option value="games"><option value="electronics"><option value="llm">
{% for d in directions %}<option value="{{ d }}">{% endfor %}
</datalist>
</div>
<div class="col-auto">
@@ -67,13 +62,31 @@
</form>
<script>
const addCrawler = document.querySelector('form[action="/sources/add"] select[name="crawler"]');
const addChannel = document.getElementById('add_channel');
if (addCrawler && addChannel) {
function toggleChannelRequired() {
addChannel.required = addCrawler.value === 'telegram';
const addAddress = document.getElementById('add_address');
const addSlug = document.querySelector('form[action="/sources/add"] input[name="slug"]');
const addName = document.querySelector('form[action="/sources/add"] input[name="name"]');
if (addCrawler && addAddress) {
function toggleAddress() {
addAddress.placeholder = addCrawler.value === 'telegram'
? '@handle, t.me/name, https://t.me/name'
: 'https://.../rss или http://...';
}
addCrawler.addEventListener('change', toggleChannelRequired);
toggleChannelRequired();
addCrawler.addEventListener('change', toggleAddress);
toggleAddress();
}
async function fetchSourcePreview() {
const addr = addAddress.value.trim();
if (!addr) { alert('Введите адрес'); return; }
try {
const resp = await fetch(`/sources/preview?address=${encodeURIComponent(addr)}&crawler=${addCrawler.value}`);
const data = await resp.json();
if (data.error) { alert(data.error); return; }
if (data.slug) addSlug.value = data.slug;
if (data.name) { addName.value = data.name; }
else if (addName.value === '') addName.placeholder = 'Название не найдено — введите вручную';
} catch (e) { alert('Ошибка запроса: ' + e); }
}
</script>
</div>