mirror of
https://gitverse.ru/kpa39l/vesti.git
synced 2026-09-29 09:55:03 +00:00
feat(sources): форма добавления источника — одно поле Address, авто-название из TG/RSS (address-field-enhancement)
- parse_address(address, crawler) в sources/sources.py: telegram (t.me/name, @name, https://t.me/s/name) → slug/channel/url; rss → feed_url/url/slug из домена. - GET /sources/preview: telegram — название через Telethon get_entity (title), rss — через feedparser (feed.title); slug и name возвращаются в JSON. - _form_source: единое поле address вместо url/channel/feed_url; slug из адреса. - /sources: direction datalist из DIRECTIONS + уникальные значения из БД, свободный ввод остаётся. - Форма: поле Address + кнопка «Найти» (JS fetch → заполнение slug/name), direction datalist динамический. Также в sources.py входит незакоммиченная ранее веб-CRUD-логика (upsert_source_yaml, delete_source_yaml, sync_yaml_to_db_and_back и др.) — работала на проде, теперь в git.
This commit is contained in:
+77
-9
@@ -25,7 +25,7 @@ except ImportError:
|
||||
# CRUD источников: работа с sources.yaml (источник истины) + БД синк
|
||||
from sources.sources import (
|
||||
db_source_to_yaml,
|
||||
add_or_update_source, delete_source, set_source_enabled,
|
||||
add_or_update_source, delete_source, set_source_enabled, parse_address,
|
||||
)
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
@@ -857,32 +857,100 @@ def sources_page(request: Request):
|
||||
FROM sources s LEFT JOIN rss_state rs ON rs.slug=s.slug
|
||||
ORDER BY s.enabled DESC, s.priority, s.slug"""
|
||||
).fetchall()
|
||||
# direction: хардкод-список + реальные значения из БД (для datalist)
|
||||
dir_rows = conn.execute(
|
||||
"SELECT DISTINCT direction FROM sources WHERE direction IS NOT NULL AND direction != '' ORDER BY direction"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
directions = sorted(set(DIRECTIONS) | {r[0] for r in dir_rows})
|
||||
html = tpl.get_template("sources.html").render(
|
||||
request=request, sources=sources,
|
||||
request=request, sources=sources, directions=directions,
|
||||
did=request.query_params.get("did", ""),
|
||||
)
|
||||
return HTMLResponse(html)
|
||||
|
||||
|
||||
@app.get("/sources/preview")
|
||||
def sources_preview(request: Request):
|
||||
"""GET /sources/preview?address=...&crawler=... → JSON {slug, name, ...}.
|
||||
|
||||
telegram: название канала через Telethon get_entity (title).
|
||||
rss: название фида через feedparser (feed.title).
|
||||
Ошибки — {"error": "..."} со статусом 200 (чтобы JS читал).
|
||||
Note: sync def (не async) — внутри asyncio.run() для Telethon,
|
||||
из async-цикла FastAPI его вызвать нельзя.
|
||||
"""
|
||||
_require_auth(request)
|
||||
address = (request.query_params.get("address") or "").strip()
|
||||
crawler = (request.query_params.get("crawler") or "telegram").strip()
|
||||
if not address:
|
||||
return {"error": "адрес пуст"}
|
||||
parsed = parse_address(address, crawler)
|
||||
if not parsed:
|
||||
return {"error": f"не могу разобрать адрес для crawler={crawler}"}
|
||||
name = None
|
||||
try:
|
||||
if crawler == "telegram":
|
||||
from crawler.telegram_crawler import make_client
|
||||
from config import TG_SESSION_DIR
|
||||
import asyncio
|
||||
|
||||
async def _get_title():
|
||||
client = make_client(TG_SESSION_DIR)
|
||||
await client.start()
|
||||
try:
|
||||
ent = await client.get_entity(parsed["channel"])
|
||||
return getattr(ent, "title", None)
|
||||
finally:
|
||||
await client.disconnect()
|
||||
|
||||
name = asyncio.run(_get_title())
|
||||
else:
|
||||
import feedparser
|
||||
try:
|
||||
d = feedparser.parse(parsed["feed_url"])
|
||||
if d.bozo and not d.feed.get("title"):
|
||||
return {"error": f"фид не читается: {getattr(d, 'bozo_exception', '')}", **parsed}
|
||||
name = d.feed.get("title") or None
|
||||
except Exception as e:
|
||||
return {"error": f"RSS: {e}", **parsed}
|
||||
except Exception as e:
|
||||
return {"error": f"не удалось получить название: {e}", **parsed}
|
||||
out = {"slug": parsed["slug"], "name": name}
|
||||
out.update(parsed)
|
||||
return out
|
||||
|
||||
|
||||
def _form_source(form) -> dict:
|
||||
"""Собирает источник из формы (str поля, None для пустых)."""
|
||||
"""Собирает источник из формы (str поля, None для пустых).
|
||||
|
||||
Поле `address` — единое: парсится по типу crawler в url/channel/feed_url.
|
||||
slug: если не заполнен (или ручной) — берётся из адреса.
|
||||
"""
|
||||
def s(k, default=None):
|
||||
v = form.get(k)
|
||||
return (v or "").strip() if v else default
|
||||
return {
|
||||
"slug": s("slug"),
|
||||
crawler = s("crawler", "telegram")
|
||||
parsed = parse_address(s("address") or "", crawler)
|
||||
src = {
|
||||
"slug": s("slug") or parsed.get("slug"),
|
||||
"name": s("name"),
|
||||
"url": s("url"),
|
||||
"channel": s("channel"),
|
||||
"crawler": s("crawler", "telegram"),
|
||||
"feed_url": s("feed_url"),
|
||||
"crawler": crawler,
|
||||
"direction": s("direction"),
|
||||
"lang": s("lang", "ru"),
|
||||
"priority": s("priority", "P1"),
|
||||
"enabled": bool(form.get("enabled")),
|
||||
"own": bool(form.get("own")),
|
||||
}
|
||||
if crawler == "telegram":
|
||||
src["channel"] = parsed.get("channel") or s("channel")
|
||||
src["url"] = parsed.get("url") or s("url")
|
||||
src["feed_url"] = s("feed_url")
|
||||
else:
|
||||
src["feed_url"] = parsed.get("feed_url") or s("feed_url")
|
||||
src["url"] = parsed.get("url") or s("url")
|
||||
src["channel"] = s("channel")
|
||||
return src
|
||||
|
||||
|
||||
@app.post("/sources/add")
|
||||
|
||||
+30
-17
@@ -23,13 +23,13 @@
|
||||
<input type="text" name="name" class="form-control form-control-sm" placeholder="My Channel">
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<label class="form-label mb-0 small">URL</label>
|
||||
<input type="text" name="url" class="form-control form-control-sm" placeholder="https://t.me/... или RSS">
|
||||
<label class="form-label mb-0 small">Address</label>
|
||||
<input type="text" name="address" class="form-control form-control-sm"
|
||||
placeholder="@handle, t.me/name, https://t.me/name или https://.../rss" id="add_address">
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<label class="form-label mb-0 small">channel (telegram)</label>
|
||||
<input type="text" name="channel" class="form-control form-control-sm"
|
||||
placeholder="@handle или t.me/..." id="add_channel">
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary" id="btn_fetch"
|
||||
onclick="fetchSourcePreview()">🔎 Найти</button>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<label class="form-label mb-0 small">crawler</label>
|
||||
@@ -38,16 +38,11 @@
|
||||
<option value="rss">rss</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<label class="form-label mb-0 small">feed_url (rss)</label>
|
||||
<input type="text" name="feed_url" class="form-control form-control-sm" placeholder="https://.../rss">
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<label class="form-label mb-0 small">direction</label>
|
||||
<input type="text" name="direction" class="form-control form-control-sm" placeholder="linux" list="dirs">
|
||||
<datalist id="dirs">
|
||||
<option value="linux"><option value="tech"><option value="politics">
|
||||
<option value="games"><option value="electronics"><option value="llm">
|
||||
{% for d in directions %}<option value="{{ d }}">{% endfor %}
|
||||
</datalist>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
@@ -67,13 +62,31 @@
|
||||
</form>
|
||||
<script>
|
||||
const addCrawler = document.querySelector('form[action="/sources/add"] select[name="crawler"]');
|
||||
const addChannel = document.getElementById('add_channel');
|
||||
if (addCrawler && addChannel) {
|
||||
function toggleChannelRequired() {
|
||||
addChannel.required = addCrawler.value === 'telegram';
|
||||
const addAddress = document.getElementById('add_address');
|
||||
const addSlug = document.querySelector('form[action="/sources/add"] input[name="slug"]');
|
||||
const addName = document.querySelector('form[action="/sources/add"] input[name="name"]');
|
||||
|
||||
if (addCrawler && addAddress) {
|
||||
function toggleAddress() {
|
||||
addAddress.placeholder = addCrawler.value === 'telegram'
|
||||
? '@handle, t.me/name, https://t.me/name'
|
||||
: 'https://.../rss или http://...';
|
||||
}
|
||||
addCrawler.addEventListener('change', toggleChannelRequired);
|
||||
toggleChannelRequired();
|
||||
addCrawler.addEventListener('change', toggleAddress);
|
||||
toggleAddress();
|
||||
}
|
||||
|
||||
async function fetchSourcePreview() {
|
||||
const addr = addAddress.value.trim();
|
||||
if (!addr) { alert('Введите адрес'); return; }
|
||||
try {
|
||||
const resp = await fetch(`/sources/preview?address=${encodeURIComponent(addr)}&crawler=${addCrawler.value}`);
|
||||
const data = await resp.json();
|
||||
if (data.error) { alert(data.error); return; }
|
||||
if (data.slug) addSlug.value = data.slug;
|
||||
if (data.name) { addName.value = data.name; }
|
||||
else if (addName.value === '') addName.placeholder = 'Название не найдено — введите вручную';
|
||||
} catch (e) { alert('Ошибка запроса: ' + e); }
|
||||
}
|
||||
</script>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user