diff --git a/QWEN.md b/QWEN.md index ce24a53..f8e616e 100644 --- a/QWEN.md +++ b/QWEN.md @@ -154,6 +154,20 @@ Chronicle — единая экосистема: 3. **Фокус**: Помощь в реализации Phase 0 (Telegram Archiver) и Phase 1 (лендинг + бренд) 4. **Стиль**: Технический, но с элементами маркетинга (проект для инвесторов и пользователей) +### ⚠️ ПРАВИЛА ПУША В GIT + +**ЗАПРЕЩЕНО пушить без проверки:** +1. ❌ Не пушить код, который не был запущен/протестирован +2. ❌ Не пушить без проверки синтаксиса (python -m py_compile) +3. ❌ Не пушить без установки зависимостей и проверки импортов +4. ❌ Не пушить без проверки работоспособности CLI/API + +**Разрешено пушить только после:** +1. ✅ `pip install -r requirements.txt` — зависимости установлены +2. ✅ `python -m py_compile` — синтаксис валиден +3. ✅ Запуск CLI или API — работает без ошибок +4. ✅ Проверка эндпойнтов (для API) — возвращают 200 OK + **Ключевые темы для обсуждения:** - Архитектура парсера Telegram (MTProto / API) - Конвертация в Markdown для Hugo diff --git a/telegram-archiver/.gitignore b/telegram-archiver/.gitignore index db344d0..543ccde 100644 --- a/telegram-archiver/.gitignore +++ b/telegram-archiver/.gitignore @@ -23,25 +23,16 @@ wheels/ *.egg # Virtual environments +.venv/ venv/ env/ ENV/ -.venv +.env -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# Session files (Telegram auth) +# Session files (Telegram auth) - CRITICAL: never commit! *.session *.session-journal -# Environment variables -.env - # Archives output archives/ */2big2get.md @@ -50,6 +41,13 @@ archives/ *.log logs/ +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + # OS .DS_Store Thumbs.db @@ -59,3 +57,7 @@ desktop.ini .pytest_cache/ .coverage htmlcov/ +.tox/ + +# mypy +.mypy_cache/ diff --git a/telegram-archiver/.pre-commit-config.yaml b/telegram-archiver/.pre-commit-config.yaml new file mode 100644 index 0000000..5169ac1 --- /dev/null +++ b/telegram-archiver/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +"""Pre-commit configuration.""" + +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.14 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + additional_dependencies: + - pydantic>=2.5.3 + - pydantic-settings>=2.1.0 + - fastapi>=0.109.0 + - telethon>=1.34.0 + args: [--ignore-missing-imports] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + args: ['--maxkb=10000'] diff --git a/telegram-archiver/DEVELOPMENT.md b/telegram-archiver/DEVELOPMENT.md new file mode 100644 index 0000000..be6a136 --- /dev/null +++ b/telegram-archiver/DEVELOPMENT.md @@ -0,0 +1,119 @@ +# Telegram Archiver - Development Configuration + +## Virtual Environment Setup + +This project uses `uv` for virtual environment and package management. + +### Setup (First Time) + +```bash +# Create virtual environment +uv venv + +# Activate virtual environment +# Windows (PowerShell): +.venv\Scripts\Activate.ps1 + +# Windows (cmd): +.venv\Scripts\activate + +# macOS / Linux: +source .venv/bin/activate + +# Install dependencies +uv pip install -r requirements.txt + +# Or install from pyproject.toml (recommended for development): +uv pip install -e ".[dev]" +``` + +### Running the Application + +#### CLI Mode + +```bash +# Activate venv first +python -m telegram-archiver --channel dedinit + +# Or with options: +python -m telegram-archiver -c dedinit -l 10 -o ./test-archive +``` + +#### API Mode + +```bash +# Start the server +uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 + +# Open browser to http://localhost:8000/docs for Swagger UI +``` + +### Development Tools + +#### Linting + +```bash +ruff check . +ruff format . +``` + +#### Type Checking + +```bash +mypy app/ +``` + +#### Testing + +```bash +# Create tests directory first +mkdir tests +touch tests/__init__.py + +# Run tests +pytest +pytest --cov=app +``` + +#### Pre-commit Hooks + +```bash +# Install pre-commit hooks +pre-commit install + +# Run all hooks manually +pre-commit run --all-files +``` + +### Code Quality Checklist + +Before committing: + +- [ ] `ruff check .` passes +- [ ] `mypy app/` passes (or has explicit ignores) +- [ ] Tests pass: `pytest` +- [ ] Manual test: CLI or API works +- [ ] `.env` file NOT committed (check .gitignore) +- [ ] `.session` files NOT committed (check .gitignore) + +### Troubleshooting + +**Import errors:** +```bash +# Make sure venv is activated +# Reinstall dependencies +uv pip install -r requirements.txt +``` + +**Telethon auth issues:** +```bash +# Delete session file and re-authenticate +rm telegram-archiver.session +# Run again, will prompt for code +``` + +**Port already in use:** +```bash +# Use different port +uvicorn app.main:app --port 8001 +``` diff --git a/telegram-archiver/README.md b/telegram-archiver/README.md index 0a84583..52e5beb 100644 --- a/telegram-archiver/README.md +++ b/telegram-archiver/README.md @@ -29,11 +29,40 @@ ## Установка -### 1. Клонирование и зависимости +### 1. Установка uv (менеджер пакетов и виртуальных окружений) + +```bash +# Windows +pip install uv + +# macOS / Linux +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +### 2. Создание виртуального окружения ```bash cd telegram-archiver -pip install -r requirements.txt +uv venv +``` + +Активация виртуального окружения: + +```bash +# Windows (PowerShell) +.venv\Scripts\Activate.ps1 + +# Windows (cmd) +.venv\Scripts\activate + +# macOS / Linux +source .venv/bin/activate +``` + +### 3. Установка зависимостей + +```bash +uv pip install -r requirements.txt ``` ### 2. Получение Telegram API ключей diff --git a/telegram-archiver/app/main.py b/telegram-archiver/app/main.py index 241f6a8..a13c1ff 100644 --- a/telegram-archiver/app/main.py +++ b/telegram-archiver/app/main.py @@ -22,7 +22,7 @@ from app.models import ( from app.telethon_client import TelethonArchiver from app.archiver import ChannelArchiver from app.logger import setup_logging, get_logger -from config import get_settings, Settings +from config import get_settings, get_required_settings, Settings # Initialize logger logger = get_logger(__name__) @@ -38,16 +38,22 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: # Startup logger.info("Starting Telegram Archiver...") - settings = get_settings() - setup_logging( - log_level=settings.log_level, - enable_otel=settings.otel_enabled, - ) - - archiver_client = TelethonArchiver(settings) - await archiver_client.connect() - - logger.info("Telegram Archiver started successfully") + + try: + settings = get_required_settings() + setup_logging( + log_level=settings.log_level, + enable_otel=settings.otel_enabled, + ) + + archiver_client = TelethonArchiver(settings) + await archiver_client.connect() + + logger.info("Telegram Archiver started successfully") + except Exception as e: + logger.warning(f"Could not initialize Telegram client: {e}") + logger.warning("Server will start but /archive endpoint will not work") + logger.warning("Create .env file with API_ID, API_HASH, PHONE to enable archiving") yield @@ -90,13 +96,16 @@ async def archive_channel(request: ArchiveRequest) -> ArchiveResponse: Large files (>MAX_FILE_SIZE) are skipped and logged in 2big2get.md """ if not archiver_client: - raise HTTPException(status_code=503, detail="Archiver not initialized") + raise HTTPException( + status_code=503, + detail="Archiver not initialized. Check server logs. Make sure .env file exists with API_ID, API_HASH, PHONE." + ) try: logger.info(f"Archive request for channel: {request.channel}") # Get settings and override output_dir if specified - settings = get_settings() + settings = get_required_settings() if request.output_dir: settings.output_dir = Path(request.output_dir) diff --git a/telegram-archiver/config.py b/telegram-archiver/config.py index 1291a83..8886485 100644 --- a/telegram-archiver/config.py +++ b/telegram-archiver/config.py @@ -4,9 +4,17 @@ Uses pydantic-settings for .env file parsing and validation. """ from pydantic_settings import BaseSettings, SettingsConfigDict -from pydantic import Field +from pydantic import Field, ValidationError from pathlib import Path from typing import Optional +import logging + +logger = logging.getLogger(__name__) + + +class SettingsError(Exception): + """Raised when settings cannot be loaded.""" + pass class Settings(BaseSettings): @@ -19,10 +27,10 @@ class Settings(BaseSettings): extra="ignore", ) - # Telegram API credentials - api_id: int = Field(..., description="Telegram API ID from my.telegram.org") - api_hash: str = Field(..., description="Telegram API Hash from my.telegram.org") - phone: str = Field(..., description="Phone number for Telegram account") + # Telegram API credentials - required for archiving to work + api_id: Optional[int] = Field(None, description="Telegram API ID from my.telegram.org") + api_hash: Optional[str] = Field(None, description="Telegram API Hash from my.telegram.org") + phone: Optional[str] = Field(None, description="Phone number for Telegram account") # Session configuration session_name: str = Field( @@ -63,8 +71,18 @@ class Settings(BaseSettings): """Return full path to session file.""" return Path(__file__).parent / f"{self.session_name}.session" + @property + def is_configured(self) -> bool: + """Check if credentials are configured.""" + return bool(self.api_id and self.api_hash and self.phone) + def validate_credentials(self) -> bool: - """Validate that required credentials are present.""" + """Validate that required credentials are present and valid.""" + if not self.is_configured: + raise SettingsError( + "Missing Telegram credentials. " + "Set API_ID, API_HASH, and PHONE in .env file or environment variables." + ) if self.api_id <= 0: raise ValueError("API_ID must be a positive integer") if len(self.api_hash) < 10: @@ -75,13 +93,19 @@ class Settings(BaseSettings): # Global settings instance -settings: Optional[Settings] = None +_settings: Optional[Settings] = None def get_settings() -> Settings: """Get or create settings instance.""" - global settings - if settings is None: - settings = Settings() - settings.validate_credentials() + global _settings + if _settings is None: + _settings = Settings() + return _settings + + +def get_required_settings() -> Settings: + """Get settings and ensure credentials are configured.""" + settings = get_settings() + settings.validate_credentials() return settings diff --git a/telegram-archiver/pyproject.toml b/telegram-archiver/pyproject.toml new file mode 100644 index 0000000..3b21e1e --- /dev/null +++ b/telegram-archiver/pyproject.toml @@ -0,0 +1,99 @@ +[project] +name = "telegram-archiver" +version = "1.0.0" +description = "Archive Telegram channels to Markdown bundles with media" +readme = "README.md" +requires-python = ">=3.11" +license = {text = "MIT"} +authors = [ + {name = "Evgeny Storozhenko", email = "dedinit"} +] +keywords = ["telegram", "archive", "markdown", "hugo", "telethon"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Environment :: Web Environment", + "Framework :: FastAPI", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Communications :: Chat", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Multimedia", +] + +dependencies = [ + "fastapi>=0.109.0", + "uvicorn[standard]>=0.27.0", + "click>=8.1.7", + "telethon>=1.34.0", + "pydantic>=2.5.3", + "pydantic-settings>=2.1.0", + "python-dotenv>=1.0.0", + "pyyaml>=6.0.1", + "opentelemetry-api>=1.22.0", + "opentelemetry-sdk>=1.22.0", + "opentelemetry-instrumentation-fastapi>=0.43b0", + "httpx>=0.26.0", + "aiofiles>=23.2.1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "pytest-cov>=4.1.0", + "ruff>=0.1.0", + "mypy>=1.8.0", + "pre-commit>=3.6.0", +] + +[project.scripts] +telegram-archiver = "app.main:cli_main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["app"] + +[tool.ruff] +target-version = "py311" +line-length = 88 +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade +] +ignore = [ + "E501", # line too long (handled by formatter) + "B008", # do not perform function calls in argument defaults +] + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true + +[[tool.mypy.overrides]] +module = "telethon.*" +ignore_missing_imports = true + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] +addopts = "-v --tb=short" diff --git a/telegram-archiver/tests/__init__.py b/telegram-archiver/tests/__init__.py new file mode 100644 index 0000000..e0ad345 --- /dev/null +++ b/telegram-archiver/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for Telegram Archiver.""" diff --git a/telegram-archiver/tests/test_config.py b/telegram-archiver/tests/test_config.py new file mode 100644 index 0000000..7a574c5 --- /dev/null +++ b/telegram-archiver/tests/test_config.py @@ -0,0 +1,73 @@ +"""Tests for configuration.""" + +import pytest +import os +from pathlib import Path + +from config import Settings + + +class TestSettings: + """Test Settings model.""" + + def test_settings_from_env(self, monkeypatch): + """Test loading settings from environment.""" + monkeypatch.setenv("API_ID", "12345678") + monkeypatch.setenv("API_HASH", "abcdef1234567890") + monkeypatch.setenv("PHONE", "+79991234567") + monkeypatch.setenv("MAX_FILE_SIZE", "104857600") + + settings = Settings() + + assert settings.api_id == 12345678 + assert settings.api_hash == "abcdef1234567890" + assert settings.phone == "+79991234567" + assert settings.max_file_size == 104857600 + + def test_settings_default_values(self, monkeypatch): + """Test default setting values.""" + monkeypatch.setenv("API_ID", "12345678") + monkeypatch.setenv("API_HASH", "abcdef1234567890") + monkeypatch.setenv("PHONE", "+79991234567") + + settings = Settings() + + assert settings.session_name == "telegram-archiver" + assert settings.max_file_size == 209715200 # 200 MB default + assert settings.log_level == "INFO" + assert settings.otel_enabled is False + + def test_settings_session_path(self, monkeypatch): + """Test session_path property.""" + monkeypatch.setenv("API_ID", "12345678") + monkeypatch.setenv("API_HASH", "abcdef1234567890") + monkeypatch.setenv("PHONE", "+79991234567") + monkeypatch.setenv("SESSION_NAME", "test_session") + + settings = Settings() + session_path = settings.session_path + + assert session_path.name == "test_session.session" + assert session_path.suffix == ".session" + + def test_settings_validate_credentials_invalid(self, monkeypatch): + """Test credential validation.""" + monkeypatch.setenv("API_ID", "0") # Invalid + monkeypatch.setenv("API_HASH", "abcdef1234567890") + monkeypatch.setenv("PHONE", "+79991234567") + + settings = Settings() + + with pytest.raises(ValueError, match="API_ID must be a positive integer"): + settings.validate_credentials() + + def test_settings_validate_phone_format(self, monkeypatch): + """Test phone number format validation.""" + monkeypatch.setenv("API_ID", "12345678") + monkeypatch.setenv("API_HASH", "abcdef1234567890") + monkeypatch.setenv("PHONE", "79991234567") # Missing + + + settings = Settings() + + with pytest.raises(ValueError, match="PHONE must start with +"): + settings.validate_credentials() diff --git a/telegram-archiver/tests/test_models.py b/telegram-archiver/tests/test_models.py new file mode 100644 index 0000000..548a200 --- /dev/null +++ b/telegram-archiver/tests/test_models.py @@ -0,0 +1,126 @@ +"""Tests for Pydantic models.""" + +import pytest +from datetime import datetime + +from app.models import PostData, MediaFile, MediaType, ArchiveRequest + + +class TestMediaFile: + """Test MediaFile model.""" + + def test_create_media_file(self): + """Test creating a MediaFile instance.""" + media = MediaFile( + filename="test.jpg", + type=MediaType.PHOTO, + caption="Test caption", + size=1024, + ) + assert media.filename == "test.jpg" + assert media.type == "photo" + assert media.caption == "Test caption" + assert media.size == 1024 + assert media.is_too_large is False + + def test_media_file_too_large(self): + """Test MediaFile with is_too_large flag.""" + media = MediaFile( + filename="large.mp4", + type="video", + size=500_000_000, + is_too_large=True, + ) + assert media.is_too_large is True + + +class TestPostData: + """Test PostData model.""" + + def test_create_post_data(self): + """Test creating a PostData instance.""" + post = PostData( + message_id=12345, + date=datetime(2024, 2, 19, 14, 30, 0), + text="Test message", + author="Test Channel", + channel_username="test_channel", + ) + assert post.message_id == 12345 + assert post.text == "Test message" + assert post.author == "Test Channel" + assert post.has_media() is False + + def test_post_with_media(self): + """Test PostData with media files.""" + post = PostData( + message_id=12345, + date=datetime.now(), + text="Test", + author="Channel", + channel_username="channel", + media_files=[ + MediaFile(filename="photo.jpg", type="photo", size=1024) + ], + ) + assert post.has_media() is True + assert post.has_downloaded_media() is True + + def test_post_bundle_dir(self): + """Test bundle_dir property.""" + post = PostData( + message_id=99999, + date=datetime.now(), + text="Test", + author="Channel", + channel_username="channel", + ) + assert post.bundle_dir == "99999" + assert post.index_path == "99999/index.md" + + def test_post_with_reply_and_repost(self): + """Test PostData with reply_to and repost_from.""" + post = PostData( + message_id=12345, + date=datetime.now(), + text="Test", + author="Channel", + channel_username="channel", + reply_to=12340, + repost_from=10000, + repost_channel="Original Channel", + ) + assert post.reply_to == 12340 + assert post.repost_from == 10000 + assert post.repost_channel == "Original Channel" + + +class TestArchiveRequest: + """Test ArchiveRequest model.""" + + def test_minimal_request(self): + """Test minimal archive request.""" + req = ArchiveRequest(channel="dedinit") + assert req.channel == "dedinit" + assert req.limit is None + assert req.force is False + + def test_full_request(self): + """Test archive request with all options.""" + req = ArchiveRequest( + channel="test_channel", + output_dir="/tmp/archive", + limit=100, + from_message_id=5000, + force=True, + ) + assert req.channel == "test_channel" + assert req.output_dir == "/tmp/archive" + assert req.limit == 100 + assert req.from_message_id == 5000 + assert req.force is True + + def test_channel_with_at(self): + """Test channel username with @ prefix.""" + req = ArchiveRequest(channel="@dedinit") + assert req.channel == "@dedinit"