Add telegram-archiver microservice (MVP)

telegram-archiver/app/main.py

@@ -0,0 +1,237 @@
+"""
+Telegram Archiver - Main Application Entry Point.
+
+Provides both FastAPI REST API and CLI interfaces for archiving
+Telegram channels to local filesystem bundles.
+"""
+
+import asyncio
+import time
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import AsyncGenerator
+
+from fastapi import FastAPI, HTTPException, BackgroundTasks
+from fastapi.responses import JSONResponse
+
+from app.models import (
+    ArchiveRequest,
+    ArchiveResponse,
+    HealthCheck,
+)
+from app.telethon_client import TelethonArchiver
+from app.archiver import ChannelArchiver
+from app.logger import setup_logging, get_logger
+from config import get_settings, Settings
+
+# Initialize logger
+logger = get_logger(__name__)
+
+# Global archiver client (shared across requests)
+archiver_client: TelethonArchiver | None = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
+    """Application lifespan manager."""
+    global archiver_client
+
+    # Startup
+    logger.info("Starting Telegram Archiver...")
+    settings = get_settings()
+    setup_logging(
+        log_level=settings.log_level,
+        enable_otel=settings.otel_enabled,
+    )
+
+    archiver_client = TelethonArchiver(settings)
+    await archiver_client.connect()
+
+    logger.info("Telegram Archiver started successfully")
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down Telegram Archiver...")
+    if archiver_client:
+        await archiver_client.disconnect()
+    logger.info("Telegram Archiver shutdown complete")
+
+
+# Create FastAPI app
+app = FastAPI(
+    title="Telegram Archiver",
+    description="Archive Telegram channels to local filesystem with media",
+    version="1.0.0",
+    lifespan=lifespan,
+)
+
+
+@app.get("/health", response_model=HealthCheck)
+async def health_check() -> HealthCheck:
+    """Health check endpoint."""
+    is_connected = archiver_client is not None and archiver_client._connected
+    return HealthCheck(
+        status="healthy" if is_connected else "disconnected",
+        version="1.0.0",
+        session_active=is_connected,
+    )
+
+
+@app.post("/archive", response_model=ArchiveResponse)
+async def archive_channel(request: ArchiveRequest) -> ArchiveResponse:
+    """
+    Archive a Telegram channel to local filesystem.
+
+    Creates a bundle directory for each message with:
+    - index.md: Message content in Markdown with front-matter
+    - Media files: Photos, videos, documents (if under size limit)
+
+    Large files (>MAX_FILE_SIZE) are skipped and logged in 2big2get.md
+    """
+    if not archiver_client:
+        raise HTTPException(status_code=503, detail="Archiver not initialized")
+
+    try:
+        logger.info(f"Archive request for channel: {request.channel}")
+
+        # Get settings and override output_dir if specified
+        settings = get_settings()
+        if request.output_dir:
+            settings.output_dir = Path(request.output_dir)
+
+        # Create archiver for this request
+        channel_archiver = ChannelArchiver(archiver_client, settings)
+
+        # Run archive process
+        result = await channel_archiver.archive_channel(
+            channel=request.channel,
+            limit=request.limit,
+            from_message_id=request.from_message_id,
+            force=request.force,
+        )
+
+        logger.info(f"Archive completed: {result['posts_archived']} posts")
+
+        return ArchiveResponse(**result)
+
+    except Exception as e:
+        logger.error(f"Archive failed: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.get("/")
+async def root() -> dict:
+    """Root endpoint with API info."""
+    return {
+        "name": "Telegram Archiver",
+        "version": "1.0.0",
+        "description": "Archive Telegram channels to local filesystem",
+        "endpoints": {
+            "health": "GET /health",
+            "archive": "POST /archive",
+            "docs": "GET /docs",
+        },
+    }
+
+
+# CLI entry point
+def cli_main() -> None:
+    """Command-line interface entry point."""
+    import click
+
+    @click.command()
+    @click.option(
+        "--channel",
+        "-c",
+        required=True,
+        help="Telegram channel username (with or without @)",
+    )
+    @click.option(
+        "--output-dir",
+        "-o",
+        type=click.Path(),
+        default=None,
+        help="Output directory for archived channel",
+    )
+    @click.option(
+        "--limit",
+        "-l",
+        type=int,
+        default=None,
+        help="Limit number of messages to archive",
+    )
+    @click.option(
+        "--from-message-id",
+        "-f",
+        type=int,
+        default=None,
+        help="Start archiving from this message ID",
+    )
+    @click.option(
+        "--force",
+        is_flag=True,
+        help="Force re-download of already archived posts",
+    )
+    @click.option(
+        "--env-file",
+        type=click.Path(exists=True),
+        default=".env",
+        help="Path to .env file",
+    )
+    def archive(
+        channel: str,
+        output_dir: str | None,
+        limit: int | None,
+        from_message_id: int | None,
+        force: bool,
+        env_file: str,
+    ) -> None:
+        """Archive a Telegram channel to local filesystem."""
+        # Load settings
+        settings = get_settings()
+
+        if output_dir:
+            settings.output_dir = Path(output_dir)
+
+        # Setup logging
+        setup_logging(log_level=settings.log_level)
+
+        # Run archive
+        async def run_archive() -> None:
+            async with TelethonArchiver(settings) as client:
+                archiver = ChannelArchiver(client, settings)
+                result = await archiver.archive_channel(
+                    channel=channel,
+                    limit=limit,
+                    from_message_id=from_message_id,
+                    force=force,
+                )
+
+                # Print summary
+                click.echo("\n" + "=" * 50)
+                click.echo("Archive Summary")
+                click.echo("=" * 50)
+                click.echo(f"Channel: {result['channel_title']} ({result['channel']})")
+                click.echo(f"Posts archived: {result['posts_archived']}")
+                click.echo(f"Posts skipped: {result['posts_skipped']}")
+                click.echo(f"Media downloaded: {result['media_downloaded']}")
+                click.echo(f"Media skipped (large): {result['media_skipped_large']}")
+                click.echo(f"Output path: {result['output_path']}")
+                click.echo(f"Log file: {result['log_file']}")
+                if result['large_files_report']:
+                    click.echo(f"Large files report: {result['large_files_report']}")
+                click.echo(f"Duration: {result['duration_seconds']:.2f}s")
+                click.echo("=" * 50)
+
+        asyncio.run(run_archive())
+
+    archive()
+
+
+if __name__ == "__main__":
+    # For development: run with uvicorn
+    # uvicorn app.main:app --reload
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000)