Lesson 16.40: Deploying, Scaling & Operating Production AI

# Version Control & Git
.git
.gitignore

# Environment & Secrets (Never bake into images)
.env
.env.*
*.pem
*.key

# Python Artifacts & Cache
__pycache__/
*.py[cod]
*$py.class
.pytest_cache/
.coverage
htmlcov/
.venv/
env/
venv/

# Local Testing & Development Uploads
tests/
docs/
*.sqlite3
uploads/
scratch/

# ==========================================
# Byteion AI Production Dockerfile
# Base: Python 3.12 Slim (Debian Bookworm)
# ==========================================
FROM python:3.12-slim AS runtime

# Step 1: Environment Variables for Python optimization
ENV PYTHONDONTWRITEBYTECODE=1     PYTHONUNBUFFERED=1     PYTHONPATH=/app     PORT=8000

# Step 2: Install minimal OS dependencies & curl for healthcheck
RUN apt-get update && apt-get install -y --no-install-recommends     curl     libpq-dev     && rm -rf /var/lib/apt/lists/*

# Step 3: Create dedicated non-root user (Security Standard)
RUN groupadd -g 1000 appgroup &&     useradd -u 1000 -g appgroup -m -s /bin/bash appuser

WORKDIR /app

# Step 4: Layer Caching - Install dependencies first
COPY requirements.txt .
RUN pip install --no-cache-dir --upgrade pip &&     pip install --no-cache-dir -r requirements.txt

# Step 5: Copy application code and set proper ownership
COPY . /app
RUN chown -R appuser:appgroup /app

# Step 6: Drop root privileges
USER appuser

# Step 7: Expose port for FastAPI
EXPOSE 8000

# Step 8: Default CMD for FastAPI (Can be overridden by Celery worker in compose)
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4", "--proxy-headers", "--forwarded-allow-ips=*"]

from fastapi import APIRouter, status, Response
import redis.asyncio as aioredis
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from app.database import get_db_engine

router = APIRouter(prefix="/health", tags=["Health"])

@router.get("/live", status_code=status.HTTP_200_OK)
async def liveness_check():
    # Liveness probe: Returns 200 immediately to confirm event loop is healthy.
    return {"status": "alive"}

@router.get("/ready")
async def readiness_check(response: Response):
    # Readiness probe: Checks local critical infrastructure (Postgres + Redis).
    # NOTE: NEVER call external LLM APIs (OpenAI/Anthropic) here!
    checks = {"database": False, "redis": False}
    
    # 1. Check PostgreSQL Connection
    try:
        engine = get_db_engine()
        async with engine.connect() as conn:
            await conn.execute(text("SELECT 1"))
            checks["database"] = True
    except Exception as exc:
        checks["database_error"] = str(exc)

    # 2. Check Redis Ping
    try:
        r = aioredis.from_url("redis://redis:6379/0", socket_timeout=2.0)
        pong = await r.ping()
        checks["redis"] = (pong is True)
        await r.aclose()
    except Exception as exc:
        checks["redis_error"] = str(exc)

    # If any critical local dependency fails, return 503 Service Unavailable
    if not (checks["database"] and checks["redis"]):
        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
        return {"status": "unready", "checks": checks}

    return {"status": "ready", "checks": checks}

services:
  # -------------------------------------------------------------
  # 1. Stateless FastAPI Ingress Service
  # -------------------------------------------------------------
  api:
    image: byteion-ai:latest
    build:
      context: .
      dockerfile: Dockerfile
    restart: unless-stopped
    ports:
      # Bind exclusively to 127.0.0.1 (Reverse proxy terminates public traffic)
      - "127.0.0.1:8000:8000"
    environment:
      - DATABASE_URL=postgresql+asyncpg://postgres:${POSTGRES_PASSWORD}@db:5432/byteion_ai
      - REDIS_URL=redis://:${REDIS_PASSWORD}@redis:6379/0
      - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/1
      - OPENAI_API_KEY=${OPENAI_API_KEY}
      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
    depends_on:
      db:
        condition: service_healthy
      redis:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8000/health/ready || exit 1"]
      interval: 15s
      timeout: 5s
      retries: 3
      start_period: 10s
    deploy:
      resources:
        limits:
          cpus: '2.0'
          memory: 1G
    networks:
      - byteion-internal

  # -------------------------------------------------------------
  # 2. Asynchronous Celery Background Worker Service
  # -------------------------------------------------------------
  worker:
    image: byteion-ai:latest
    restart: unless-stopped
    command: >
      celery -A app.worker.celery_app worker
      --loglevel=INFO
      --queues=heavy_queue,fast_queue
      --concurrency=4
      --prefetch-multiplier=1
    environment:
      - DATABASE_URL=postgresql+asyncpg://postgres:${POSTGRES_PASSWORD}@db:5432/byteion_ai
      - REDIS_URL=redis://:${REDIS_PASSWORD}@redis:6379/0
      - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/1
      - OPENAI_API_KEY=${OPENAI_API_KEY}
    depends_on:
      redis:
        condition: service_healthy
      db:
        condition: service_healthy
    # Give active PDF extraction jobs 60 seconds to safely drain before SIGKILL
    stop_grace_period: 60s
    deploy:
      resources:
        limits:
          cpus: '2.0'
          memory: 2G
    networks:
      - byteion-internal

  # -------------------------------------------------------------
  # 3. Persistent PostgreSQL 17 Database with pgvector
  # -------------------------------------------------------------
  db:
    image: pgvector/pgvector:pg17
    restart: unless-stopped
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_DB=byteion_ai
    volumes:
      - pgdata:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres -d byteion_ai"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - byteion-internal
    # Notice: NO public ports exposed! Accessible only within internal network.

  # -------------------------------------------------------------
  # 4. In-Memory Redis 7 Broker & Semantic Cache
  # -------------------------------------------------------------
  redis:
    image: redis:7-alpine
    restart: unless-stopped
    command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD}
    volumes:
      - redisdata:/data
    healthcheck:
      test: ["CMD-SHELL", "redis-cli -a ${REDIS_PASSWORD} ping | grep PONG"]
      interval: 10s
      timeout: 3s
      retries: 3
    networks:
      - byteion-internal

# ---------------------------------------------------------------
# Persistent Volumes & Isolated Bridge Networks
# ---------------------------------------------------------------
volumes:
  pgdata:
  redisdata:

networks:
  byteion-internal:
    driver: bridge
    internal: false

from celery import Celery
import os

celery_app = Celery(
    "byteion_worker",
    broker=os.getenv("CELERY_BROKER_URL", "redis://redis:6379/1"),
    backend=os.getenv("CELERY_RESULT_BACKEND", "redis://redis:6379/2")
)

celery_app.conf.update(
    # 1. Critical for AI: Prevent worker from hoarding heavy PDF jobs
    worker_prefetch_multiplier=1,
    
    # 2. Acknowledge task only AFTER successful execution (survive OOM crashes)
    task_acks_late=True,
    
    # 3. If worker container dies or gets SIGKILLed, re-queue task immediately
    task_reject_on_worker_lost=True,
    
    # 4. Hard and Soft time limits (seconds) to prevent infinite hallucination loops
    task_time_limit=300,        # 5 minutes hard limit (SIGKILL)
    task_soft_time_limit=270,   # 4.5 minutes soft limit (SoftTimeLimitExceeded)
    
    # 5. Route specific tasks to dedicated queues
    task_routes={
        "app.worker.tasks.process_document_pdf": {"queue": "heavy_queue"},
        "app.worker.tasks.generate_batch_embeddings": {"queue": "heavy_queue"},
        "app.worker.tasks.invalidate_user_cache": {"queue": "fast_queue"},
        "app.worker.tasks.send_webhook_event": {"queue": "fast_queue"},
    }
)

name: Byteion AI CI/CD Pipeline

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  # -------------------------------------------------------------
  # Job 1: Lint, Unit Tests & AI Golden Dataset Evaluation
  # -------------------------------------------------------------
  test-and-eval:
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis:7-alpine
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
          cache: 'pip'
      
      - name: Install dependencies
        run: |
          pip install --upgrade pip
          pip install -r requirements.txt pytest ruff httpx

      - name: Lint and Code Style (Ruff)
        run: ruff check .

      - name: Run Unit & Integration Tests (Mock Providers)
        run: pytest tests/unit tests/integration -v

      - name: AI Golden Dataset Regression Gate (Lessons 33-35)
        run: python scripts/run_golden_eval.py --min-recall 0.85 --min-groundedness 0.90

  # -------------------------------------------------------------
  # Job 2: Build Immutable Image & Security Scan
  # -------------------------------------------------------------
  build-and-push:
    needs: test-and-eval
    if: github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: actions/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: {{ github.actor }}
          password: {{ secrets.GITHUB_TOKEN }}

      - name: Build and Push Docker Image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: |
            ghcr.io/byteion/byteion-ai:latest
            ghcr.io/byteion/byteion-ai:{{ github.sha }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  # -------------------------------------------------------------
  # Job 3: Isolated Database Migration & Production Deploy
  # -------------------------------------------------------------
  deploy-production:
    needs: build-and-push
    runs-on: ubuntu-latest
    steps:
      - name: Trigger Isolated Database Migration
        run: |
          echo "Executing one-off Alembic migration container before rolling update..."
          # ssh deploy_user@prod.byteion.com 'docker compose run --rm api alembic upgrade head'

      - name: Rolling Update Containers
        run: |
          echo "Pulling new image and executing rolling container restart..."
          # ssh deploy_user@prod.byteion.com 'docker compose pull && docker compose up -d'