# ==========================================
# Testhide — Production Docker Compose
# Download this file, place it in your deployment directory,
# create .env next to it from .env.example.
#
# Images:  https://hub.docker.com/u/thuesdays
# Docs:    https://testhide.com/installation/
# License: https://testhide.com/pricing/
# ==========================================
#
# 12 services:
#   * config-init           — one-shot config sync (image entrypoint bypass)
#   * frontend              — Nginx + Angular SPA (TLS termination)
#   * backend ×2            — Python REST API + WebSocket (gunicorn)
#   * ai-api                — LLM / FAISS / CLIP inference (HTTP only)
#   * ai-worker             — Training, vectorization, Edge AI (background)
#   * sidecar-docker        — SEC-004 Docker socket isolation
#   * mongo                 — Database
#   * redis                 — Cache + pub/sub
#   * redisinsight          — Optional Redis admin (proxied via nginx)
#   * prometheus            — Metrics (loopback bind by default)
#   * grafana               — Dashboards + alerts (HTTPS, port 3000)
#   * tempo                 — Distributed tracing (OTel)
#   * loki                  — Log aggregation
#   * alloy                 — Log/metric collector → Loki + Prometheus
#
# Required .env vars (see .env.example for the full list):
#   PUBLIC_URL, JWT_SECRET, MONGO_USER, MONGO_PASS, MONGO_DATA_PATH,
#   REDIS_PASSWORD, CERT_FILE, CERT_KEY, GRAFANA_ADMIN_PASSWORD.
# ==========================================

name: testhide

services:

  # ── Config-init (one-shot, syncs baked configs to named volumes) ─
  # Runs once on every `docker compose up -d --force-recreate`.
  # The testhide-backend image bakes the entire /app/baked_configs/
  # directory (grafana, loki, prometheus, tempo, alloy configs).
  # This sidecar copies them into named docker volumes that the
  # observability services mount read-only at runtime.
  #
  # No host repo or bind-mount needed — configs travel inside the image.
  config-init:
    image: thuesdays/testhide-backend:latest
    container_name: testhide-config-init
    # Bypass the image ENTRYPOINT (model downloads / DB migrations) —
    # config-init only needs the sync script.
    entrypoint: []
    command: ["bash", "/app/scripts/sync-configs-to-volumes.sh"]
    environment:
      - SKIP_LLM_CHECK=true
      - LOAD_AI_MODELS=false
      - RUN_AI_WORKER=false
      - DISABLE_CRON=true
      - SKIP_BUILD_RECOVERY=true
    volumes:
      - grafana_dashboards:/mnt/grafana_dashboards
      - grafana_provisioning:/mnt/grafana_provisioning
      - loki_config:/mnt/loki_config
      - prometheus_config:/mnt/prometheus_config
      - tempo_config:/mnt/tempo_config
      - alloy_config:/mnt/alloy_config
    restart: "no"

  # ── Frontend (Nginx + Angular SPA, TLS termination) ─────────────
  frontend:
    container_name: testhide_frontend
    image: thuesdays/testhide-frontend:latest
    restart: unless-stopped
    ports:
      - "80:80"
      - "443:443"
      - "7771:7771"
    env_file:
      - .env
    environment:
      - CERT_FILE=${CERT_FILE}
      - CERT_KEY=${CERT_KEY}
    volumes:
      - ./ssl:/etc/ssl:ro
      - testhide_static_data:/usr/share/nginx/html/static:ro
    depends_on:
      - backend
      - ai-api
    networks:
      - testhide-net
    deploy:
      resources:
        limits:
          cpus: "0.25"
          memory: 256m

  # ── Backend (Python API + WebSocket, 2 replicas) ─────────────────
  backend:
    # No container_name — replicas need unique names
    image: thuesdays/testhide-backend:latest
    restart: unless-stopped
    env_file:
      - .env
    environment:
      - LOAD_AI_MODELS=false
      - RUN_AI_WORKER=false
      - GUNICORN_WORKERS=${GUNICORN_WORKERS:-2}
      - COMPOSE_PROJECT_NAME=testhide
      - ENVIRONMENT=production
      - SERVICE_NAME=testhide
      # Backend must NEVER train (mem_limit=2.5g → OOM). Safety guards:
      # (1) trigger_retraining() checks RUN_AI_WORKER=false.
      # (2) absurdly high threshold so counter never triggers.
      - BUILD_RCA_RETRAIN_THRESHOLD=9999999
      - LOG_LEVEL=${LOG_LEVEL:-INFO}
      - DOCKER_API_VERSION=1.41
      # SEC-004 — Docker socket lives in sidecar-docker, not here.
      # Sidecar auth token is auto-derived from JWT_SECRET (HMAC-SHA256).
      - SIDECAR_URL=http://sidecar-docker:8081
      # CORS — comma-separated list of allowed origins.
      # Default = your public URL (set PUBLIC_URL in .env).
      - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-${PUBLIC_URL}}
    volumes:
      - testhide_static_data:/app/static
      - ./releases:/app/releases
      - ./monitoring_scripts:/app/monitoring_scripts
      # Optional: mount your corporate LDAP CA (uncomment if needed).
      # - ./ssl/ldap-ca.pem:/etc/ssl/ldap-ca.pem:ro
      - ./sandbox_data:/app/sandbox_data
      # SSH keys persist across rebuilds AND are shared between replicas.
      # Without this mount, every deploy generates new keys → agents lose access.
      - ./ssh_keys:/app/ssh_keys
      # Compose file mount lets the API restart child containers.
      - ./docker-compose.yaml:/app/docker-compose.yaml:ro
      - testhide_hf_cache:/app/hf_cache
    depends_on:
      mongo:
        condition: service_healthy
      redis:
        condition: service_healthy
      sidecar-docker:
        condition: service_healthy
    networks:
      - testhide-net
      - testhide-internal
    healthcheck:
      disable: true
    mem_limit: 2.5g
    memswap_limit: 2.5g
    deploy:
      replicas: 2
      resources:
        limits:
          cpus: "2.0"
          memory: 2.5g
        reservations:
          cpus: "0.5"
          memory: 1g

  # ── AI API (LLM / CLIP / FAISS inference — HTTP only) ────────────
  ai-api:
    container_name: testhide_ai_api
    image: thuesdays/testhide-backend:latest
    restart: unless-stopped
    env_file:
      - .env
    environment:
      - LOAD_AI_MODELS=true
      - RUN_AI_WORKER=false
      - DISABLE_CRON=true
      - SKIP_BUILD_RECOVERY=true
      - GUNICORN_WORKERS=${AI_API_GUNICORN_WORKERS:-2}
      - ENVIRONMENT=production
      - SERVICE_NAME=testhide-ai-api
      - LOG_LEVEL=${LOG_LEVEL:-INFO}
      - HF_HOME=/app/hf_cache
      - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-0}
      - TRANSFORMERS_OFFLINE=${TRANSFORMERS_OFFLINE:-0}
      - RC_MODEL_NAME=${RC_MODEL_NAME:-distilbert-base-uncased}
      - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-${PUBLIC_URL}}
    volumes:
      - testhide_static_data:/app/static
      - ./sandbox_data:/app/sandbox_data
      - ./releases:/app/releases
      - ./monitoring_scripts:/app/monitoring_scripts
      # - ./ssl/ldap-ca.pem:/etc/ssl/ldap-ca.pem:ro
      - testhide_hf_cache:/app/hf_cache
    depends_on:
      mongo:
        condition: service_healthy
      redis:
        condition: service_healthy
    networks:
      - testhide-net
    healthcheck:
      disable: true
    # Elastic memory contract with ai-worker:
    #   mem_reservation 1.5 GB = guaranteed minimum (inference models in RAM).
    #   mem_limit 6 GB so ai-api grows into idle headroom when worker is quiet.
    #   While training is active (Redis key ai:training:in_progress),
    #   ai-api releases heavy caches → drops to ~1.5 GB working set.
    #   Sum of mem_limit across api+worker can exceed physical RAM —
    #   intentional, since the two are anti-correlated.
    mem_limit: 6g
    memswap_limit: 6g
    mem_reservation: 1500m
    deploy:
      replicas: 1
      resources:
        limits:
          cpus: "4.0"
          memory: 6g
        reservations:
          cpus: "0.5"
          memory: 1500m

  # ── AI Worker (training, vectorization, Edge AI) ─────────────────
  ai-worker:
    container_name: testhide_ai_worker
    image: thuesdays/testhide-backend:latest
    restart: unless-stopped
    env_file:
      - .env
    environment:
      - LOAD_AI_MODELS=true
      - RUN_AI_WORKER=true
      - DISABLE_CRON=true
      - SKIP_BUILD_RECOVERY=true
      - GUNICORN_WORKERS=1
      - ENVIRONMENT=production
      - SERVICE_NAME=testhide-ai-worker
      - LOG_LEVEL=${LOG_LEVEL:-INFO}
      # Memory safety: soft limit fires before Docker hard-kills at mem_limit.
      - AI_MEMORY_LIMIT_MB=${AI_MEMORY_LIMIT_MB:-7500}
      - AI_HARD_EXIT_PERCENT=${AI_HARD_EXIT_PERCENT:-88}
      # LLM context window: 8192 saves ~1.5 GB KV-cache vs 32768.
      - AI_LLM_CTX=${AI_LLM_CTX:-8192}
      # Pandas merge batch peak: ~24 MB per batch at 150 rows.
      - AI_MERGE_BATCH_SIZE=${AI_MERGE_BATCH_SIZE:-150}
      - AI_RETRAIN_QUEUE_BUSY_THRESHOLD=${AI_RETRAIN_QUEUE_BUSY_THRESHOLD:-200}
      - AI_LOCK_STALE_SECS=${AI_LOCK_STALE_SECS:-300}
      # Worker parallelism — defaults tuned for 6 CPU cores.
      - AI_ASSIST_INFER_WORKERS=${AI_ASSIST_INFER_WORKERS:-4}
      - AI_ASSIST_IO_WORKERS=${AI_ASSIST_IO_WORKERS:-4}
      - AI_ASSIST_LLM_WORKERS=${AI_ASSIST_LLM_WORKERS:-1}
      # ─── AI Pipeline streaming (Phase 1 + Phase 2 §3.7) ───
      - AI_DATASET_SHARD_ROWS=${AI_DATASET_SHARD_ROWS:-50000}
      - AI_DATASET_MAX_ROWS=${AI_DATASET_MAX_ROWS:-2000000}
      - AI_COMPACT_AFTER_SHARDS=${AI_COMPACT_AFTER_SHARDS:-32}
      - AI_BUDGET_SAFETY_FACTOR=${AI_BUDGET_SAFETY_FACTOR:-0.5}
      - AI_BUDGET_MIN_CHUNK_ROWS=${AI_BUDGET_MIN_CHUNK_ROWS:-16}
      - AI_BUDGET_MAX_CHUNK_ROWS=${AI_BUDGET_MAX_CHUNK_ROWS:-100000}
      - AI_TRAINING_LOCK_TTL_SEC=${AI_TRAINING_LOCK_TTL_SEC:-3600}
      # Vector sidecar memmap (Phase 2 §3.7). 1 = enabled (default).
      # On first deploy migrate_vectors_to_sidecar.py runs idempotently.
      - AI_VECTORS_SIDECAR_ENABLED=${AI_VECTORS_SIDECAR_ENABLED:-1}
      # YOLO settings (CPU training is the default).
      - AI_YOLO_FORCE_CPU=${AI_YOLO_FORCE_CPU:-true}
      - AI_YOLO_EPOCHS=${AI_YOLO_EPOCHS:-5}
      - AI_YOLO_MIN_ANNOTATIONS=${AI_YOLO_MIN_ANNOTATIONS:-5}
      - HF_HOME=/app/hf_cache
      - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-0}
      - TRANSFORMERS_OFFLINE=${TRANSFORMERS_OFFLINE:-0}
      - RC_MODEL_NAME=${RC_MODEL_NAME:-distilbert-base-uncased}
      - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-${PUBLIC_URL}}
    volumes:
      - testhide_static_data:/app/static
      - ./sandbox_data:/app/sandbox_data
      - ./releases:/app/releases
      - ./monitoring_scripts:/app/monitoring_scripts
      # - ./ssl/ldap-ca.pem:/etc/ssl/ldap-ca.pem:ro
      - testhide_hf_cache:/app/hf_cache
    depends_on:
      mongo:
        condition: service_healthy
      redis:
        condition: service_healthy
    networks:
      testhide-net:
        aliases:
          - ai-worker
    healthcheck:
      disable: true
    # Elastic memory contract with ai-api (see ai-api comment).
    # 5 GB reservation, 12 GB limit. Soft limit AI_MEMORY_LIMIT_MB=7500
    # + AI_HARD_EXIT_PERCENT=88 fire before Docker OOM kill.
    mem_limit: 12g
    memswap_limit: 12g
    mem_reservation: 5g
    deploy:
      replicas: 1
      resources:
        limits:
          cpus: "6.0"
          memory: 12g
        reservations:
          cpus: "1.0"
          memory: 5g

  # ── MongoDB 8 ───────────────────────────────────────────────────
  mongo:
    container_name: testhide_mongo
    image: 'mongo:8.2.3'
    restart: unless-stopped
    command: ["mongod", "--auth", "--wiredTigerCacheSizeGB", "2"]
    environment:
      - MONGO_INITDB_ROOT_USERNAME=${MONGO_USER}
      - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASS}
    ports:
      - "27017:27017"
    volumes:
      - ${MONGO_DATA_PATH}:/data/db
    networks:
      - testhide-net
    healthcheck:
      test: ["CMD", "mongosh", "--norc", "--quiet",
             "-u", "${MONGO_USER}", "-p", "${MONGO_PASS}",
             "--authenticationDatabase", "admin",
             "--eval", "db.adminCommand('ping')"]
      interval: 30s
      timeout: 5s
      retries: 5
    mem_limit: 3g
    memswap_limit: 3g
    deploy:
      resources:
        limits:
          cpus: "1.5"
          memory: 3g
        reservations:
          cpus: "0.5"
          memory: 1.5g

  # ── Redis 7 ────────────────────────────────────────────────────
  redis:
    container_name: testhide_redis
    image: redis:7-alpine
    restart: unless-stopped
    command: ["redis-server",
              "--maxmemory", "512mb",
              "--maxmemory-policy", "allkeys-lru",
              "--appendonly", "no",
              "--requirepass", "${REDIS_PASSWORD}"]
    networks:
      - testhide-net
    healthcheck:
      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
      interval: 10s
      timeout: 3s
      retries: 5
    mem_limit: 768m
    memswap_limit: 768m
    deploy:
      resources:
        limits:
          cpus: "0.5"
          memory: 768m

  # ── Redis Insight (optional admin UI, proxied via nginx) ────────
  redisinsight:
    container_name: testhide_redisinsight
    image: redis/redisinsight:latest
    restart: unless-stopped
    depends_on:
      redis:
        condition: service_healthy
    networks:
      - testhide-net
    mem_limit: 256m
    memswap_limit: 256m
    deploy:
      resources:
        limits:
          cpus: "0.25"
          memory: 256m

  # ── Docker Socket Sidecar (SEC-004) ─────────────────────────────
  # Privileged sidecar that holds /var/run/docker.sock.
  # Backend talks to it over testhide-internal (no external port).
  # Auth token auto-derived from JWT_SECRET via HMAC-SHA256.
  sidecar-docker:
    container_name: testhide_sidecar_docker
    image: thuesdays/testhide-sidecar:latest
    restart: unless-stopped
    environment:
      - JWT_SECRET=${JWT_SECRET}
      - SIDECAR_ALLOWED_IMAGES=${SIDECAR_ALLOWED_IMAGES:-thuesdays/testhide-agent:latest,thuesdays/testhide-backend:latest}
      - SIDECAR_ALLOWED_EXEC_PATTERNS=${SIDECAR_ALLOWED_EXEC_PATTERNS:-^python\s,^bash\s,^sh\s}
      - SIDECAR_ALLOWED_NETWORKS=${SIDECAR_ALLOWED_NETWORKS:-testhide-internal,bridge,testhide-net}
      - SIDECAR_PORT=8081
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - testhide-internal
    mem_limit: 128m
    memswap_limit: 128m
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:8081/health"]
      interval: 30s
      timeout: 3s
      retries: 3
      start_period: 10s
    logging:
      driver: json-file
      options:
        max-size: "10m"
        max-file: "5"

  # ── Observability: Prometheus (metrics) ─────────────────────────
  # Default bind: 127.0.0.1:9090 (loopback only). Override via
  # PROMETHEUS_PORT_BIND in .env. Production should NEVER use 0.0.0.0:9090.
  prometheus:
    image: prom/prometheus:v2.51.0
    container_name: testhide-prometheus
    restart: unless-stopped
    depends_on:
      config-init:
        condition: service_completed_successfully
    volumes:
      - prometheus_config:/etc/prometheus:ro
      - prometheus_data:/prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=30d
      - --web.enable-lifecycle
      - --web.enable-remote-write-receiver
    ports:
      - "${PROMETHEUS_PORT_BIND:-127.0.0.1:9090}:9090"
    networks:
      - testhide-net
    mem_limit: 512m
    memswap_limit: 512m
    deploy:
      resources:
        limits:
          cpus: "0.5"
          memory: 512m

  # ── Observability: Grafana (dashboards, HTTPS) ──────────────────
  grafana:
    image: grafana/grafana:10.4.2
    container_name: testhide-grafana
    restart: unless-stopped
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
      GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-${PUBLIC_URL}:3000}
      GF_SERVER_PROTOCOL: https
      GF_SERVER_CERT_FILE: /etc/grafana/ssl/server.crt
      GF_SERVER_CERT_KEY: /etc/grafana/ssl/server.key
      GF_FEATURE_TOGGLES_ENABLE: traceqlEditor
      GF_AUTH_ANONYMOUS_ENABLED: "false"
    volumes:
      - grafana_provisioning:/etc/grafana/provisioning:ro
      - grafana_dashboards:/var/lib/grafana/dashboards:ro
      - grafana_data:/var/lib/grafana
      # Grafana SSL certs — defaults reuse the main testhide cert.
      # Override via GRAFANA_CERT_HOST_CRT / _KEY in .env if separate.
      - ${GRAFANA_CERT_HOST_CRT:-./ssl/testhide.crt}:/etc/grafana/ssl/server.crt:ro
      - ${GRAFANA_CERT_HOST_KEY:-./ssl/testhide.key}:/etc/grafana/ssl/server.key:ro
    ports:
      - "3000:3000"
    depends_on:
      config-init:
        condition: service_completed_successfully
      prometheus:
        condition: service_started
      tempo:
        condition: service_started
      loki:
        condition: service_started
    networks:
      - testhide-net
    mem_limit: 256m
    memswap_limit: 256m
    deploy:
      resources:
        limits:
          cpus: "0.25"
          memory: 256m

  # ── Observability: Tempo (distributed traces via OTel) ──────────
  tempo:
    image: grafana/tempo:2.4.1
    container_name: testhide-tempo
    restart: unless-stopped
    command: ["-config.file=/etc/tempo/tempo.yml"]
    depends_on:
      config-init:
        condition: service_completed_successfully
    volumes:
      - tempo_config:/etc/tempo:ro
      - tempo_data:/tmp/tempo
    ports:
      - "4317:4317"   # OTLP gRPC
      - "4318:4318"   # OTLP HTTP
      - "3200:3200"   # Tempo HTTP API
    networks:
      - testhide-net
    mem_limit: 512m
    memswap_limit: 512m
    deploy:
      resources:
        limits:
          cpus: "0.5"
          memory: 512m

  # ── Observability: Loki (log aggregation) ───────────────────────
  loki:
    image: grafana/loki:2.9.4
    container_name: testhide-loki
    restart: unless-stopped
    command: -config.file=/etc/loki/loki.yml
    depends_on:
      config-init:
        condition: service_completed_successfully
    volumes:
      - loki_config:/etc/loki:ro
      - loki_data:/loki
    ports:
      - "3100:3100"
    networks:
      - testhide-net
    mem_limit: 256m
    memswap_limit: 256m
    deploy:
      resources:
        limits:
          cpus: "0.25"
          memory: 256m

  # ── Observability: Grafana Alloy (log/metric collector) ─────────
  alloy:
    image: grafana/alloy:v1.5.1
    container_name: testhide-alloy
    restart: unless-stopped
    volumes:
      - alloy_config:/etc/alloy:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      # Share static volume so Alloy tails testhide.log / testhide_worker.log.
      - testhide_static_data:/var/log/testhide:ro
    ports:
      - "12345:12345"
    depends_on:
      config-init:
        condition: service_completed_successfully
      loki:
        condition: service_started
      prometheus:
        condition: service_started
    networks:
      - testhide-net
    mem_limit: 256m
    memswap_limit: 256m
    deploy:
      resources:
        limits:
          cpus: "0.25"
          memory: 256m

# ── Volumes ─────────────────────────────────────────────────────
volumes:
  testhide_static_data:
    name: testhide_static_data
  testhide_hf_cache:
    name: testhide_hf_cache
  prometheus_data:
    name: testhide_prometheus_data
  grafana_data:
    name: testhide_grafana_data
  tempo_data:
    name: testhide_tempo_data
  loki_data:
    name: testhide_loki_data
  # Config volumes — populated by config-init on every deploy.
  # The repo is source of truth; --force-recreate refreshes them.
  grafana_dashboards:
    name: testhide_grafana_dashboards
  grafana_provisioning:
    name: testhide_grafana_provisioning
  loki_config:
    name: testhide_loki_config
  prometheus_config:
    name: testhide_prometheus_config
  tempo_config:
    name: testhide_tempo_config
  alloy_config:
    name: testhide_alloy_config

# ── Networks ────────────────────────────────────────────────────
networks:
  testhide-net:
    name: testhide-net
  testhide-internal:
    driver: bridge
    internal: true   # No external internet access from this network
