docker-compose.yml

services:
  postgres:
    image: postgres:16
    container_name: postgres
    environment:
      POSTGRES_USER: ${POSTGRES_USER:-airflow}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-change-me-strong-password}
      POSTGRES_DB: ${POSTGRES_DB:-airflow}
    ports:
      - "5432:5432"
    volumes:
      - etl-postgres-data:/var/lib/postgresql/data
    networks:
      - etl-network
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-airflow} -d ${POSTGRES_DB:-airflow}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s

  airflow:
    build:
      context: ./airflow
      dockerfile: Dockerfile
    container_name: airflow
    environment:
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://${POSTGRES_USER:-airflow}:${POSTGRES_PASSWORD:-change-me-strong-password}@postgres/${POSTGRES_DB:-airflow}
    depends_on:
      postgres:
        condition: service_healthy
      statsd-exporter:
        condition: service_started
    volumes:
      - etl-containers-shared-data:/app/data         # Shared Volume between containers for data
      - etl-airflow-data:/opt/airflow                # Volume for persistence of Airflow data
      - ./airflow/dags:/opt/airflow/dags             # Mount the local folder 'dags' in the container
      - ./preparator:/opt/airflow/preparator         # Mount the local folder 'preparator' in the container
    ports:
      - "8080:8080"
    networks:
      - etl-network
    restart: always

  # exporter metrics Airflow -> Prometheus
  statsd-exporter:
    image: prom/statsd-exporter:latest
    container_name: statsd-exporter
    depends_on:
    - postgres
    ports:
      - "9102:9102"
    networks:
      - etl-network

#  Microservice monitoring
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    volumes:
      - etl-prometheus-data:/etc/prometheus                         
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml  
    ports:
      - "9090:9090"
    networks:
      - etl-network
    restart: always
    healthcheck:
      test: ["CMD-SHELL", "wget -qO- http://localhost:9090/-/healthy || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 15s
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/etc/prometheus/data
      - --storage.tsdb.retention.time=15d
      - --web.enable-lifecycle

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-change-me-strong-password}
    volumes:
      - etl-grafana-data:/var/lib/grafana
      - ./prometheus/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./prometheus/grafana/dashboards:/etc/grafana/provisioned-dashboards:ro
    ports:
      - "3000:3000"
    networks:
      - etl-network
    depends_on:
      - prometheus
    restart: always
    healthcheck:
      test: ["CMD-SHELL", "wget -qO- http://localhost:3000/api/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:latest
    container_name: cadvisor
    privileged: true
    devices:
      - /dev/kmsg:/dev/kmsg
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /cgroup:/cgroup:ro
    ports:
      - "8088:8080"
    networks:
      - etl-network
    command:
      - --docker_only=true
      - --housekeeping_interval=15s
      - --disable_metrics=advtcp,cpu_topology,cpuset,hugetlb,memory_numa,process,referenced_memory,resctrl,sched,tcp,udp
    restart: unless-stopped

# Microservices
  extract-csv-service:
    build:
      context: ./services
      dockerfile: extract-csv-service/Dockerfile
    container_name: extract-csv-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5001:5001"
    networks:
      - etl-network
    restart: always

  clean-nan-service:
    build:
      context: ./services
      dockerfile: clean-nan-service/Dockerfile
    container_name: clean-nan-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5002:5002"
    networks:
      - etl-network
    restart: always

  delete-columns-service:
    build:
      context: ./services
      dockerfile: delete-columns-service/Dockerfile
    container_name: delete-columns-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5004:5004"
    networks:
      - etl-network
    restart: always

  extract-sql-service:
    build:
      context: ./services
      dockerfile: extract-sql-service/Dockerfile
    container_name: extract-sql-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5005:5005"
    networks:
      - etl-network
    restart: always

  extract-api-service:
    build:
      context: ./services
      dockerfile: extract-api-service/Dockerfile
    container_name: extract-api-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5006:5006"
    networks:
      - etl-network
    restart: always

  extract-excel-service:
    build:
      context: ./services
      dockerfile: extract-excel-service/Dockerfile
    container_name: extract-excel-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5007:5007"
    networks:
      - etl-network
    restart: always

  join-datasets-service:
    build:
      context: ./services
      dockerfile: join-datasets-service/Dockerfile
    container_name: join-datasets-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5008:5008"
    networks:
      - etl-network
    restart: always

  load-data-service:
    build:
      context: ./services
      dockerfile: load-data-service/Dockerfile
    container_name: load-data-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5009:5009"
    networks:
      - etl-network
    restart: always

  data-quality-service:
    build:
      context: ./services
      dockerfile: data-quality-service/Dockerfile
    container_name: data-quality-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5010:5010"
    networks:
      - etl-network
    restart: always

  outlier-detection-service:
    build:
      context: ./services
      dockerfile: outlier-detection-service/Dockerfile
    container_name: outlier-detection-service
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "5011:5011"
    networks:
      - etl-network
    restart: always

  text-completion-llm-service:
    build:
      context: ./services
      dockerfile: text-completion-llm-service/Dockerfile
    container_name: text-completion-llm-service
    volumes:
      - etl-containers-shared-data:/app/data
      # Mount local HuggingFace model cache (set HF_MODELS_PATH in .env)
      - type: bind
        source: "${HF_MODELS_PATH:-./hf_models}"
        target: "/root/.cache/huggingface"
    ports:
      - "5012:5012"
    networks:
      - etl-network
    restart: always

  streamlit-app:
    build:
      context: .
      dockerfile: streamlit_app/Dockerfile
    container_name: streamlit-app
    environment:
      - LLM_PROVIDER=${LLM_PROVIDER:-openai}
      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
      - OPENAI_MODEL=${OPENAI_MODEL:-gpt-4o-mini}
      - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-}
      - OPENROUTER_MODEL=${OPENROUTER_MODEL:-meta-llama/llama-3.1-8b-instruct:free}
    volumes:
      - etl-containers-shared-data:/app/data
    ports:
      - "8501:8501"
    networks:
      - etl-network
    depends_on:
      extract-csv-service:
        condition: service_healthy
      load-data-service:
        condition: service_healthy
    restart: always

networks:
  etl-network:
    driver: bridge

volumes:
  etl-containers-shared-data:
  etl-postgres-data:
  etl-airflow-data:
  etl-grafana-data:
  etl-prometheus-data: