Dockerization

[wj-turner]

docker-compose.yml

version: '3.7'

services:

  # Apache Kafka
  kafka:
    image: confluentinc/cp-kafka:latest
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka:9092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
    ports:
      - "9092:9092"
      - "9094:9094"

  # ZooKeeper (used by Kafka)
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181

  # Apache NiFi
  nifi:
    image: apache/nifi:latest
    ports:
      - "8080:8080"

  # Redis
  redis:
    image: redis:latest
    ports:
      - "6379:6379"

  # Apache Airflow
  airflow:
    image: apache/airflow:latest
    depends_on:
      - postgres_airflow
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres_airflow/airflow
    ports:
      - "8088:8088"

  # PostgreSQL for Apache Airflow metadata
  postgres_airflow:
    image: postgres:latest
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow

  # TimescaleDB
  timescaledb:
    image: timescale/timescaledb:latest-pg12
    environment:
      POSTGRES_USER: timescale
      POSTGRES_PASSWORD: timescale
      POSTGRES_DB: forex_data
    ports:
      - "5432:5432"

networks:
  default:
    external:
      name: my_network