docker-compose.yml

version: '3'

services:
  spark-master:
    image: spark-image 
    container_name: spark-master
    entrypoint: ['./entrypoint.sh', 'master']
    build:
      context: ./data-processing-spark/1-lab-setup/containers/spark
      dockerfile: Dockerfile 
    ports:
      - "4040:4040"
      - "9090:8080"
      - "7077:7077"
      - "3000:3000"
    volumes:
      - ./capstone:/opt/spark/work-dir/capstone
      - ./data-processing-spark:/opt/spark/work-dir/data-processing-spark
      - ./etl:/opt/spark/work-dir/etl
      - spark-logs:/opt/spark/spark-events
      - tpch-data:/opt/spark/tpch-dbgen
    environment:
      UPSTREAM_DRIVERNAME: postgresql
      UPSTREAM_HOST: upstream
      UPSTREAM_PORT: '5432'
      UPSTREAM_USERNAME: sdeuser
      UPSTREAM_PASSWORD: sdepassword
      UPSTREAM_DATABASE: upstreamdb
    env_file:
      - .env.spark
  
  spark-history-server:
    container_name: spark-history
    image: spark-image
    entrypoint: ['./entrypoint.sh', 'history']
    depends_on:
      - spark-master
    env_file:
      - .env.spark
    volumes:
      - spark-logs:/opt/spark/spark-events
    ports:
      - '18080:18080'
  
  spark-worker:
    image: spark-image
    entrypoint: ['./entrypoint.sh', 'worker']
    volumes:
      - ./capstone:/opt/spark/work-dir/capstone
      - ./data-processing-spark:/opt/spark/work-dir/data-processing-spark
      - spark-logs:/opt/spark/spark-events
      - tpch-data:/opt/spark/tpch-dbgen
      - ./etl:/opt/spark/work-dir/etl
    env_file:
      - .env.spark
    environment:
      UPSTREAM_DRIVERNAME: postgresql
      UPSTREAM_HOST: upstream
      UPSTREAM_PORT: '5432'
      UPSTREAM_USERNAME: sdeuser
      UPSTREAM_PASSWORD: sdepassword
      UPSTREAM_DATABASE: upstreamdb
 
  minio:
    image: 'minio/minio:RELEASE.2023-07-21T21-12-44Z'
    hostname: minio
    container_name: minio
    ports:
      - '9000:9000'  
      - '9001:9001'
    environment:
      MINIO_ACCESS_KEY: minio
      MINIO_SECRET_KEY: minio123
    command: server --console-address ":9001" /data

  upstream:
    image: postgres:15
    container_name: upstream
    environment:
      POSTGRES_USER: sdeuser
      POSTGRES_PASSWORD: sdepassword
      POSTGRES_DB: upstreamdb
    ports:
      - "5432:5432"
    volumes:
      - ./data-processing-spark/1-lab-setup/containers/upstream/1-upstream-data.sql:/docker-entrypoint-initdb.d/init.sql

  createbuckets:
    image: minio/mc
    container_name: createbuckets
    depends_on:
      - minio
    entrypoint: >
      /bin/sh -c " /usr/bin/mc config host add myminio http://minio:9000 minio minio123; /usr/bin/mc rm -r --force myminio/tpch; /usr/bin/mc mb myminio/tpch; /usr/bin/mc policy download myminio/tpch; /usr/bin/mc rm -r --force myminio/rainforest; /usr/bin/mc mb myminio/rainforest; /usr/bin/mc policy download myminio/rainforest; exit 0; "


volumes:
  spark-logs:
  tpch-data: