docker-compose.yml

version: '3'
services:
  spark-submit:
    image: metorikku/metorikku
    container_name: spark-submit
    ports:
      - "4040:4040"
    environment:
      - SPARK_MASTER_HOST=spark-master
      - SPARK_MASTER_PORT=7077
      - SPARK_WEBUI_PORT=8080
      - SPARK_UI_PORT=4040
      - MIN_WORKERS=2
      - MAX_RETRIES=300
      - SUBMIT_COMMAND=spark-submit --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/movies.yaml
      - JSON_LOG=true
      - JSON_LOG_LEVEL=WARN
      - GRAPHITE_HOST=graphite.local
      - GRAPHITE_PORT=3001
    entrypoint:
      - /scripts/entrypoint-submit.sh
  spark-master:
    image: metorikku/metorikku
    container_name: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
      - "6066:6066"
    environment:
      - SPARK_MASTER_HOST=spark-master
      - SPARK_MASTER_PORT=7077
      - SPARK_WEBUI_PORT=8080
      - JSON_LOG=true
      - GRAPHITE_HOST=graphite.local
      - GRAPHITE_PORT=3001
    entrypoint:
      - /scripts/entrypoint-master.sh
  spark-worker-1:
    image: metorikku/metorikku
    container_name: spark-worker-1
    ports:
      - "7078:7078"
      - "8081:8081"
    environment:
      - SPARK_MASTER_HOST=spark-master
      - SPARK_MASTER_PORT=7077
      - SPARK_WORKER_PORT=7078
      - SPARK_WORKER_WEBUI_PORT=8081
      - OUTPUT_EXECUTOR_LOGS_TO_STDOUT=true
      - JSON_LOG=true
      - GRAPHITE_HOST=graphite.local
      - GRAPHITE_PORT=3001
    entrypoint:
      - /scripts/entrypoint-worker.sh
  spark-worker-2:
    image: metorikku/metorikku
    container_name: spark-worker-2
    ports:
      - "7079:7079"
      - "8082:8082"
    environment:
      - SPARK_MASTER_HOST=spark-master
      - SPARK_MASTER_PORT=7077
      - SPARK_WORKER_PORT=7079
      - SPARK_WORKER_WEBUI_PORT=8082
      - OUTPUT_EXECUTOR_LOGS_TO_STDOUT=true
      - JSON_LOG=true
      - GRAPHITE_HOST=graphite.local
      - GRAPHITE_PORT=3001
    entrypoint:
      - /scripts/entrypoint-worker.sh