ElasticSearch-related docker compose and docker image updates (tempor…

…alio#311)

* Merge `temporal-auto-setup` functionality into `temporal-server` docker image
* Add 'docker-images' target to Makefile
* Switch `docker-compose-es.yml` to `temporalio/server`, and add `admin-tools`
* Upgrade docker-compose-es cassandra to 6.8.1, cap elasticsearch memory

Dockerfile

@@ -38,7 +38,7 @@ RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSI
 # Alpine base image
 FROM alpine:3.11 AS alpine
 
-RUN apk add --update --no-cache ca-certificates tzdata bash curl
+RUN apk add --update --no-cache ca-certificates tzdata bash curl vim
 
 # set up nsswitch.conf for Go's "netgo" implementation
 # https://github.com/gliderlabs/docker-alpine/issues/367#issuecomment-424546457
@@ -50,6 +50,9 @@ SHELL ["/bin/bash", "-c"]
 # Temporal server
 FROM alpine AS temporal-server
 
+RUN apk add --update --no-cache ca-certificates py-pip mysql-client \
+    && pip install cqlsh
+
 ENV TEMPORAL_HOME /etc/temporal
 RUN mkdir -p /etc/temporal
 
@@ -64,24 +67,19 @@ COPY docker/entrypoint.sh /docker-entrypoint.sh
 COPY config/dynamicconfig /etc/temporal/config/dynamicconfig
 COPY docker/config_template.yaml /etc/temporal/config
 COPY docker/start-temporal.sh /start-temporal.sh
+COPY docker/start.sh /start.sh
 
 WORKDIR /etc/temporal
 
 ENV SERVICES="history,matching,frontend,worker"
 
 EXPOSE 7933 7934 7935 7939 6933 6934 6935 6939 7233 7234 7235 7239
 ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD /start-temporal.sh
+CMD /start.sh
 
 # All-in-one Temporal server
 FROM temporal-server AS temporal-auto-setup
-
-RUN apk add --update --no-cache ca-certificates py-pip mysql-client
-RUN pip install cqlsh
-
-COPY docker/start.sh /start.sh
-
-CMD /start.sh
+ENV AUTO_SETUP="true"
 
 # Temporal CLI
 FROM alpine AS temporal-tctl

Makefile

@@ -13,6 +13,13 @@ clean: clean-bins clean-proto clean-test-results
 
 # Update proto submodule from remote and rebuild proto files.
 update-proto: clean-proto update-proto-submodule protoc update-proto-go proto-mock gomodtidy
+
+# Build all docker images.
+docker-images:
+	docker build --build-arg "TARGET=server" -t temporalio/server:$(DOCKER_IMAGE_TAG) .
+	docker build --build-arg "TARGET=tctl" -t temporalio/tctl:$(DOCKER_IMAGE_TAG) .
+	docker build --build-arg "TARGET=auto-setup" -t temporalio/auto-setup:$(DOCKER_IMAGE_TAG) .
+	docker build --build-arg "TARGET=admin-tools" -t temporalio/admin-tools:$(DOCKER_IMAGE_TAG) .
 ########################################################################
 
 .PHONY: proto
@@ -52,6 +59,8 @@ INTEG_TEST_NDC_OUT_DIR := hostndc
 GO_BUILD_LDFLAGS_CMD      := $(abspath ./scripts/go-build-ldflags.sh)
 GO_BUILD_LDFLAGS          := $(shell $(GO_BUILD_LDFLAGS_CMD) LDFLAG)
 
+DOCKER_IMAGE_TAG := $(shell (whoami | tr -d ' '))-local
+
 ifndef PERSISTENCE_TYPE
 override PERSISTENCE_TYPE := cassandra
 endif
@@ -364,4 +373,4 @@ go-generate:
 
 gomodtidy:
 	@printf $(COLOR) "go mod tidy..."
-	@go mod tidy
+	@go mod tidy

docker/docker-compose-es.yml

@@ -4,13 +4,6 @@ services:
     image: cassandra:3.11
     ports:
       - "9042:9042"
-  statsd:
-    image: graphiteapp/graphite-statsd
-    ports:
-      - "8080:80"
-      - "2003:2003"
-      - "8125:8125"
-      - "8126:8126"
   zookeeper:
     image: wurstmeister/zookeeper:3.4.6
     ports:
@@ -26,13 +19,14 @@ services:
       KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.1
+    image: elasticsearch:6.8.8
     ports:
       - "9200:9200"
     environment:
       - discovery.type=single-node
+      - ES_JAVA_OPTS=-Xms100m -Xmx100m
   temporal:
-    image: temporalio/auto-setup:0.21.1
+    image: temporalio/server:${SERVER_TAG:-0.21.1}
     ports:
      - "7233:7233"
      - "7234:7234"
@@ -43,22 +37,29 @@ services:
      - "6935:6935"
      - "6939:6939"
     environment:
+      - "AUTO_SETUP=true"
       - "CASSANDRA_SEEDS=cassandra"
-      - "STATSD_ENDPOINT=statsd:8125"
       - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development_es.yaml"
       - "ENABLE_ES=true"
       - "ES_SEEDS=elasticsearch"
       - "KAFKA_SEEDS=kafka"
     depends_on:
       - cassandra
-      - statsd
       - kafka
       - elasticsearch
+  temporal-admin-tools:
+    image: temporalio/admin-tools:${ADMIN_TOOLS_TAG:-0.21.1}
+    stdin_open: true
+    tty: true
+    environment:
+      - "TEMPORAL_CLI_ADDRESS=temporal:7233"
+    depends_on:
+      - temporal
   temporal-web:
     image: temporalio/web:0.21.1
     environment:
       - "TEMPORAL_GRPC_ENDPOINT=temporal:7233"
     ports:
       - "8088:8088"
     depends_on:
-      - temporal
+      - temporal

docker/start.sh

@@ -80,10 +80,6 @@ setup_schema() {
         echo 'setup cassandra schema'
         setup_cassandra_schema
     fi
-
-    if [ "$ENABLE_ES" == "true" ]; then
-        setup_es_template
-    fi
 }
 
 wait_for_cassandra() {
@@ -138,10 +134,6 @@ wait_for_db() {
     else
         wait_for_cassandra
     fi
-
-    if [ "$ENABLE_ES" == "true" ]; then
-        wait_for_es
-    fi
 }
 
 register_default_namespace() {
@@ -156,13 +148,24 @@ register_default_namespace() {
     echo "Default namespace registration complete."
 }
 
-wait_for_db
-if [ "$SKIP_SCHEMA_SETUP" != true ]; then
-    setup_schema
+auto_setup() {
+    wait_for_db
+    if [ "$SKIP_SCHEMA_SETUP" != true ]; then
+        setup_schema
+    fi
+
+    if [ "$SKIP_DEFAULT_NAMESPACE_CREATION" != true ]; then
+        register_default_namespace &
+    fi
+}
+
+if [ "$AUTO_SETUP" = "true" ]; then
+	auto_setup
 fi
 
-if [ "$SKIP_DEFAULT_NAMESPACE_CREATION" != true ]; then
-    register_default_namespace &
+if [ "$ENABLE_ES" == "true" ]; then
+    wait_for_es
+    setup_es_template
 fi
 
 bash /start-temporal.sh