forked from cadence-workflow/cadence
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Pinot docker files, table config and schema (cadence-workflow#5163)
* Initial checkin for pinot config files
- Loading branch information
Showing
5 changed files
with
306 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
version: '3' | ||
services: | ||
cassandra: | ||
image: cassandra:3.11 | ||
ports: | ||
- "9042:9042" | ||
zookeeper: | ||
image: zookeeper:3.5.8 | ||
container_name: zookeeper | ||
ports: | ||
- "2181:2181" | ||
environment: | ||
ZOOKEEPER_CLIENT_PORT: 2181 | ||
ZOOKEEPER_TICK_TIME: 2000 | ||
pinot-controller: | ||
image: apachepinot/pinot:0.11.0 | ||
command: "StartController -zkAddress zookeeper:2181" | ||
container_name: pinot-controller | ||
restart: unless-stopped | ||
ports: | ||
- "9000:9000" | ||
environment: | ||
JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log" | ||
depends_on: | ||
- zookeeper | ||
pinot-broker: | ||
image: apachepinot/pinot:0.11.0 | ||
command: "StartBroker -zkAddress zookeeper:2181" | ||
restart: unless-stopped | ||
container_name: "pinot-broker" | ||
ports: | ||
- "8099:8099" | ||
environment: | ||
JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log" | ||
depends_on: | ||
- pinot-controller | ||
pinot-server: | ||
image: apachepinot/pinot:0.11.0 | ||
command: "StartServer -zkAddress zookeeper:2181" | ||
restart: unless-stopped | ||
container_name: "pinot-server" | ||
ports: | ||
- "8098:8098" | ||
environment: | ||
JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log" | ||
depends_on: | ||
- pinot-broker | ||
kafka: | ||
image: wurstmeister/kafka:2.13-2.8.1 | ||
depends_on: | ||
- zookeeper | ||
ports: | ||
- "9092:9092" | ||
expose: | ||
- "9093" | ||
environment: | ||
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/kafka | ||
KAFKA_BROKER_ID: 0 | ||
KAFKA_ADVERTISED_HOST_NAME: kafka | ||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,OUTSIDE://localhost:9092 | ||
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 | ||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,112 @@ | ||
version: '3' | ||
services: | ||
cassandra: | ||
image: cassandra:3.11 | ||
ports: | ||
- "9042:9042" | ||
prometheus: | ||
image: prom/prometheus:latest | ||
volumes: | ||
- ./prometheus:/etc/prometheus | ||
command: | ||
- '--config.file=/etc/prometheus/prometheus.yml' | ||
ports: | ||
- '9090:9090' | ||
kafka: | ||
image: wurstmeister/kafka:2.13-2.8.1 | ||
depends_on: | ||
- zookeeper | ||
ports: | ||
- "9092:9092" | ||
expose: | ||
- "9093" | ||
environment: | ||
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/kafka | ||
KAFKA_BROKER_ID: 0 | ||
KAFKA_ADVERTISED_HOST_NAME: kafka | ||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,OUTSIDE://localhost:9092 | ||
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 | ||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT | ||
zookeeper: | ||
image: zookeeper:3.5.8 | ||
container_name: zookeeper | ||
ports: | ||
- "2181:2181" | ||
environment: | ||
ZOOKEEPER_CLIENT_PORT: 2181 | ||
ZOOKEEPER_TICK_TIME: 2000 | ||
pinot-controller: | ||
image: apachepinot/pinot:0.11.0 | ||
command: "StartController -zkAddress zookeeper:2181" | ||
container_name: pinot-controller | ||
restart: unless-stopped | ||
ports: | ||
- "9000:9000" | ||
environment: | ||
JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log" | ||
depends_on: | ||
- zookeeper | ||
pinot-broker: | ||
image: apachepinot/pinot:0.11.0 | ||
command: "StartBroker -zkAddress zookeeper:2181" | ||
restart: unless-stopped | ||
container_name: "pinot-broker" | ||
ports: | ||
- "8099:8099" | ||
environment: | ||
JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log" | ||
depends_on: | ||
- pinot-controller | ||
pinot-server: | ||
image: apachepinot/pinot:0.11.0 | ||
command: "StartServer -zkAddress zookeeper:2181" | ||
restart: unless-stopped | ||
container_name: "pinot-server" | ||
ports: | ||
- "8098:8098" | ||
environment: | ||
JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log" | ||
depends_on: | ||
- pinot-broker | ||
cadence: | ||
image: ubercadence/server:master-auto-setup | ||
ports: | ||
- "8000:8000" | ||
- "8001:8001" | ||
- "8002:8002" | ||
- "8003:8003" | ||
- "7933:7933" | ||
- "7934:7934" | ||
- "7935:7935" | ||
- "7939:7939" | ||
- "7833:7833" | ||
environment: | ||
- "CASSANDRA_SEEDS=cassandra" | ||
- "PROMETHEUS_ENDPOINT_0=0.0.0.0:8000" | ||
- "PROMETHEUS_ENDPOINT_1=0.0.0.0:8001" | ||
- "PROMETHEUS_ENDPOINT_2=0.0.0.0:8002" | ||
- "PROMETHEUS_ENDPOINT_3=0.0.0.0:8003" | ||
- "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development_es.yaml" | ||
- "ENABLE_ES=true" | ||
- "ES_SEEDS=elasticsearch" | ||
- "KAFKA_SEEDS=kafka" | ||
depends_on: | ||
- cassandra | ||
- prometheus | ||
- kafka | ||
- elasticsearch | ||
cadence-web: | ||
image: ubercadence/web:latest | ||
environment: | ||
- "CADENCE_TCHANNEL_PEERS=cadence:7933" | ||
ports: | ||
- "8088:8088" | ||
depends_on: | ||
- cadence | ||
grafana: | ||
image: grafana/grafana | ||
user: "1000" | ||
depends_on: | ||
- prometheus | ||
ports: | ||
- '3000:3000' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
There are 3 ways to create the tables in Pinot, need to make sure Pinot server is running. To start pinot server, use `docker compose -f ./docker/dev/cassandra-pinot-kafka.yml up`. | ||
Pinot has 4 components need to be started in order, so some components may fail to start for the first time. You can try again or restart the failed components in docker. | ||
|
||
1. Manually add schema and realtime table in the UI, the configs can be found in the cadence-visibility-config file. | ||
|
||
2. Use Pinot launcher script to add schema and table, this requires to download Apache Pinot. | ||
|
||
`sh pinot-admin.sh AddTable \ | ||
-schemaFile [path_to_cadence]/cadence/Schema/Pinot/cadence-visibility-schema.json \ | ||
-tableConfigFile [path_to_cadence]/cadence/Schema/Pinot/cadence-visibility-config.json -exec` | ||
|
||
3. Use docker to execute the launcher script. | ||
|
||
`docker exec -it pinot-controller bin/pinot-admin.sh AddTable \ | ||
-tableConfigFile /Schema/Pinot/cadence-visibility-schema.json \ | ||
-schemaFile /Schema/Pinot/cadence-visibility-config.json -exec` | ||
|
||
The result can be checked at http://localhost:9000/#/tables |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
{ | ||
"tableName": "cadence-visibility-pinot", | ||
"tableType": "REALTIME", | ||
"segmentsConfig": { | ||
"timeColumnName": "StartTime", | ||
"timeType": "MILLISECONDS", | ||
"schemaName": "cadence-visibility-pinot", | ||
"replicasPerPartition": "1" | ||
}, | ||
"tenants": {}, | ||
"tableIndexConfig": { | ||
"loadMode": "MMAP", | ||
"streamConfigs": { | ||
"streamType": "kafka", | ||
"stream.kafka.consumer.type": "lowlevel", | ||
"stream.kafka.topic.name": "cadence-visibility-pinot", | ||
"stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", | ||
"stream.kafka.hlc.zk.connect.string": "zookeeper:2181/kafka", | ||
"stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", | ||
"stream.kafka.zk.broker.url": "zookeeper:2181/kafka", | ||
"stream.kafka.broker.list": "kafka:9093", | ||
"realtime.segment.flush.threshold.size": 30, | ||
"realtime.segment.flush.threshold.rows": 30, | ||
"realtime.segment.flush.threshold.time": "24h", | ||
"realtime.segment.flush.threshold.segment.size": "50M" | ||
} | ||
}, | ||
"routing": { | ||
"instanceSelectorType": "strictReplicaGroup" | ||
}, | ||
"upsertConfig": { | ||
"mode": "FULL" | ||
}, | ||
"metadata": {} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
{ | ||
"schemaName": "cadence-visibility-pinot", | ||
"primaryKeyColumns": ["DocID"], | ||
"dimensionFieldSpecs": [ | ||
{ | ||
"name": "DocID", | ||
"dataType": "STRING" | ||
}, | ||
{ | ||
"name": "DomainID", | ||
"dataType": "STRING" | ||
}, | ||
{ | ||
"name": "WorkflowID", | ||
"dataType": "STRING" | ||
}, | ||
{ | ||
"name": "RunID", | ||
"dataType": "STRING" | ||
}, | ||
{ | ||
"name": "WorkflowType", | ||
"dataType": "STRING" | ||
}, | ||
{ | ||
"name": "CloseStatus", | ||
"dataType": "INT" | ||
}, | ||
{ | ||
"name": "HistoryLength", | ||
"dataType": "INT" | ||
}, | ||
{ | ||
"name": "KafkaKey", | ||
"dataType": "STRING" | ||
}, | ||
{ | ||
"name": "TaskList", | ||
"dataType": "STRING" | ||
}, | ||
{ | ||
"name": "IsCron", | ||
"dataType": "BOOLEAN" | ||
}, | ||
{ | ||
"name": "NumCluster", | ||
"dataType": "INT" | ||
}, | ||
{ | ||
"name": "ShardID", | ||
"dataType": "INT" | ||
}, | ||
{ | ||
"name": "Attr", | ||
"dataType": "STRING" | ||
} | ||
], | ||
"dateTimeFieldSpecs": [{ | ||
"name": "StartTime", | ||
"dataType": "LONG", | ||
"format" : "1:MILLISECONDS:EPOCH", | ||
"granularity": "1:MILLISECONDS" | ||
},{ | ||
"name": "CloseTime", | ||
"dataType": "LONG", | ||
"format" : "1:MILLISECONDS:EPOCH", | ||
"granularity": "1:MILLISECONDS" | ||
},{ | ||
"name": "UpdateTime", | ||
"dataType": "LONG", | ||
"format" : "1:MILLISECONDS:EPOCH", | ||
"granularity": "1:MILLISECONDS" | ||
},{ | ||
"name": "ExecutionTime", | ||
"dataType": "LONG", | ||
"format" : "1:MILLISECONDS:EPOCH", | ||
"granularity": "1:MILLISECONDS" | ||
}] | ||
} |