Add pinot integration test (cadence-workflow#6044)

* add pinot integration test for PR * add pinot integration test into pipeline * still not working * running new kafka, but container doesn't connect with integration test * all components started, but kafka doesn't publish or consume * make sure pinot uses v7 es * update pinot test pipeline-pull-request cmd * update timeout time * delete cmd in docker-compose-pinot.yml * test pinot integration test * comment out local run cmd * longer timeout time for pinot integration test * make fmt * revert unnecessary changes * change back a kafka setting * cleanup dead code * also add pinot integration test into pipeline-master
thecodekaizen · Jun 18, 2024 · 91c09ef · 91c09ef
1 parent 333658f
commit 91c09ef
Show file tree

Hide file tree

Showing 7 changed files with 280 additions and 11 deletions.
diff --git a/.buildkite/pipeline-master.yml b/.buildkite/pipeline-master.yml
@@ -120,6 +120,29 @@ steps:
           run: integration-test-cassandra
           config: docker/buildkite/docker-compose-es7.yml
 
+  - label: ":golang: integration test with cassandra with pinot"
+    artifact_paths:
+      - ".build/coverage/*.out"
+    retry:
+      automatic:
+        limit: 1
+    plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                  - |-
+                    # ensure that we are not rebuilding binaries and not regenerating code
+                    make .just-build
+                    go test -timeout 600s -run ^TestPinotIntegrationSuite -tags pinotintegration -count 1 -v github.com/uber/cadence/host
+      - docker-compose#v3.0.0:
+          run: integration-test-cassandra-pinot
+          config: docker/buildkite/docker-compose-pinot.yml
+
+
   - label: ":golang: integration test with cassandra with OpenSearch v2"
     artifact_paths:
       - ".build/coverage/*.out"

diff --git a/.buildkite/pipeline-pull-request.yml b/.buildkite/pipeline-pull-request.yml
@@ -124,6 +124,28 @@ steps:
           run: integration-test-cassandra
           config: docker/buildkite/docker-compose-es7.yml
 
+  - label: ":golang: integration test with cassandra with pinot"
+    artifact_paths:
+      - ".build/coverage/*.out"
+    retry:
+      automatic:
+        limit: 1
+    plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                  - |-
+                    # ensure that we are not rebuilding binaries and not regenerating code
+                    make .just-build
+                    go test -timeout 600s -run ^TestPinotIntegrationSuite -tags pinotintegration -count 1 -v github.com/uber/cadence/host
+      - docker-compose#v3.0.0:
+          run: integration-test-cassandra-pinot
+          config: docker/buildkite/docker-compose-pinot.yml
+
   - label: ":golang: integration test with cassandra with OpenSearch v2"
     artifact_paths:
       - ".build/coverage/*.out"

diff --git a/docker/buildkite/docker-compose-pinot.yml b/docker/buildkite/docker-compose-pinot.yml
@@ -0,0 +1,208 @@
+version: "3.5"
+
+services:
+  cassandra:
+    image: cassandra:4.1.1
+    ports:
+      - "9042:9042"
+    environment:
+      - "MAX_HEAP_SIZE=256M"
+      - "HEAP_NEWSIZE=128M"
+    networks:
+      services-network:
+        aliases:
+          - cassandra
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
+  prometheus:
+    image: prom/prometheus:latest
+    networks:
+      services-network:
+        aliases:
+          - prometheus
+    volumes:
+      - ./prometheus:/etc/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - '9090:9090'
+
+  kafka:
+    image: docker.io/bitnami/kafka:3.7
+    restart: unless-stopped
+    container_name: "kafka"
+    ports:
+      - "9092:9092"
+      - "9093:9093"
+    environment:
+      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9093,OUTSIDE://:9092
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9093,OUTSIDE://kafka:9092
+      - KAFKA_CFG_BROKER_ID=0
+      - ALLOW_PLAINTEXT_LISTENER=yes
+      # Topic settings
+      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+    depends_on:
+      - zookeeper
+    networks:
+      services-network:
+        aliases:
+          - kafka
+
+  # will be deleted later when we get rid of ES usages
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3
+    # version here must be the same as the one in testdata/integration_pinot_cluster
+    ports:
+      - "9200:9200"
+    networks:
+      services-network:
+        aliases:
+          - elasticsearch
+    environment:
+      - discovery.type=single-node
+
+  zookeeper:
+    image: zookeeper:3.5.8
+    restart: always
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - '2181:2181'
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=2181
+      - ZOOKEEPER_TICK_TIME=2000
+    networks:
+      services-network:
+        aliases:
+          - zookeeper
+
+  pinot-controller:
+    image: apachepinot/pinot:0.12.1
+    command: "StartController -zkAddress zookeeper:2181 -controllerPort 9001"
+    container_name: pinot-controller
+    restart: unless-stopped
+    ports:
+      - "9001:9001"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log"
+    depends_on:
+      - zookeeper
+    networks:
+      services-network:
+        aliases:
+          - pinot-controller
+  pinot-broker:
+    image: apachepinot/pinot:0.12.1
+    command: "StartBroker -zkAddress zookeeper:2181"
+    restart: unless-stopped
+    container_name: "pinot-broker"
+    ports:
+      - "8099:8099"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log"
+    depends_on:
+      - pinot-controller
+    networks:
+      services-network:
+        aliases:
+          - pinot-broker
+  pinot-server:
+    image: apachepinot/pinot:0.12.1
+    command: "StartServer -zkAddress zookeeper:2181"
+    restart: unless-stopped
+    container_name: "pinot-server"
+    ports:
+      - "8098:8098"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log"
+    depends_on:
+      - pinot-broker
+    networks:
+      services-network:
+        aliases:
+          - pinot-server
+
+  pinot-admin:
+    image: apachepinot/pinot:latest
+    container_name: pinot-admin
+    depends_on:
+      - pinot-controller
+      - pinot-broker
+      - pinot-server
+    entrypoint: ["/bin/sh", "-c", "cp /schema/pinot/create_pinot_table.sh /opt/pinot/create_pinot_table.sh && chmod +x /opt/pinot/create_pinot_table.sh && /opt/pinot/create_pinot_table.sh"]
+    volumes:
+      - ../../schema:/schema  # Ensure the schema files and script are available in the container
+    networks:
+      services-network:
+        aliases:
+          - pinot-server
+
+  integration-test-cassandra-pinot:
+    build:
+      context: ../../
+      dockerfile: ./docker/buildkite/Dockerfile
+#    When run integration test locally, uncomment this.
+#    command:
+#      - /bin/sh
+#      - -e
+#      - -c
+#      - >
+#        go test -timeout 600s
+#        -run ^TestPinotIntegrationSuite$
+#        -tags pinotintegration
+#        -count 1
+#        -v
+#        github.com/uber/cadence/host
+#        | tee test.log
+#    ports:
+#      - "7933:7933"
+#      - "7934:7934"
+#      - "7935:7935"
+#      - "7939:7939"
+    environment:
+      - "CASSANDRA=1"
+      - "CASSANDRA_SEEDS=cassandra"
+      - "ES_SEEDS=elasticsearch"
+      - "TEST_TAG=pinotintegration"
+      - "ES_VERSION=v7"
+      - "KAFKA_SEEDS=kafka"
+      - "PINOT_SEEDS=pinot-broker"
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_BUILD_ID
+      - BUILDKITE_BUILD_NUMBER
+    depends_on:
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      kafka:
+        condition: service_started
+      pinot-controller:
+        condition: service_started
+      pinot-broker:
+        condition: service_started
+      pinot-server:
+        condition: service_started
+      pinot-admin:
+        condition: service_started
+    volumes:
+      - ../../:/cadence
+    networks:
+      services-network:
+        aliases:
+          - integration-test
+
+networks:
+  services-network:
+    enable_ipv6: true
+    ipam:
+      config:
+        - subnet: 2001:0DB8::/112
+    name: services-network
+    driver: bridge
diff --git a/host/pinot_test.go b/host/pinot_test.go
@@ -18,8 +18,8 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 
-//go:build pinotintegration
-// +build pinotintegration
+//go:build !race && pinotintegration
+// +build !race,pinotintegration
 
 // to run locally, make sure kafka and pinot is running,
 // then run cmd `go test -v ./host -run TestPinotIntegrationSuite -tags pinotintegration`
@@ -134,7 +134,7 @@ func (s *PinotIntegrationSuite) SetupSuite() {
 		Table:       tableName,
 		ServiceName: "",
 	}
-	s.pinotClient = pinotutils.CreatePinotClient(s.Suite, pinotConfig, s.Logger)
+	s.pinotClient = pinotutils.CreatePinotClient(&s.Suite, pinotConfig, s.Logger)
 }
 
 func (s *PinotIntegrationSuite) SetupTest() {

diff --git a/host/testcluster.go b/host/testcluster.go
@@ -199,18 +199,20 @@ func NewPinotTestCluster(t *testing.T, options *TestClusterConfig, logger log.Lo
 	var pinotClient pnt.GenericClient
 	if options.WorkerConfig.EnableIndexer {
 		var err error
+		// if options.PinotConfig.Migration.Enabled {
 		esClient, err = elasticsearch.NewGenericClient(options.ESConfig, logger)
 		if err != nil {
 			return nil, err
 		}
-		pConfig.AdvancedVisibilityStore = "pinot-visibility"
-		pinotBroker := options.PinotConfig.Broker
-		pinotRawClient, err := pinot.NewFromBrokerList([]string{pinotBroker})
-		if err != nil || pinotRawClient == nil {
-			return nil, err
-		}
-		pinotClient = pnt.NewPinotClient(pinotRawClient, logger, options.PinotConfig)
+		// }
+	}
+	pConfig.AdvancedVisibilityStore = "pinot-visibility"
+	pinotBroker := options.PinotConfig.Broker
+	pinotRawClient, err := pinot.NewFromBrokerList([]string{pinotBroker})
+	if err != nil || pinotRawClient == nil {
+		return nil, err
 	}
+	pinotClient = pnt.NewPinotClient(pinotRawClient, logger, options.PinotConfig)
 
 	scope := tally.NewTestScope("integration-test", nil)
 	metricsClient := metrics.NewClient(scope, metrics.ServiceIdx(0))

diff --git a/host/testdata/integration_pinot_cluster.yaml b/host/testdata/integration_pinot_cluster.yaml
@@ -31,12 +31,13 @@ workerconfig:
   enablereplicator: false
   enableindexer: true
 esconfig:
+  version: "v7"
   url:
     scheme: "http"
     host: "${ES_SEEDS}:9200"
   indices:
     visibility: test-visibility-
 pinotconfig:
-  broker: "localhost:8099"
+  broker: "${PINOT_SEEDS}:8099"
   cluster: pinot-test
   table: "cadence_visibility_pinot"
diff --git a/schema/pinot/create_pinot_table.sh b/schema/pinot/create_pinot_table.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Wait for Pinot components to start
+sleep 30
+
+# Add the table
+/opt/pinot/bin/pinot-admin.sh AddTable \
+  -schemaFile /schema/pinot/cadence-visibility-schema.json \
+  -tableConfigFile /schema/pinot/cadence-visibility-config.json \
+  -controllerProtocol http \
+  -controllerHost pinot-controller \
+  -controllerPort 9001 \
+  -exec