Merge branch 'master' into add-self-balancing

README.md

@@ -77,6 +77,7 @@ You need to allocate Docker 8GB when running these. Avoid allocating all your ma
   - [Industry themes (e.g. banking Next Best Offer)](industry-themes)
   - [Self-Balancing Clusters Demo](self-balancing)
   - [Tiered Storage Demo](tiered-storage)
+  - [Cluster Linking Demo](cluster-linking)
   - [Distributed tracing](distributed-tracing)
   - [Analysing Sonos data in Kafka](sonos) ([✍️ blog](https://rmoff.net/2020/01/21/monitoring-sonos-with-ksqldb-influxdb-and-grafana/))
   - [Analysing Wi-Fi pcap data with Kafka](wifi-fun)

cluster-linking/README.md

@@ -0,0 +1,8 @@
+# Cluster Linking Demo
+
+Cluster Linking allows two or more clusters to directly connect at the broker level. It eliminates the need to monitor, secure, and scale 
+another distributed system (e.g. Kafka Connect) to replicate data from one cluster to another--all while preserving offsets to help rationalize about the state of the system.
+
+For information on how to run this demo, please reference the Confluent documentation [here](https://docs.confluent.io/current/multi-dc-deployments/cluster-linking/index.html).
+
+Note: Cluster Linking is currently in preview in CP 6.0.

cluster-linking/docker-compose.yml

@@ -0,0 +1,85 @@
+version: "3"
+
+services:
+  zookeeper-west:
+    image: confluentinc/cp-zookeeper:latest
+    hostname: zookeeper-west
+    container_name: zookeeper-west
+    networks:
+      - n1
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_SERVERS: zookeeper-west:2888:3888
+
+  broker-west:
+    image: confluentinc/cp-server:latest
+    hostname: broker-west
+    container_name: broker-west
+    networks:
+      - n1
+    ports:
+      - "9091:9091"
+      - "8091:8091"
+    volumes:
+      - ./config:/etc/kafka/demo
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_BROKER_RACK: "west"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://broker-west:19091,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9091
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper-west:2181"
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker-west:19091
+      KAFKA_CONFLUENT_CLUSTER_LINK_ENABLE: "true"
+      KAFKA_JMX_PORT: 8091
+    depends_on:
+      - zookeeper-west
+
+  zookeeper-east:
+    image: confluentinc/cp-zookeeper:latest
+    hostname: zookeeper-east
+    container_name: zookeeper-east
+    networks:
+      - n1
+    ports:
+      - "2182:2182"
+    environment:
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_CLIENT_PORT: 2182
+      ZOOKEEPER_SERVERS: zookeeper-east:2888:3888
+
+  broker-east:
+    image: confluentinc/cp-server:latest
+    hostname: broker-east
+    container_name: broker-east
+    networks:
+      - n1
+    ports:
+      - "9092:9092"
+      - "8092:8092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_BROKER_RACK: "east"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://broker-east:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper-east:2182"
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker-east:19092
+      KAFKA_CONFLUENT_CLUSTER_LINK_ENABLE: "true"
+      KAFKA_JMX_PORT: 8092
+    depends_on:
+      - zookeeper-east
+
+networks:
+  n1:

cluster-linking/scripts/2-create-links-topics.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+echo -e "\n==> Create West Demo Topic"
+docker-compose exec broker-west kafka-topics  --create \
+	--bootstrap-server broker-west:19091 \
+	--topic west-trades \
+	--partitions 1 \
+	--config min.insync.replicas=1
+
+sleep 2
+
+echo -e "\n==> Create East -> West link"
+docker-compose exec broker-east bash -c 'echo "{\"groupFilters\": [{\"name\": \"*\",\"patternType\": \"LITERAL\",\"filterType\": \"INCLUDE\"}]}" > groupFilters.json'
+docker-compose exec broker-east kafka-cluster-links \
+	--bootstrap-server broker-east:19092 \
+	--create \
+	--link-name west-cluster-link \
+	--config bootstrap.servers=broker-west:19091,consumer.offset.sync.enable=true,consumer.offset.sync.ms=10000 \
+	--consumer-group-filters-json-file groupFilters.json
+
+sleep 2
+
+echo -e "\n==> Create an east mirror of west-trades"
+
+docker-compose exec broker-east kafka-topics --create \
+	--bootstrap-server broker-east:19092 \
+	--topic west-trades \
+	--mirror-topic west-trades \
+	--link-name west-cluster-link \
+	--replication-factor 1

cluster-linking/scripts/3-list-links-and-lag.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+echo -e "\n==> List cluster links\n"
+
+docker-compose exec broker-east kafka-cluster-links \
+	--bootstrap-server broker-east:19092 \
+	--list
+
+echo -e "\n==> Link Metrics\n"
+
+for metric in MaxLag
+do
+    echo -e "\n\n==> Monitor $metric \n"
+
+    for link in west-cluster-link
+    do
+        LAG=$(docker-compose exec broker-east kafka-run-class kafka.tools.JmxTool --jmx-url service:jmx:rmi:///jndi/rmi://localhost:8092/jmxrmi \
+        --object-name kafka.server.link:type=ClusterLinkFetcherManager,name=$metric,clientId=ClusterLink,link-name=$link \
+        --one-time true | tail -n 1 | awk -F, '{print $2;}' | head -c 1)
+        echo "$link: $LAG"
+    done
+
+done

cluster-linking/scripts/4-run-consumer.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo -e "\n\n==> Consume from east cluster, west-trades \n"
+
+docker-compose exec broker-east kafka-console-consumer \
+    --bootstrap-server broker-east:19092 \
+    --topic west-trades \
+    --from-beginning

cluster-linking/scripts/5-run-producer.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo -e "\n\n==> Produce: West -> East west-trades \n"
+
+docker-compose exec broker-west bash -c 'seq 1 1000 | kafka-console-producer \
+    --broker-list broker-west:19091 \
+    --topic west-trades'

cluster-linking/scripts/6-setup-consumer.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+docker-compose exec broker-west bash -c 'echo enable.auto.commit=true > consumer.properties'
+docker-compose exec broker-west bash -c 'echo group.id=someGroup >> consumer.properties'
+
+echo -e "\n\n==> Consume from west cluster, west-trades and commit offsets (source cluster) \n"
+docker-compose exec broker-west kafka-console-consumer \
+    --bootstrap-server broker-west:19091 \
+    --topic west-trades \
+    --from-beginning \
+    --consumer.config consumer.properties

cluster-linking/scripts/7-migrate-one-cg.sh

@@ -0,0 +1,39 @@
+!/bin/bash
+
+echo -e "\n==> Stop migrating the consumer group someGroup via the west link"
+docker-compose exec broker-east bash -c 'echo "consumer.offset.group.filters={\"groupFilters\": [{\"name\": \"*\",\"patternType\": \"LITERAL\",\"filterType\": \"INCLUDE\"},{\"name\":\"someGroup\",\"patternType\":\"LITERAL\",\"filterType\":\"EXCLUDE\"}]}" > newGroupFilters.properties'
+docker-compose exec broker-east kafka-configs \
+  --bootstrap-server broker-east:19092 \
+  --alter \
+  --cluster-link west-cluster-link \
+  --add-config-file newGroupFilters.properties
+
+sleep 2
+
+echo -e "\n==> Produce 100 more messages to the source topic"
+docker-compose exec broker-west bash -c 'seq 1001 2000 | kafka-console-producer \
+    --broker-list broker-west:19091 \
+    --topic west-trades'
+
+docker-compose exec broker-east bash -c 'echo enable.auto.commit=true > consumer.properties'
+docker-compose exec broker-east bash -c 'echo group.id=someGroup >> consumer.properties'
+
+echo -e "\n\n==> Consume from east cluster, west-trades and commit offsets (destination cluster) \n"
+docker-compose exec broker-east kafka-console-consumer \
+    --bootstrap-server broker-east:19092 \
+    --topic west-trades \
+    --consumer.config consumer.properties
+
+sleep 2
+
+echo -e "\n\n==> Monitor that the consumer offsets have correctly been migrated \n"
+echo -e "\n\n==> West Cluster \n"
+docker-compose exec broker-west kafka-consumer-groups  \
+    --bootstrap-server broker-west:19091 \
+    --describe \
+    --group someGroup
+echo -e "\n\n==> East Cluster \n"
+docker-compose exec broker-east kafka-consumer-groups  \
+    --bootstrap-server broker-east:19092 \
+    --describe \
+    --group someGroup

cluster-linking/scripts/8-stop-link.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+echo -e "\n==> Using replica status to see mirrored topic"
+
+docker-compose exec broker-east kafka-replica-status \
+	--bootstrap-server=broker-east:19092 \
+	--include-linked
+
+echo -e "\n==> Stop west-link"
+
+docker-compose exec broker-east kafka-topics --alter --mirror-action stop \
+	--bootstrap-server=broker-east:19092 \
+  --topic west-trades
+
+echo -e "\n==> Monitor the change in mirrored topic status"
+
+docker-compose exec broker-east kafka-replica-status \
+	--bootstrap-server=broker-east:19092 \
+	--include-linked

cluster-linking/scripts/9-shutdown.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker container stop pumba-latency
+docker-compose down -v --remove-orphans
+sleep 1