Add ksql-workshop with Windows notes

ksql-workshop/.gitattributes

@@ -0,0 +1,4 @@
+viour
+* text=auto
+# Bash shell scripts need just LF
+*.sh text eol=lf

ksql-workshop/README.md

@@ -0,0 +1,32 @@
+![image](../images/confluent-logo-300-2.png)
+
+# Overview
+
+This KSQL ratings demo showcases Kafka stream processing using KSQL. This demo was initially created for a KSQL workshop.
+
+----
+
+**You can follow the [step-by-step guide](ksql-workshop.adoc) workshop instructions here. These instructions are based on Docker instead of your local Confluent Platform install.**
+
+----
+
+As with the other demos in this repo, you may run the entire demo end-to-end with `./start.sh`, and it runs on your local Confluent Platform install instead of Docker.
+
+![image](images/ksql_workshop_01.png)
+
+
+# Prerequisites
+
+* [Common demo prerequisites](https://github.com/confluentinc/examples#prerequisites)
+* [Confluent Platform 5.0](https://www.confluent.io/download/)
+* If you are running the [step-by-step guide](ksql-workshop.adoc)
+  * Docker
+  * Docker Compose
+  * 8GB+ RAM
+
+# What Should I see?
+
+After you run `./start.sh`:
+
+* If you are running Confluent Platform, open your browser and navigate to the Control Center web interface Monitoring -> Data streams tab at http://localhost:9021/monitoring/streams to see throughput and latency performance of the KSQL queries
+* If you are running Confluent Platform, use Control Center to view and create KSQL queries. Otherwise, run the KSQL CLI `ksql http://localhost:8088`.

ksql-workshop/data/customers.json

@@ -0,0 +1,10 @@
+{"id":1,"first_name":"Annemarie","last_name":"Arent","email":"[email protected]","gender":"Female","club_status":"platinum","comments":"Organized web-enabled ability"}
+{"id":2,"first_name":"Merilyn","last_name":"Doughartie","email":"[email protected]","gender":"Female","club_status":"platinum","comments":"Optimized local definition"}
+{"id":3,"first_name":"Mirilla","last_name":"Orwin","email":"[email protected]","gender":"Female","club_status":"gold","comments":"Extended system-worthy superstructure"}
+{"id":4,"first_name":"Shirl","last_name":"Barnfather","email":"[email protected]","gender":"Female","club_status":"bronze","comments":"Vision-oriented global extranet"}
+{"id":5,"first_name":"Munmro","last_name":"Igounet","email":"[email protected]","gender":"Male","club_status":"gold","comments":"Streamlined bandwidth-monitored firmware"}
+{"id":6,"first_name":"Allyn","last_name":"Langlois","email":"[email protected]","gender":"Male","club_status":"gold","comments":"Expanded explicit product"}
+{"id":7,"first_name":"Pepi","last_name":"Medgwick","email":"[email protected]","gender":"Female","club_status":"bronze","comments":"Public-key high-level leverage"}
+{"id":8,"first_name":"Gianina","last_name":"Mixhel","email":"[email protected]","gender":"Female","club_status":"gold","comments":"Extended neutral function"}
+{"id":9,"first_name":"Mirella","last_name":"Bellin","email":"[email protected]","gender":"Female","club_status":"gold","comments":"Inverse user-facing encryption"}
+{"id":10,"first_name":"Deborah","last_name":"Thackray","email":"[email protected]","gender":"Female","club_status":"silver","comments":"Assimilated value-added standardization"}

ksql-workshop/data/customers.sql

@@ -0,0 +1,41 @@
+GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'replicator' IDENTIFIED BY 'replpass';
+GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT  ON *.* TO 'debezium' IDENTIFIED BY 'dbz';
+
+# Create the database that we'll use to populate data and watch the effect in the binlog
+CREATE DATABASE demo;
+GRANT ALL PRIVILEGES ON demo.* TO 'mysqluser'@'%';
+
+use demo;
+
+create table CUSTOMERS (
+        id INT PRIMARY KEY,
+        first_name VARCHAR(50),
+        last_name VARCHAR(50),
+        email VARCHAR(50),
+        gender VARCHAR(50),
+	club_status VARCHAR(8),
+        comments VARCHAR(90),
+        create_ts timestamp DEFAULT CURRENT_TIMESTAMP ,
+        update_ts timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+);
+
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (1, 'Rica', 'Blaisdell', '[email protected]', 'Female', 'bronze', 'Universal optimal hierarchy');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (2, 'Ruthie', 'Brockherst', '[email protected]', 'Female', 'platinum', 'Reverse-engineered tangible interface');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (3, 'Mariejeanne', 'Cocci', '[email protected]', 'Female', 'bronze', 'Multi-tiered bandwidth-monitored capability');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (4, 'Hashim', 'Rumke', '[email protected]', 'Male', 'platinum', 'Self-enabling 24/7 firmware');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (5, 'Hansiain', 'Coda', '[email protected]', 'Male', 'platinum', 'Centralized full-range approach');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (6, 'Robinet', 'Leheude', '[email protected]', 'Female', 'platinum', 'Virtual upward-trending definition');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (7, 'Fay', 'Huc', '[email protected]', 'Female', 'bronze', 'Operative composite capacity');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (8, 'Patti', 'Rosten', '[email protected]', 'Female', 'silver', 'Integrated bandwidth-monitored instruction set');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (9, 'Even', 'Tinham', '[email protected]', 'Male', 'silver', 'Virtual full-range info-mediaries');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (10, 'Brena', 'Tollerton', '[email protected]', 'Female', 'silver', 'Diverse tangible methodology');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (11, 'Alexandro', 'Peeke-Vout', '[email protected]', 'Male', 'gold', 'Ameliorated value-added orchestration');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (12, 'Sheryl', 'Hackwell', '[email protected]', 'Female', 'gold', 'Self-enabling global parallelism');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (13, 'Laney', 'Toopin', '[email protected]', 'Female', 'platinum', 'Phased coherent alliance');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (14, 'Isabelita', 'Talboy', '[email protected]', 'Female', 'gold', 'Cloned transitional synergy');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (15, 'Rodrique', 'Silverton', '[email protected]', 'Male', 'gold', 'Re-engineered static application');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (16, 'Clair', 'Vardy', '[email protected]', 'Male', 'bronze', 'Expanded bottom-line Graphical User Interface');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (17, 'Brianna', 'Paradise', '[email protected]', 'Female', 'bronze', 'Open-source global toolset');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (18, 'Waldon', 'Keddey', '[email protected]', 'Male', 'gold', 'Business-focused multi-state functionalities');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (19, 'Josiah', 'Brockett', '[email protected]', 'Male', 'gold', 'Realigned didactic info-mediaries');
+insert into CUSTOMERS (id, first_name, last_name, email, gender, club_status, comments) values (20, 'Anselma', 'Rook', '[email protected]', 'Female', 'gold', 'Cross-group 24/7 application');

ksql-workshop/docker-compose.yml

@@ -0,0 +1,154 @@
+---
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.1.0
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  kafka:
+    image: confluentinc/cp-enterprise-kafka:5.1.0
+    depends_on:
+      - zookeeper
+    ports:
+    # Exposes 9092 for external connections to the broker
+    # Use kafka:29092 for connections internal on the docker network
+    # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
+      - 9092:9092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
+      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
+      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
+      CONFLUENT_METRICS_ENABLE: 'true'
+      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry:5.1.0
+    depends_on:
+      - zookeeper
+      - kafka
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
+
+  kafka-connect-cp:
+    image: confluentinc/cp-kafka-connect:5.1.0
+    depends_on:
+      - zookeeper
+      - kafka
+      - schema-registry
+    ports:
+      - 18083:18083
+    environment:
+      CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
+      CONNECT_REST_PORT: 18083
+      CONNECT_GROUP_ID: kafka-connect-cp
+      CONNECT_CONFIG_STORAGE_TOPIC: docker-kafka-connect-cp-configs
+      CONNECT_OFFSET_STORAGE_TOPIC: docker-kafka-connect-cp-offsets
+      CONNECT_STATUS_STORAGE_TOPIC: docker-kafka-connect-cp-status
+      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
+      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
+      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
+      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
+      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect-cp"
+      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
+      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
+      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
+      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
+      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
+      CONNECT_PLUGIN_PATH: '/usr/share/java'
+
+  ksql-server:
+    image: confluentinc/cp-ksql-server:5.1.0
+    depends_on:
+      - kafka
+      - schema-registry
+    environment:
+      KSQL_CUB_KAFKA_TIMEOUT: 120
+      KSQL_BOOTSTRAP_SERVERS: kafka:29092
+      KSQL_LISTENERS: http://0.0.0.0:8088
+      KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry:8081
+      KSQL_KSQL_SERVICE_ID: confluent_rmoff_01
+
+  # Runs the Kafka KSQL data generator for ratings
+  datagen-ratings:
+    image: confluentinc/ksql-examples:5.1.0
+    depends_on:
+      - kafka
+      - schema-registry
+    command: "bash -c 'echo Waiting for Kafka to be ready... && \
+                       cub kafka-ready -b kafka:29092 1 300 && \
+                       echo Waiting for Confluent Schema Registry to be ready... && \
+                       cub sr-ready schema-registry 8081 300 && \
+                       echo Waiting a few seconds for topic creation to finish... && \
+                       sleep 20 && \
+                       ksql-datagen \
+                          quickstart=ratings \
+                          format=avro \
+                          topic=ratings \
+                          maxInterval=500 \
+                          bootstrap-server=kafka:29092 \
+                          schemaRegistryUrl=http://schema-registry:8081'"
+
+# Other systems
+  mysql:
+    image: debezium/example-mysql:0.8
+    environment:
+     - MYSQL_ROOT_PASSWORD=debezium
+     - MYSQL_USER=mysqluser
+     - MYSQL_PASSWORD=mysqlpw
+    volumes:
+     - ${PWD}/data/customers.sql:/docker-entrypoint-initdb.d/z99_dump.sql
+
+  connect-debezium:
+    image: debezium/connect:0.8
+    depends_on:
+     - kafka
+     - mysql
+     - schema-registry
+    ports:
+     - 8083:8083
+    environment:
+      BOOTSTRAP_SERVERS: kafka:29092
+      GROUP_ID: connect-debezium
+      CONFIG_STORAGE_TOPIC: docker-connect-debezium-configs
+      OFFSET_STORAGE_TOPIC: docker-connect-debezium-offsets
+      STATUS_STORAGE_TOPIC: docker-connect-debezium-status
+      KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
+      VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
+      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
+      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
+    volumes:
+     - ${PWD}/scripts:/scripts
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.3.0
+    ports:
+      - 9200:9200
+    environment:
+      xpack.security.enabled: "false"
+      ES_JAVA_OPTS: "-Xms1g -Xmx1g"
+    volumes:
+     - ${PWD}/scripts:/scripts
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.3.0
+    depends_on:
+      - elasticsearch
+    ports:
+      - 5601:5601
+    environment:
+      xpack.security.enabled: "false"
+      discovery.type: "single-node"