ksqlDB twitter demo

brianj-kim · Dec 6, 2019 · 899c151 · 899c151
1 parent 8ff101b
commit 899c151
Show file tree

Hide file tree

Showing 5 changed files with 260 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -28,3 +28,4 @@ python_notify
 rsa_key.pem
 rsa_key.pub
 slack_creds.sh
+credentials.properties
diff --git a/ksqldb-twitter/credentials.properties.example b/ksqldb-twitter/credentials.properties.example
@@ -0,0 +1,4 @@
+TWITTER_CONSUMERKEY=
+TWITTER_CONSUMERSECRET=
+TWITTER_ACCESSTOKEN=
+TWITTER_ACCESSTOKENSECRET=
diff --git a/ksqldb-twitter/docker-compose.yml b/ksqldb-twitter/docker-compose.yml
@@ -0,0 +1,139 @@
+---
+version: '3'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.4.0-beta1
+    container_name: zookeeper
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  kafka:
+    image: confluentinc/cp-enterprise-kafka:5.4.0-beta1
+    container_name: kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,HOST://localhost:9092
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry:5.4.0-beta1
+    ports:
+      - 8081:8081
+    container_name: schema-registry
+    depends_on:
+      - kafka
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:29092
+      SCHEMA_REGISTRY_CUB_KAFKA_TIMEOUT: 300
+
+  ksqldb-server:
+    image: confluentinc/ksqldb-server:0.6.0
+    hostname: ksqldb-server
+    container_name: ksqldb-server
+    depends_on:
+      - kafka
+      - kafka-connect-01
+    ports:
+      - "8088:8088"
+    environment:
+      KSQL_LISTENERS: http://0.0.0.0:8088
+      KSQL_BOOTSTRAP_SERVERS: kafka:29092
+      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
+      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
+      KSQL_KSQL_CONNECT_URL: http://kafka-connect-01:8083
+      KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry:8081
+
+  ksqldb-cli:
+    image: confluentinc/ksqldb-cli:0.6.0
+    container_name: ksqldb-cli
+    depends_on:
+      - ksqldb-server
+    entrypoint: /bin/sh
+    tty: true
+
+  kafka-connect-01:
+    image: confluentinc/cp-kafka-connect:5.4.0-beta1
+    container_name: kafka-connect-01
+    depends_on:
+      - kafka
+      - schema-registry
+    ports:
+      - 8083:8083
+    environment:
+      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
+      CONNECT_CUB_KAFKA_TIMEOUT: 300
+      CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
+      CONNECT_REST_ADVERTISED_HOST_NAME: 'kafka-connect-01'
+      CONNECT_REST_PORT: 8083
+      CONNECT_GROUP_ID: kafka-connect-group-01
+      CONNECT_CONFIG_STORAGE_TOPIC: _kafka-connect-group-01-configs
+      CONNECT_OFFSET_STORAGE_TOPIC: _kafka-connect-group-01-offsets
+      CONNECT_STATUS_STORAGE_TOPIC: _kafka-connect-group-01-status
+      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
+      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
+      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
+      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
+      CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
+      CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
+      CONNECT_LOG4J_ROOT_LOGLEVEL: 'INFO'
+      CONNECT_LOG4J_LOGGERS: 'org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR'
+#      CONNECT_LOG4J_LOGGERS: 'org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR,org.eclipse.jetty.server=DEBUG'
+      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1'
+      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1'
+      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1'
+      CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/,/data/connect-jars'
+      # External secrets config
+      # See https://docs.confluent.io/current/connect/security.html#externalizing-secrets
+      CONNECT_CONFIG_PROVIDERS: 'file'
+      CONNECT_CONFIG_PROVIDERS_FILE_CLASS: 'org.apache.kafka.common.config.provider.FileConfigProvider'
+    volumes:
+      - ${PWD}/credentials.properties:/data/credentials.properties
+    command: 
+      # In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option'
+      - bash 
+      - -c 
+      - |
+        echo "Installing connector plugins"
+        confluent-hub install --no-prompt jcustenborder/kafka-connect-twitter:0.3.33
+        confluent-hub install --no-prompt neo4j/kafka-connect-neo4j:1.0.2
+        #
+        echo "Launching Kafka Connect worker"
+        /etc/confluent/docker/run & 
+        #
+        sleep infinity
+
+  postgres:
+    # *-----------------------------*
+    # To connect to the DB:
+    #   docker exec -it postgres bash -c 'psql -U $POSTGRES_USER $POSTGRES_DB'
+    # *-----------------------------*
+    image: postgres:11
+    ports: 
+      - 5432:5432
+    environment:
+     - POSTGRES_USER=postgres
+     - POSTGRES_PASSWORD=postgres
+    volumes:
+     - ./data/postgres:/docker-entrypoint-initdb.d/
+
+  neo4j:
+    image: neo4j:3.5-enterprise
+    container_name: neo4j
+    ports:
+    - "7474:7474"
+    - "7687:7687"
+    environment:
+      NEO4J_AUTH: neo4j/connect
+      NEO4J_dbms_memory_heap_max__size: 8G
+      NEO4J_ACCEPT_LICENSE_AGREEMENT: 'yes'
diff --git a/ksqldb-twitter/readme.adoc b/ksqldb-twitter/readme.adoc
@@ -0,0 +1,18 @@
+= ksqlDB Twitter demo
+
+1. Get your API credentials from https://developer.twitter.com/
+2. Copy `credentials.properties.example` to `credentials.properties` and update it with your API credentials
+3. `docker-compose up -d`
+4. See <blog> for details. 
+
+ksqlDB CLI: 
+
+    docker exec -it ksqldb-cli bash -c 'echo -e "\n\n⏳ Waiting for ksqlDB to be available before launching CLI\n"; while : ; do curl_status=$(curl -s -o /dev/null -w %{http_code} http://ksqldb-server:8088/info) ; echo -e $(date) " ksqlDB server listener HTTP state: " $curl_status " (waiting for 200)" ; if [ $curl_status -eq 200 ] ; then  break ; fi ; sleep 5 ; done ; ksql http://ksqlDB-server:8088'
+
+Postgres CLI: 
+
+    docker exec -it postgres bash -c 'psql -U $POSTGRES_USER $POSTGRES_DB'
+
+Neo4j GUI: 
+
+    http://localhost:7474/browser/)
diff --git a/ksqldb-twitter/sample_tweet.json b/ksqldb-twitter/sample_tweet.json
@@ -0,0 +1,98 @@
+{
+    "CreatedAt": 1575044236000,
+    "Id": 1200448653137195008,
+    "Text": "RT @IwantJC4PM: \"When Labour wins, local government wins! And that's who I'll be voting for, the Labour Party.\"\nSimon. #Binman #GE2019 http\u2026",
+    "Source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+    "Truncated": false,
+    "InReplyToStatusId": -1,
+    "InReplyToUserId": -1,
+    "InReplyToScreenName": null,
+    "GeoLocation": null,
+    "Place": null,
+    "Favorited": false,
+    "Retweeted": false,
+    "FavoriteCount": 0,
+    "User": {
+        "Id": 952885954657816576,
+        "Name": "SpanishDan #FBPE �",
+        "ScreenName": "SpanishDan1",
+        "Location": "Tarragona.",
+        "Description": "Brexit is just wrong-Labour leadership are wrong, Tories are mendacious opportunists.Searching for social democrats-tweet me if you see any..",
+        "ContributorsEnabled": false,
+        "ProfileImageURL": "http://pbs.twimg.com/profile_images/1175044385215582208/dbNJWnAP_normal.jpg",
+        "BiggerProfileImageURL": "http://pbs.twimg.com/profile_images/1175044385215582208/dbNJWnAP_bigger.jpg",
+        "MiniProfileImageURL": "http://pbs.twimg.com/profile_images/1175044385215582208/dbNJWnAP_mini.jpg",
+        "OriginalProfileImageURL": "http://pbs.twimg.com/profile_images/1175044385215582208/dbNJWnAP.jpg",
+        "ProfileImageURLHttps": "https://pbs.twimg.com/profile_images/1175044385215582208/dbNJWnAP_normal.jpg",
+        "BiggerProfileImageURLHttps": "https://pbs.twimg.com/profile_images/1175044385215582208/dbNJWnAP_bigger.jpg",
+        "MiniProfileImageURLHttps": "https://pbs.twimg.com/profile_images/1175044385215582208/dbNJWnAP_mini.jpg",
+        "OriginalProfileImageURLHttps": "https://pbs.twimg.com/profile_images/1175044385215582208/dbNJWnAP.jpg",
+        "DefaultProfileImage": false,
+        "URL": null,
+        "Protected": false,
+        "FollowersCount": 8287,
+        "ProfileBackgroundColor": "F5F8FA",
+        "ProfileTextColor": "333333",
+        "ProfileLinkColor": "1DA1F2",
+        "ProfileSidebarFillColor": "DDEEF6",
+        "ProfileSidebarBorderColor": "C0DEED",
+        "ProfileUseBackgroundImage": true,
+        "DefaultProfile": true,
+        "ShowAllInlineMedia": false,
+        "FriendsCount": 5081,
+        "CreatedAt": 1516020690000,
+        "FavouritesCount": 241716,
+        "UtcOffset": -1,
+        "TimeZone": null,
+        "ProfileBackgroundImageURL": "",
+        "ProfileBackgroundImageUrlHttps": "",
+        "ProfileBannerURL": "https://pbs.twimg.com/profile_banners/952885954657816576/1516021351/web",
+        "ProfileBannerRetinaURL": "https://pbs.twimg.com/profile_banners/952885954657816576/1516021351/web_retina",
+        "ProfileBannerIPadURL": "https://pbs.twimg.com/profile_banners/952885954657816576/1516021351/ipad",
+        "ProfileBannerIPadRetinaURL": "https://pbs.twimg.com/profile_banners/952885954657816576/1516021351/ipad_retina",
+        "ProfileBannerMobileURL": "https://pbs.twimg.com/profile_banners/952885954657816576/1516021351/mobile",
+        "ProfileBannerMobileRetinaURL": "https://pbs.twimg.com/profile_banners/952885954657816576/1516021351/mobile_retina",
+        "ProfileBackgroundTiled": false,
+        "Lang": null,
+        "StatusesCount": 287468,
+        "GeoEnabled": true,
+        "Verified": false,
+        "Translator": false,
+        "ListedCount": 14,
+        "FollowRequestSent": false,
+        "WithheldInCountries": []
+    },
+    "Retweet": true,
+    "Contributors": [],
+    "RetweetCount": 0,
+    "RetweetedByMe": false,
+    "CurrentUserRetweetId": -1,
+    "PossiblySensitive": false,
+    "Lang": "en",
+    "WithheldInCountries": [],
+    "HashtagEntities": [
+        {
+            "Text": "Binman",
+            "Start": 119,
+            "End": 126
+        },
+        {
+            "Text": "GE2019",
+            "Start": 127,
+            "End": 134
+        }
+    ],
+    "UserMentionEntities": [
+        {
+            "Name": "David",
+            "Id": 3095107056,
+            "Text": "IwantJC4PM",
+            "ScreenName": "IwantJC4PM",
+            "Start": 3,
+            "End": 14
+        }
+    ],
+    "MediaEntities": [],
+    "SymbolEntities": [],
+    "URLEntities": []
+}