Skip to content

Commit

Permalink
Updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@rmoff
rmoff committed Apr 8, 2020
1 parent 8cd3fee commit 1d8bf15
Show file tree
Hide file tree
Showing 9 changed files with 788 additions and 52 deletions.
717 changes: 717 additions & 0 deletions wifi-fun/Learning all about WiFi data with Apache Kafka.adoc
View file

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions wifi-fun/capture-data.sh
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
#!/bin/bash

cd /home/pi
source $(dirname "$0")/.env

while [ 1 -eq 1 ];do
date
sudo tshark -i mon0 \
-b duration:3600 -b files:2 -w /tmp/wireshark_tmp.pcap \
tshark -i mon0 \
-b duration:3600 -b files:12 -w /pcap_data/mon0 \
-T ek \
-l \
-e wlan.fc.type -e wlan.fc.type_subtype -e wlan_radio.channel \
Expand Down
118 changes: 68 additions & 50 deletions wifi-fun/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ services:
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
# volumes:
# - ./data/container_data/zk-data:/var/lib/zookeeper/data
# - ./data/container_data/zk-txn-logs:/var/lib/zookeeper/log
volumes:
- ./data/container_data/zk-data:/var/lib/zookeeper/data
- ./data/container_data/zk-txn-logs:/var/lib/zookeeper/log

kafka-1:
image: confluentinc/cp-kafka:5.4.1
Expand All @@ -27,8 +27,9 @@ services:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:39092,HOST://localhost:9092
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
# volumes:
# - ./data/container_data/kafka-1-data:/var/lib/kafka/data
KAFKA_LOG_RETENTION_HOURS: 4320
volumes:
- ./data/container_data/kafka-1-data:/var/lib/kafka/data

kafka-2:
image: confluentinc/cp-kafka:5.4.1
Expand All @@ -45,9 +46,10 @@ services:
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:49092,HOST://0.0.0.0:19092
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:49092,HOST://localhost:19092
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_LOG_RETENTION_HOURS: 4320
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
# volumes:
# - ./data/container_data/kafka-2-data:/var/lib/kafka/data
volumes:
- ./data/container_data/kafka-2-data:/var/lib/kafka/data

kafka-3:
image: confluentinc/cp-kafka:5.4.1
Expand All @@ -64,9 +66,10 @@ services:
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:59092,HOST://0.0.0.0:29092
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:59092,HOST://localhost:29092
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_LOG_RETENTION_HOURS: 4320
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
# volumes:
# - ./data/container_data/kafka-3-data:/var/lib/kafka/data
volumes:
- ./data/container_data/kafka-3-data:/var/lib/kafka/data

schema-registry:
image: confluentinc/cp-schema-registry:5.4.1
Expand All @@ -83,62 +86,71 @@ services:
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:39092,PLAINTEXT://kafka-2:49092,PLAINTEXT://kafka-3:59092
SCHEMA_REGISTRY_CUB_KAFKA_TIMEOUT: 300

ksqldb:
image: confluentinc/ksqldb-server:0.8.1
hostname: ksqldb
container_name: ksqldb
kafka-connect-01:
image: confluentinc/cp-kafka-connect:5.4.0
container_name: kafka-connect-01
depends_on:
- kafka-1
# - kafka-connect-01
- kafka-2
- kafka-3
- schema-registry
ports:
- "8088:8088"
- "8083:8083"
- 8083:8083
environment:
KSQL_LISTENERS: http://0.0.0.0:8088
KSQL_BOOTSTRAP_SERVERS: kafka-1:39092,kafka-2:49092,kafka-3:59092
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry:8081
KSQL_STREAMS_PRODUCER_MAX_BLOCK_MS: 9223372036854775807
# Setting KSQL_KSQL_CONNECT_WORKER_CONFIG enables embedded Kafka Connect
#KSQL_KSQL_CONNECT_URL: http://kafka-connect-01:8083
KSQL_KSQL_CONNECT_WORKER_CONFIG: "/connect/connect.properties"
# Kafka Connect config below
KSQL_CONNECT_BOOTSTRAP_SERVERS: kafka-1:39092,kafka-2:49092,kafka-3:59092
KSQL_CONNECT_REST_ADVERTISED_HOST_NAME: 'ksqldb'
KSQL_CONNECT_REST_PORT: 8083
KSQL_CONNECT_GROUP_ID: ksqldb-kafka-connect-group-01
KSQL_CONNECT_CONFIG_STORAGE_TOPIC: _ksqldb-kafka-connect-group-01-configs
KSQL_CONNECT_OFFSET_STORAGE_TOPIC: _ksqldb-kafka-connect-group-01-offsets
KSQL_CONNECT_STATUS_STORAGE_TOPIC: _ksqldb-kafka-connect-group-01-status
KSQL_CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
KSQL_CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
KSQL_CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
KSQL_CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
KSQL_CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
KSQL_CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/,/data/connect-jars'
# If you want to use the Confluent Hub installer to d/l component, but make them available
# when running this offline, spin up the stack once and then run :
# docker cp ksqldb:/usr/share/confluent-hub-components ./data/connect-jars
volumes:
- $PWD/data:/data
CONNECT_BOOTSTRAP_SERVERS: kafka-1:39092,kafka-2:49092,kafka-3:59092
CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect-01"
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: kafka-connect-01
CONNECT_CONFIG_STORAGE_TOPIC: _kafka-connect-01-configs
CONNECT_OFFSET_STORAGE_TOPIC: _kafka-connect-01-offsets
CONNECT_STATUS_STORAGE_TOPIC: _kafka-connect-01-status
CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/'
# In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option'
command:
# In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option'
- bash
- -c
- |
echo "Installing connector plugins"
mkdir -p /usr/share/confluent-hub-components/
confluent-hub install --no-prompt --component-dir /usr/share/confluent-hub-components/ confluentinc/kafka-connect-elasticsearch:5.4.1
confluent-hub install --no-prompt --component-dir /usr/share/confluent-hub-components/ confluentinc/kafka-connect-jdbc:5.4.1
confluent-hub install --no-prompt --component-dir /usr/share/confluent-hub-components/ neo4j/kafka-connect-neo4j:1.0.2
confluent-hub install --no-prompt --component-dir /usr/share/confluent-hub-components/ debezium/debezium-connector-mongodb:0.10.0
#
echo "Launching ksqlDB"
/usr/bin/docker/run &
echo "Launching Kafka Connect worker"
/etc/confluent/docker/run &
#
sleep infinity
ksqldb:
# *-----------------------------*
# To connect to ksqlDB CLI
# docker exec --interactive --tty ksqldb ksql http://localhost:8088
# *-----------------------------*
image: confluentinc/ksqldb-server:0.8.1
hostname: ksqldb
container_name: ksqldb
depends_on:
- kafka-1
ports:
- "8088:8088"
environment:
KSQL_LISTENERS: http://0.0.0.0:8088
KSQL_BOOTSTRAP_SERVERS: kafka-1:39092,kafka-2:49092,kafka-3:59092
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry:8081
KSQL_STREAMS_PRODUCER_MAX_BLOCK_MS: 9223372036854775807
KSQL_KSQL_CONNECT_URL: http://kafka-connect:8083

replicator:
image: confluentinc/cp-enterprise-replicator:5.4.1
container_name: replicator
Expand Down Expand Up @@ -200,6 +212,8 @@ services:
# Init the replicaset
mongo admin --eval "rs.initiate();"
sleep infinity
volumes:
- ./data/container_data/mongodb:/data/db

neo4j:
image: neo4j:3.5-enterprise
Expand All @@ -212,6 +226,9 @@ services:
NEO4J_AUTH: neo4j/connect
NEO4J_dbms_memory_heap_max__size: 8G
NEO4J_ACCEPT_LICENSE_AGREEMENT: 'yes'
volumes:
- ./data/container_data/neo4j/data:/data
- ./data/container_data/neo4j/logs:/logs

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
Expand All @@ -223,8 +240,8 @@ services:
xpack.security.enabled: "false"
ES_JAVA_OPTS: "-Xms1g -Xmx1g"
discovery.type: "single-node"
# volumes:
# - ./data/container_data/elasticserarch:/usr/share/elasticsearch/data
volumes:
- ./data/container_data/elasticserarch:/usr/share/elasticsearch/data
command:
- bash
- -c
Expand Down Expand Up @@ -287,6 +304,7 @@ services:
- POSTGRES_PASSWORD=postgres
volumes:
- ./data/postgres:/docker-entrypoint-initdb.d/
- ./data/container_data/postgres:/var/lib/postgresql/data

kafkacat:
image: edenhill/kafkacat:1.5.0
Expand Down
Binary file added wifi-fun/images/kib_doc_count01.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added wifi-fun/images/kib_probe_01.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added wifi-fun/images/kib_ra_da01.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added wifi-fun/images/kib_type_count01.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added wifi-fun/images/mac_wifi01.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added wifi-fun/images/wireshark_probe00.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit 1d8bf15

Please sign in to comment.