forked from confluentinc/demo-scene
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request confluentinc#200 from gianlucanatali/gn-pacman-v3
Streaming Pacman V2 - ksqlDB only
- Loading branch information
Showing
90 changed files
with
882 additions
and
4,911 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,32 +1,7 @@ | ||
.vscode/**/*.* | ||
|
||
terraform/aws/cert.pem | ||
terraform/aws/ccloud.auto.tfvars | ||
terraform/aws/cloud.auto.tfvars | ||
terraform/aws/alexa.auto.tfvars | ||
terraform/aws/.terraform | ||
terraform/aws/terraform.tfstate | ||
terraform/aws/terraform.tfstate.backup | ||
terraform/aws/.terraform.tfstate.lock.info | ||
stack-configs | ||
logs | ||
|
||
terraform/gcp/cert.pem | ||
terraform/gcp/ccloud.auto.tfvars | ||
terraform/gcp/cloud.auto.tfvars | ||
terraform/gcp/credentials.json | ||
terraform/gcp/.terraform | ||
terraform/gcp/terraform.tfstate | ||
terraform/gcp/terraform.tfstate.backup | ||
terraform/gcp/.terraform.tfstate.lock.info | ||
|
||
terraform/azr/cert.pem | ||
terraform/azr/ccloud.auto.tfvars | ||
terraform/azr/cloud.auto.tfvars | ||
terraform/azr/.terraform | ||
terraform/azr/terraform.tfstate | ||
terraform/azr/terraform.tfstate.backup | ||
terraform/azr/.terraform.tfstate.lock.info | ||
|
||
redis-sink/go.sum | ||
scoreboard/go.sum | ||
pipeline/ksql-server.properties | ||
scoreboard/ccloud.properties | ||
streaming-pacman_start.log | ||
streaming-pacman_stop.log |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
# This fill will contains user credentials and keys | ||
demo.cfg |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
export AWS_ACCESS_KEY="<AWS_ACCESS_KEY>" | ||
export AWS_SECRET_KEY="<AWS_SECRET_KEY>" | ||
|
||
# These are optional configs | ||
# export S3_BUCKET_NAME="ksqldbpacman" | ||
# export SCHEMA_REGISTRY_GEO="eu" | ||
# export CLUSTER_REGION="eu-west-2" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
#!/bin/bash | ||
|
||
################################################################# | ||
# Initialization | ||
################################################################# | ||
|
||
PRJ_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" | ||
UTILS_DIR="${PRJ_DIR}/utils" | ||
export EXAMPLE="streaming-pacman" | ||
|
||
# Source library | ||
source $UTILS_DIR/helper.sh | ||
source $UTILS_DIR/ccloud_library.sh | ||
|
||
# Source demo-specific configurations | ||
source config/demo.cfg | ||
|
||
################################################################# | ||
# Source CCloud configurations | ||
################################################################# | ||
DELTA_CONFIGS_DIR=delta_configs | ||
source $DELTA_CONFIGS_DIR/env.delta | ||
|
||
################################################################# | ||
# Confluent Cloud ksqlDB application | ||
################################################################# | ||
echo -e "\nConfluent Cloud ksqlDB application\n" | ||
ccloud::validate_ksqldb_up "$KSQLDB_ENDPOINT" || exit 1 | ||
|
||
# Create required topics and ACLs | ||
for TOPIC in $TOPICS_TO_CREATE | ||
do | ||
echo -e "\n# Create new Kafka topic $TOPIC" | ||
ccloud kafka topic create "$TOPIC" | ||
done | ||
|
||
ksqlDBAppId=$(ccloud ksql app list | grep "$KSQLDB_ENDPOINT" | awk '{print $1}') | ||
ccloud ksql app configure-acls $ksqlDBAppId $TOPICS_TO_CREATE | ||
|
||
for TOPIC in $TOPICS_TO_CREATE | ||
do | ||
ccloud kafka acl create --allow --service-account $(ccloud service-account list | grep $ksqlDBAppId | awk '{print $1;}') --operation WRITE --topic $TOPIC | ||
done | ||
|
||
# Submit KSQL queries | ||
echo -e "\nSubmit KSQL queries\n" | ||
properties='"ksql.streams.auto.offset.reset":"earliest","ksql.streams.cache.max.bytes.buffering":"0"' | ||
while read ksqlCmd; do | ||
echo -e "\n$ksqlCmd\n" | ||
response=$(curl -X POST $KSQLDB_ENDPOINT/ksql \ | ||
-H "Content-Type: application/vnd.ksql.v1+json; charset=utf-8" \ | ||
-u $KSQLDB_BASIC_AUTH_USER_INFO \ | ||
--silent \ | ||
-d @<(cat <<EOF | ||
{ | ||
"ksql": "$ksqlCmd", | ||
"streamsProperties": {$properties} | ||
} | ||
EOF | ||
)) | ||
echo $response | ||
if [[ ! "$response" =~ "SUCCESS" ]]; then | ||
echo -e "\nERROR: KSQL command '$ksqlCmd' did not include \"SUCCESS\" in the response. Please troubleshoot." | ||
exit 1 | ||
fi | ||
done <statements.sql | ||
echo -e "\nSleeping 20 seconds after submitting KSQL queries\n" | ||
sleep 20 | ||
|
||
exit 0 |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.