Skip to content

Commit

Permalink
Merge branch '7.0.x' into 7.1.x
Browse files Browse the repository at this point in the history
  • Loading branch information
ybyzek committed Nov 29, 2021
2 parents b28b4d1 + b0a67ee commit cb485bc
Show file tree
Hide file tree
Showing 8 changed files with 33 additions and 25 deletions.
2 changes: 1 addition & 1 deletion ccloud-observability/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export EXAMPLE="ccloud-observability"
ccloud::create_ccloud_stack false \
&& print_code_pass -c "ccloud::create_ccloud_stack false"

SERVICE_ACCOUNT_ID=$(confluent kafka cluster list -o json | jq -r '.[0].name' | awk -F'-' '{print $4 "-" $5;}')
SERVICE_ACCOUNT_ID=$(ccloud:get_service_account_from_current_cluster_name)
CONFIG_FILE=stack-configs/java-service-account-$SERVICE_ACCOUNT_ID.config
export CONFIG_FILE=$CONFIG_FILE
ccloud::validate_ccloud_config $CONFIG_FILE || exit 1
Expand Down
2 changes: 1 addition & 1 deletion ccloud/ccloud-stack/ccloud_stack_create.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ ccloud::create_ccloud_stack $enable_ksqldb || exit 1

echo
echo "Validating..."
SERVICE_ACCOUNT_ID=$(confluent kafka cluster list -o json | jq -r '.[0].name' | awk -F'-' '{print $4 "-" $5;}')
SERVICE_ACCOUNT_ID=$(ccloud:get_service_account_from_current_cluster_name)
CONFIG_FILE=stack-configs/java-service-account-$SERVICE_ACCOUNT_ID.config
ccloud::validate_ccloud_config $CONFIG_FILE || exit 1
ccloud::generate_configs $CONFIG_FILE > /dev/null
Expand Down
16 changes: 8 additions & 8 deletions cloud-etl/create_ksqldb_app.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ source ../utils/ccloud_library.sh
source config/demo.cfg

#################################################################
# Source CCloud configurations
# Source Confluent Cloud configurations
#################################################################
DELTA_CONFIGS_DIR=delta_configs
source $DELTA_CONFIGS_DIR/env.delta
Expand All @@ -25,13 +25,13 @@ ccloud::validate_credentials_ksqldb "$KSQLDB_ENDPOINT" "$CONFIG_FILE" "$KSQLDB_B

# Create required topics and ACLs
echo -e "Create output topics $KAFKA_TOPIC_NAME_OUT1 and $KAFKA_TOPIC_NAME_OUT2, and ACLs to allow the ksqlDB application to run\n"
ccloud kafka topic create $KAFKA_TOPIC_NAME_OUT1
ccloud kafka topic create $KAFKA_TOPIC_NAME_OUT2
ksqlDBAppId=$(ccloud ksql app list | grep "$KSQLDB_ENDPOINT" | awk '{print $1}')
ccloud ksql app configure-acls $ksqlDBAppId $KAFKA_TOPIC_NAME_IN $KAFKA_TOPIC_NAME_OUT1 $KAFKA_TOPIC_NAME_OUT2
SERVICE_ACCOUNT_ID=$(ccloud kafka cluster list -o json | jq -r '.[0].name' | awk -F'-' '{print $4;}')
ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --topic $KAFKA_TOPIC_NAME_OUT1
ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --topic $KAFKA_TOPIC_NAME_OUT2
confluent kafka topic create $KAFKA_TOPIC_NAME_OUT1
confluent kafka topic create $KAFKA_TOPIC_NAME_OUT2
ksqlDBAppId=$(confluent ksql app list | grep "$KSQLDB_ENDPOINT" | awk '{print $1}')
confluent ksql app configure-acls $ksqlDBAppId $KAFKA_TOPIC_NAME_IN $KAFKA_TOPIC_NAME_OUT1 $KAFKA_TOPIC_NAME_OUT2
SERVICE_ACCOUNT_ID=$(ccloud:get_service_account_from_current_cluster_name)
confluent kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --topic $KAFKA_TOPIC_NAME_OUT1
confluent kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --topic $KAFKA_TOPIC_NAME_OUT2

# Submit KSQL queries
echo -e "\nSubmit KSQL queries\n"
Expand Down
10 changes: 5 additions & 5 deletions cloud-etl/docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -186,11 +186,11 @@ Because this example interacts with real resources in Kinesis or RDS PostgreSQL,
- ``AZBLOB_STORAGE_ACCOUNT``
- ``AZBLOB_CONTAINER``

#. Log in to |ccloud| with the command ``ccloud login --save``, and use your |ccloud| username and password. The ``--save`` argument saves your Confluent Cloud user login credentials or refresh token (in the case of SSO) to the local ``netrc`` file.
#. Log in to |ccloud| with the command ``confluent login --save``, and use your |ccloud| username and password. The ``--save`` argument saves your Confluent Cloud user login credentials or refresh token (in the case of SSO) to the local ``netrc`` file.

.. code:: shell
ccloud login --save
confluent login --save
Run
~~~
Expand Down Expand Up @@ -247,7 +247,7 @@ Run
Connectors
~~~~~~~~~~

#. The example automatically created |kconnect-long| connectors using the |ccloud| CLI command ``ccloud connector create`` that included passing in connector configuration files from the :devx-examples:`connector configuration directory|cloud-etl/connectors/`:
#. The example automatically created |kconnect-long| connectors using the |ccloud| CLI command ``confluent connector create`` that included passing in connector configuration files from the :devx-examples:`connector configuration directory|cloud-etl/connectors/`:

- :devx-examples:`AWS Kinesis source connector configuration file|cloud-etl/connectors/kinesis.json`
- :devx-examples:`PostgreSQL source connector configuration file|cloud-etl/connectors/rds.json`
Expand All @@ -271,7 +271,7 @@ Connectors

.. code:: bash
ccloud connector list
confluent connector list
Your output should resemble:

Expand All @@ -287,7 +287,7 @@ Connectors

.. code:: bash
ccloud connector describe lcc-vnrqp
confluent connector describe lcc-vnrqp
Your output should resemble:

Expand Down
12 changes: 6 additions & 6 deletions cloud-etl/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ source ../utils/ccloud_library.sh
ccloud::prompt_continue_ccloud_demo || exit 1

ccloud::validate_version_cli $CLI_MIN_VERSION \
&& print_pass "ccloud version ok" \
&& print_pass "confluent version ok" \
|| exit 1
ccloud::validate_logged_in_cli \
&& print_pass "logged into ccloud CLI" \
&& print_pass "logged into confluent CLI" \
|| exit 1
check_python \
&& print_pass "python installed" \
Expand All @@ -36,9 +36,9 @@ export EXAMPLE="cloud-etl"
echo
echo ====== Create new Confluent Cloud stack
ccloud::create_ccloud_stack true
SERVICE_ACCOUNT_ID=$(ccloud kafka cluster list -o json | jq -r '.[0].name' | awk -F'-' '{print $4;}')
SERVICE_ACCOUNT_ID=$(ccloud:get_service_account_from_current_cluster_name)
if [[ "$SERVICE_ACCOUNT_ID" == "" ]]; then
echo "ERROR: Could not determine SERVICE_ACCOUNT_ID from 'ccloud kafka cluster list'. Please troubleshoot, destroy stack, and try again to create the stack."
echo "ERROR: Could not determine SERVICE_ACCOUNT_ID from 'confluent kafka cluster list'. Please troubleshoot, destroy stack, and try again to create the stack."
exit 1
fi
CONFIG_FILE=stack-configs/java-service-account-$SERVICE_ACCOUNT_ID.config
Expand All @@ -47,7 +47,7 @@ ccloud::validate_ccloud_config $CONFIG_FILE \
&& print_pass "$CONFIG_FILE ok" \
|| exit 1

echo ====== Generate CCloud configurations
echo ====== Generate Confluent Cloud configurations
ccloud::generate_configs $CONFIG_FILE

DELTA_CONFIGS_DIR=delta_configs
Expand All @@ -73,7 +73,7 @@ ccloud::create_acls_connector $SERVICE_ACCOUNT_ID
#################################################################
# Create input topic and create source connector
#################################################################
ccloud kafka topic create $KAFKA_TOPIC_NAME_IN
confluent kafka topic create $KAFKA_TOPIC_NAME_IN
export AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id --profile $AWS_PROFILE)
export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key --profile $AWS_PROFILE)
if [[ "${DATA_SOURCE}" == "rds" ]]; then
Expand Down
4 changes: 2 additions & 2 deletions cloud-etl/stop.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ fi
# Delete connectors
for f in connectors/*.json; do
connector=$(cat $f | jq -r .name)
connectorId=$(ccloud connector list | grep $connector | awk '{print $1}')
connectorId=$(confluent connector list | grep $connector | awk '{print $1}')
if [[ "$connectorId" != "" ]]; then
echo "Deleting connector $connector with id $connectorId"
ccloud connector delete $connectorId
confluent connector delete $connectorId
fi
done

Expand Down
2 changes: 1 addition & 1 deletion cp-quickstart/start-cloud.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ export EXAMPLE="cp-quickstart"
ccloud::create_ccloud_stack true \
&& print_code_pass -c "cccloud::create_ccloud_stack true"

SERVICE_ACCOUNT_ID=$(confluent kafka cluster list -o json | jq -r '.[0].name' | awk -F'-' '{print $4 "-" $5;}')
SERVICE_ACCOUNT_ID=$(ccloud:get_service_account_from_current_cluster_name)
CONFIG_FILE=stack-configs/java-service-account-$SERVICE_ACCOUNT_ID.config
export CONFIG_FILE=$CONFIG_FILE
ccloud::validate_ccloud_config $CONFIG_FILE || exit 1
Expand Down
10 changes: 9 additions & 1 deletion utils/ccloud_library.sh
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,14 @@ function ccloud::create_service_account() {
return 0
}

function ccloud:get_service_account_from_current_cluster_name() {
SERVICE_ACCOUNT_ID=$(confluent kafka cluster list -o json | jq -r '.[0].name' | awk -F'-' '{print $4 "-" $5;}')

echo $SERVICE_ACCOUNT_ID

return 0
}

function ccloud::enable_schema_registry() {
SCHEMA_REGISTRY_CLOUD=$1
SCHEMA_REGISTRY_GEO=$2
Expand Down Expand Up @@ -1197,7 +1205,7 @@ function ccloud::generate_configs() {
KSQLDB_BASIC_AUTH_USER_INFO=$( grep "^ksql.basic.auth.user.info" $CONFIG_FILE | awk -F'=' '{print $2;}' )

################################################################################
# Build configuration file with CCloud connection parameters and
# Build configuration file with Confluent Cloud connection parameters and
# Confluent Monitoring Interceptors for Streams Monitoring in Confluent Control Center
################################################################################
INTERCEPTORS_CONFIG_FILE=$DEST/interceptors-ccloud.config
Expand Down

0 comments on commit cb485bc

Please sign in to comment.