Single connect worker, install Ora pre-reqs automagically w/out needi…

…ng Oracle container, add kafkacat

no-more-silos-oracle/docker-compose/docker-compose.yml → no-more-silos-oracle/docker-compose.yml

@@ -14,9 +14,23 @@ services:
     depends_on:
       - zookeeper
     ports:
-    # Exposes 9092 for external connections to the broker
-    # Use kafka:29092 for connections internal on the docker network
+    # "`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-
+    # An important note about accessing Kafka from clients on other machines:
+    # -----------------------------------------------------------------------
+    #
+    # The config used here exposes port 9092 for _external_ connections to the broker
+    # i.e. those from _outside_ the docker network. This could be from the host machine
+    # running docker, or maybe further afield if you've got a more complicated setup.
+    # If the latter is true, you will need to change the value 'localhost' in
+    # KAFKA_ADVERTISED_LISTENERS to one that is resolvable to the docker host from those
+    # remote clients
+    #
+    # For connections _internal_ to the docker network, such as from other services
+    # and components, use kafka:29092.
+    #
     # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
+    # "`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-
+    #
       - 9092:9092
     environment:
       KAFKA_BROKER_ID: 1
@@ -28,6 +42,7 @@ services:
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
 
   schema-registry:
     image: confluentinc/cp-schema-registry:5.5.0
@@ -42,44 +57,63 @@ services:
       SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
 
   kafka-connect:
-    image: confluentinc/cp-kafka-connect:5.5.0
+    image: confluentinc/cp-kafka-connect-base:5.5.0
     container_name: kafka-connect
     depends_on:
       - zookeeper
       - kafka
       - schema-registry
     ports:
-      - 18083:18083
+      - 8083:8083
     environment:
       CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
-      CONNECT_REST_PORT: 18083
+      CONNECT_REST_PORT: 8083
       CONNECT_GROUP_ID: compose-connect-group
       CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
       CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
       CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
-      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
-      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
+      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
       CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
       CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
       CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
       CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
+      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
       CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
-      CONNECT_PLUGIN_PATH: '/usr/share/java'
-    volumes:
-      - $PWD/scripts:/scripts
-      - db-leach:/db-leach/
+      CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/'
+      #
+      # Make sure you set this for the correct path that the library files for InstantClient have been installed into
+      LD_LIBRARY_PATH: '/usr/share/java/debezium-connector-oracle/instantclient_19_6//'
     command: 
       - bash 
       - -c 
       - |
-        cp /db-leach/jdbc/lib/ojdbc8.jar /usr/share/java/kafka-connect-jdbc
-        sleep infinity &
-        /etc/confluent/docker/run
+        echo "Installing connector plugins"
+        confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:5.5.0
+        #
+        echo "Downloading JDBC drivers"
+        cd /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib
+        curl https://maven.xwiki.org/externals/com/oracle/jdbc/ojdbc8/12.2.0.1/ojdbc8-12.2.0.1.jar -o ojdbc8-12.2.0.1.jar
+        # -----------
+        # Install Debezium Oracle connector and required libraries
+        # -- Download
+        wget "https://oss.sonatype.org/service/local/artifact/maven/redirect?r=snapshots&g=io.debezium&a=debezium-connector-oracle&v=LATEST&c=plugin&e=tar.gz" -O /tmp/dbz-ora.tgz
+        tar -xvf /tmp/dbz-ora.tgz --directory /usr/share/java/
+        # -- Install the required library files
+        apt-get update
+        apt-get install -y --force-yes unzip libaio1
+        wget "https://download.oracle.com/otn_software/linux/instantclient/19600/instantclient-basiclite-linux.x64-19.6.0.0.0dbru.zip" -O /tmp/ic.zip
+        unzip /tmp/ic.zip -d /usr/share/java/debezium-connector-oracle/
+        # -----------
+        # Launch the Kafka Connect worker
+        /etc/confluent/docker/run &
+        #
+        # Don't exit
+        sleep infinity
 
   ksqldb:
     image: confluentinc/ksqldb-server:0.9.0
@@ -114,63 +148,19 @@ services:
       - $PWD/oradata/recovery_area:/opt/oracle/oradata/recovery_area
       - $PWD/ora-setup-scripts:/opt/oracle/scripts/setup
       - $PWD/ora-startup-scripts:/opt/oracle/scripts/startup
-      - db-leach:/opt/oracle/product/12.2.0.1/dbhome_1/
-
-  connect-debezium:
-    image: debezium/connect:0.9
-    container_name: connect-debezium
-    depends_on:
-     - kafka
-     - oracle
-     - schema-registry
-    ports:
-     - 8083:8083
-    environment:
-     - BOOTSTRAP_SERVERS=kafka:29092
-     - GROUP_ID=1
-     - CONFIG_STORAGE_TOPIC=my_connect_configs
-     - OFFSET_STORAGE_TOPIC=my_connect_offsets
-     - STATUS_STORAGE_TOPIC=my_connect_status
-     - LD_LIBRARY_PATH=/kafka/libs/instant_client
-     - KEY_CONVERTER=io.confluent.connect.avro.AvroConverter
-     - VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter
-     - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL=http://schema-registry:8081
-     - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL=http://schema-registry:8081
-    volumes:
-      - $PWD/scripts:/scripts
-      - db-leach:/db-leach/
-    command: 
-      - bash 
-      - -c 
-      - |
-        cp /db-leach/jdbc/lib/ojdbc8.jar /kafka/libs
-        cp /db-leach/rdbms/jlib/xstreams.jar /kafka/libs
-        mkdir /kafka/libs/instant_client
-        cp /db-leach/bin/adrci /kafka/libs/instant_client/
-        cp /db-leach/bin/genezi /kafka/libs/instant_client/
-        cp /db-leach/bin/uidrvci /kafka/libs/instant_client/
-        cp /db-leach/inventory/Scripts/ext/lib/libociei.so /kafka/libs/instant_client/
-        cp /db-leach/jdbc/lib/ojdbc8.jar /kafka/libs/instant_client/
-        cp /db-leach/lib/libclntsh.so.12.1 /kafka/libs/instant_client/
-        cp /db-leach/lib/libclntshcore.so.12.1 /kafka/libs/instant_client/
-        cp /db-leach/lib/libipc1.so /kafka/libs/instant_client/
-        cp /db-leach/lib/libmql1.so /kafka/libs/instant_client/
-        cp /db-leach/lib/libnnz12.so /kafka/libs/instant_client/
-        cp /db-leach/lib/libocci.so.12.1 /kafka/libs/instant_client/
-        cp /db-leach/lib/libocijdbc12.so /kafka/libs/instant_client/
-        cp /db-leach/lib/libons.so /kafka/libs/instant_client/
-        cp /db-leach/lib/liboramysql12.so /kafka/libs/instant_client/
-        cp /db-leach/rdbms/jlib/xstreams.jar /kafka/libs/instant_client/
-        cp /db-leach/lib/stubs/libaio.so /kafka/libs/instant_client/
-        cp /db-leach/lib/stubs/libaio.so.1 /kafka/libs/instant_client/
-        sleep infinity &
-        /docker-entrypoint.sh start
 
   # swingbench:
   #   image: domgiles/swingbench:latest
 
-volumes:
-    db-leach: {}
-
-
-
+  kafkacat:
+    image: edenhill/kafkacat:1.5.0
+    container_name: kafkacat
+    links:
+      - kafka
+      - schema-registry
+    entrypoint: 
+      - /bin/sh 
+      - -c 
+      - |
+        apk add jq; 
+        while [ 1 -eq 1 ];do sleep 60;done

no-more-silos-oracle/no-more-silos-oracle.adoc

@@ -66,6 +66,34 @@ DBZXOUT - Propagation Send/Rcv        399      6031 3386              CX01
 
 If you don't see these running (particularly the `Capture`) then refer to link:debezium-xstream-system-output.adoc[]
 
+Check that Kafka Connect is running:
+
+[source,bash]
+----
+bash -c ' \
+echo -e "\n\n=============\nWaiting for Kafka Connect to start listening on localhost ⏳\n=============\n"
+while [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors) -ne 200 ] ; do
+  echo -e "\t" $(date) " Kafka Connect listener HTTP state: " $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors) " (waiting for 200)"
+  sleep 5
+done
+echo -e $(date) "\n\n--------------\n\o/ Kafka Connect is ready! Listener HTTP state: " $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors) "\n--------------\n"
+'
+----
+
+Check that required connectors are loaded
+
+[source,bash]
+----
+curl -s localhost:8083/connector-plugins|jq '.[].class'|egrep 'OracleConnector|JdbcSourceConnector'
+----
+
+[source,bash]
+----
+"io.confluent.connect.jdbc.JdbcSourceConnector"
+"io.debezium.connector.oracle.OracleConnector"
+----
+
+
 === Run ksqlDB CLI and SQL*Plus
 
 Optionally, use something like `screen` or `tmux` to have these both easily to hand. Or multiple Terminal tabs. Whatever works for you :)
@@ -128,20 +156,18 @@ SELECT ID, FIRST_NAME, LAST_NAME, CREATE_TS, UPDATE_TS FROM CUSTOMERS;
 
 === Check status of connectors
 
-* Debezium
-+
-[source,bash]
 ----
-$ curl -s "http://localhost:8083/connectors"| jq '.[]'| xargs -I{connector_name} curl -s "http://localhost:8083/connectors/"{connector_name}"/status"| jq -c -M '[.name,.connector.state,.tasks[].state]|join(":|:")'| column -s : -t| sed 's/\"//g'| sort
-ora-source-debezium-xstream  |  RUNNING  |  RUNNING
+curl -s "http://localhost:8083/connectors?expand=info&expand=status" | \
+       jq '. | to_entries[] | [ .value.info.type, .key, .value.status.connector.state,.value.status.tasks[].state,.value.info.config."connector.class"]|join(":|:")' | \
+       column -s : -t| sed 's/\"//g'| sort
 ----
-curl -s "http://localhost:8083/connectors/ora-source-debezium-xstream/status"
-* JDBC
-+
+
+Expected:
+
 [source,bash]
 ----
-$ curl -s "http://localhost:18083/connectors"| jq '.[]'| xargs -I{connector_name} curl -s "http://localhost:18083/connectors/"{connector_name}"/status"| jq -c -M '[.name,.connector.state,.tasks[].state]|join(":|:")'| column -s : -t| sed 's/\"//g'| sort
-ora-source-jdbc  |  RUNNING  |  RUNNING
+source  |  ora-source-debezium-xstream  |  RUNNING  |  RUNNING  |  io.debezium.connector.oracle.OracleConnector
+source  |  ora-source-jdbc              |  RUNNING  |  RUNNING  |  io.confluent.connect.jdbc.JdbcSourceConnector
 ----
 
 === Show Kafka topic has been created & populated
@@ -206,6 +232,13 @@ Pretty-print the source data to show why nested
 echo '{"before": {"ID": 42, "FIRST_NAME": "Rick", "LAST_NAME": "Astley", "EMAIL": null, "GENDER": null, "CLUB_STATUS": "Bronze", "COMMENTS": null, "CREATE_TS": 1544000706681769, "UPDATE_TS": 1544000706000000}, "after": {"ID": 42, "FIRST_NAME": "Rick", "LAST_NAME": "Astley", "EMAIL": null, "GENDER": null, "CLUB_STATUS": "Platinum", "COMMENTS": null, "CREATE_TS": 1544000706681769, "UPDATE_TS": 1544000742000000}, "source": {"version": "0.9.0.Alpha2", "connector": "oracle", "name": "asgard", "ts_ms": 1544000742000, "txId": "6.26.734", "scn": 2796831, "snapshot": false}, "op": "u", "ts_ms": 1544000745823, "messagetopic": "asgard.DEBEZIUM.CUSTOMERS", "messagesource": "Debezium CDC from Oracle on asgard"}'|jq '.'
 ----
 
+or use kafkacat
+
+[source,bash]
+----
+docker exec kafkacat kafkacat -b kafka:29092 -t asgard.DEBEZIUM.CUSTOMERS -C -u -q -o-1 -c1 -r http://schema-registry:8081 -s key=s -s value=avro|jq '.'
+----
+
 
 Look at before & after: