update aws deployment for 2.6.0 (apache#7668)

### Motivation
I tried to follow the Aws deployment guide at https://pulsar.apache.org/docs/en/deploy-aws/ but found it was pretty outdated - It was trying to install pulsar 2.1.0-incubating. This PR updates it to install 2.6.0.

### Modifications

* Updated the pulsar version to 2.6.0
  * Fixed download location for 2.6.0
  * Updated config files for 2.6.0
  * Fixed connector installation for 2.6.0
  * Fixed Ansible's yum warning about installing multiple packages

deployment/terraform-ansible/aws/setup-disk.yaml

@@ -30,7 +30,7 @@
         src: "{{ item.src }}"
         fstype: xfs
         opts: defaults,noatime,nodiscard
-        state: present
+        state: mounted
       with_items:
         - { path: "/mnt/journal", src: "/dev/nvme0n1" }
         - { path: "/mnt/storage", src: "/dev/nvme1n1" }

deployment/terraform-ansible/deploy-pulsar.yaml

@@ -28,20 +28,21 @@
         state: directory
       with_items: ["/opt/pulsar"]
     - name: Install RPM packages
-      yum: pkg={{ item }} state=latest
-      with_items:
-        - wget
-        - java
-        - sysstat
-        - vim
+      yum:
+        state: latest
+        name:
+          - wget
+          - java
+          - sysstat
+          - vim
     - set_fact:
-        zookeeper_servers: "{{ groups['zookeeper']|map('extract', hostvars, ['ansible_default_ipv4', 'address'])|map('regex_replace', '(.*)', '\\1:2181') | join(',') }}"
-        service_url: "pulsar://{{ hostvars[groups['proxy'][0]].public_ip }}:6650/"
-        http_url: "http://{{ hostvars[groups['proxy'][0]].public_ip }}:8080/"
-        pulsar_version: "2.1.0-incubating"
+        zookeeper_servers: "{{ groups['zookeeper']|map('extract', hostvars, ['ansible_default_ipv4', 'address'])|map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}"
+        service_url: "{{ pulsar_service_url }}"
+        http_url: "{{ pulsar_web_url }}"
+        pulsar_version: "2.6.0"
     - name: Download Pulsar binary package
       unarchive:
-        src: http://archive.apache.org/dist/incubator/pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz
+        src: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz
         remote_src: yes
         dest: /opt/pulsar
         extra_opts: ["--strip-components=1"]
@@ -123,12 +124,45 @@
   connection: ssh
   become: true
   tasks:
-    - name: Download Pulsar IO package
-      unarchive:
-        src: http://archive.apache.org/dist/incubator/pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-io-connectors-{{ pulsar_version }}-bin.tar.gz
-        remote_src: yes
-        dest: /opt/pulsar
-        extra_opts: ["--strip-components=1"]
+    - name: Create connectors directory
+      file:
+        path: "/opt/pulsar/{{ item }}"
+        state: directory
+      loop:
+        - connectors
+    - name: Download Pulsar IO packages
+      get_url:
+        url: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/connectors/pulsar-io-{{ item }}-{{ pulsar_version }}.nar
+        dest: /opt/pulsar/connectors/pulsar-io-{{ item }}-{{ pulsar_version }}.nar
+      loop:
+#        - aerospike
+#        - canal
+#        - cassandra
+#        - data-generator
+#        - debezium-mongodb
+#        - debezium-mysql
+#        - debezium-postgres
+#        - dynamodb
+#        - elastic-search
+#        - file
+#        - flume
+#        - hbase
+#        - hdfs2
+#        - hdfs3
+#        - influxdb
+#        - jdbc-clickhouse
+#        - jdbc-mariadb
+#        - jdbc-postgres
+#        - jdbc-sqlite
+        - kafka
+#        - kafka-connect-adaptor
+#        - kinesis
+#        - mongo
+#        - netty
+#        - rabbitmq
+#        - redis
+#        - solr
+#        - twitter
     - name: Set up broker
       template:
         src: "../templates/broker.conf"

deployment/terraform-ansible/templates/bookkeeper.conf

@@ -52,8 +52,8 @@ minUsableSizeForIndexFileCreation=1073741824
 
 # Configure a specific hostname or IP address that the bookie should use to advertise itself to
 # clients. If not set, bookie will advertised its own IP address or hostname, depending on the
-# listeningInterface and `seHostNameAsBookieID settings.
-# advertisedAddress=
+# listeningInterface and useHostNameAsBookieID settings.
+advertisedAddress=
 
 # Whether the bookie allowed to use a loopback interface as its primary
 # interface(i.e. the interface it uses to establish its identity)?
@@ -92,7 +92,7 @@ flushInterval=60000
 
 # Whether the bookie should use its hostname to register with the
 # co-ordination service(eg: Zookeeper service).
-# When false, bookie will use its ipaddress for the registration.
+# When false, bookie will use its ip address for the registration.
 # Defaults to false.
 useHostNameAsBookieID=false
 
@@ -224,18 +224,18 @@ maxPendingAddRequestsPerThread=10000
 auditorPeriodicBookieCheckInterval=86400
 
 # The number of entries that a replication will rereplicate in parallel.
-rereplicationEntryBatchSize=5000
+rereplicationEntryBatchSize=100
 
 # Auto-replication
 # The grace period, in seconds, that the replication worker waits before fencing and
 # replicating a ledger fragment that's still being written to upon bookie failure.
-# openLedgerRereplicationGracePeriod=30
+openLedgerRereplicationGracePeriod=30
 
 # Whether the bookie itself can start auto-recovery service also or not
 autoRecoveryDaemonEnabled=true
 
 # How long to wait, in seconds, before starting auto recovery of a lost bookie
-# lostBookieRecoveryDelay=0
+lostBookieRecoveryDelay=0
 
 #############################################################################
 ## Netty server settings
@@ -268,28 +268,34 @@ serverTcpNoDelay=true
 # The Recv ByteBuf allocator max buf size.
 # byteBufAllocatorSizeMax=1048576
 
+# The maximum netty frame size in bytes. Any message received larger than this will be rejected. The default value is 1G.
+nettyMaxFrameSizeBytes=5253120
+
 #############################################################################
 ## Journal settings
 #############################################################################
 
 # The journal format version to write.
-# Available formats are 1-5:
+# Available formats are 1-6:
 # 1: no header
 # 2: a header section was added
 # 3: ledger key was introduced
 # 4: fencing key was introduced
 # 5: expanding header to 512 and padding writes to align sector size configured by `journalAlignmentSize`
-# By default, it is `4`. If you'd like to enable `padding-writes` feature, you can set journal version to `5`.
+# 6: persisting explicitLac is introduced
+# By default, it is `6`.
+# If you'd like to disable persisting ExplicitLac, you can set this config to < `6` and also
+# fileInfoFormatVersionToWrite should be set to 0. If there is mismatch then the serverconfig is considered invalid.
 # You can disable `padding-writes` by setting journal version back to `4`. This feature is available in 4.5.0
 # and onward versions.
-# journalFormatVersionToWrite=4
+journalFormatVersionToWrite=5
 
 # Max file size of journal file, in mega bytes
 # A new journal file will be created when the old one reaches the file size limitation
 journalMaxSizeMB=2048
 
 # Max number of old journal file to kept
-# Keep a number of old journal files would help data recovery in specia case
+# Keep a number of old journal files would help data recovery in special case
 journalMaxBackups=5
 
 # How much space should we pre-allocate at a time in the journal.
@@ -345,7 +351,7 @@ ledgerStorageClass=org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage
 # For example:
 # ledgerDirectories=/tmp/bk1-data,/tmp/bk2-data
 #
-# Ideally ledger dirs and journal dir are each in a differet device,
+# Ideally ledger dirs and journal dir are each in a different device,
 # which reduce the contention between random i/o and sequential write.
 # It is possible to run with a single disk, but performance will be significantly lower.
 ledgerDirectories=data/bookkeeper/ledgers
@@ -360,7 +366,7 @@ ledgerDirectories=data/bookkeeper/ledgers
 auditorPeriodicCheckInterval=604800
 
 # Whether sorted-ledger storage enabled (default true)
-# sortedLedgerStorageEnabled=ture
+# sortedLedgerStorageEnabled=true
 
 # The skip list data size limitation (default 64MB) in EntryMemTable
 # skipListSizeLimit=67108864L
@@ -376,9 +382,19 @@ auditorPeriodicCheckInterval=604800
 # to gain performance according your requirements.
 openFileLimit=0
 
+# The fileinfo format version to write.
+#  Available formats are 0-1:
+#   0: Initial version
+#   1: persisting explicitLac is introduced
+# By default, it is `1`.
+# If you'd like to disable persisting ExplicitLac, you can set this config to 0 and
+# also journalFormatVersionToWrite should be set to < 6. If there is mismatch then the
+# serverconfig is considered invalid.
+fileInfoFormatVersionToWrite=0
+
 # Size of a index page in ledger cache, in bytes
 # A larger index page can improve performance writing page to disk,
-# which is efficent when you have small number of ledgers and these
+# which is efficient when you have small number of ledgers and these
 # ledgers have similar number of entries.
 # If you have large number of ledgers and each ledger has fewer entries,
 # smaller index page would improve memory usage.
@@ -391,7 +407,7 @@ openFileLimit=0
 # pageLimit*pageSize should not more than JVM max memory limitation,
 # otherwise you would got OutOfMemoryException.
 # In general, incrementing pageLimit, using smaller index page would
-# gain bettern performance in lager number of ledgers with fewer entries case
+# gain better performance in lager number of ledgers with fewer entries case
 # If pageLimit is -1, bookie server will use 1/3 of JVM memory to compute
 # the limitation of number of index pages.
 pageLimit=0
@@ -405,7 +421,7 @@ pageLimit=0
 # and garbage collected. Try to read 'BookKeeper Internals' for detail info.
 # ledgerManagerFactoryClass=org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory
 
-# @Drepcated - `ledgerManagerType` is deprecated in favor of using `ledgerManagerFactoryClass`.
+# @Deprecated - `ledgerManagerType` is deprecated in favor of using `ledgerManagerFactoryClass`.
 # ledgerManagerType=hierarchical
 
 # Root Zookeeper path to store ledger metadata
@@ -429,7 +445,7 @@ entryLogFilePreallocationEnabled=true
 # happens on log rotation.
 # Flushing in smaller chunks but more frequently reduces spikes in disk
 # I/O. Flushing too frequently may also affect performance negatively.
-# flushEntrylogBytes=0
+flushEntrylogBytes=268435456
 
 # The number of bytes we should use as capacity for BufferedReadChannel. Default is 512 bytes.
 readBufferSizeBytes=4096
@@ -462,6 +478,7 @@ minorCompactionThreshold=0.2
 
 # Interval to run minor compaction, in seconds
 # If it is set to less than zero, the minor compaction is disabled.
+# Note: should be greater than gcWaitTime.
 minorCompactionInterval=3600
 
 # Set the maximum number of entries which can be compacted without flushing.
@@ -484,6 +501,7 @@ majorCompactionThreshold=0.5
 
 # Interval to run major compaction, in seconds
 # If it is set to less than zero, the major compaction is disabled.
+# Note: should be greater than gcWaitTime.
 majorCompactionInterval=86400
 
 # Throttle compaction by bytes or by entries.
@@ -521,7 +539,7 @@ readOnlyModeEnabled=true
 # Whether the bookie is force started in read only mode or not
 # forceReadOnlyBookie=false
 
-# Persiste the bookie status locally on the disks. So the bookies can keep their status upon restarts
+# Persist the bookie status locally on the disks. So the bookies can keep their status upon restarts
 # @Since 4.6
 # persistBookieStatusEnabled=false
 
@@ -531,7 +549,7 @@ readOnlyModeEnabled=true
 
 # For each ledger dir, maximum disk space which can be used.
 # Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will
-# be written to that partition. If all ledger dir partions are full, then bookie
+# be written to that partition. If all ledger dir partitions are full, then bookie
 # will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will
 # shutdown.
 # Valid values should be in between 0 and 1 (exclusive).
@@ -590,6 +608,16 @@ zkEnableSecurity=false
 ## Server parameters
 #############################################################################
 
+# The flag enables/disables starting the admin http server. Default value is 'false'.
+httpServerEnabled=false
+
+# The http server port to listen on. Default value is 8080.
+# Use `8000` as the port to keep it consistent with prometheus stats provider
+httpServerPort=8000
+
+# The http server class
+httpServerClass=org.apache.bookkeeper.http.vertx.VertxHttpServer
+
 # Configure a list of server components to enable and load on a bookie server.
 # This provides the plugin run extra services along with a bookie server.
 #
@@ -605,12 +633,15 @@ zkEnableSecurity=false
 
 # Size of Write Cache. Memory is allocated from JVM direct memory.
 # Write cache is used to buffer entries before flushing into the entry log
-# For good performance, it should be big enough to hold a sub
-dbStorage_writeCacheMaxSizeMb=512
+# For good performance, it should be big enough to hold a substantial amount
+# of entries in the flush interval
+#  By default it will be allocated to 1/4th of the available direct memory
+dbStorage_writeCacheMaxSizeMb=
 
 # Size of Read cache. Memory is allocated from JVM direct memory.
 # This read cache is pre-filled doing read-ahead whenever a cache miss happens
-dbStorage_readAheadCacheMaxSizeMb=256
+#  By default it will be allocated to 1/4th of the available direct memory
+dbStorage_readAheadCacheMaxSizeMb=
 
 # How many entries to pre-fill in cache after a read cache miss
 dbStorage_readAheadCacheBatchSize=1000
@@ -622,8 +653,8 @@ dbStorage_readAheadCacheBatchSize=1000
 # Size of RocksDB block-cache. For best performance, this cache
 # should be big enough to hold a significant portion of the index
 # database which can reach ~2GB in some cases
-# Default is 256 MBytes
-dbStorage_rocksDB_blockCacheSize=268435456
+# Default is to use 10% of the direct memory size
+dbStorage_rocksDB_blockCacheSize=
 
 # Other RocksDB specific tunables
 dbStorage_rocksDB_writeBufferSizeMB=64