[deploy][documentation] Fix terraform-ansible script and update docum…

…entation (apache#2357)

* [documentation] Improve documentation on bare mental deployment

 ### Motiviation

Fixes apache#2329

 ### Changes

- How to enable state storage for stateful functions
- How to enable function worker
- How to install builtin connectors
- Instructions to test functions

* Improve ansible script

* Fix the documentation

* update 2.1.0 documentation

deployment/terraform-ansible/aws/instances.tf

@@ -30,17 +30,41 @@ resource "aws_instance" "zookeeper" {
   }
 }
 
-resource "aws_instance" "pulsar" {
+resource "aws_instance" "bookie" {
   ami                    = "${var.aws_ami}"
-  instance_type          = "${var.instance_types["pulsar"]}"
+  instance_type          = "${var.instance_types["bookie"]}"
   key_name               = "${aws_key_pair.default.id}"
   subnet_id              = "${aws_subnet.default.id}"
   vpc_security_group_ids = ["${aws_security_group.default.id}"]
-  count                  = "${var.num_pulsar_brokers}"
+  count                  = "${var.num_bookie_nodes}"
 
   tags {
-    Name = "pulsar-${count.index + 1}"
+    Name = "bookie-${count.index + 1}"
   }
+}
 
-  associate_public_ip_address = true
+resource "aws_instance" "broker" {
+  ami                    = "${var.aws_ami}"
+  instance_type          = "${var.instance_types["broker"]}"
+  key_name               = "${aws_key_pair.default.id}"
+  subnet_id              = "${aws_subnet.default.id}"
+  vpc_security_group_ids = ["${aws_security_group.default.id}"]
+  count                  = "${var.num_broker_nodes}"
+
+  tags {
+    Name = "broker-${count.index + 1}"
+  }
+}
+
+resource "aws_instance" "proxy" {
+  ami                    = "${var.aws_ami}"
+  instance_type          = "${var.instance_types["proxy"]}"
+  key_name               = "${aws_key_pair.default.id}"
+  subnet_id              = "${aws_subnet.default.id}"
+  vpc_security_group_ids = ["${aws_security_group.default.id}"]
+  count                  = "${var.num_proxy_nodes}"
+
+  tags {
+    Name = "proxy-${count.index + 1}"
+  }
 }

deployment/terraform-ansible/aws/network.tf

@@ -91,7 +91,7 @@ resource "aws_route" "internet_access" {
 /* Load balancer */
 resource "aws_elb" "default" {
   name            = "pulsar-elb"
-  instances       = ["${aws_instance.pulsar.*.id}"]
+  instances       = ["${aws_instance.proxy.*.id}"]
   security_groups = ["${aws_security_group.elb.id}"]
   subnets         = ["${aws_subnet.default.id}"]
 

deployment/terraform-ansible/aws/output.tf

@@ -30,5 +30,5 @@ output "pulsar_web_url" {
 }
 
 output "pulsar_ssh_host" {
-  value = "${aws_instance.pulsar.0.public_ip}"
+  value = "${aws_instance.proxy.0.public_ip}"
 }

deployment/terraform-ansible/aws/setup-disk.yaml

@@ -18,7 +18,7 @@
 #
 
 - name: Disk setup
-  hosts: pulsar
+  hosts: bookie
   connection: ssh
   become: true
   tasks:

deployment/terraform-ansible/aws/terraform.tfvars

@@ -22,10 +22,14 @@ region              = "us-west-2"
 availability_zone   = "us-west-2a"
 aws_ami             = "ami-9fa343e7"
 num_zookeeper_nodes = 3
-num_pulsar_brokers  = 3
+num_bookie_nodes    = 3
+num_broker_nodes    = 2
+num_proxy_nodes     = 1
 base_cidr_block     = "10.0.0.0/16"
 
 instance_types      = {
-  "pulsar"    = "i3.xlarge"
-  "zookeeper" = "t2.small"
+  "zookeeper"   = "t2.small"
+  "bookie"      = "i3.xlarge"
+  "broker"      = "c5.2xlarge"
+  "proxy"       = "c5.2xlarge"
 }

deployment/terraform-ansible/aws/variables.tf

@@ -49,8 +49,16 @@ variable "num_zookeeper_nodes" {
   description = "The number of EC2 instances running ZooKeeper"
 }
 
-variable "num_pulsar_brokers" {
-  description = "The number of EC2 instances running a Pulsar broker plus a BookKeeper bookie"
+variable "num_bookie_nodes" {
+  description = "The number of EC2 instances running BookKeeper"
+}
+
+variable "num_broker_nodes" {
+  description = "The number of EC2 instances running Pulsar brokers"
+}
+
+variable "num_proxy_nodes" {
+  description = "The number of EC2 instances running Pulsar proxies"
 }
 
 variable "instance_types" {

deployment/terraform-ansible/deploy-pulsar.yaml

@@ -18,7 +18,7 @@
 #
 
 - name: Pulsar setup
-  hosts: all
+  hosts: zookeeper, bookie, broker, proxy
   connection: ssh
   become: true
   tasks:
@@ -36,19 +36,19 @@
         - vim
     - set_fact:
         zookeeper_servers: "{{ groups['zookeeper']|map('extract', hostvars, ['ansible_default_ipv4', 'address'])|map('regex_replace', '(.*)', '\\1:2181') | join(',') }}"
-        service_url: "pulsar://{{ hostvars[groups['pulsar'][0]].public_ip }}:6650/"
-        http_url: "http://{{ hostvars[groups['pulsar'][0]].public_ip }}:8080/"
+        service_url: "pulsar://{{ hostvars[groups['proxy'][0]].public_ip }}:6650/"
+        http_url: "http://{{ hostvars[groups['proxy'][0]].public_ip }}:8080/"
         pulsar_version: "2.1.0-incubating"
-
     - name: Download Pulsar binary package
       unarchive:
         src: http://archive.apache.org/dist/incubator/pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz
         remote_src: yes
         dest: /opt/pulsar
         extra_opts: ["--strip-components=1"]
     - set_fact:
-        max_heap_memory: "24g"
-        max_direct_memory: "24g"
+        max_heap_memory: "12g"
+        max_direct_memory: "12g"
+        cluster_name: "local"
     - name: Add pulsar_env.sh configuration file
       template:
         src: "../templates/pulsar_env.sh"
@@ -63,7 +63,6 @@
         zid: "{{ groups['zookeeper'].index(inventory_hostname) }}"
         max_heap_memory: "512m"
         max_direct_memory: "1g"
-        cluster_name: "local"
     - name: Create ZooKeeper data directory
       file:
         path: "/opt/pulsar/{{ item }}"
@@ -103,54 +102,79 @@
         chdir: /opt/pulsar
       when: groups['zookeeper'][0] == inventory_hostname
 
-- name: Set up Bookkeeper
-  hosts: pulsar
+- name: Set up bookies
+  hosts: bookie
   connection: ssh
   become: true
   tasks:
     - template:
         src: "../templates/bookkeeper.conf"
         dest: "/opt/pulsar/conf/bookkeeper.conf"
+    - template:
+        src: "../templates/bookkeeper.service"
+        dest: "/etc/systemd/system/bookkeeper.service"
+    - systemd:
+        state: restarted
+        daemon_reload: yes
+        name: "bookkeeper"
 
-- name: Set up Pulsar
-  hosts: pulsar
+- name: Set up brokers
+  hosts: broker
   connection: ssh
   become: true
   tasks:
+    - name: Download Pulsar IO package
+      unarchive:
+        src: http://archive.apache.org/dist/incubator/pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-io-connectors-{{ pulsar_version }}-bin.tar.gz
+        remote_src: yes
+        dest: /opt/pulsar
+        extra_opts: ["--strip-components=1"]
     - name: Set up broker
       template:
         src: "../templates/broker.conf"
         dest: "/opt/pulsar/conf/broker.conf"
+    - name: Set up function worker
+      template:
+        src: "../templates/functions_worker.yml"
+        dest: "/opt/pulsar/conf/functions_worker.yml"
     - template:
-        src: "../templates/pulsar.service"
-        dest: "/etc/systemd/system/pulsar.service"
+        src: "../templates/pulsar.broker.service"
+        dest: "/etc/systemd/system/pulsar.broker.service"
     - systemd:
         state: restarted
         daemon_reload: yes
-        name: "pulsar"
+        name: "pulsar.broker"
 
-- name: Pulsar multi-tenancy setup
-  hosts: pulsar
+- name: Set up Pulsar Proxy
+  hosts: proxy
   connection: ssh
   become: true
   tasks:
-    - name: Create default property and namespace
-      shell: |
-        bin/pulsar-admin tenants create public \
-          --allowed-clusters local \
-          --admin-roles all
-        bin/pulsar-admin namespaces create public/local/default
-      args:
-        chdir: /opt/pulsar
-      when: groups['zookeeper'][0] == inventory_hostname
+    - name: Set up proxy
+      template:
+        src: "../templates/proxy.conf"
+        dest: "/opt/pulsar/conf/proxy.conf"
+    - template:
+        src: "../templates/pulsar.proxy.service"
+        dest: "/etc/systemd/system/pulsar.proxy.service"
+    - systemd:
+        state: restarted
+        daemon_reload: yes
+        name: "pulsar.proxy"
 
 - name:  Hosts addresses
   hosts: localhost
   become: false
   tasks:
     - debug:
-        msg: "Zookeeper servers {{ item }}"
+        msg: "Zookeeper Server {{ item }}"
       with_items: "{{ groups['zookeeper'] }}"
     - debug:
-        msg: "Pulsar/BookKeeper servers {{ item }}"
-      with_items: "{{ groups['pulsar'] }}"
+        msg: "Bookie {{ item }}"
+      with_items: "{{ groups['bookie'] }}"
+    - debug:
+        msg: "Broker {{ item }}"
+      with_items: "{{ groups['broker'] }}"
+    - debug:
+        msg: "Proxy {{ item }}"
+      with_items: "{{ groups['proxy'] }}"

deployment/terraform-ansible/templates/bookkeeper.service

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[Unit]
+Description=BookKeeper
+After=network.target
+
+[Service]
+ExecStart=/opt/pulsar/bin/pulsar bookie
+WorkingDirectory=/opt/pulsar
+RestartSec=1s
+Restart=on-failure
+Type=simple
+
+[Install]
+WantedBy=multi-user.target
+

deployment/terraform-ansible/templates/broker.conf

@@ -41,10 +41,10 @@ webServicePortTls=8443
 bindAddress=0.0.0.0
 
 # Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used.
-advertisedAddress={{ hostvars[inventory_hostname].public_ip }}
+advertisedAddress={{ hostvars[inventory_hostname].private_ip }}
 
 # Name of the cluster to which this broker belongs to
-clusterName=local
+clusterName={{ cluster_name }}
 
 # Enable cluster's failure-domain which can distribute brokers into logical region
 failureDomainsEnabled=false
@@ -468,7 +468,7 @@ exposeTopicLevelMetricsInPrometheus=true
 ### --- Functions --- ###
 
 # Enable Functions Worker Service in Broker
-functionsWorkerEnabled=false
+functionsWorkerEnabled=true
 
 ### --- Broker Web Stats --- ###
 

deployment/terraform-ansible/templates/functions_worker.yml

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+workerId: standalone
+workerHostname: localhost
+workerPort: 6750
+
+connectorsDirectory: ./connectors
+
+functionMetadataTopicName: metadata
+clusterCoordinationTopicName: coordinate
+pulsarFunctionsNamespace: public/functions
+pulsarFunctionsCluster: {{ cluster_name }}
+pulsarServiceUrl: pulsar://localhost:6650
+pulsarWebServiceUrl: http://localhost:8080
+numFunctionPackageReplicas: 1
+downloadDirectory: /tmp/pulsar_functions
+#threadContainerFactory:
+#  threadGroupName: "Thread Function Container Group"
+processContainerFactory:
+  logDirectory:
+
+schedulerClassName: "org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler"
+functionAssignmentTopicName: "assignments"
+failureCheckFreqMs: 30000
+rescheduleTimeoutMs: 60000
+initialBrokerReconnectMaxRetries: 60
+assignmentWriteMaxRetries: 60
+instanceLivenessCheckFreqMs: 30000
+metricsSamplingPeriodSec: 60