Provide an Ansible playbook for AWS with documentation (apache#920)

.gitignore

@@ -41,3 +41,9 @@ target/
 
 # Generated website
 generated-site/
+
+# Ansible and Terraform artifacts
+deployment/terraform-ansible/deploy-pulsar.retry
+deployment/terraform-ansible/aws/terraform*
+deployment/terraform-ansible/aws/.terraform/
+deployment/terraform-ansible/aws/.terraform.tfstate.lock.info

deployment/terraform-ansible/aws/ansible.cfg

@@ -0,0 +1,9 @@
+[defaults]
+private_key_file=~/.ssh/pulsar_aws
+host_key_checking=false
+user='ec2-user'
+
+[privilege_escalation]
+become=True
+become_method='sudo'
+become_user='root'

deployment/terraform-ansible/aws/instances.tf

@@ -0,0 +1,27 @@
+resource "aws_instance" "zookeeper" {
+  ami                    = "${var.aws_ami}"
+  instance_type          = "${var.instance_types["zookeeper"]}"
+  key_name               = "${aws_key_pair.default.id}"
+  subnet_id              = "${aws_subnet.default.id}"
+  vpc_security_group_ids = ["${aws_security_group.default.id}"]
+  count                  = "${var.num_zookeeper_nodes}"
+
+  tags {
+    Name = "zookeeper-${count.index + 1}"
+  }
+}
+
+resource "aws_instance" "pulsar" {
+  ami                         = "${var.aws_ami}"
+  instance_type               = "${var.instance_types["pulsar"]}"
+  key_name                    = "${aws_key_pair.default.id}"
+  subnet_id                   = "${aws_subnet.default.id}"
+  vpc_security_group_ids      = ["${aws_security_group.default.id}"]
+  count                       = "${var.num_pulsar_brokers}"
+
+  tags {
+    Name = "pulsar-${count.index + 1}"
+  }
+
+  associate_public_ip_address = true
+}

deployment/terraform-ansible/aws/keys.tf

@@ -0,0 +1,9 @@
+resource "random_id" "key_pair_name" {
+  byte_length = 4
+  prefix      = "${var.key_name_prefix}-"
+}
+
+resource "aws_key_pair" "default" {
+  key_name   = "${random_id.key_pair_name.hex}"
+  public_key = "${file(var.public_key_path)}"
+}

deployment/terraform-ansible/aws/network.tf

@@ -0,0 +1,98 @@
+resource "aws_vpc" "pulsar_vpc" {
+  cidr_block           = "${var.base_cidr_block}"
+  enable_dns_support   = true
+  enable_dns_hostnames = true
+
+  tags {
+    Name = "Pulsar-VPC"
+  }
+}
+
+resource "aws_subnet" "default" {
+  vpc_id                  = "${aws_vpc.pulsar_vpc.id}"
+  cidr_block              = "${cidrsubnet(var.base_cidr_block, 8, 2)}"
+  availability_zone       = "${var.availability_zone}"
+  map_public_ip_on_launch = true
+
+  tags {
+    Name = "Pulsar-Subnet"
+  }
+}
+
+resource "aws_route_table" "default" {
+  vpc_id = "${aws_vpc.pulsar_vpc.id}"
+
+  tags {
+    Name = "Pulsar-Route-Table"
+  }
+}
+
+resource "aws_route" "default" {
+  route_table_id         = "${aws_route_table.default.id}"
+  destination_cidr_block = "0.0.0.0/0"
+  nat_gateway_id         = "${aws_nat_gateway.default.id}"
+}
+
+resource "aws_route_table_association" "default" {
+  subnet_id      = "${aws_subnet.default.id}"
+  route_table_id = "${aws_vpc.pulsar_vpc.main_route_table_id}"
+}
+
+/* Misc */
+resource "aws_eip" "default" {
+  vpc = true
+  depends_on = ["aws_internet_gateway.default"]
+}
+
+resource "aws_internet_gateway" "default" {
+  vpc_id = "${aws_vpc.pulsar_vpc.id}"
+
+  tags {
+    Name = "Pulsar-Internet-Gateway"
+  }
+}
+
+resource "aws_nat_gateway" "default" {
+  allocation_id = "${aws_eip.default.id}"
+  subnet_id     = "${aws_subnet.default.id}"
+  depends_on    = ["aws_internet_gateway.default"]
+
+  tags {
+    Name = "Pulsar-NAT-Gateway"
+  }
+}
+
+/* Public internet route */
+resource "aws_route" "internet_access" {
+  route_table_id         = "${aws_vpc.pulsar_vpc.main_route_table_id}"
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id             = "${aws_internet_gateway.default.id}"
+}
+
+/* Load balancer */
+resource "aws_elb" "default" {
+  name               = "pulsar-elb"
+  instances          = ["${aws_instance.pulsar.*.id}"]
+  security_groups    = ["${aws_security_group.elb.id}"]
+  subnets            = ["${aws_subnet.default.id}"]
+
+  listener {
+    instance_port     = 8080
+    instance_protocol = "http"
+    lb_port           = 8080
+    lb_protocol       = "http"
+  }
+
+  listener {
+    instance_port     = 6650
+    instance_protocol = "tcp"
+    lb_port           = 6650
+    lb_protocol       = "tcp"
+  }
+
+  cross_zone_load_balancing = false
+
+  tags {
+    Name = "Pulsar-Load-Balancer"
+  }
+}

deployment/terraform-ansible/aws/output.tf

@@ -0,0 +1,15 @@
+output "dns_name" {
+  value = "${aws_elb.default.dns_name}"
+}
+
+output "pulsar_service_url" {
+  value = "pulsar://${aws_elb.default.dns_name}:6650"
+}
+
+output "pulsar_web_url" {
+  value = "http://${aws_elb.default.dns_name}:8080"
+}
+
+output "pulsar_ssh_host" {
+  value = "${aws_instance.pulsar.0.public_ip}"
+}

deployment/terraform-ansible/aws/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = "${var.region}"
+  version = "1.5"
+}

deployment/terraform-ansible/aws/security.tf

@@ -0,0 +1,58 @@
+resource "aws_security_group" "elb" {
+  name   = "pulsar-elb"
+  vpc_id = "${aws_vpc.pulsar_vpc.id}"
+
+  ingress {
+    from_port   = 6650
+    to_port     = 6650
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port   = 8080
+    to_port     = 8080
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "default" {
+  name   = "pulsar-terraform"
+  vpc_id = "${aws_vpc.pulsar_vpc.id}"
+
+  # SSH access from anywhere
+  ingress {
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  # All ports open within the VPC
+  ingress {
+    from_port   = 0
+    to_port     = 65535
+    protocol    = "tcp"
+    cidr_blocks = ["${var.base_cidr_block}"]
+  }
+
+  # outbound internet access
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags {
+    Name = "Pulsar-Security-Group"
+  }
+}

deployment/terraform-ansible/aws/variables.tf

@@ -0,0 +1,43 @@
+variable "public_key_path" {
+  description = <<DESCRIPTION
+Path to the SSH public key to be used for authentication.
+Ensure this keypair is added to your local SSH agent so provisioners can
+connect.
+
+Example: ~/.ssh/my_keys.pub
+Default: ~/.ssh/id_rsa.pub
+DESCRIPTION
+}
+
+variable "key_name_prefix" {
+  description = "The prefix for the randomly generated name for the AWS key pair to be used for SSH connections (e.g. `pulsar-terraform-ssh-keys-0a1b2cd3`)"
+  default     = "pulsar-terraform-ssh-keys"
+}
+
+variable "region" {
+  description = "The AWS region in which the Pulsar cluster will be deployed"
+}
+
+variable "availability_zone" {
+  description = "The AWS availability zone in which the cluster will run"
+}
+
+variable "aws_ami" {
+  description = "The AWS AMI to be used by the Pulsar cluster"
+}
+
+variable "num_zookeeper_nodes" {
+  description = "The number of EC2 instances running ZooKeeper"
+}
+
+variable "num_pulsar_brokers" {
+  description = "The number of EC2 instances running a Pulsar broker plus a BookKeeper bookie"
+}
+
+variable "instance_types" {
+  type = "map"
+}
+
+variable "base_cidr_block" {
+  description = "The baseline CIDR block to be used by network assets for the Pulsar cluster"
+}