Add consul health checks for zookeeper / mesos / marathon. Handle set…

…ting weave docker subnet properly when we have more than 1 slave active

Vagrantfile

@@ -24,6 +24,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     "all:children" => ["mesos_masters", "mesos_slaves"],
     "zookeeper_servers:children" => ["mesos_masters"],
     "consul_servers:children" => ["mesos_masters"],
+    "weave_servers:children" => ["mesos_slaves"],
   }
 
   # Mesos master nodes

bootstrap/digitalocean/config-default.sh

@@ -12,3 +12,5 @@ ATLAS_INFRASTRUCTURE=${ATLAS_INFRASTRUCTURE:-capgemini/infrastructure}
 MASTER_SIZE=${MASTER_SIZE:-512mb}
 SLAVE_SIZE=${SLAVE_SIZE:-512mb}
 NUM_SLAVES=${NUM_SLAVES:-1}
+MASTER_IMAGE=${MASTER_IMAGE:-11147507}
+SLAVE_IMAGE=${SLAVE_IMAGE:-11147507}

bootstrap/digitalocean/util.sh

@@ -33,9 +33,6 @@ apollo_down() {
     terraform destroy -var "do_token=${DIGITALOCEAN_API_TOKEN}" \
       -var "key_file=${DIGITALOCEAN_SSH_KEY}" \
       -var "ssh_fingerprint=${DIGITALOCEAN_SSH_FINGERPRINT}" \
-      -var "instance_size.master=${MASTER_SIZE}" \
-      -var "instance_size.slave=${SLAVE_SIZE}" \
-      -var "slaves=${NUM_SLAVES}" \
       -var "region=${DIGITALOCEAN_REGION}"
   popd
 }
@@ -47,6 +44,8 @@ terraform_apply() {
       -var "ssh_fingerprint=${DIGITALOCEAN_SSH_FINGERPRINT}" \
       -var "instance_size.master=${MASTER_SIZE}" \
       -var "instance_size.slave=${SLAVE_SIZE}" \
+      -var "image.master=${MASTER_IMAGE}" \
+      -var "image.slave=${SLAVE_IMAGE}" \
       -var "slaves=${NUM_SLAVES}" \
       -var "region=${DIGITALOCEAN_REGION}"
   popd

inventory/digitalocean/inventory

@@ -11,3 +11,6 @@ mesos_masters
 
 [consul_servers:children]
 mesos_masters
+
+[weave_servers:children]
+mesos_slaves

roles/consul/templates/consul.json.j2

@@ -10,9 +10,5 @@
   "bootstrap_expect": {{ consul_bootstrap_expect }},
 {% endif %}
   "data_dir": "/var/lib/consul",
-  "ui_dir": "/opt/consul-ui",
-  "service": {
-    "name": "consul",
-    "tags": ["consul"]
-  }
+  "ui_dir": "/opt/consul-ui"
 }

roles/marathon/defaults/main.yml

@@ -1,2 +1,4 @@
 ---
 # defaults file for marathon
+marathon_port: 8080
+consul_dir: /etc/consul.d

roles/marathon/meta/main.yml

@@ -120,7 +120,8 @@ galaxy_info:
   #- packaging
   - system
   #- web
-dependencies: []
+dependencies:
+  - role: consul
   # List your role dependencies here, one per line. Only
   # dependencies available via galaxy should be listed here.
   # Be sure to remove the '[]' above if you add dependencies

roles/marathon/tasks/main.yml

@@ -5,3 +5,11 @@
 
 - name: ensure marathon is running (and enable it at boot)
   service: name=marathon state=started enabled=yes
+
+- name: Set Marathon consul service definition
+  sudo: yes
+  template:
+    src: marathon-consul.j2
+    dest: "{{ consul_dir }}/marathon.json"
+  notify:
+    - Restart consul

roles/marathon/templates/marathon-consul.j2

@@ -0,0 +1,11 @@
+{
+  "service": {
+    "name": "marathon",
+    "tags": [ "marathon" ],
+    "port": {{ marathon_port }},
+    "check": {
+      "script": "curl --silent --show-error  --fail --dump-header /dev/stderr --retry 2 http://127.0.0.1:{{ marathon_port }}/ping",
+      "interval": "10s"
+    }
+  }
+}

roles/mesos/defaults/main.yml

@@ -2,3 +2,6 @@
 # defaults file for mesos
 mesos_zk_port: 2181
 mesos_zookeeper_group: zookeeper_servers
+mesos_master_port: 5050
+consul_dir: /etc/consul.d
+mesos_endpoint: "{{ ansible_default_ipv4.address }}"

roles/mesos/meta/main.yml

@@ -120,7 +120,8 @@ galaxy_info:
   #- packaging
   - system
   #- web
-dependencies: []
+dependencies:
+  - role: consul
   # List your role dependencies here, one per line. Only
   # dependencies available via galaxy should be listed here.
   # Be sure to remove the '[]' above if you add dependencies

roles/mesos/tasks/main.yml

@@ -43,6 +43,15 @@
     - Start mesos master
   when: mesos_install_mode == "master"
 
+- name: Set Mesos Master consul service definition
+  sudo: yes
+  template:
+    src: mesos-master-consul.j2
+    dest: "{{ consul_dir }}/mesos-master.json"
+  notify:
+    - Restart consul
+  when: mesos_install_mode == "master"
+
 # Tasks for Slave nodes
 - name: Set Mesos Slave hostname
   copy:

roles/mesos/templates/mesos-master-consul.j2

@@ -0,0 +1,11 @@
+{
+  "service": {
+    "name": "mesos-master",
+    "port": {{ mesos_master_port }},
+    "tags": [ "mesos" ],
+    "check": {
+      "script": "curl --silent --show-error --fail --dump-header /dev/stderr --retry 2 http://{{ mesos_endpoint }}:{{ mesos_master_port }}/master/health",
+      "interval": "10s"
+    }
+  }
+}

roles/weave/defaults/main.yml

@@ -1,4 +1,11 @@
 ---
 # defaults file for weave
 weave_bridge: "10.2.0.1/16"
-weave_docker_subnet: "10.2.1.0/24"
+weave_server_group: weave_servers
+weave_docker_subnet: "
+    {%- for host in groups[weave_server_group] -%}
+      {%- if host == 'default' or host == inventory_hostname or host == ansible_fqdn or host in ansible_all_ipv4_addresses -%}
+        10.2.{{ loop.index }}.0/24
+      {%- endif -%}
+    {%- endfor -%}
+"

roles/zookeeper/defaults/main.yml

@@ -11,3 +11,4 @@ zookeeper_id: "
       {%- endif -%}
     {%- endfor -%}
 "
+consul_dir: /etc/consul.d

roles/zookeeper/meta/main.yml

@@ -120,7 +120,8 @@ galaxy_info:
   #- packaging
   - system
   #- web
-dependencies: []
+dependencies:
+  - role: consul
   # List your role dependencies here, one per line. Only
   # dependencies available via galaxy should be listed here.
   # Be sure to remove the '[]' above if you add dependencies

roles/zookeeper/tasks/main.yml

@@ -10,3 +10,11 @@
   sudo: yes
   notify:
     - Start zookeeper
+
+- name: Set Zookeeper consul service definition
+  sudo: yes
+  template:
+    src: zookeeper-consul.j2
+    dest: "{{ consul_dir }}/zookeeper.json"
+  notify:
+    - Restart consul

roles/zookeeper/templates/zookeeper-consul.j2

@@ -0,0 +1,11 @@
+{
+  "service": {
+    "name": "zookeeper",
+    "port": {{ zookeeper_client_port }},
+    "tags": [ "zookeeper" ],
+    "check": {
+      "script": "echo ruok | nc 127.0.0.1 {{ zookeeper_client_port }} | grep imok",
+      "interval": "10s"
+    }
+  }
+}

terraform/digitalocean/outputs.tf

@@ -7,6 +7,6 @@ output "master.2.ip" {
 output "master.3.ip" {
   value = "${digitalocean_droplet.mesos-master.2.ipv4_address}"
 }
-output "slave.1.ip" {
-  value = "${digitalocean_droplet.mesos-slave.0.ipv4_address}"
+output "slave_ips" {
+   value = "${join(",", digitalocean_droplet.mesos-slave.*.ipv4_address)}"
 }