Merge pull request deis#1833 from gabrtv/fleet-http

New Fleet scheduler implementation using HTTP API

Makefile

@@ -4,10 +4,8 @@
 
 include includes.mk
 
-list-units = list-units --fields unit,state,load,active,sub,desc,machine
-
 define check_for_errors
-	@if $(FLEETCTL) $(list-units) -no-legend | awk '(($$2 == "launched") && ($$5 == "failed"))' | egrep -q "deis-.+service"; then \
+	@if $(FLEETCTL) list-units -no-legend | awk '($$4 == "failed")' | egrep -q "deis-.+service"; then \
 		echo "\033[0;31mOne or more services failed! Check which services by running 'make status'\033[0m" ; \
 		echo "\033[0;31mYou can get detailed output with 'fleetctl status deis-servicename.service'\033[0m" ; \
 		echo "\033[0;31mThis usually indicates an error with Deis - please open an issue on GitHub or ask for help in IRC\033[0m" ; \
@@ -16,8 +14,8 @@ define check_for_errors
 endef
 
 define deis_units
-	$(shell $(FLEETCTL) $(list-units) -no-legend=true | \
-	  awk '($$2 ~ "$(1)" && ($$4 ~ "$(2)"))' | \
+	$(shell $(FLEETCTL) list-units -no-legend=true | \
+	  awk '($$4 ~ "$(1)")' | \
 	  sed -n 's/\(deis-.*\.service\).*/\1/p' | tr '\n' ' ')
 endef
 
@@ -51,7 +49,7 @@ install: check-fleet install-routers install-data-containers
 install-data-containers: check-fleet
 	@$(foreach T, $(DATA_CONTAINER_TEMPLATES), \
 		UNIT=`basename $(T)` ; \
-		EXISTS=`$(FLEETCTL) $(list-units) | grep $$UNIT` ; \
+		EXISTS=`$(FLEETCTL) list-units | grep $$UNIT` ; \
 		if [ "$$EXISTS" != "" ]; then \
 		  echo $$UNIT already loaded. Skipping... ; \
 		else \
@@ -87,43 +85,43 @@ start: check-fleet start-warning start-routers
 	@# logger cache database
 	$(call echo_yellow,"Waiting for deis-cache to start...")
 	$(FLEETCTL) start -no-block $(START_UNITS)
-	@until $(FLEETCTL) $(list-units) | egrep -q "deis-cache\..+(running|failed)"; \
+	@until $(FLEETCTL) list-units | egrep -q "deis-cache\..+(running|failed)"; \
 		do sleep 2; \
-			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) $(list-units) | \
-			grep "deis-cache\." | awk '{printf "%-10s (%s)    \r", $$4, $$5}'; \
+			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) list-units | \
+			grep "deis-cache\." | awk '{printf "%-10s (%s)    \r", $$3, $$4}'; \
 			sleep 8; \
 		done
 	$(call check_for_errors)
 
 	@# registry
 	$(call echo_yellow,"Waiting for deis-registry to start...")
 	$(FLEETCTL) start -no-block registry/systemd/*.service
-	@until $(FLEETCTL) $(list-units) | egrep -q "deis-registry\..+(running|failed)"; \
+	@until $(FLEETCTL) list-units | egrep -q "deis-registry\..+(running|failed)"; \
 		do sleep 2; \
-			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) $(list-units) | \
-			grep "deis-registry\." | awk '{printf "%-10s (%s)    \r", $$4, $$5}'; \
+			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) list-units | \
+			grep "deis-registry\." | awk '{printf "%-10s (%s)    \r", $$3, $$4}'; \
 			sleep 8; \
 		done
 	$(call check_for_errors)
 
 	@# controller
 	$(call echo_yellow,"Waiting for deis-controller to start...")
 	$(FLEETCTL) start -no-block controller/systemd/*.service
-	@until $(FLEETCTL) $(list-units) | egrep -q "deis-controller\..+(running|failed)"; \
+	@until $(FLEETCTL) list-units | egrep -q "deis-controller\..+(running|failed)"; \
 		do sleep 2; \
-			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) $(list-units) | \
-			grep "deis-controller\." | awk '{printf "%-10s (%s)    \r", $$4, $$5}'; \
+			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) list-units | \
+			grep "deis-controller\." | awk '{printf "%-10s (%s)    \r", $$3, $$4}'; \
 			sleep 8; \
 		done
 	$(call check_for_errors)
 
 	@# builder
 	$(call echo_yellow,"Waiting for deis-builder to start...")
 	$(FLEETCTL) start -no-block builder/systemd/*.service
-	@until $(FLEETCTL) $(list-units) | egrep -q "deis-builder\..+(running|failed)"; \
+	@until $(FLEETCTL) list-units | egrep -q "deis-builder\..+(running|failed)"; \
 		do sleep 2; \
-			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) $(list-units) | \
-			grep "deis-builder\." | awk '{printf "%-10s (%s)    \r", $$4, $$5}'; \
+			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) list-units | \
+			grep "deis-builder\." | awk '{printf "%-10s (%s)    \r", $$3, $$4}'; \
 			sleep 8; \
 		done
 	$(call check_for_errors)
@@ -133,11 +131,11 @@ start: check-fleet start-warning start-routers
 start-routers: check-fleet start-warning
 	$(call echo_yellow,"Waiting for 1 of $(DEIS_NUM_ROUTERS) deis-routers to start...")
 	$(foreach R,$(ROUTER_UNITS),$(FLEETCTL) start -no-block $(R);)
-	@until $(FLEETCTL) $(list-units) | egrep -q "deis-router.+(running)"; \
+	@until $(FLEETCTL) list-units | egrep -q "deis-router.+(running)"; \
 		do sleep 2; \
-			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) $(list-units) | \
+			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) list-units | \
 			grep "deis-router" | head -n 1 | \
-			awk '{printf "%-10s (%s)    \r", $$4, $$5}'; \
+			awk '{printf "%-10s (%s)    \r", $$3, $$4}'; \
 			sleep 8; \
 		done
 	$(call check_for_errors)
@@ -146,10 +144,10 @@ start-warning:
 	$(call echo_cyan,"Deis components may take a long time to start the first time they are initialized.")
 
 status: check-fleet
-	$(FLEETCTL) $(list-units)
+	$(FLEETCTL) list-units
 
 stop: check-fleet
-	$(FLEETCTL) stop -block-attempts=600 $(strip $(call deis_units,launched,active))
+	$(FLEETCTL) stop -block-attempts=600 $(strip $(call deis_units,active))
 
 test: test-components test-integration
 
@@ -166,5 +164,5 @@ test-smoke:
 	$(MAKE) -C tests/ test-smoke
 
 uninstall: check-fleet stop
-	$(FLEETCTL) unload -block-attempts=600 $(call deis_units,launched,.)
-	$(FLEETCTL) destroy $(strip $(call deis_units,.,.))
+	$(FLEETCTL) unload -block-attempts=600 $(call deis_units,.)
+	$(FLEETCTL) destroy $(strip $(call deis_units,.))

README.md

@@ -24,7 +24,7 @@ to the [latest documentation](http://docs.deis.io/en/latest/).
 ## Install prerequisites
 On your workstation:
 * Install [Vagrant v1.6+](http://www.vagrantup.com/downloads.html) and [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
-* Install the fleetctl client: Install v0.6.2 from the [fleet GitHub page](https://github.com/coreos/fleet/releases/tag/v0.6.2).
+* Install the fleetctl client: Install v0.8.0 from the [fleet GitHub page](https://github.com/coreos/fleet/releases/tag/v0.8.0).
 
 A single-node cluster launched with Vagrant will consume about 5 GB of RAM on
 the host machine. Please be sure you have sufficient free memory before

Vagrantfile

@@ -11,7 +11,7 @@ else
   cpus = 1
 end
 
-COREOS_VERSION = "402.2.0"
+COREOS_VERSION = "438.0.0"
 
 Vagrant.configure("2") do |config|
   config.vm.box = "coreos-#{COREOS_VERSION}"

builder/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -yq \
     openssh-server git \
     aufs-tools iptables lxc \
     curl \
-    lxc-docker-1.1.2
+    lxc-docker-1.2.0
 
 # configure ssh server
 RUN rm /etc/ssh/ssh_host_*

contrib/bare-metal/README.md

@@ -49,7 +49,7 @@ coreos-install -C alpha -c /tmp/config -d /dev/sda
 ```
 
 This will install the current [CoreOS](https://coreos.com/) release to disk. If you want to install the recommended [CoreOS](https://coreos.com/) version check the [Deis changelog](../../CHANGELOG.md)
-and specify that version by appending the `-V` parameter to the install command, e.g. `-V 402.2.0`.
+and specify that version by appending the `-V` parameter to the install command, e.g. `-V 438.0.0`.
 
 After the installation has finished reboot your server. Once your machine is back up you should be able to log in as the `core` user using the `deis` ssh key.
 

contrib/coreos/user-data

@@ -22,6 +22,16 @@ coreos:
     command: start
   - name: fleet.service
     command: start
+  - name: fleet.socket
+    command: start
+    content: |
+      [Socket]
+      # Talk to the API over a Unix domain socket (default)
+      ListenStream=/var/run/fleet.sock
+      Service=fleet.service
+
+      [Install]
+      WantedBy=sockets.target
   - name: stop-update-engine.service
     command: start
     content: |

contrib/digitalocean/update-coreos

@@ -2,7 +2,7 @@
 
 set -e
 
-COREOS_VERSION=402.2.0
+COREOS_VERSION=438.0.0
 BASE_URL="http://storage.core-os.net/coreos/amd64-usr/$COREOS_VERSION"
 
 KERNEL="/boot/coreos/vmlinuz"

contrib/ec2/deis.template.json

@@ -84,14 +84,14 @@
 
   "Mappings" : {
     "CoreOSAMIs" : {
-      "us-east-1"      : { "PV" : "ami-b85786d0", "HVM" : "ami-a65786ce" },
-      "us-west-2"      : { "PV" : "ami-cfafd5ff", "HVM" : "ami-cdafd5fd" },
-      "us-west-1"      : { "PV" : "ami-45151800", "HVM" : "ami-bb1a17fe" },
-      "eu-west-1"      : { "PV" : "ami-72865b05", "HVM" : "ami-74865b03" },
-      "ap-southeast-1" : { "PV" : "ami-88e4bcda", "HVM" : "ami-ce376c9c" },
-      "ap-southeast-2" : { "PV" : "ami-3ba8ce01", "HVM" : "ami-0d98fe37" },
-      "ap-northeast-1" : { "PV" : "ami-47cb9246", "HVM" : "ami-49cb9248" },
-      "sa-east-1"      : { "PV" : "ami-e1f65efc", "HVM" : "ami-e7f65efa" }
+      "us-east-1"      : { "PV" : "ami-28973b40", "HVM" : "ami-56973b3e" },
+      "us-west-2"      : { "PV" : "ami-afc2839f", "HVM" : "ami-adc2839d" },
+      "us-west-1"      : { "PV" : "ami-b71811f2", "HVM" : "ami-b11811f4" },
+      "eu-west-1"      : { "PV" : "ami-70f35707", "HVM" : "ami-7ef35709" },
+      "ap-southeast-1" : { "PV" : "ami-54446006", "HVM" : "ami-5a446008" },
+      "ap-southeast-2" : { "PV" : "ami-c32142f9", "HVM" : "ami-cd2142f7" },
+      "ap-northeast-1" : { "PV" : "ami-83cee382", "HVM" : "ami-85cee384" },
+      "sa-east-1"      : { "PV" : "ami-51d8724c", "HVM" : "ami-57d8724a" }
     },
     "SubnetConfig" : {
       "VPC"     : { "CIDR" : "10.21.0.0/16" },

contrib/gce/README.md

@@ -112,10 +112,10 @@ Table of resources:
 +--------+---------------+--------+---------+
 ```
 
-Launch 3 instances using `coreos-alpha-402-2-0-v20140807` image. You can choose another starting CoreOS image from the listing output of `gcloud compute images list`:
+Launch 3 instances using `coreos-alpha-438-0-0-v20140913` image. You can choose another starting CoreOS image from the listing output of `gcloud compute images list`:
 
 ```console
-$ for num in 1 2 3; do gcutil addinstance --image projects/coreos-cloud/global/images/coreos-alpha-402-2-0-v20140807 --persistent_boot_disk --zone us-central1-a --machine_type n1-standard-2 --tags deis --metadata_from_file user-data:gce-user-data --disk cored${num},deviceName=coredocker --authorized_ssh_keys=core:~/.ssh/deis.pub,core:~/.ssh/google_compute_engine.pub core${num}; done
+$ for num in 1 2 3; do gcutil addinstance --image projects/coreos-cloud/global/images/coreos-alpha-438-0-0-v20140913 --persistent_boot_disk --zone us-central1-a --machine_type n1-standard-2 --tags deis --metadata_from_file user-data:gce-user-data --disk cored${num},deviceName=coredocker --authorized_ssh_keys=core:~/.ssh/deis.pub,core:~/.ssh/google_compute_engine.pub core${num}; done
 
 Table of resources:
 

contrib/rackspace/provision-rackspace-cluster.sh

@@ -42,7 +42,8 @@ $CONTRIB_DIR/util/check-user-data.sh
 
 i=1 ; while [[ $i -le $DEIS_NUM_INSTANCES ]] ; do \
     echo_yellow "Provisioning deis-$i..."
-    supernova production boot --image 513f96f3-20e4-4865-b039-d2ca3944af4e --flavor $FLAVOR --key-name $1 --user-data ../coreos/user-data --no-service-net --nic net-id=$NETWORK_ID --config-drive true deis-$i ; \
+    # this image is CoreOS 423.0.0
+    supernova production boot --image e90d62b9-0f01-4da9-9e06-79b8ddf42a70 --flavor $FLAVOR --key-name $1 --user-data ../coreos/user-data --no-service-net --nic net-id=$NETWORK_ID --config-drive true deis-$i ; \
     ((i = i + 1)) ; \
 done
 

controller/api/models.py

@@ -402,7 +402,7 @@ def _command_announceable(self):
 
     @transition(field=state, source=INITIALIZED, target=CREATED)
     def create(self):
-        image = self.release.image
+        image = self.release.image + ':v' + str(self.release.version)
         kwargs = {'memory': self.release.config.memory,
                   'cpu': self.release.config.cpu,
                   'tags': self.release.config.tags}
@@ -427,7 +427,7 @@ def deploy(self, new_release):
         self.release = new_release
         # deploy new container
         new_job_id = self._job_id
-        image = self.release.image
+        image = self.release.image + ':v' + str(self.release.version)
         c_type = self.type
         kwargs = {'memory': self.release.config.memory,
                   'cpu': self.release.config.cpu,
@@ -453,7 +453,8 @@ def destroy(self):
 
     def run(self, command):
         """Run a one-off command"""
-        rc, output = self._scheduler.run(self._job_id, self.release.image, command)
+        image = self.release.image + ':v' + str(self.release.version)
+        rc, output = self._scheduler.run(self._job_id, image, command)
         return rc, output
 
 

controller/requirements.txt

@@ -14,6 +14,7 @@ django-yamlfield==0.5
 djangorestframework==2.3.14
 docker-py==0.4.0
 gunicorn==18.0
+paramiko==1.14.1
 psycopg2==2.5.2
 python-etcd==0.3.0
 PyYAML==3.11