feat(store): add store-volume and store-metadata

These components enable the logger and controller to mount
a shared filesystem backed by CephFS.

controller/api/static/500.html

@@ -26,7 +26,7 @@
   <div class="container">
     <div class="row">
     	<h1>Internal Server Error</h1>
-    	<p>See /var/log/deis for details</p>
+    	<p>See /data for details</p>
     </div> <!-- .row -->
   </div> <!-- .main_section container-->
 

controller/bin/boot

@@ -51,8 +51,8 @@ done
 
 cd /app
 
-# give the deis user permissions to write to the volume mount
-chown -R deis:deis /var/log/deis
+mkdir -p /data/logs
+chmod 777 /data/logs
 
 # run an idempotent database migration
 sudo -E -u deis ./manage.py syncdb --migrate --noinput

controller/build.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
 if [[ -z $DOCKER_BUILD ]]; then
-  echo 
+  echo
   echo "Note: this script is intended for use by the Dockerfile and not as a way to build the controller locally"
-  echo 
+  echo
   exit 1
 fi
 
@@ -28,9 +28,6 @@ mkdir -p /app && chown -R deis:deis /app
 # create directory for confd templates
 mkdir -p /templates && chown -R deis:deis /templates
 
-# create directory for logs
-mkdir -p /var/log/deis && chown -R deis:deis /var/log/deis
-
 # install dependencies
 pip install -r /app/requirements.txt
 

controller/templates/confd_settings.py

@@ -38,7 +38,7 @@
 CACHE_URL = 'redis://{{ .deis_cache_host }}:{{ .deis_cache_port }}/0'
 
 # move log directory out of /app/deis
-DEIS_LOG_DIR = '/var/log/deis'
+DEIS_LOG_DIR = '/data/logs'
 
 {{ if .deis_controller_registrationEnabled }}
 REGISTRATION_ENABLED = bool({{ .deis_controller_registrationEnabled }})

controller/tests/controller_test.go

@@ -29,10 +29,13 @@ func TestController(t *testing.T) {
 		"/deis/domains",
 	}
 	tag, etcdPort := utils.BuildTag(), utils.RandomPort()
+
+	//start etcd container
 	etcdName := "deis-etcd-" + tag
 	cli, stdout, stdoutPipe := dockercli.NewClient()
 	dockercli.RunTestEtcd(t, etcdName, etcdPort)
 	defer cli.CmdRm("-f", etcdName)
+
 	handler := etcdutils.InitEtcd(setdir, setkeys, etcdPort)
 	etcdutils.PublishEtcd(t, handler)
 	mock.RunMockDatabase(t, tag, etcdPort, utils.RandomPort())

deisctl/README.md

@@ -79,8 +79,6 @@ $ deisctl install platform
 ■ ● ▴ Installing Deis...
 ▴ ■ ●
 
-Data containers...
-deis-logger-data.service: loaded
 Logging subsystem...
 deis-logger.service: loaded
 Storage subsystem...
@@ -105,8 +103,6 @@ $ deisctl start platform
 ■ ● ▴ Starting Deis...
 ▴ ■ ●
 
-Data containers...
-deis-logger-data.service: exited
 Logging subsystem...
 deis-logger.service: running
 deis-logspout.service: running
@@ -186,7 +182,6 @@ deis-builder.service		f936b7a5.../172.17.8.100	loaded	active	running
 deis-cache.service  		f936b7a5.../172.17.8.100	loaded	active	running
 deis-controller.service	    f936b7a5.../172.17.8.100	loaded	active	running
 deis-database.service		f936b7a5.../172.17.8.100	loaded	active	running
-deis-logger-data.service	f936b7a5.../172.17.8.100	loaded	active	exited
 deis-logger.service	    	f936b7a5.../172.17.8.100	loaded	active	running
 deis-registry.service		f936b7a5.../172.17.8.100	loaded	active	running
 [email protected]		f936b7a5.../172.17.8.100	loaded	active	running

deisctl/backend/fleet/create.go

@@ -2,7 +2,6 @@ package fleet
 
 import (
 	"fmt"
-	"strings"
 	"sync"
 
 	"github.com/coreos/fleet/job"
@@ -62,11 +61,7 @@ func (c *FleetClient) createUnitFile(target string) (unitName string, uf *unit.U
 	if err != nil {
 		return
 	}
-	if strings.HasSuffix(component, "-data") {
-		unitName, uf, err = c.createDataUnit(component)
-	} else {
-		unitName, uf, err = c.createServiceUnit(component, num)
-	}
+	unitName, uf, err = c.createServiceUnit(component, num)
 	if err != nil {
 		return unitName, uf, err
 	}
@@ -85,21 +80,3 @@ func (c *FleetClient) createServiceUnit(component string, num int) (name string,
 	}
 	return name, uf, nil
 }
-
-// Create data container unit
-func (c *FleetClient) createDataUnit(component string) (name string, uf *unit.UnitFile, err error) {
-	name, err = formatUnitName(component, 0)
-	if err != nil {
-		return
-	}
-	machineID, err := randomMachineID(c)
-	if err != nil {
-		return
-	}
-	uf, err = NewDataUnit(component, machineID)
-	if err != nil {
-		return
-	}
-	return name, uf, nil
-
-}

deisctl/backend/fleet/fleet.go

@@ -18,16 +18,3 @@ func NewClient() (*FleetClient, error) {
 	cAPI = client
 	return &FleetClient{Fleet: client}, nil
 }
-
-// randomMachineID return a random machineID from the Fleet cluster
-func randomMachineID(c *FleetClient) (machineID string, err error) {
-	machineState, err := c.Fleet.Machines()
-	if err != nil {
-		return "", err
-	}
-	var machineIDs []string
-	for _, ms := range machineState {
-		machineIDs = append(machineIDs, ms.ID)
-	}
-	return randomValue(machineIDs), nil
-}

deisctl/backend/fleet/start.go

@@ -2,7 +2,6 @@ package fleet
 
 import (
 	"fmt"
-	"strings"
 	"sync"
 	"time"
 
@@ -34,12 +33,7 @@ func doStart(c *FleetClient, target string, wg *sync.WaitGroup, outchan chan str
 	}
 
 	requestState := "launched"
-	var desiredState string
-	if strings.Contains(name, "-data.service") {
-		desiredState = "exited"
-	} else {
-		desiredState = "running"
-	}
+	desiredState := "running"
 
 	if err := c.Fleet.SetUnitTargetState(name, requestState); err != nil {
 		errchan <- err

deisctl/backend/fleet/unit.go

@@ -24,18 +24,9 @@ func (c *FleetClient) Units(target string) (units []string, err error) {
 	if err != nil {
 		return
 	}
-	if strings.HasSuffix(target, "-data") {
-		for _, u := range allUnits {
-			if strings.Contains(u.Name, target) {
-				units = []string{u.Name}
-				return
-			}
-		}
-	} else {
-		for _, u := range allUnits {
-			if strings.Contains(u.Name, target) && !strings.HasSuffix(u.Name, "-data.service") {
-				units = append(units, u.Name)
-			}
+	for _, u := range allUnits {
+		if strings.Contains(u.Name, target) {
+			units = append(units, u.Name)
 		}
 	}
 	if len(units) == 0 {
@@ -84,22 +75,6 @@ func NewUnit(component string) (uf *unit.UnitFile, err error) {
 	return
 }
 
-// NewDataUnit takes a component type and returns a Fleet unit
-// that is hard-scheduled to a machine ID
-func NewDataUnit(component string, machineID string) (uf *unit.UnitFile, err error) {
-	template, err := readTemplate(component)
-	if err != nil {
-		return
-	}
-	// replace CHANGEME with random machineID
-	replaced := strings.Replace(string(template), "CHANGEME", machineID, 1)
-	uf, err = unit.NewUnitFile(replaced)
-	if err != nil {
-		return
-	}
-	return
-}
-
 // formatUnitName returns a properly formatted systemd service name
 // using the given component type and number
 func formatUnitName(component string, num int) (unitName string, err error) {

deisctl/backend/fleet/utils.go

@@ -42,8 +42,7 @@ func countUnits(units []string) (count []int, err error) {
 	for _, unit := range units {
 		_, n, err := splitJobName(unit)
 		if err != nil {
-			// FIXME: assume data container for now
-			continue
+			return []int{}, err
 		}
 		count = append(count, n)
 	}
@@ -67,7 +66,7 @@ func splitJobName(component string) (c string, num int, err error) {
 
 func splitTarget(target string) (component string, num int, err error) {
 	// see if we were provided a specific target
-	r := regexp.MustCompile(`^([a-z-]+)(\-data|@\d+)?$`)
+	r := regexp.MustCompile(`^([a-z-]+)(@\d+)?$`)
 	match := r.FindStringSubmatch(target)
 	// check for failed match
 	if len(match) != 3 {
@@ -77,9 +76,6 @@ func splitTarget(target string) (component string, num int, err error) {
 	if match[2] == "" {
 		component = match[1]
 		return component, 0, nil
-	} else if match[2] == "-data" {
-		component = match[1] + "-data"
-		return component, 0, nil
 	}
 	num, err = strconv.Atoi(match[2][1:])
 	if err != nil {

deisctl/cmd/cmd.go

@@ -27,12 +27,6 @@ const (
 	PlatformInstallCommand string = "platform"
 )
 
-var (
-	DefaultDataContainers = []string{
-		"logger-data",
-	}
-)
-
 func ListUnits(b backend.Backend) error {
 	err := b.ListUnits()
 	return err
@@ -110,7 +104,6 @@ func StartPlatform(b backend.Backend) error {
 
 	outchan <- utils.DeisIfy("Starting Deis...")
 
-	startDataContainers(b, &wg, outchan, errchan)
 	startDefaultServices(b, &wg, outchan, errchan)
 
 	wg.Wait()
@@ -122,12 +115,6 @@ func StartPlatform(b backend.Backend) error {
 	return nil
 }
 
-func startDataContainers(b backend.Backend, wg *sync.WaitGroup, outchan chan string, errchan chan error) {
-	outchan <- fmt.Sprintf("Data containers...")
-	b.Start(DefaultDataContainers, wg, outchan, errchan)
-	wg.Wait()
-}
-
 func startDefaultServices(b backend.Backend, wg *sync.WaitGroup, outchan chan string, errchan chan error) {
 
 	// create separate channels for background tasks
@@ -147,7 +134,14 @@ func startDefaultServices(b backend.Backend, wg *sync.WaitGroup, outchan chan st
 
 	// wait for groups to come up
 	outchan <- fmt.Sprintf("Storage subsystem...")
-	b.Start([]string{"store-daemon", "store-monitor", "store-gateway"}, wg, outchan, errchan)
+	b.Start([]string{"store-monitor"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Start([]string{"store-daemon"}, wg, outchan, errchan)
+	wg.Wait()
+	b.Start([]string{"store-metadata"}, wg, outchan, errchan)
+	wg.Wait()
+
+	b.Start([]string{"store-volume", "store-gateway"}, wg, outchan, errchan)
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Control plane...")
@@ -221,7 +215,7 @@ func stopDefaultServices(b backend.Backend, wg *sync.WaitGroup, outchan chan str
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Storage subsystem...")
-	b.Stop([]string{"store-gateway", "store-monitor", "store-daemon"}, wg, outchan, errchan)
+	b.Stop([]string{"store-gateway", "store-volume", "store-metadata", "store-monitor", "store-daemon"}, wg, outchan, errchan)
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Logging subsystem...")
@@ -289,7 +283,6 @@ func InstallPlatform(b backend.Backend) error {
 
 	outchan <- utils.DeisIfy("Installing Deis...")
 
-	installDataContainers(b, &wg, outchan, errchan)
 	installDefaultServices(b, &wg, outchan, errchan)
 
 	wg.Wait()
@@ -301,20 +294,14 @@ func InstallPlatform(b backend.Backend) error {
 	return nil
 }
 
-func installDataContainers(b backend.Backend, wg *sync.WaitGroup, outchan chan string, errchan chan error) {
-	outchan <- fmt.Sprintf("Data containers...")
-	b.Create(DefaultDataContainers, wg, outchan, errchan)
-	wg.Wait()
-}
-
 func installDefaultServices(b backend.Backend, wg *sync.WaitGroup, outchan chan string, errchan chan error) {
 
 	outchan <- fmt.Sprintf("Logging subsystem...")
 	b.Create([]string{"logger", "logspout"}, wg, outchan, errchan)
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Storage subsystem...")
-	b.Create([]string{"store-daemon", "store-monitor", "store-gateway"}, wg, outchan, errchan)
+	b.Create([]string{"store-daemon", "store-monitor", "store-metadata", "store-volume", "store-gateway"}, wg, outchan, errchan)
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Control plane...")
@@ -385,7 +372,7 @@ func uninstallAllServices(b backend.Backend, wg *sync.WaitGroup, outchan chan st
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Storage subsystem...")
-	b.Destroy([]string{"store-gateway", "store-monitor", "store-daemon"}, wg, outchan, errchan)
+	b.Destroy([]string{"store-gateway", "store-volume", "store-metadata", "store-monitor", "store-daemon"}, wg, outchan, errchan)
 	wg.Wait()
 
 	outchan <- fmt.Sprintf("Logging subsystem...")
@@ -490,14 +477,15 @@ Options:
 		"deis-controller.service",
 		"deis-database.service",
 		"deis-logger.service",
-		"deis-logger-data.service",
 		"deis-logspout.service",
 		"deis-publisher.service",
 		"deis-registry.service",
 		"deis-router.service",
 		"deis-store-daemon.service",
 		"deis-store-gateway.service",
+		"deis-store-metadata.service",
 		"deis-store-monitor.service",
+		"deis-store-volume.serviice",
 	}
 	for _, unit := range units {
 		src := rootURL + tag + "/deisctl/units/" + unit

deisctl/units/deis-controller.service

@@ -1,21 +1,17 @@
 [Unit]
 Description=deis-controller
-Requires=deis-logger.service
-After=deis-logger.service
+Requires=deis-store-volume.service
+After=deis-store-volume.service
 
 [Service]
 EnvironmentFile=/etc/environment
 TimeoutStartSec=20m
-ExecStartPre=/bin/sh -c "echo Waiting for deis-logger to start... ; until docker inspect deis-logger >/dev/null 2>&1; do sleep 2; done"
 ExecStartPre=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/controller` && docker history $IMAGE >/dev/null || docker pull $IMAGE"
 ExecStartPre=/bin/sh -c "docker inspect deis-controller >/dev/null && docker rm -f deis-controller || true"
-ExecStart=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/controller` && docker run --name deis-controller -v /var/run/fleet.sock:/var/run/fleet.sock --rm -p 8000:8000 -e EXTERNAL_PORT=8000 -e HOST=$COREOS_PRIVATE_IPV4 --volumes-from=deis-logger $IMAGE"
+ExecStart=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/controller` && docker run --name deis-controller --rm -p 8000:8000 -e EXTERNAL_PORT=8000 -e HOST=$COREOS_PRIVATE_IPV4 -v /var/run/fleet.sock:/var/run/fleet.sock -v /var/lib/deis/store:/data $IMAGE"
 ExecStopPost=-/usr/bin/docker rm -f deis-controller
 Restart=on-failure
 RestartSec=5
 
 [Install]
 WantedBy=multi-user.target
-
-[X-Fleet]
-MachineOf=deis-logger.service