Check if the agent can safely be rebooted on startup

Since the jobs directory may be mounted on tmpfs, it may not be present
after reboot. To stop the agent starting without starting its jobs we
make sure to crash it on subsequent reboots.

[#161569005](https://www.pivotaltracker.com/story/show/161569005)

Signed-off-by: Joshua Aresty <[email protected]>

agent/agent.go

@@ -20,6 +20,12 @@ const (
 	agentLogTag = "agent"
 )
 
+//go:generate counterfeiter . CanRebooter
+
+type CanRebooter interface {
+	CanReboot() (bool, error)
+}
+
 type Agent struct {
 	logger            boshlog.Logger
 	mbusHandler       boshhandler.Handler
@@ -31,6 +37,7 @@ type Agent struct {
 	settingsService   boshsettings.Service
 	uuidGenerator     boshuuid.Generator
 	timeService       clock.Clock
+	canRebooter       CanRebooter
 }
 
 func New(
@@ -44,6 +51,7 @@ func New(
 	settingsService boshsettings.Service,
 	uuidGenerator boshuuid.Generator,
 	timeService clock.Clock,
+	canRebooter CanRebooter,
 ) Agent {
 	return Agent{
 		logger:            logger,
@@ -56,10 +64,19 @@ func New(
 		settingsService:   settingsService,
 		uuidGenerator:     uuidGenerator,
 		timeService:       timeService,
+		canRebooter:       canRebooter,
 	}
 }
 
 func (a Agent) Run() error {
+	bootable, err := a.canRebooter.CanReboot()
+	if err != nil {
+		return bosherr.WrapError(err, "Failed to check if agent can be rebooted")
+	}
+	if !bootable {
+		return bosherr.Error("Refusing to boot")
+	}
+
 	errCh := make(chan error, 1)
 
 	a.actionDispatcher.ResumePreviouslyDispatchedTasks()
@@ -75,10 +92,7 @@ func (a Agent) Run() error {
 		}
 	}()
 
-	select {
-	case err := <-errCh:
-		return err
-	}
+	return <-errCh
 }
 
 func (a Agent) subscribeActionDispatcher(errCh chan error) {

agent/agent_test.go

@@ -10,6 +10,7 @@ import (
 	. "github.com/cloudfoundry/bosh-agent/agent"
 
 	"code.cloudfoundry.org/clock/fakeclock"
+	"github.com/cloudfoundry/bosh-agent/agent/agentfakes"
 	boshalert "github.com/cloudfoundry/bosh-agent/agent/alert"
 	boshas "github.com/cloudfoundry/bosh-agent/agent/applier/applyspec"
 	fakeas "github.com/cloudfoundry/bosh-agent/agent/applier/applyspec/fakes"
@@ -37,8 +38,10 @@ func init() {
 			settingsService  *fakesettings.FakeSettingsService
 			uuidGenerator    *fakeuuid.FakeGenerator
 			timeService      *fakeclock.FakeClock
-			agent            Agent
 			vitalService     *vitalsfakes.FakeService
+			canRebooter      *agentfakes.FakeCanRebooter
+
+			agent Agent
 		)
 
 		BeforeEach(func() {
@@ -52,6 +55,8 @@ func init() {
 			uuidGenerator = &fakeuuid.FakeGenerator{}
 			timeService = fakeclock.NewFakeClock(time.Now())
 			vitalService = &vitalsfakes.FakeService{}
+			canRebooter = &agentfakes.FakeCanRebooter{}
+			canRebooter.CanRebootReturns(true, nil) // bootable by default
 
 			platform.GetVitalsServiceReturns(vitalService)
 
@@ -66,6 +71,7 @@ func init() {
 				settingsService,
 				uuidGenerator,
 				timeService,
+				canRebooter,
 			)
 		})
 
@@ -98,7 +104,6 @@ func init() {
 			Context("when heartbeats can be sent", func() {
 				BeforeEach(func() {
 					handler.KeepOnRunning()
-
 				})
 
 				BeforeEach(func() {
@@ -145,6 +150,7 @@ func init() {
 						settingsService,
 						uuidGenerator,
 						timeService,
+						canRebooter,
 					)
 
 					// Immediately exit after sending initial heartbeat
@@ -189,6 +195,28 @@ func init() {
 					}
 					Expect(jobSupervisor.GetHealthRecorded()).To(BeNumerically(">=", 3))
 				})
+
+				Context("when the agent may not be rebooted", func() {
+					BeforeEach(func() {
+						canRebooter.CanRebootReturns(false, nil)
+					})
+
+					It("stops the boot process and returns an error", func() {
+						err := agent.Run()
+						Expect(err).To(HaveOccurred())
+					})
+				})
+
+				Context("when the checking if the agent may be rebooted returns an error", func() {
+					BeforeEach(func() {
+						canRebooter.CanRebootReturns(true, errors.New("disaster"))
+					})
+
+					It("stops the boot process and returns an error", func() {
+						err := agent.Run()
+						Expect(err).To(HaveOccurred())
+					})
+				})
 			})
 
 			Context("when the agent fails to get job spec for a heartbeat", func() {

agent/bootonce/bootonce_suite_test.go

@@ -0,0 +1,13 @@
+package bootonce_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestBootonce(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Bootonce Suite")
+}

agent/bootonce/reboot_checker.go

@@ -0,0 +1,53 @@
+package bootonce
+
+import (
+	"path/filepath"
+
+	boshsettings "github.com/cloudfoundry/bosh-agent/settings"
+	boshdir "github.com/cloudfoundry/bosh-agent/settings/directories"
+	boshsys "github.com/cloudfoundry/bosh-utils/system"
+)
+
+type RebootChecker struct {
+	settings    boshsettings.Service
+	fs          boshsys.FileSystem
+	dirProvider boshdir.Provider
+}
+
+func NewRebootChecker(
+	settings boshsettings.Service,
+	fs boshsys.FileSystem,
+	dirProvider boshdir.Provider,
+) *RebootChecker {
+	return &RebootChecker{
+		settings:    settings,
+		fs:          fs,
+		dirProvider: dirProvider,
+	}
+}
+
+func (r *RebootChecker) CanReboot() (bool, error) {
+	if !r.tmpFsFeatureEnabled() {
+		return true, nil
+	}
+
+	path := filepath.Join(r.dirProvider.BoshDir(), "bootonce")
+	return checkAndMark(r.fs, path)
+}
+
+func (r *RebootChecker) tmpFsFeatureEnabled() bool {
+	settings := r.settings.GetSettings()
+	return settings.Env.Bosh.JobDir.TmpFs
+}
+
+func checkAndMark(fs boshsys.FileSystem, path string) (bool, error) {
+	if fs.FileExists(path) {
+		return false, nil
+	}
+
+	return true, touch(fs, path)
+}
+
+func touch(fs boshsys.FileSystem, path string) error {
+	return fs.WriteFile(path, nil)
+}