[Fix] reset the desired capacity to remove the spread

aws/mocks/mock_asg.go

@@ -35,6 +35,8 @@ type ASGClient struct {
 
 	DescribeLoadBalancerTargetGroupsOutput *autoscaling.DescribeLoadBalancerTargetGroupsOutput
 	DescribeLoadBalancersOutput            *autoscaling.DescribeLoadBalancersOutput
+
+	SetDesiredCapacityLastInput *autoscaling.SetDesiredCapacityInput
 }
 
 func (m *ASGClient) init() {
@@ -245,3 +247,8 @@ func (m *ASGClient) DescribeLoadBalancers(input *autoscaling.DescribeLoadBalance
 	}
 	return &autoscaling.DescribeLoadBalancersOutput{}, nil
 }
+
+func (m *ASGClient) SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error) {
+	m.SetDesiredCapacityLastInput = input
+	return nil, nil
+}

deployer/handlers.go

@@ -186,6 +186,14 @@ func CleanUpSuccess(awsc aws.Clients) DeployHandler {
 			return nil, &errors.LockError{err.Error()}
 		}
 
+		if err := release.ResetDesiredCapacity(
+			awsc.ASGClient(release.AwsRegion, release.AwsAccountID, assumedRole),
+		); err != nil {
+			// We ignore this error as failing to reset the capacity should not cause a massive issue
+			// Log the error in case
+			fmt.Printf("IGNORED: %v \n", err)
+		}
+
 		release.RemoveHalt(awsc.S3Client(release.AwsRegion, nil, nil)) // Delete Halt
 
 		release.Success = to.Boolp(true) // Wait till the end to mark success

deployer/models/release_resources.go

@@ -208,6 +208,25 @@ func (release *Release) SuccessfulTearDown(asgc aws.ASGAPI, cwc aws.CWAPI) error
 	return nil
 }
 
+// ResetDesiredCapacity resets the ASGs to the desired capacity that would exist without `spread`
+// This is due to a situation where each successive deploy would ratchet up the desired capacity
+func (release *Release) ResetDesiredCapacity(asgc aws.ASGAPI) error {
+	errors := []error{}
+	for _, service := range release.Services {
+		err := service.SetDesiredCapacity(asgc)
+		// Continue to set capacities even when error is detected
+		if err != nil {
+			errors = append(errors, err)
+		}
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("Error ResetDesiredCapacity: %v", errors)
+	}
+
+	return nil
+}
+
 // Failure
 
 // DetachForFailure detach new ASGs

deployer/models/release_resources_test.go

@@ -3,6 +3,7 @@ package models
 import (
 	"testing"
 
+	"github.com/coinbase/step/utils/to"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -82,3 +83,26 @@ func Test_Release_UnsuccessfulTearDown_Works(t *testing.T) {
 	awsc := MockAwsClients(r)
 	assert.NoError(t, r.UnsuccessfulTearDown(awsc.ASG, awsc.CW))
 }
+
+func Test_Release_ResetDesiredCapacity_Works(t *testing.T) {
+	// func (release *Release) ResetDesiredCapacity(asgc aws.ASGAPI) error {
+	r := MockRelease(t)
+	MockPrepareRelease(r)
+
+	awsc := MockAwsClients(r)
+	s := r.Services["web"]
+	s.CreatedASG = to.Strp("name")
+
+	s.PreviousDesiredCapacity = to.Int64p(6)
+
+	a := s.Autoscaling
+
+	a.MinSize = to.Int64p(int64(4))
+	a.MaxSize = to.Int64p(int64(10))
+	a.Spread = to.Float64p(float64(0.8))
+
+	assert.NoError(t, r.ResetDesiredCapacity(awsc.ASG))
+
+	assert.Equal(t, int64(6), *awsc.ASG.SetDesiredCapacityLastInput.DesiredCapacity)
+
+}

deployer/models/service.go

@@ -23,12 +23,13 @@ import (
 // web: .....|.
 // gray targets, red terminated, yellow unhealthy, green healthy
 type HealthReport struct {
-	TargetHealthy  *int     `json:"target_healthy,omitempty"`  // Number of instances aimed to to Launch
-	TargetLaunched *int     `json:"target_launched,omitempty"` // Number of instances aimed to to Launch
-	Healthy        *int     `json:"healthy,omitempty"`         // Number of instances that are healthy
-	Launching      *int     `json:"launching,omitempty"`       // Number of instances that have been created
-	Terminating    *int     `json:"terminating,omitempty"`     // Number of instances that are Terminating
-	TerminatingIDs []string `json:"terminating_ids,omitempty"` // Instance IDs that are Terminating
+	TargetHealthy   *int     `json:"target_healthy,omitempty"`   // Number of instances aimed to to Launch
+	TargetLaunched  *int     `json:"target_launched,omitempty"`  // Number of instances aimed to to Launch
+	Healthy         *int     `json:"healthy,omitempty"`          // Number of instances that are healthy
+	Launching       *int     `json:"launching,omitempty"`        // Number of instances that have been created
+	Terminating     *int     `json:"terminating,omitempty"`      // Number of instances that are Terminating
+	TerminatingIDs  []string `json:"terminating_ids,omitempty"`  // Instance IDs that are Terminating
+	DesiredCapacity *int     `json:"desired_capacity,omitempty"` // The final desired capacity goal
 }
 
 // TYPES
@@ -179,6 +180,11 @@ func (service *Service) target() int {
 	return service.Autoscaling.TargetHealthy(service.PreviousDesiredCapacity)
 }
 
+func (service *Service) desiredCapacity() int {
+	// This returns the desired capacity without spread included
+	return service.Autoscaling.DesiredCapacity(service.PreviousDesiredCapacity)
+}
+
 func (service *Service) maxTerminations() int {
 	return service.Autoscaling.MaxTerminationsInt()
 }
@@ -220,12 +226,13 @@ func (service *Service) setHealthy(instances aws.Instances) {
 	terming := instances.TerminatingIDs()
 
 	service.HealthReport = &HealthReport{
-		TargetHealthy:  to.Intp(service.target()),
-		TargetLaunched: to.Intp(service.targetCapacity()),
-		Healthy:        to.Intp(len(healthy)),
-		Terminating:    to.Intp(len(terming)),
-		TerminatingIDs: terming,
-		Launching:      to.Intp(len(instances)),
+		TargetHealthy:   to.Intp(service.target()),
+		TargetLaunched:  to.Intp(service.targetCapacity()),
+		Healthy:         to.Intp(len(healthy)),
+		Terminating:     to.Intp(len(terming)),
+		TerminatingIDs:  terming,
+		Launching:       to.Intp(len(instances)),
+		DesiredCapacity: to.Intp(service.desiredCapacity()),
 	}
 
 	// The Service is Healthy if
@@ -586,3 +593,15 @@ func (service *Service) UpdateHealthy(asgc aws.ASGAPI, elbc aws.ELBAPI, albc aws
 	service.setHealthy(all)
 	return nil
 }
+
+//////////
+// Update Resources
+//////////
+func (service *Service) SetDesiredCapacity(asgc aws.ASGAPI) error {
+	_, err := asgc.SetDesiredCapacity(&autoscaling.SetDesiredCapacityInput{
+		AutoScalingGroupName: service.CreatedASG,
+		DesiredCapacity:      to.Int64p(int64(service.desiredCapacity())),
+	})
+
+	return err
+}

deployer/models/service_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/coinbase/odin/aws/mocks"
 	"github.com/coinbase/step/utils/to"
 	"github.com/stretchr/testify/assert"
 )
@@ -73,3 +74,34 @@ func Test_Service_PlacementgroupValidation(t *testing.T) {
 	assert.NoError(t, err)
 
 }
+
+func Test_Service_SetDesiredCapacity_Works(t *testing.T) {
+	service := Service{
+		Autoscaling: &AutoScalingConfig{
+			MinSize: to.Int64p(int64(4)),
+			MaxSize: to.Int64p(int64(10)),
+			Spread:  to.Float64p(float64(0.8)),
+		},
+		PreviousDesiredCapacity: to.Int64p(6),
+	}
+
+	awsc := mocks.MockAWS()
+
+	assert.NoError(t, service.SetDesiredCapacity(awsc.ASG))
+	assert.Equal(t, int64(6), *awsc.ASG.SetDesiredCapacityLastInput.DesiredCapacity)
+}
+
+func Test_Service_CapacityValues(t *testing.T) {
+	service := Service{
+		Autoscaling: &AutoScalingConfig{
+			MinSize: to.Int64p(int64(10)),
+			MaxSize: to.Int64p(int64(50)),
+			Spread:  to.Float64p(float64(0.8)),
+		},
+		PreviousDesiredCapacity: to.Int64p(20),
+	}
+
+	assert.Equal(t, 10, service.target())          // The number of instances we want healthy
+	assert.Equal(t, 20, service.desiredCapacity()) // The final number of instances
+	assert.Equal(t, 36, service.targetCapacity())  // The number of launched instances
+}