bump asts mod and hack asts for volumeMode update (pingcap#5273)

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/onsi/gomega v1.10.2
 	github.com/openshift/generic-admission-server v1.14.1-0.20210422140326-da96454c926d
 	github.com/pingcap/TiProxy/lib v0.0.0-20230201020701-df06ec482c69
-	github.com/pingcap/advanced-statefulset/client v1.17.1-0.20230724024631-026a4f2a911d
+	github.com/pingcap/advanced-statefulset/client v1.17.1-0.20230830071059-cfaedeea6cb3
 	github.com/pingcap/errors v0.11.4
 	github.com/pingcap/kvproto v0.0.0-20200927054727-1290113160f0
 	github.com/pingcap/tidb-operator/pkg/apis v1.6.0-alpha.6

go.sum

@@ -601,8 +601,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pingcap/TiProxy/lib v0.0.0-20230201020701-df06ec482c69 h1:UY87I8QfUUbJHrkbZ51eQpopIdQc34D2vRpLZILWyCg=
 github.com/pingcap/TiProxy/lib v0.0.0-20230201020701-df06ec482c69/go.mod h1:AR0IJ6SdStz1dkxG0gi47RxzQWxbPKACQ7JG1VY6Dnc=
-github.com/pingcap/advanced-statefulset/client v1.17.1-0.20230724024631-026a4f2a911d h1:CI92Lc6WlUQbT0cmOcuPavG+UvmFWbjKUYItJjiABEQ=
-github.com/pingcap/advanced-statefulset/client v1.17.1-0.20230724024631-026a4f2a911d/go.mod h1:iJ3ur9e2ZW1/AjiBiPACpvj0rC/T28mU4sBGO7gjORM=
+github.com/pingcap/advanced-statefulset/client v1.17.1-0.20230830071059-cfaedeea6cb3 h1:/GofeTlMzVNnrtkU0XbTeiAwubdro18H5ROMgx43KGE=
+github.com/pingcap/advanced-statefulset/client v1.17.1-0.20230830071059-cfaedeea6cb3/go.mod h1:iJ3ur9e2ZW1/AjiBiPACpvj0rC/T28mU4sBGO7gjORM=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=

pkg/manager/utils/statefulset.go

@@ -19,6 +19,7 @@ import (
 	"github.com/pingcap/tidb-operator/pkg/apis/label"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/controller"
+	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/util"
 
 	apps "k8s.io/api/apps/v1"
@@ -116,7 +117,8 @@ func UpdateStatefulSet(setCtl controller.StatefulSetControlInterface, object run
 
 	// Check if an upgrade is needed.
 	// If not, early return.
-	if util.StatefulSetEqual(*newSet, *oldSet) && !isOrphan {
+	stsEqual, podTemplateEqual := util.StatefulSetEqual(*newSet, *oldSet)
+	if stsEqual && !isOrphan {
 		return nil
 	}
 
@@ -127,7 +129,7 @@ func UpdateStatefulSet(setCtl controller.StatefulSetControlInterface, object run
 	set.Spec.UpdateStrategy = newSet.Spec.UpdateStrategy
 	set.Labels = newSet.Labels
 	set.Annotations = newSet.Annotations
-	set.Spec.Template = newSet.Spec.Template
+	set.Spec.Template = *newSet.Spec.Template.DeepCopy() // do a copy to avoid changing the original TidbCluster CR
 	if isOrphan {
 		set.OwnerReferences = newSet.OwnerReferences
 	}
@@ -153,6 +155,14 @@ func UpdateStatefulSet(setCtl controller.StatefulSetControlInterface, object run
 		return err
 	}
 
+	// this should be done after SetStatefulSetLastAppliedConfigAnnotation,
+	// then this volumeMode set by defaults won't be stored in the annotation
+	if podTemplateEqual {
+		// if the pod template is not equal,
+		// there is no need to hack the volume mode as it can be updated in this round
+		hackEphemeralVolumeMode(oldSet, &set)
+	}
+
 	// commit to k8s
 	_, err = setCtl.UpdateStatefulSet(object, &set)
 	return err
@@ -163,3 +173,33 @@ func SetUpgradePartition(set *apps.StatefulSet, upgradeOrdinal int32) {
 	set.Spec.UpdateStrategy.RollingUpdate = &apps.RollingUpdateStatefulSetStrategy{Partition: &upgradeOrdinal}
 	klog.Infof("set %s/%s partition to %d", set.GetNamespace(), set.GetName(), upgradeOrdinal)
 }
+
+// hackEphemeralVolumeMode appends exstings ephemeral volume mode to asts so that no unexpected rolling update will be triggered.
+// before https://github.com/pingcap/advanced-statefulset/pull/96, some asts may have volume mode in ephemeral volume,
+// but after that, no volume mode in ephemeral volume will be set by defaults,
+// so we need to append the volume mode to the new asts spec if it exists in old spec.
+func hackEphemeralVolumeMode(oldSts *apps.StatefulSet, newSts *apps.StatefulSet) {
+	if !features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) {
+		// only need this hack for AdvancedStatefulSet
+		return
+	}
+
+	volumeModels := make(map[string]corev1.PersistentVolumeMode)
+	for _, volume := range oldSts.Spec.Template.Spec.Volumes {
+		if volume.Ephemeral != nil && volume.Ephemeral.VolumeClaimTemplate != nil &&
+			volume.Ephemeral.VolumeClaimTemplate.Spec.VolumeMode != nil {
+			volumeModels[volume.Name] = *volume.Ephemeral.VolumeClaimTemplate.Spec.VolumeMode
+		}
+	}
+	if len(volumeModels) == 0 {
+		return // no need to hack
+	}
+
+	for _, volume := range newSts.Spec.Template.Spec.Volumes {
+		if volumeMode, ok := volumeModels[volume.Name]; ok && volume.Ephemeral != nil && volume.Ephemeral.VolumeClaimTemplate != nil &&
+			volume.Ephemeral.VolumeClaimTemplate.Spec.VolumeMode == nil {
+			volume.Ephemeral.VolumeClaimTemplate.Spec.VolumeMode = &volumeMode
+			klog.Infof("hack volume mode %s for volume %s in sts %s/%s", volumeMode, volume.Name, newSts.Namespace, newSts.Name)
+		}
+	}
+}

pkg/util/util.go

@@ -386,7 +386,7 @@ func MatchLabelFromStoreLabels(storeLabels []*metapb.StoreLabel, componentLabel
 }
 
 // statefulSetEqual compares the new Statefulset's spec with old Statefulset's last applied config
-func StatefulSetEqual(new apps.StatefulSet, old apps.StatefulSet) bool {
+func StatefulSetEqual(new apps.StatefulSet, old apps.StatefulSet) (equal bool, podTemplateEqual bool) {
 	// The annotations in old sts may include LastAppliedConfigAnnotation
 	tmpAnno := map[string]string{}
 	for k, v := range old.Annotations {
@@ -395,24 +395,25 @@ func StatefulSetEqual(new apps.StatefulSet, old apps.StatefulSet) bool {
 		}
 	}
 	if !apiequality.Semantic.DeepEqual(new.Annotations, tmpAnno) {
-		return false
+		return false, false
 	}
 	oldConfig := apps.StatefulSetSpec{}
 	if lastAppliedConfig, ok := old.Annotations[LastAppliedConfigAnnotation]; ok {
 		err := json.Unmarshal([]byte(lastAppliedConfig), &oldConfig)
 		if err != nil {
 			klog.Errorf("unmarshal Statefulset: [%s/%s]'s applied config failed,error: %v", old.GetNamespace(), old.GetName(), err)
-			return false
+			return false, false
 		}
 		// oldConfig.Template.Annotations may include LastAppliedConfigAnnotation to keep backward compatiability
 		// Please check detail in https://github.com/pingcap/tidb-operator/pull/1489
 		tmpTemplate := oldConfig.Template.DeepCopy()
 		delete(tmpTemplate.Annotations, LastAppliedConfigAnnotation)
+		podTemplateEqual = apiequality.Semantic.DeepEqual(*tmpTemplate, new.Spec.Template)
 		return apiequality.Semantic.DeepEqual(oldConfig.Replicas, new.Spec.Replicas) &&
-			apiequality.Semantic.DeepEqual(*tmpTemplate, new.Spec.Template) &&
-			apiequality.Semantic.DeepEqual(oldConfig.UpdateStrategy, new.Spec.UpdateStrategy)
+			apiequality.Semantic.DeepEqual(oldConfig.UpdateStrategy, new.Spec.UpdateStrategy) && // this will be changed when scaling
+			podTemplateEqual, podTemplateEqual
 	}
-	return false
+	return false, false
 }
 
 // ResolvePVCFromPod parses pod volumes definition, and returns all PVCs mounted by this pod