manager: fix tiproxy scale & delete-slots (pingcap#4917)

hack/localtest/operator.yaml

@@ -0,0 +1,5 @@
+operatorImage: "xx/tidb-operator:latest"
+features:
+  - AdvancedStatefulSet=true
+advancedStatefulset:
+  create: true

hack/localtest/test.sh

@@ -12,7 +12,8 @@ if [ -n "$BUILD" ]; then
 	make DOCKER_REPO=xx operator-docker
 fi
 
-kubectl create -f $BASE/../../manifests/crd.yaml || kubectl replace -f $BASE/../../manifests/crd.yaml
+kubectl replace -f $BASE/../../manifests/crd.yaml
+kubectl replace -f $BASE/../../manifests/advanced-statefulset-crd.v1.yaml
 
 kubectl delete --force namespace $namespace || true
 
@@ -34,5 +35,5 @@ if [ -n "$ENABLE_SSL" ]; then
 	kubectl create secret generic ${cluster}-cluster-client-secret --namespace=${namespace} --from-file=tls.crt=client.pem --from-file=tls.key=client-key.pem --from-file=ca.crt=ca.pem
 fi
 
-helm install operator $BASE/../../charts/tidb-operator/ --namespace ${namespace} --set "operatorImage=xx/tidb-operator:latest"
+helm install operator $BASE/../../charts/tidb-operator/ --namespace ${namespace} -f $BASE/operator.yaml
 kubectl apply -f $BASE/cluster.yaml --namespace ${namespace}

pkg/apis/label/label.go

@@ -116,6 +116,8 @@ const (
 	AnnTiKVDeleteSlots = "tikv.tidb.pingcap.com/delete-slots"
 	// AnnTiFlashDeleteSlots is annotation key of tiflash delete slots.
 	AnnTiFlashDeleteSlots = "tiflash.tidb.pingcap.com/delete-slots"
+	// AnnTiProxyDeleteSlots is annotation key of tiproxy delete slots.
+	AnnTiProxyDeleteSlots = "tiproxy.tidb.pingcap.com/delete-slots"
 	// AnnDMMasterDeleteSlots is annotation key of dm-master delete slots.
 	AnnDMMasterDeleteSlots = "dm-master.tidb.pingcap.com/delete-slots"
 	// AnnDMWorkerDeleteSlots is annotation key of dm-worker delete slots.

pkg/apis/pingcap/v1alpha1/tidbcluster.go

@@ -480,6 +480,8 @@ func (tc *TidbCluster) getDeleteSlots(component string) (deleteSlots sets.Int32)
 		key = label.AnnTiKVDeleteSlots
 	} else if component == label.TiFlashLabelVal {
 		key = label.AnnTiFlashDeleteSlots
+	} else if component == label.TiProxyLabelVal {
+		key = label.AnnTiProxyDeleteSlots
 	} else {
 		return
 	}

pkg/apis/pingcap/v1alpha1/validation/validation.go

@@ -107,7 +107,7 @@ func ValidateTidbMonitor(monitor *v1alpha1.TidbMonitor) field.ErrorList {
 func validateAnnotations(anns map[string]string, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 	allErrs = append(allErrs, apivalidation.ValidateAnnotations(anns, fldPath)...)
-	for _, key := range []string{label.AnnPDDeleteSlots, label.AnnTiDBDeleteSlots, label.AnnTiKVDeleteSlots, label.AnnTiFlashDeleteSlots} {
+	for _, key := range []string{label.AnnPDDeleteSlots, label.AnnTiDBDeleteSlots, label.AnnTiKVDeleteSlots, label.AnnTiFlashDeleteSlots, label.AnnTiProxyDeleteSlots} {
 		allErrs = append(allErrs, validateDeleteSlots(anns, key, fldPath.Child(key))...)
 	}
 	return allErrs

pkg/manager/member/tiproxy_scaler.go

@@ -18,7 +18,6 @@ import (
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/controller"
-	"github.com/pingcap/tidb-operator/pkg/util"
 	apps "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
@@ -58,10 +57,6 @@ func (s *tiproxyScaler) ScaleOut(meta metav1.Object, oldSet *apps.StatefulSet, n
 	resetReplicas(newSet, oldSet)
 
 	klog.Infof("scaling out tiproxy statefulset %s/%s, ordinal: %d (replicas: %d, delete slots: %v)", oldSet.Namespace, oldSet.Name, ordinal, replicas, deleteSlots.List())
-	_, err := s.deleteDeferDeletingPVC(tc, v1alpha1.TiProxyMemberType, ordinal)
-	if err != nil {
-		return err
-	}
 
 	setReplicasAndDeleteSlots(newSet, replicas, deleteSlots)
 	return nil
@@ -79,30 +74,13 @@ func (s *tiproxyScaler) ScaleIn(meta metav1.Object, oldSet *apps.StatefulSet, ne
 	tcName := tc.GetName()
 	_, ordinal, replicas, deleteSlots := scaleOne(oldSet, newSet)
 	resetReplicas(newSet, oldSet)
-	podName := ordinalPodName(v1alpha1.TiProxyMemberType, tcName, ordinal)
 
 	if !tc.Status.TiProxy.Synced {
 		return fmt.Errorf("TidbCluster: %s/%s's pd status sync failed, can't scale in now", ns, tcName)
 	}
 
 	klog.Infof("scaling in tiproxy statefulset %s/%s, ordinal: %d (replicas: %d, delete slots: %v)", oldSet.Namespace, oldSet.Name, ordinal, replicas, deleteSlots.List())
 
-	pod, err := s.deps.PodLister.Pods(ns).Get(podName)
-	if err != nil {
-		return fmt.Errorf("tiproxyScaler.ScaleIn: failed to get pod %s/%s for tiproxy in tc %s/%s, error: %s", ns, podName, ns, tcName, err)
-	}
-
-	pvcs, err := util.ResolvePVCFromPod(pod, s.deps.PVCLister)
-	if err != nil {
-		return fmt.Errorf("tiproxyScaler.ScaleIn: failed to get pvcs for pod %s/%s in tc %s/%s, error: %s", ns, pod.Name, ns, tcName, err)
-	}
-
-	for _, pvc := range pvcs {
-		if err := addDeferDeletingAnnoToPVC(tc, pvc, s.deps.PVCControl); err != nil {
-			return err
-		}
-	}
-
 	setReplicasAndDeleteSlots(newSet, replicas, deleteSlots)
 	return nil
 }

pkg/manager/member/utils.go

@@ -196,6 +196,8 @@ func getStsAnnotations(tcAnns map[string]string, component string) map[string]st
 		key = label.AnnTiKVDeleteSlots
 	case label.TiFlashLabelVal:
 		key = label.AnnTiFlashDeleteSlots
+	case label.TiProxyLabelVal:
+		key = label.AnnTiProxyDeleteSlots
 	case label.DMMasterLabelVal:
 		key = label.AnnDMMasterDeleteSlots
 	case label.DMWorkerLabelVal:

pkg/upgrader/upgrader.go

@@ -137,7 +137,7 @@ func deleteSlotAnns(tc *v1alpha1.TidbCluster) map[string]string {
 	if tc == nil || tc.Annotations == nil {
 		return anns
 	}
-	for _, key := range []string{label.AnnPDDeleteSlots, label.AnnTiDBDeleteSlots, label.AnnTiKVDeleteSlots, label.AnnTiFlashDeleteSlots} {
+	for _, key := range []string{label.AnnPDDeleteSlots, label.AnnTiDBDeleteSlots, label.AnnTiKVDeleteSlots, label.AnnTiFlashDeleteSlots, label.AnnTiProxyDeleteSlots} {
 		if v, ok := tc.Annotations[key]; ok {
 			anns[key] = v
 		}

pkg/util/util.go

@@ -114,6 +114,9 @@ func GetPodOrdinals(tc *v1alpha1.TidbCluster, memberType v1alpha1.MemberType) (s
 	} else if memberType == v1alpha1.TiFlashMemberType {
 		ann = label.AnnTiFlashDeleteSlots
 		replicas = tc.Spec.TiFlash.Replicas
+	} else if memberType == v1alpha1.TiProxyMemberType {
+		ann = label.AnnTiProxyDeleteSlots
+		replicas = tc.Spec.TiProxy.Replicas
 	} else {
 		return nil, fmt.Errorf("unknown member type %v", memberType)
 	}