chore: wait to create pod rather than exit with Failed status when po…

…d scheduling is blocked during instance rebuilding. (apecloud#7361)
aure-lee · May 16, 2024 · df81dff · df81dff
1 parent 5a4a1ae
commit df81dff
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 6 deletions.
diff --git a/controllers/apps/operations/rebuild_instance.go b/controllers/apps/operations/rebuild_instance.go
@@ -774,7 +774,7 @@ func (r rebuildInstanceOpsHandler) instanceIsAvailable(
 	if !targetPod.DeletionTimestamp.IsZero() {
 		return false, nil
 	}
-	isFailed, isTimeout, _ := intctrlutil.IsPodFailedAndTimedOut(targetPod)
+	isFailed, isTimeout, _ := intctrlutil.IsPodFailedAndTimedOut(targetPod, true)
 	if isFailed && isTimeout {
 		return false, intctrlutil.NewFatalError(fmt.Sprintf(`the new instance "%s" is failed, please check it`, targetPod.Name))
 	}

diff --git a/pkg/controller/instanceset/reconciler_status.go b/pkg/controller/instanceset/reconciler_status.go
@@ -170,7 +170,7 @@ func buildFailureCondition(its *workloads.InstanceSet, pods []*corev1.Pod) (*met
 			continue
 		}
 		// KubeBlocks says the Pod is 'Failed'
-		isFailed, isTimedOut, _ := intctrlutil.IsPodFailedAndTimedOut(pod)
+		isFailed, isTimedOut, _ := intctrlutil.IsPodFailedAndTimedOut(pod, false)
 		if isFailed && isTimedOut {
 			failureNames = append(failureNames, pod.Name)
 		}

diff --git a/pkg/controllerutil/pod_utils.go b/pkg/controllerutil/pod_utils.go
@@ -550,9 +550,11 @@ func GetPodContainer(pod *corev1.Pod, containerName string) *corev1.Container {
 }
 
 // IsPodFailedAndTimedOut checks if the pod is failed and timed out.
-func IsPodFailedAndTimedOut(pod *corev1.Pod) (bool, bool, string) {
-	if isFailed, isTimedOut, message := isPodScheduledFailedAndTimedOut(pod); isFailed {
-		return isFailed, isTimedOut, message
+func IsPodFailedAndTimedOut(pod *corev1.Pod, ignoreScheduledCheck bool) (bool, bool, string) {
+	if !ignoreScheduledCheck {
+		if isFailed, isTimedOut, message := isPodScheduledFailedAndTimedOut(pod); isFailed {
+			return isFailed, isTimedOut, message
+		}
 	}
 	initContainerFailed, message := isAnyContainerFailed(pod.Status.InitContainerStatuses)
 	if initContainerFailed {

diff --git a/pkg/dataprotection/action/action_stateful.go b/pkg/dataprotection/action/action_stateful.go
@@ -211,6 +211,6 @@ func (s *StatefulSetAction) stsIsFailed(ctx ActionContext) bool {
 		Namespace: s.ObjectMeta.Namespace}, pod); err != nil {
 		return false
 	}
-	isFailed, isTimeout, _ := intctrlutil.IsPodFailedAndTimedOut(pod)
+	isFailed, isTimeout, _ := intctrlutil.IsPodFailedAndTimedOut(pod, false)
 	return isFailed && isTimeout
 }