Merge pull request openshift#22678 from smarterclayton/monitor

When pod restarts with exitCode 1, monitor should print term message
FyreNet · Apr 26, 2019 · 3552587 · 3552587
2 parents 624d076 + f86f369
commit 3552587
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 2 deletions.
diff --git a/pkg/monitor/operator.go b/pkg/monitor/operator.go
@@ -66,7 +66,7 @@ func startClusterOperatorMonitoring(ctx context.Context, m Recorder, client conf
 					conditions = append(conditions, Condition{
 						Level:   Info,
 						Locator: locateClusterOperator(co),
-						Message: fmt.Sprintf("versions: %v", strings.Join(changes, ",")),
+						Message: fmt.Sprintf("versions: %v", strings.Join(changes, ", ")),
 					})
 				}
 			}
@@ -299,6 +299,7 @@ func findOperatorVersionChange(old, new []configv1.OperandVersion) []string {
 				continue
 			}
 			changed = append(changed, fmt.Sprintf("%s %s -> %s", new[i].Name, old[p].Version, new[i].Version))
+			break
 		}
 	}
 	return changed

diff --git a/pkg/monitor/pod.go b/pkg/monitor/pod.go
@@ -105,6 +105,25 @@ func startPodMonitoring(ctx context.Context, m Recorder, client kubernetes.Inter
 					Locator: locatePod(pod),
 					Message: fmt.Sprintf("pod moved to the Unknown phase"),
 				})
+			case new == corev1.PodFailed && old != corev1.PodFailed:
+				for _, s := range pod.Status.InitContainerStatuses {
+					if t := s.State.Terminated; t != nil && t.ExitCode != 0 {
+						conditions = append(conditions, Condition{
+							Level:   Error,
+							Locator: locatePodContainer(pod, s.Name),
+							Message: fmt.Sprintf("container exited with code %d: %s", t.ExitCode, t.Message),
+						})
+					}
+				}
+				for _, s := range pod.Status.ContainerStatuses {
+					if t := s.State.Terminated; t != nil && t.ExitCode != 0 {
+						conditions = append(conditions, Condition{
+							Level:   Error,
+							Locator: locatePodContainer(pod, s.Name),
+							Message: fmt.Sprintf("init container exited with code %d: %s", t.ExitCode, t.Message),
+						})
+					}
+				}
 			}
 			return conditions
 		},
@@ -136,21 +155,49 @@ func startPodMonitoring(ctx context.Context, m Recorder, client kubernetes.Inter
 				if previous == nil {
 					continue
 				}
+				if t := s.State.Terminated; t != nil && previous.State.Terminated == nil && t.ExitCode != 0 {
+					conditions = append(conditions, Condition{
+						Level:   Error,
+						Locator: locatePodContainer(pod, s.Name),
+						Message: fmt.Sprintf("container exited with code %d: %s", t.ExitCode, t.Message),
+					})
+				}
 				if s.RestartCount != previous.RestartCount {
 					conditions = append(conditions, Condition{
 						Level:   Warning,
 						Locator: locatePodContainer(pod, s.Name),
 						Message: "container restarted",
 					})
 				}
-				if previous.Ready && !s.Ready {
+				if s.State.Terminated == nil && previous.Ready && !s.Ready {
 					conditions = append(conditions, Condition{
 						Level:   Warning,
 						Locator: locatePodContainer(pod, s.Name),
 						Message: "container stopped being ready",
 					})
 				}
 			}
+			for i := range pod.Status.InitContainerStatuses {
+				s := &pod.Status.InitContainerStatuses[i]
+				previous := findContainerStatus(oldPod.Status.InitContainerStatuses, s.Name, i)
+				if previous == nil {
+					continue
+				}
+				if t := s.State.Terminated; t != nil && previous.State.Terminated == nil && t.ExitCode != 0 {
+					conditions = append(conditions, Condition{
+						Level:   Error,
+						Locator: locatePodContainer(pod, s.Name),
+						Message: fmt.Sprintf("init container exited with code %d: %s", t.ExitCode, t.Message),
+					})
+				}
+				if s.RestartCount != previous.RestartCount {
+					conditions = append(conditions, Condition{
+						Level:   Warning,
+						Locator: locatePodContainer(pod, s.Name),
+						Message: "init container restarted",
+					})
+				}
+			}
 			return conditions
 		},
 	}