Skip to content

Commit

Permalink
Migrate oom watcher not relying on cAdviosr any more, it is part of plan
Browse files Browse the repository at this point in the history
of removing cAdvisor from k8s.
For more informations about this plan, please refer:
kubernetes#68522
  • Loading branch information
WanLinghao committed Mar 5, 2019
1 parent 6af2e9a commit b2ce446
Show file tree
Hide file tree
Showing 7 changed files with 167 additions and 34 deletions.
5 changes: 2 additions & 3 deletions pkg/kubelet/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ go_library(
"kubelet_pods.go",
"kubelet_resources.go",
"kubelet_volumes.go",
"oom_watcher.go",
"pod_container_deletor.go",
"pod_workers.go",
"reason_cache.go",
Expand Down Expand Up @@ -72,6 +71,7 @@ go_library(
"//pkg/kubelet/network/dns:go_default_library",
"//pkg/kubelet/nodelease:go_default_library",
"//pkg/kubelet/nodestatus:go_default_library",
"//pkg/kubelet/oom:go_default_library",
"//pkg/kubelet/pleg:go_default_library",
"//pkg/kubelet/pod:go_default_library",
"//pkg/kubelet/preemption:go_default_library",
Expand Down Expand Up @@ -144,7 +144,6 @@ go_library(
"//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
"//third_party/forked/golang/expansion:go_default_library",
"//vendor/github.com/golang/groupcache/lru:go_default_library",
"//vendor/github.com/google/cadvisor/events:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
Expand All @@ -168,7 +167,6 @@ go_test(
"kubelet_test.go",
"kubelet_volumes_linux_test.go",
"kubelet_volumes_test.go",
"oom_watcher_test.go",
"pod_container_deletor_test.go",
"pod_workers_test.go",
"reason_cache_test.go",
Expand Down Expand Up @@ -297,6 +295,7 @@ filegroup(
"//pkg/kubelet/network:all-srcs",
"//pkg/kubelet/nodelease:all-srcs",
"//pkg/kubelet/nodestatus:all-srcs",
"//pkg/kubelet/oom:all-srcs",
"//pkg/kubelet/pleg:all-srcs",
"//pkg/kubelet/pod:all-srcs",
"//pkg/kubelet/preemption:all-srcs",
Expand Down
5 changes: 3 additions & 2 deletions pkg/kubelet/kubelet.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
"k8s.io/kubernetes/pkg/kubelet/network/dns"
"k8s.io/kubernetes/pkg/kubelet/nodelease"
oomwatcher "k8s.io/kubernetes/pkg/kubelet/oom"
"k8s.io/kubernetes/pkg/kubelet/pleg"
kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
"k8s.io/kubernetes/pkg/kubelet/preemption"
Expand Down Expand Up @@ -469,7 +470,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,

containerRefManager := kubecontainer.NewRefManager()

oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
oomWatcher := oomwatcher.NewOOMWatcher(kubeDeps.Recorder)

clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS))
for _, ipEntry := range kubeCfg.ClusterDNS {
Expand Down Expand Up @@ -1088,7 +1089,7 @@ type Kubelet struct {
os kubecontainer.OSInterface

// Watcher of out of memory events.
oomWatcher OOMWatcher
oomWatcher oomwatcher.OOMWatcher

// Monitor resource usage
resourceAnalyzer serverstats.ResourceAnalyzer
Expand Down
82 changes: 82 additions & 0 deletions pkg/kubelet/oom/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "go_default_library",
srcs = [
"oom_watcher_linux.go",
"oom_watcher_unsupported.go",
"types.go",
],
importpath = "k8s.io/kubernetes/pkg/kubelet/oom",
visibility = ["//visibility:public"],
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//vendor/github.com/google/cadvisor/utils/oomparser:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"//conditions:default": [],
}),
)

go_test(
name = "go_default_test",
srcs = ["oom_watcher_linux_test.go"],
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
"//conditions:default": [],
}),
)

filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)

filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
43 changes: 18 additions & 25 deletions pkg/kubelet/oom_watcher.go → pkg/kubelet/oom/oom_watcher_linux.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// +build linux

/*
Copyright 2015 The Kubernetes Authors.
Expand All @@ -14,61 +16,52 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet
package oom

import (
"github.com/google/cadvisor/events"
cadvisorapi "github.com/google/cadvisor/info/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/tools/record"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
)

// OOMWatcher defines the interface of OOM watchers.
type OOMWatcher interface {
Start(ref *v1.ObjectReference) error
}
"github.com/google/cadvisor/utils/oomparser"
)

type realOOMWatcher struct {
cadvisor cadvisor.Interface
recorder record.EventRecorder
}

var _ OOMWatcher = &realOOMWatcher{}

// NewOOMWatcher creates and initializes a OOMWatcher based on parameters.
func NewOOMWatcher(cadvisor cadvisor.Interface, recorder record.EventRecorder) OOMWatcher {
func NewOOMWatcher(recorder record.EventRecorder) OOMWatcher {
return &realOOMWatcher{
cadvisor: cadvisor,
recorder: recorder,
}
}

const systemOOMEvent = "SystemOOM"

// Watches cadvisor for system oom's and records an event for every system oom encountered.
// Watches for system oom's and records an event for every system oom encountered.
func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error {
request := events.Request{
EventType: map[cadvisorapi.EventType]bool{
cadvisorapi.EventOom: true,
},
ContainerName: "/",
IncludeSubcontainers: false,
}
eventChannel, err := ow.cadvisor.WatchEvents(&request)
oomLog, err := oomparser.New()
if err != nil {
return err
}
outStream := make(chan *oomparser.OomInstance, 10)
go oomLog.StreamOoms(outStream)

go func() {
defer runtime.HandleCrash()

for event := range eventChannel.GetChannel() {
klog.V(2).Infof("Got sys oom event from cadvisor: %v", event)
ow.recorder.PastEventf(ref, metav1.Time{Time: event.Timestamp}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
for event := range outStream {
if event.ContainerName == "/" {
klog.V(1).Infof("Got sys oom event: %v", event)
ow.recorder.PastEventf(ref, metav1.Time{Time: event.TimeOfDeath}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
}
}
klog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor")
klog.Errorf("Unexpectedly stopped receiving OOM notifications")
}()
return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet
package oom

import (
"testing"
Expand All @@ -23,14 +23,12 @@ import (

"k8s.io/api/core/v1"
"k8s.io/client-go/tools/record"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
)

func TestBasic(t *testing.T) {
fakeRecorder := &record.FakeRecorder{}
mockCadvisor := &cadvisortest.Fake{}
node := &v1.ObjectReference{}
oomWatcher := NewOOMWatcher(mockCadvisor, fakeRecorder)
oomWatcher := NewOOMWatcher(fakeRecorder)
assert.NoError(t, oomWatcher.Start(node))

// TODO: Improve this test once cadvisor exports events.EventChannel as an interface
Expand Down
35 changes: 35 additions & 0 deletions pkg/kubelet/oom/oom_watcher_unsupported.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// +build !linux

/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oom

import (
"k8s.io/api/core/v1"
"k8s.io/client-go/tools/record"
)

type oomWatcherUnsupported struct{}

var _ OOMWatcher = new(oomWatcherUnsupported)

func NewOOMWatcher(_ record.EventRecorder) OOMWatcher {
return &oomWatcherUnsupported{}
}

func (ow *oomWatcherUnsupported) Start(_ *v1.ObjectReference) error {
return nil
}
25 changes: 25 additions & 0 deletions pkg/kubelet/oom/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oom

import (
"k8s.io/api/core/v1"
)

// OOMWatcher defines the interface of OOM watchers.
type OOMWatcher interface {
Start(ref *v1.ObjectReference) error
}

0 comments on commit b2ce446

Please sign in to comment.