Skip to content

Commit

Permalink
Merge pull request kubernetes#54721 from runcom/reset-remove-dockerism
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm: reset: use crictl to reset containers

@luxas PTAL



Signed-off-by: Antonio Murdaca <[email protected]>



**What this PR does / why we need it**:

This patch makes kubeadm to try and reset containers using `crictl` first instead of docker. The reason is that kubeadm reset is ineffective with new container runtimes using the CRI (like CRI-O).
This patch uses `crictl` first and falls back to `docker` in case `crictl` isn't available. 

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

Fix kubernetes/kubeadm#508

**Special notes for your reviewer**:

**Release note**:

```release-note
kubeadm: reset: use crictl to reset containers
```
  • Loading branch information
Kubernetes Submit Queue authored Nov 4, 2017
2 parents 3e24536 + bb0cd27 commit 5691a8d
Show file tree
Hide file tree
Showing 3 changed files with 187 additions and 12 deletions.
3 changes: 3 additions & 0 deletions cmd/kubeadm/app/cmd/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)

Expand All @@ -87,6 +88,8 @@ go_test(
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/exec/testing:go_default_library",
],
)

Expand Down
66 changes: 54 additions & 12 deletions cmd/kubeadm/app/cmd/reset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,23 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/util/initsystem"
utilsexec "k8s.io/utils/exec"
)

var (
crictlParamsFormat = "%s -r %s sandboxes --quiet | xargs -r %s -r %s rms"
)

// NewCmdReset returns the "kubeadm reset" command
func NewCmdReset(out io.Writer) *cobra.Command {
var skipPreFlight bool
var certsDir string
var criSocketPath string
cmd := &cobra.Command{
Use: "reset",
Short: "Run this to revert any changes made to this host by 'kubeadm init' or 'kubeadm join'.",
Run: func(cmd *cobra.Command, args []string) {
r, err := NewReset(skipPreFlight, certsDir)
r, err := NewReset(skipPreFlight, certsDir, criSocketPath)
kubeadmutil.CheckErr(err)
kubeadmutil.CheckErr(r.Run(out))
},
Expand All @@ -56,16 +62,22 @@ func NewCmdReset(out io.Writer) *cobra.Command {
"The path to the directory where the certificates are stored. If specified, clean this directory.",
)

cmd.PersistentFlags().StringVar(
&criSocketPath, "cri-socket", "/var/run/dockershim.sock",
"The path to the CRI socket to use with crictl when cleaning up containers.",
)

return cmd
}

// Reset defines struct used for kubeadm reset command
type Reset struct {
certsDir string
certsDir string
criSocketPath string
}

// NewReset instantiate Reset struct
func NewReset(skipPreFlight bool, certsDir string) (*Reset, error) {
func NewReset(skipPreFlight bool, certsDir, criSocketPath string) (*Reset, error) {
if !skipPreFlight {
fmt.Println("[preflight] Running pre-flight checks.")

Expand All @@ -77,7 +89,8 @@ func NewReset(skipPreFlight bool, certsDir string) (*Reset, error) {
}

return &Reset{
certsDir: certsDir,
certsDir: certsDir,
criSocketPath: criSocketPath,
}, nil
}

Expand Down Expand Up @@ -105,15 +118,11 @@ func (r *Reset) Run(out io.Writer) error {
fmt.Printf("[reset] Failed to unmount mounted directories in /var/lib/kubelet: %s\n", string(umountOutputBytes))
}

fmt.Println("[reset] Removing kubernetes-managed containers.")
dockerCheck := preflight.ServiceCheck{Service: "docker", CheckIfActive: true}
if _, errors := dockerCheck.Check(); len(errors) == 0 {
fmt.Println("[reset] Removing kubernetes-managed containers.")
if err := exec.Command("sh", "-c", "docker ps -a --filter name=k8s_ -q | xargs -r docker rm --force --volumes").Run(); err != nil {
fmt.Println("[reset] Failed to stop the running containers.")
}
} else {
fmt.Println("[reset] Docker doesn't seem to be running. Skipping the removal of running Kubernetes containers.")
}
execer := utilsexec.New()

reset(execer, dockerCheck, r.criSocketPath)

dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"}

Expand Down Expand Up @@ -141,6 +150,39 @@ func (r *Reset) Run(out io.Writer) error {
return nil
}

func reset(execer utilsexec.Interface, dockerCheck preflight.Checker, criSocketPath string) {
crictlPath, err := execer.LookPath("crictl")
if err == nil {
resetWithCrictl(execer, dockerCheck, criSocketPath, crictlPath)
} else {
resetWithDocker(execer, dockerCheck)
}
}

func resetWithDocker(execer utilsexec.Interface, dockerCheck preflight.Checker) {
if _, errors := dockerCheck.Check(); len(errors) == 0 {
if err := execer.Command("sh", "-c", "docker ps -a --filter name=k8s_ -q | xargs -r docker rm --force --volumes").Run(); err != nil {
fmt.Println("[reset] Failed to stop the running containers.")
}
} else {
fmt.Println("[reset] Docker doesn't seem to be running. Skipping the removal of running Kubernetes containers.")
}
}

func resetWithCrictl(execer utilsexec.Interface, dockerCheck preflight.Checker, criSocketPath, crictlPath string) {
if criSocketPath != "" {
fmt.Printf("[reset] Cleaning up running containers using crictl with socket %s\n", criSocketPath)
cmd := fmt.Sprintf(crictlParamsFormat, crictlPath, criSocketPath, crictlPath, criSocketPath)
if err := execer.Command("sh", "-c", cmd).Run(); err != nil {
fmt.Println("[reset] Failed to stop the running containers using crictl. Trying using docker instead.")
resetWithDocker(execer, dockerCheck)
}
} else {
fmt.Println("[reset] CRI socket path not provided for crictl. Trying docker instead.")
resetWithDocker(execer, dockerCheck)
}
}

// cleanDir removes everything in a directory, but not the directory itself
func cleanDir(filePath string) error {
// If the directory doesn't even exist there's nothing to do, and we do
Expand Down
130 changes: 130 additions & 0 deletions cmd/kubeadm/app/cmd/reset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,17 @@ limitations under the License.
package cmd

import (
"errors"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"

kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
"k8s.io/utils/exec"
fakeexec "k8s.io/utils/exec/testing"
)

func assertExists(t *testing.T, path string) {
Expand Down Expand Up @@ -181,3 +185,129 @@ func TestConfigDirCleaner(t *testing.T) {
}
}
}

type fakeDockerChecker struct {
warnings []error
errors []error
}

func (c *fakeDockerChecker) Check() (warnings, errors []error) {
return c.warnings, c.errors
}

func newFakeDockerChecker(warnings, errors []error) preflight.Checker {
return &fakeDockerChecker{warnings: warnings, errors: errors}
}

func TestResetWithDocker(t *testing.T) {
fcmd := fakeexec.FakeCmd{
RunScript: []fakeexec.FakeRunAction{
func() ([]byte, []byte, error) { return nil, nil, nil },
func() ([]byte, []byte, error) { return nil, nil, errors.New("docker error") },
func() ([]byte, []byte, error) { return nil, nil, nil },
},
}
fexec := fakeexec.FakeExec{
CommandScript: []fakeexec.FakeCommandAction{
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
resetWithDocker(&fexec, newFakeDockerChecker(nil, nil))
if fcmd.RunCalls != 1 {
t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls)
}
resetWithDocker(&fexec, newFakeDockerChecker(nil, nil))
if fcmd.RunCalls != 2 {
t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls)
}
resetWithDocker(&fexec, newFakeDockerChecker(nil, []error{errors.New("test error")}))
if fcmd.RunCalls != 2 {
t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls)
}
}

func TestResetWithCrictl(t *testing.T) {
fcmd := fakeexec.FakeCmd{
RunScript: []fakeexec.FakeRunAction{
func() ([]byte, []byte, error) { return nil, nil, nil },
func() ([]byte, []byte, error) { return nil, nil, nil },
func() ([]byte, []byte, error) { return nil, nil, errors.New("crictl error") },
func() ([]byte, []byte, error) { return nil, nil, nil },
},
}
fexec := fakeexec.FakeExec{
CommandScript: []fakeexec.FakeCommandAction{
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}

resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "", "crictl")
if fcmd.RunCalls != 1 {
t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls)
}
if !strings.Contains(fcmd.RunLog[0][2], "docker") {
t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0])
}

resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "/test.sock", "crictl")
if fcmd.RunCalls != 2 {
t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls)
}
if !strings.Contains(fcmd.RunLog[1][2], "crictl") {
t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0])
}

resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "/test.sock", "crictl")
if fcmd.RunCalls != 4 {
t.Errorf("expected 4 calls to Run, got %d", fcmd.RunCalls)
}
if !strings.Contains(fcmd.RunLog[2][2], "crictl") {
t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0])
}
if !strings.Contains(fcmd.RunLog[3][2], "docker") {
t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0])
}

resetWithCrictl(&fexec, newFakeDockerChecker(nil, []error{errors.New("test error")}), "", "crictl")
if fcmd.RunCalls != 4 {
t.Errorf("expected 4 calls to Run, got %d", fcmd.RunCalls)
}
}

func TestReset(t *testing.T) {
fcmd := fakeexec.FakeCmd{
RunScript: []fakeexec.FakeRunAction{
func() ([]byte, []byte, error) { return nil, nil, nil },
func() ([]byte, []byte, error) { return nil, nil, nil },
},
}
fexec := fakeexec.FakeExec{
CommandScript: []fakeexec.FakeCommandAction{
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
LookPathFunc: func(cmd string) (string, error) { return cmd, nil },
}

reset(&fexec, newFakeDockerChecker(nil, nil), "/test.sock")
if fcmd.RunCalls != 1 {
t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls)
}
if !strings.Contains(fcmd.RunLog[0][2], "crictl") {
t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0])
}

fexec.LookPathFunc = func(cmd string) (string, error) { return "", errors.New("no crictl") }
reset(&fexec, newFakeDockerChecker(nil, nil), "/test.sock")
if fcmd.RunCalls != 2 {
t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls)
}
if !strings.Contains(fcmd.RunLog[1][2], "docker") {
t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0])
}
}

0 comments on commit 5691a8d

Please sign in to comment.