Move layer mount refcounts to mountedLayer

Instead of implementing refcounts at each graphdriver, implement this in
the layer package which is what the engine actually interacts with now.
This means interacting directly with the graphdriver is no longer
explicitly safe with regard to Get/Put calls being refcounted.

In addition, with the containerd, layers may still be mounted after
a daemon restart since we will no longer explicitly kill containers when
we shutdown or startup engine.
Because of this ref counts would need to be repopulated.

Signed-off-by: Brian Goff <[email protected]>

daemon/commit.go

@@ -222,6 +222,7 @@ func (daemon *Daemon) exportContainerRw(container *container.Container) (archive
 
 	archive, err := container.RWLayer.TarStream()
 	if err != nil {
+		daemon.Unmount(container) // logging is already handled in the `Unmount` function
 		return nil, err
 	}
 	return ioutils.NewReadCloserWrapper(archive, func() error {

daemon/graphdriver/aufs/aufs.go

@@ -29,6 +29,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 	"sync"
 	"syscall"
@@ -64,21 +65,13 @@ func init() {
 	graphdriver.Register("aufs", Init)
 }
 
-type data struct {
-	referenceCount int
-	path           string
-}
-
 // Driver contains information about the filesystem mounted.
-// root of the filesystem
-// sync.Mutex to protect against concurrent modifications
-// active maps mount id to the count
 type Driver struct {
-	root       string
-	uidMaps    []idtools.IDMap
-	gidMaps    []idtools.IDMap
-	sync.Mutex // Protects concurrent modification to active
-	active     map[string]*data
+	root          string
+	uidMaps       []idtools.IDMap
+	gidMaps       []idtools.IDMap
+	pathCacheLock sync.Mutex
+	pathCache     map[string]string
 }
 
 // Init returns a new AUFS driver.
@@ -111,10 +104,10 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 	}
 
 	a := &Driver{
-		root:    root,
-		active:  make(map[string]*data),
-		uidMaps: uidMaps,
-		gidMaps: gidMaps,
+		root:      root,
+		uidMaps:   uidMaps,
+		gidMaps:   gidMaps,
+		pathCache: make(map[string]string),
 	}
 
 	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
@@ -228,9 +221,7 @@ func (a *Driver) Create(id, parent, mountLabel string) error {
 			}
 		}
 	}
-	a.Lock()
-	a.active[id] = &data{}
-	a.Unlock()
+
 	return nil
 }
 
@@ -259,108 +250,91 @@ func (a *Driver) createDirsFor(id string) error {
 
 // Remove will unmount and remove the given id.
 func (a *Driver) Remove(id string) error {
-	// Protect the a.active from concurrent access
-	a.Lock()
-	defer a.Unlock()
-
-	m := a.active[id]
-	if m != nil {
-		if m.referenceCount > 0 {
-			return nil
-		}
-		// Make sure the dir is umounted first
-		if err := a.unmount(m); err != nil {
-			return err
-		}
+	a.pathCacheLock.Lock()
+	mountpoint, exists := a.pathCache[id]
+	a.pathCacheLock.Unlock()
+	if !exists {
+		mountpoint = a.getMountpoint(id)
 	}
-	tmpDirs := []string{
-		"mnt",
-		"diff",
+	if err := a.unmount(mountpoint); err != nil {
+		// no need to return here, we can still try to remove since the `Rename` will fail below if still mounted
+		logrus.Debugf("aufs: error while unmounting %s: %v", mountpoint, err)
 	}
 
 	// Atomically remove each directory in turn by first moving it out of the
 	// way (so that docker doesn't find it anymore) before doing removal of
 	// the whole tree.
-	for _, p := range tmpDirs {
-		realPath := path.Join(a.rootPath(), p, id)
-		tmpPath := path.Join(a.rootPath(), p, fmt.Sprintf("%s-removing", id))
-		if err := os.Rename(realPath, tmpPath); err != nil && !os.IsNotExist(err) {
-			return err
-		}
-		defer os.RemoveAll(tmpPath)
+	tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id))
+	if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	defer os.RemoveAll(tmpMntPath)
+
+	tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id))
+	if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) {
+		return err
 	}
+	defer os.RemoveAll(tmpDiffpath)
+
 	// Remove the layers file for the id
 	if err := os.Remove(path.Join(a.rootPath(), "layers", id)); err != nil && !os.IsNotExist(err) {
 		return err
 	}
-	if m != nil {
-		delete(a.active, id)
-	}
+
+	a.pathCacheLock.Lock()
+	delete(a.pathCache, id)
+	a.pathCacheLock.Unlock()
 	return nil
 }
 
 // Get returns the rootfs path for the id.
 // This will mount the dir at it's given path
 func (a *Driver) Get(id, mountLabel string) (string, error) {
-	// Protect the a.active from concurrent access
-	a.Lock()
-	defer a.Unlock()
-
-	m := a.active[id]
-	if m == nil {
-		m = &data{}
-		a.active[id] = m
-	}
-
 	parents, err := a.getParentLayerPaths(id)
 	if err != nil && !os.IsNotExist(err) {
 		return "", err
 	}
 
+	a.pathCacheLock.Lock()
+	m, exists := a.pathCache[id]
+	a.pathCacheLock.Unlock()
+
+	if !exists {
+		m = a.getDiffPath(id)
+		if len(parents) > 0 {
+			m = a.getMountpoint(id)
+		}
+	}
+
 	// If a dir does not have a parent ( no layers )do not try to mount
 	// just return the diff path to the data
-	m.path = path.Join(a.rootPath(), "diff", id)
 	if len(parents) > 0 {
-		m.path = path.Join(a.rootPath(), "mnt", id)
-		if m.referenceCount == 0 {
-			if err := a.mount(id, m, mountLabel, parents); err != nil {
-				return "", err
-			}
+		if err := a.mount(id, m, mountLabel, parents); err != nil {
+			return "", err
 		}
 	}
-	m.referenceCount++
-	return m.path, nil
+
+	a.pathCacheLock.Lock()
+	a.pathCache[id] = m
+	a.pathCacheLock.Unlock()
+	return m, nil
 }
 
 // Put unmounts and updates list of active mounts.
 func (a *Driver) Put(id string) error {
-	// Protect the a.active from concurrent access
-	a.Lock()
-	defer a.Unlock()
-
-	m := a.active[id]
-	if m == nil {
-		// but it might be still here
-		if a.Exists(id) {
-			path := path.Join(a.rootPath(), "mnt", id)
-			err := Unmount(path)
-			if err != nil {
-				logrus.Debugf("Failed to unmount %s aufs: %v", id, err)
-			}
-		}
-		return nil
+	a.pathCacheLock.Lock()
+	m, exists := a.pathCache[id]
+	if !exists {
+		m = a.getMountpoint(id)
+		a.pathCache[id] = m
 	}
-	if count := m.referenceCount; count > 1 {
-		m.referenceCount = count - 1
-	} else {
-		ids, _ := getParentIds(a.rootPath(), id)
-		// We only mounted if there are any parents
-		if ids != nil && len(ids) > 0 {
-			a.unmount(m)
-		}
-		delete(a.active, id)
+	a.pathCacheLock.Unlock()
+
+	err := a.unmount(m)
+	if err != nil {
+		logrus.Debugf("Failed to unmount %s aufs: %v", id, err)
 	}
-	return nil
+	return err
 }
 
 // Diff produces an archive of the changes between the specified
@@ -443,43 +417,53 @@ func (a *Driver) getParentLayerPaths(id string) ([]string, error) {
 	return layers, nil
 }
 
-func (a *Driver) mount(id string, m *data, mountLabel string, layers []string) error {
+func (a *Driver) mount(id string, target string, mountLabel string, layers []string) error {
 	// If the id is mounted or we get an error return
-	if mounted, err := a.mounted(m); err != nil || mounted {
+	if mounted, err := a.mounted(target); err != nil || mounted {
 		return err
 	}
 
-	var (
-		target = m.path
-		rw     = path.Join(a.rootPath(), "diff", id)
-	)
+	rw := a.getDiffPath(id)
 
 	if err := a.aufsMount(layers, rw, target, mountLabel); err != nil {
 		return fmt.Errorf("error creating aufs mount to %s: %v", target, err)
 	}
 	return nil
 }
 
-func (a *Driver) unmount(m *data) error {
-	if mounted, err := a.mounted(m); err != nil || !mounted {
+func (a *Driver) unmount(mountPath string) error {
+	if mounted, err := a.mounted(mountPath); err != nil || !mounted {
+		return err
+	}
+	if err := Unmount(mountPath); err != nil {
 		return err
 	}
-	return Unmount(m.path)
+	return nil
 }
 
-func (a *Driver) mounted(m *data) (bool, error) {
-	var buf syscall.Statfs_t
-	if err := syscall.Statfs(m.path, &buf); err != nil {
-		return false, nil
-	}
-	return graphdriver.FsMagic(buf.Type) == graphdriver.FsMagicAufs, nil
+func (a *Driver) mounted(mountpoint string) (bool, error) {
+	return graphdriver.Mounted(graphdriver.FsMagicAufs, mountpoint)
 }
 
 // Cleanup aufs and unmount all mountpoints
 func (a *Driver) Cleanup() error {
-	for id, m := range a.active {
+	var dirs []string
+	if err := filepath.Walk(a.mntPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			return nil
+		}
+		dirs = append(dirs, path)
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	for _, m := range dirs {
 		if err := a.unmount(m); err != nil {
-			logrus.Errorf("Unmounting %s: %s", stringid.TruncateID(id), err)
+			logrus.Debugf("aufs error unmounting %s: %s", stringid.TruncateID(m), err)
 		}
 	}
 	return mountpk.Unmount(a.root)

daemon/graphdriver/aufs/aufs_test.go

@@ -200,7 +200,7 @@ func TestMountedFalseResponse(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	response, err := d.mounted(d.active["1"])
+	response, err := d.mounted(d.getDiffPath("1"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -227,7 +227,7 @@ func TestMountedTrueReponse(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	response, err := d.mounted(d.active["2"])
+	response, err := d.mounted(d.pathCache["2"])
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -293,7 +293,7 @@ func TestRemoveMountedDir(t *testing.T) {
 		t.Fatal("mntPath should not be empty string")
 	}
 
-	mounted, err := d.mounted(d.active["2"])
+	mounted, err := d.mounted(d.pathCache["2"])
 	if err != nil {
 		t.Fatal(err)
 	}

daemon/graphdriver/aufs/dirs.go

@@ -46,3 +46,19 @@ func getParentIds(root, id string) ([]string, error) {
 	}
 	return out, s.Err()
 }
+
+func (a *Driver) getMountpoint(id string) string {
+	return path.Join(a.mntPath(), id)
+}
+
+func (a *Driver) mntPath() string {
+	return path.Join(a.rootPath(), "mnt")
+}
+
+func (a *Driver) getDiffPath(id string) string {
+	return path.Join(a.diffPath(), id)
+}
+
+func (a *Driver) diffPath() string {
+	return path.Join(a.rootPath(), "diff")
+}