Add directory parameter to Rmdir and Mkdir rclone#100 rclone#831

This will enable rclone to manage directories properly in the future.
tkuennen · Nov 26, 2016 · aaa1370 · aaa1370
1 parent c41b67e
commit aaa1370
Show file tree

Hide file tree

Showing 34 changed files with 220 additions and 89 deletions.
diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go
@@ -18,6 +18,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"path"
 	"regexp"
 	"strings"
 	"sync/atomic"
@@ -607,8 +608,15 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
 }
 
 // Mkdir creates the container if it doesn't exist
-func (f *Fs) Mkdir() error {
-	return f.dirCache.FindRoot(true)
+func (f *Fs) Mkdir(dir string) error {
+	err := f.dirCache.FindRoot(true)
+	if err != nil {
+		return err
+	}
+	if dir != "" {
+		_, err = f.dirCache.FindDir(dir, true)
+	}
+	return err
 }
 
 // Move src to this remote using server side move operations.
@@ -685,16 +693,16 @@ func (f *Fs) DirMove(src fs.Fs) (err error) {
 
 // purgeCheck remotes the root directory, if check is set then it
 // refuses to do so if it has anything in
-func (f *Fs) purgeCheck(check bool) error {
-	if f.root == "" {
+func (f *Fs) purgeCheck(dir string, check bool) error {
+	root := path.Join(f.root, dir)
+	if root == "" {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(false)
+	rootID, err := dc.FindDir(dir, false)
 	if err != nil {
 		return err
 	}
-	rootID := dc.RootID()
 
 	if check {
 		// check directory is empty
@@ -730,7 +738,7 @@ func (f *Fs) purgeCheck(check bool) error {
 		return err
 	}
 
-	f.dirCache.ResetRoot()
+	f.dirCache.FlushDir(dir)
 	if err != nil {
 		return err
 	}
@@ -740,8 +748,8 @@ func (f *Fs) purgeCheck(check bool) error {
 // Rmdir deletes the root folder
 //
 // Returns an error if it isn't empty
-func (f *Fs) Rmdir() error {
-	return f.purgeCheck(true)
+func (f *Fs) Rmdir(dir string) error {
+	return f.purgeCheck(dir, true)
 }
 
 // Precision return the precision of this Fs
@@ -783,7 +791,7 @@ func (f *Fs) Hashes() fs.HashSet {
 // deleting all the files quicker than just running Remove() on the
 // result of List()
 func (f *Fs) Purge() error {
-	return f.purgeCheck(false)
+	return f.purgeCheck("", false)
 }
 
 // ------------------------------------------------------------

diff --git a/amazonclouddrive/amazonclouddrive_test.go b/amazonclouddrive/amazonclouddrive_test.go
@@ -23,6 +23,7 @@ func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
 func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
 func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
 func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
+func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
 func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
 func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
 func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }

diff --git a/b2/b2.go b/b2/b2.go
@@ -745,7 +745,11 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
 }
 
 // Mkdir creates the bucket if it doesn't exist
-func (f *Fs) Mkdir() error {
+func (f *Fs) Mkdir(dir string) error {
+	// Can't create subdirs
+	if dir != "" {
+		return nil
+	}
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_create_bucket",
@@ -784,8 +788,8 @@ func (f *Fs) Mkdir() error {
 // Rmdir deletes the bucket if the fs is at the root
 //
 // Returns an error if it isn't empty
-func (f *Fs) Rmdir() error {
-	if f.root != "" {
+func (f *Fs) Rmdir(dir string) error {
+	if f.root != "" || dir != "" {
 		return nil
 	}
 	opts := rest.Opts{
@@ -896,7 +900,7 @@ func (f *Fs) purge(oldOnly bool) error {
 	wg.Wait()
 
 	if !oldOnly {
-		checkErr(f.Rmdir())
+		checkErr(f.Rmdir(""))
 	}
 	return errReturn
 }

diff --git a/b2/b2_test.go b/b2/b2_test.go
@@ -23,6 +23,7 @@ func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
 func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
 func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
 func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
+func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
 func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
 func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
 func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }

diff --git a/cmd/mkdir/mkdir.go b/cmd/mkdir/mkdir.go
@@ -17,7 +17,7 @@ var mkdirCmd = &cobra.Command{
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDst(args)
 		cmd.Run(true, command, func() error {
-			return fs.Mkdir(fdst)
+			return fs.Mkdir(fdst, "")
 		})
 	},
 }
diff --git a/cmd/mount/fs_test.go b/cmd/mount/fs_test.go
@@ -82,7 +82,7 @@ func newRun() *Run {
 		log.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
 	}
 
-	err = r.fremote.Mkdir()
+	err = r.fremote.Mkdir("")
 	if err != nil {
 		log.Fatalf("Failed to open mkdir %q: %v", *RemoteName, err)
 	}

diff --git a/cmd/rmdir/rmdir.go b/cmd/rmdir/rmdir.go
@@ -20,7 +20,7 @@ objects in it, use purge for that.`,
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDst(args)
 		cmd.Run(true, command, func() error {
-			return fs.Rmdir(fdst)
+			return fs.Rmdir(fdst, "")
 		})
 	},
 }
diff --git a/crypt/crypt2_test.go b/crypt/crypt2_test.go
@@ -24,6 +24,7 @@ func TestFsString2(t *testing.T)            { fstests.TestFsString(t) }
 func TestFsRmdirEmpty2(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
 func TestFsRmdirNotFound2(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
 func TestFsMkdir2(t *testing.T)             { fstests.TestFsMkdir(t) }
+func TestFsMkdirRmdirSubdir2(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
 func TestFsListEmpty2(t *testing.T)         { fstests.TestFsListEmpty(t) }
 func TestFsListDirEmpty2(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
 func TestFsNewObjectNotFound2(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }

diff --git a/crypt/crypt_test.go b/crypt/crypt_test.go
@@ -24,6 +24,7 @@ func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
 func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
 func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
 func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
+func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
 func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
 func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
 func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }

diff --git a/dircache/dircache.go b/dircache/dircache.go
@@ -78,6 +78,35 @@ func (dc *DirCache) Flush() {
 	dc.cacheMu.Unlock()
 }
 
+// FlushDir flushes the map of all data starting with dir
+//
+// If dir is empty then this is equivalent to calling ResetRoot
+func (dc *DirCache) FlushDir(dir string) {
+	if dir == "" {
+		dc.ResetRoot()
+		return
+	}
+	dc.cacheMu.Lock()
+
+	// Delete the root dir
+	ID, ok := dc.cache[dir]
+	if ok {
+		delete(dc.cache, dir)
+		delete(dc.invCache, ID)
+	}
+
+	// And any sub directories
+	dir += "/"
+	for key, ID := range dc.cache {
+		if strings.HasPrefix(key, dir) {
+			delete(dc.cache, key)
+			delete(dc.invCache, ID)
+		}
+	}
+
+	dc.cacheMu.Unlock()
+}
+
 // SplitPath splits a path into directory, leaf
 //
 // Path shouldn't start or end with a /

diff --git a/drive/drive.go b/drive/drive.go
@@ -12,6 +12,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"path"
 	"strings"
 	"time"
 
@@ -592,21 +593,30 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
 }
 
 // Mkdir creates the container if it doesn't exist
-func (f *Fs) Mkdir() error {
-	return f.dirCache.FindRoot(true)
+func (f *Fs) Mkdir(dir string) error {
+	err := f.dirCache.FindRoot(true)
+	if err != nil {
+		return err
+	}
+	if dir != "" {
+		_, err = f.dirCache.FindDir(dir, true)
+	}
+	return err
 }
 
 // Rmdir deletes the container
 //
 // Returns an error if it isn't empty
-func (f *Fs) Rmdir() error {
-	err := f.dirCache.FindRoot(false)
+func (f *Fs) Rmdir(dir string) error {
+	root := path.Join(f.root, dir)
+	dc := f.dirCache
+	rootID, err := dc.FindDir(dir, false)
 	if err != nil {
 		return err
 	}
 	var children *drive.ChildList
 	err = f.pacer.Call(func() (bool, error) {
-		children, err = f.svc.Children.List(f.dirCache.RootID()).MaxResults(10).Do()
+		children, err = f.svc.Children.List(rootID).MaxResults(10).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {
@@ -616,20 +626,23 @@ func (f *Fs) Rmdir() error {
 		return errors.Errorf("directory not empty: %#v", children.Items)
 	}
 	// Delete the directory if it isn't the root
-	if f.root != "" {
+	if root != "" {
 		err = f.pacer.Call(func() (bool, error) {
 			if *driveUseTrash {
-				_, err = f.svc.Files.Trash(f.dirCache.RootID()).Do()
+				_, err = f.svc.Files.Trash(rootID).Do()
 			} else {
-				err = f.svc.Files.Delete(f.dirCache.RootID()).Do()
+				err = f.svc.Files.Delete(rootID).Do()
 			}
 			return shouldRetry(err)
 		})
 		if err != nil {
 			return err
 		}
 	}
-	f.dirCache.ResetRoot()
+	f.dirCache.FlushDir(dir)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 

diff --git a/drive/drive_test.go b/drive/drive_test.go
@@ -23,6 +23,7 @@ func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
 func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
 func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
 func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
+func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
 func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
 func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
 func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }

diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go
@@ -448,30 +448,33 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
 }
 
 // Mkdir creates the container if it doesn't exist
-func (f *Fs) Mkdir() error {
-	entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit)
+func (f *Fs) Mkdir(dir string) error {
+	root := path.Join(f.slashRoot, dir)
+	entry, err := f.db.Metadata(root, false, false, "", "", metadataLimit)
 	if err == nil {
 		if entry.IsDir {
 			return nil
 		}
 		return errors.Errorf("%q already exists as file", f.root)
 	}
-	_, err = f.db.CreateFolder(f.slashRoot)
+	_, err = f.db.CreateFolder(root)
 	return err
 }
 
 // Rmdir deletes the container
 //
 // Returns an error if it isn't empty
-func (f *Fs) Rmdir() error {
-	entry, err := f.db.Metadata(f.slashRoot, true, false, "", "", 16)
+func (f *Fs) Rmdir(dir string) error {
+	root := path.Join(f.slashRoot, dir)
+	entry, err := f.db.Metadata(root, true, false, "", "", 16)
 	if err != nil {
 		return err
 	}
 	if len(entry.Contents) != 0 {
 		return errors.New("directory not empty")
 	}
-	return f.Purge()
+	_, err = f.db.Delete(root)
+	return err
 }
 
 // Precision returns the precision

diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go
@@ -23,6 +23,7 @@ func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
 func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
 func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
 func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
+func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
 func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
 func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
 func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }

diff --git a/fs/fs.go b/fs/fs.go
@@ -135,12 +135,12 @@ type Fs interface {
 	// Mkdir makes the directory (container, bucket)
 	//
 	// Shouldn't return an error if it already exists
-	Mkdir() error
+	Mkdir(dir string) error
 
 	// Rmdir removes the directory (container, bucket) if empty
 	//
 	// Return an error if it doesn't exist or isn't empty
-	Rmdir() error
+	Rmdir(dir string) error
 }
 
 // Info provides an interface to reading information about a filesystem.

diff --git a/fs/operations.go b/fs/operations.go
@@ -707,12 +707,12 @@ func ListDir(f Fs, w io.Writer) error {
 }
 
 // Mkdir makes a destination directory or container
-func Mkdir(f Fs) error {
+func Mkdir(f Fs, dir string) error {
 	if Config.DryRun {
 		Log(f, "Not making directory as dry run is set")
 		return nil
 	}
-	err := f.Mkdir()
+	err := f.Mkdir(dir)
 	if err != nil {
 		Stats.Error()
 		return err
@@ -722,17 +722,17 @@ func Mkdir(f Fs) error {
 
 // TryRmdir removes a container but not if not empty.  It doesn't
 // count errors but may return one.
-func TryRmdir(f Fs) error {
+func TryRmdir(f Fs, dir string) error {
 	if Config.DryRun {
 		Log(f, "Not deleting as dry run is set")
 		return nil
 	}
-	return f.Rmdir()
+	return f.Rmdir(dir)
 }
 
 // Rmdir removes a container but not if not empty
-func Rmdir(f Fs) error {
-	err := TryRmdir(f)
+func Rmdir(f Fs, dir string) error {
+	err := TryRmdir(f, dir)
 	if err != nil {
 		Stats.Error()
 		return err
@@ -764,7 +764,7 @@ func Purge(f Fs) error {
 		if err != nil {
 			return err
 		}
-		err = Rmdir(f)
+		err = Rmdir(f, "")
 	}
 	if err != nil {
 		Stats.Error()