diff --git a/fs/fs.go b/fs/fs.go index 6a85a5c712b59..86af2c2735c84 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -49,6 +49,7 @@ var ( ErrorIsFile = errors.New("is a file not a directory") ErrorNotAFile = errors.New("is a not a regular file") ErrorNotDeleting = errors.New("not deleting files as there were IO errors") + ErrorNotDeletingDirs = errors.New("not deleting directories as there were IO errors") ErrorCantMoveOverlapping = errors.New("can't move files on overlapping remotes") ErrorDirectoryNotEmpty = errors.New("directory not empty") ) diff --git a/fs/sync.go b/fs/sync.go index 601d948f470f2..c7eb57ea283bc 100644 --- a/fs/sync.go +++ b/fs/sync.go @@ -4,6 +4,7 @@ package fs import ( "fmt" + "sort" "sync" "time" @@ -30,6 +31,8 @@ type syncCopyMove struct { srcFilesChan chan Object // passes src objects srcFilesResult chan error // error result of src listing dstFilesResult chan error // error result of dst listing + dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs + dstEmptyDirs []DirEntry // potentially empty directories abort chan struct{} // signal to abort the copiers checkerWg sync.WaitGroup // wait for checkers toBeChecked ObjectPairChan // checkers channel @@ -508,6 +511,46 @@ func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error { return deleteFilesWithBackupDir(toDelete, s.backupDir) } +// This deletes the empty directories in the slice passed in. It +// ignores any errors deleting directories +func deleteEmptyDirectories(f Fs, entries DirEntries) error { + if len(entries) == 0 { + return nil + } + if Stats.Errored() { + Errorf(f, "%v", ErrorNotDeletingDirs) + return ErrorNotDeletingDirs + } + + // Now delete the empty directories starting from the longest path + sort.Sort(entries) + var errorCount int + var okCount int + for i := len(entries) - 1; i >= 0; i-- { + entry := entries[i] + dir, ok := entry.(Directory) + if ok { + // TryRmdir only deletes empty directories + err := TryRmdir(f, dir.Remote()) + if err != nil { + Debugf(logDirName(f, dir.Remote()), "Failed to Rmdir: %v", err) + errorCount++ + } else { + okCount++ + } + } else { + Errorf(f, "Not a directory: %v", entry) + } + } + if errorCount > 0 { + Debugf(f, "failed to delete %d directories", errorCount) + } + if okCount > 0 { + Debugf(f, "deleted %d directories", okCount) + } + return nil +} + // renameHash makes a string with the size and the hash for rename detection // // it may return an empty string in which case no hash could be made @@ -680,7 +723,7 @@ func (s *syncCopyMove) run() error { if !ok { return } - jobs := s._run(job) + jobs := s.processJob(job) if len(jobs) > 0 { traversing.Add(len(jobs)) go func() { @@ -734,6 +777,15 @@ func (s *syncCopyMove) run() error { s.processError(s.deleteFiles(false)) } } + + // Prune empty directories + if s.deleteMode != DeleteModeOff { + if s.currentError() != nil { + Errorf(s.fdst, "%v", ErrorNotDeletingDirs) + } else { + s.processError(deleteEmptyDirectories(s.fdst, s.dstEmptyDirs)) + } + } return s.currentError() } @@ -764,6 +816,12 @@ func (s *syncCopyMove) dstOnly(dst DirEntry, job listDirJob, jobs *[]listDirJob) noSrc: true, }) } + // Record directory as it is potentially empty and needs deleting + if s.fdst.Features().CanHaveEmptyDirectories { + s.dstEmptyDirsMu.Lock() + s.dstEmptyDirs = append(s.dstEmptyDirs, dst) + s.dstEmptyDirsMu.Unlock() + } default: panic("Bad object in DirEntries") @@ -907,8 +965,12 @@ func matchListings(srcList, dstList DirEntries) (srcOnly DirEntries, dstOnly Dir return } +// processJob processes a listDirJob listing the source and +// destination directories, comparing them and returning a slice of +// more jobs +// // returns errors using processError -func (s *syncCopyMove) _run(job listDirJob) (jobs []listDirJob) { +func (s *syncCopyMove) processJob(job listDirJob) (jobs []listDirJob) { var ( srcList, dstList DirEntries srcListErr, dstListErr error diff --git a/fs/sync_test.go b/fs/sync_test.go index e09d29c3e3f88..ca8611c6d5625 100644 --- a/fs/sync_test.go +++ b/fs/sync_test.go @@ -476,32 +476,141 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) { file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) file2 := r.WriteObject("b/potato", "SMALLER BUT SAME DATE", t2) file3 := r.WriteBoth("c/non empty space", "AhHa!", t2) - fstest.CheckItems(t, r.fremote, file2, file3) - fstest.CheckItems(t, r.flocal, file1, file3) + require.NoError(t, fs.Mkdir(r.fremote, "d")) + require.NoError(t, fs.Mkdir(r.fremote, "d/e")) + + fstest.CheckListingWithPrecision( + t, + r.flocal, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + fs.Config.ModifyWindow, + ) + fstest.CheckListingWithPrecision( + t, + r.fremote, + []fstest.Item{ + file2, + file3, + }, + []string{ + "b", + "c", + "d", + "d/e", + }, + fs.Config.ModifyWindow, + ) fs.Stats.ResetCounters() err := fs.Sync(r.fremote, r.flocal) require.NoError(t, err) - fstest.CheckItems(t, r.flocal, file1, file3) - fstest.CheckItems(t, r.fremote, file1, file3) + + fstest.CheckListingWithPrecision( + t, + r.flocal, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + fs.Config.ModifyWindow, + ) + fstest.CheckListingWithPrecision( + t, + r.fremote, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + fs.Config.ModifyWindow, + ) } // Sync after removing a file and adding a file with IO Errors -func TestSyncAfterRemovingAFileAndAddingAFileWithErrors(t *testing.T) { +func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) { r := NewRun(t) defer r.Finalise() - file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1) - file2 := r.WriteObject("potato", "SMALLER BUT SAME DATE", t2) - file3 := r.WriteBoth("empty space", "", t2) - fstest.CheckItems(t, r.fremote, file2, file3) - fstest.CheckItems(t, r.flocal, file1, file3) + file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) + file2 := r.WriteObject("b/potato", "SMALLER BUT SAME DATE", t2) + file3 := r.WriteBoth("c/non empty space", "AhHa!", t2) + require.NoError(t, fs.Mkdir(r.fremote, "d")) + + fstest.CheckListingWithPrecision( + t, + r.flocal, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + fs.Config.ModifyWindow, + ) + fstest.CheckListingWithPrecision( + t, + r.fremote, + []fstest.Item{ + file2, + file3, + }, + []string{ + "b", + "c", + "d", + }, + fs.Config.ModifyWindow, + ) fs.Stats.ResetCounters() fs.Stats.Error() err := fs.Sync(r.fremote, r.flocal) assert.Equal(t, fs.ErrorNotDeleting, err) - fstest.CheckItems(t, r.flocal, file1, file3) - fstest.CheckItems(t, r.fremote, file1, file2, file3) + + fstest.CheckListingWithPrecision( + t, + r.flocal, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + fs.Config.ModifyWindow, + ) + fstest.CheckListingWithPrecision( + t, + r.fremote, + []fstest.Item{ + file1, + file2, + file3, + }, + []string{ + "a", + "b", + "c", + "d", + }, + fs.Config.ModifyWindow, + ) } // Sync test delete after