rcat: directly upload small files without streaming them

jfabaf · Sep 11, 2017 · 5781739 · 5781739
1 parent 10fa2a7
commit 5781739
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 18 deletions.
diff --git a/fs/operations.go b/fs/operations.go
@@ -1587,8 +1587,28 @@ func Rcat(fdst Fs, dstFileName string, in0 io.ReadCloser, modTime time.Time) (er
 	Stats.Transferring(dstFileName)
 	defer func() {
 		Stats.DoneTransferring(dstFileName, err == nil)
+		if err := in0.Close(); err != nil {
+			Debugf(fdst, "Rcat: failed to close source: %v", err)
+		}
 	}()
 
+	hashOption := &HashesOption{Hashes: NewHashSet()}
+
+	in := in0
+	buf := make([]byte, 100*1024)
+	if n, err := io.ReadFull(in0, buf); err != nil {
+		Debugf(fdst, "File to upload is small, uploading instead of streaming")
+		in = ioutil.NopCloser(bytes.NewReader(buf[:n]))
+		in = NewAccountSizeName(in, int64(n), dstFileName).WithBuffer()
+		if !Config.SizeOnly {
+			hashOption = &HashesOption{Hashes: HashSet(fdst.Hashes().GetOne())}
+		}
+		objInfo := NewStaticObjectInfo(dstFileName, modTime, int64(n), false, nil, nil)
+		_, err := fdst.Put(in, objInfo, hashOption)
+		return err
+	}
+	in = ioutil.NopCloser(io.MultiReader(bytes.NewReader(buf), in0))
+
 	fStreamTo := fdst
 	canStream := fdst.Features().PutStream != nil
 	if !canStream {
@@ -1606,21 +1626,11 @@ func Rcat(fdst Fs, dstFileName string, in0 io.ReadCloser, modTime time.Time) (er
 		fStreamTo = tmpLocalFs
 	}
 
-	objInfo := NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, nil)
-
-	// work out which hash to use - limit to 1 hash in common
-	var common HashSet
-	hashType := HashNone
 	if !Config.SizeOnly {
-		common = fStreamTo.Hashes().Overlap(SupportedHashes)
-		if common.Count() > 0 {
-			hashType = common.GetOne()
-			common = HashSet(hashType)
-		}
+		hashOption = &HashesOption{Hashes: HashSet(fStreamTo.Hashes().GetOne())}
 	}
-	hashOption := &HashesOption{Hashes: common}
 
-	in := NewAccountSizeName(in0, -1, dstFileName).WithBuffer()
+	in = NewAccountSizeName(in, -1, dstFileName).WithBuffer()
 
 	if Config.DryRun {
 		Logf("stdin", "Not copying as --dry-run")
@@ -1629,6 +1639,7 @@ func Rcat(fdst Fs, dstFileName string, in0 io.ReadCloser, modTime time.Time) (er
 		return err
 	}
 
+	objInfo := NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, nil)
 	tmpObj, err := fStreamTo.Features().PutStream(in, objInfo, hashOption)
 	if err == nil && !canStream {
 		err = Copy(fdst, nil, dstFileName, tmpObj)

diff --git a/fs/operations_test.go b/fs/operations_test.go
@@ -737,15 +737,23 @@ func TestRcat(t *testing.T) {
 
 	fstest.CheckListing(t, r.fremote, []fstest.Item{})
 
-	data := "this is some really nice test data"
-	path := "file_from_pipe"
+	data1 := "this is some really nice test data"
+	path1 := "small_file_from_pipe"
 
-	in := ioutil.NopCloser(strings.NewReader(data))
-	err := fs.Rcat(r.fremote, path, in, t1)
+	data2 := string(make([]byte, 100*1024+1))
+	path2 := "big_file_from_pipe"
+
+	in := ioutil.NopCloser(strings.NewReader(data1))
+	err := fs.Rcat(r.fremote, path1, in, t1)
+	require.NoError(t, err)
+
+	in = ioutil.NopCloser(strings.NewReader(data2))
+	err = fs.Rcat(r.fremote, path2, in, t2)
 	require.NoError(t, err)
 
-	file := fstest.NewItem(path, data, t1)
-	fstest.CheckItems(t, r.fremote, file)
+	file1 := fstest.NewItem(path1, data1, t1)
+	file2 := fstest.NewItem(path2, data2, t2)
+	fstest.CheckItems(t, r.fremote, file1, file2)
 }
 
 func TestRmdirs(t *testing.T) {