fs: Add context to NewFs rclone#3257 rclone#4685

This adds a context.Context parameter to NewFs and related calls.

This is necessary as part of reading config from the context -
backends need to be able to read the global config.

backend/alias/alias.go

@@ -1,6 +1,7 @@
 package alias
 
 import (
+	"context"
 	"errors"
 	"strings"
 
@@ -34,7 +35,7 @@ type Options struct {
 // NewFs constructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -47,5 +48,5 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if strings.HasPrefix(opt.Remote, name+":") {
 		return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
 	}
-	return cache.Get(fspath.JoinRootPath(opt.Remote, root))
+	return cache.Get(ctx, fspath.JoinRootPath(opt.Remote, root))
 }

backend/alias/alias_internal_test.go

@@ -68,7 +68,7 @@ func TestNewFS(t *testing.T) {
 		remoteRoot, err := filepath.Abs(filepath.FromSlash(path.Join("test/files", test.remoteRoot)))
 		require.NoError(t, err, what)
 		prepare(t, remoteRoot)
-		f, err := fs.NewFs(fmt.Sprintf("%s:%s", remoteName, test.fsRoot))
+		f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:%s", remoteName, test.fsRoot))
 		require.NoError(t, err, what)
 		gotEntries, err := f.List(context.Background(), test.fsList)
 		require.NoError(t, err, what)
@@ -90,15 +90,15 @@ func TestNewFS(t *testing.T) {
 
 func TestNewFSNoRemote(t *testing.T) {
 	prepare(t, "")
-	f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName))
+	f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:", remoteName))
 
 	require.Error(t, err)
 	require.Nil(t, f)
 }
 
 func TestNewFSInvalidRemote(t *testing.T) {
 	prepare(t, "not_existing_test_remote:")
-	f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName))
+	f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:", remoteName))
 
 	require.Error(t, err)
 	require.Nil(t, f)

backend/amazonclouddrive/amazonclouddrive.go

@@ -239,8 +239,7 @@ func filterRequest(req *http.Request) {
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	ctx := context.Background()
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)

backend/azureblob/azureblob.go

@@ -379,8 +379,7 @@ func (f *Fs) setRoot(root string) {
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	ctx := context.Background()
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)

backend/b2/b2.go

@@ -391,8 +391,7 @@ func (f *Fs) setRoot(root string) {
 }
 
 // NewFs constructs an Fs from the path, bucket:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	ctx := context.Background()
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)

backend/box/box.go

@@ -372,8 +372,7 @@ func errorHandler(resp *http.Response) error {
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	ctx := context.Background()
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)

backend/cache/cache.go

@@ -340,7 +340,7 @@ func parseRootPath(path string) (string, error) {
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -362,7 +362,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	remotePath := fspath.JoinRootPath(opt.Remote, rootPath)
-	wrappedFs, wrapErr := cache.Get(remotePath)
+	wrappedFs, wrapErr := cache.Get(ctx, remotePath)
 	if wrapErr != nil && wrapErr != fs.ErrorIsFile {
 		return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath)
 	}
@@ -479,7 +479,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 			return nil, errors.Wrapf(err, "failed to create cache directory %v", f.opt.TempWritePath)
 		}
 		f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath)
-		f.tempFs, err = cache.Get(f.opt.TempWritePath)
+		f.tempFs, err = cache.Get(ctx, f.opt.TempWritePath)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to create temp fs: %v", err)
 		}
@@ -506,7 +506,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	if doChangeNotify := wrappedFs.Features().ChangeNotify; doChangeNotify != nil {
 		pollInterval := make(chan time.Duration, 1)
 		pollInterval <- time.Duration(f.opt.ChunkCleanInterval)
-		doChangeNotify(context.Background(), f.receiveChangeNotify, pollInterval)
+		doChangeNotify(ctx, f.receiveChangeNotify, pollInterval)
 	}
 
 	f.features = (&fs.Features{

backend/cache/cache_internal_test.go

@@ -932,7 +932,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		boltDb.PurgeTempUploads()
 		_ = os.RemoveAll(path.Join(runInstance.tmpUploadDir, id))
 	}
-	f, err := cache.NewFs(remote, id, m)
+	f, err := cache.NewFs(context.Background(), remote, id, m)
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
 	require.NoError(t, err)

backend/chunker/chunker.go

@@ -223,7 +223,7 @@ It has the following fields: ver, size, nchunks, md5, sha1.`,
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -248,7 +248,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	}
 	// Look for a file first
 	remotePath := fspath.JoinRootPath(basePath, rpath)
-	baseFs, err := cache.Get(baseName + remotePath)
+	baseFs, err := cache.Get(ctx, baseName+remotePath)
 	if err != fs.ErrorIsFile && err != nil {
 		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", baseName+remotePath)
 	}
@@ -276,7 +276,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// (yet can't satisfy fstest.CheckListing, will ignore)
 	if err == nil && !f.useMeta && strings.Contains(rpath, "/") {
 		firstChunkPath := f.makeChunkName(remotePath, 0, "", "")
-		_, testErr := cache.Get(baseName + firstChunkPath)
+		_, testErr := cache.Get(ctx, baseName+firstChunkPath)
 		if testErr == fs.ErrorIsFile {
 			err = testErr
 		}

backend/crypt/crypt.go

@@ -144,7 +144,7 @@ func NewCipher(m configmap.Mapper) (*Cipher, error) {
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -166,14 +166,14 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Look for a file first
 	var wrappedFs fs.Fs
 	if rpath == "" {
-		wrappedFs, err = cache.Get(remote)
+		wrappedFs, err = cache.Get(ctx, remote)
 	} else {
 		remotePath := fspath.JoinRootPath(remote, cipher.EncryptFileName(rpath))
-		wrappedFs, err = cache.Get(remotePath)
+		wrappedFs, err = cache.Get(ctx, remotePath)
 		// if that didn't produce a file, look for a directory
 		if err != fs.ErrorIsFile {
 			remotePath = fspath.JoinRootPath(remote, cipher.EncryptDirName(rpath))
-			wrappedFs, err = cache.Get(remotePath)
+			wrappedFs, err = cache.Get(ctx, remotePath)
 		}
 	}
 	if err != fs.ErrorIsFile && err != nil {

backend/crypt/crypt_internal_test.go

@@ -33,7 +33,7 @@ func (o testWrapper) UnWrap() fs.Object {
 // Create a temporary local fs to upload things from
 
 func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
-	localFs, err := fs.TemporaryLocalFs()
+	localFs, err := fs.TemporaryLocalFs(context.Background())
 	require.NoError(t, err)
 	cleanup = func() {
 		require.NoError(t, localFs.Rmdir(context.Background(), ""))

backend/drive/drive.go

@@ -1129,8 +1129,7 @@ func newFs(name, path string, m configmap.Mapper) (*Fs, error) {
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
-	ctx := context.Background()
+func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, error) {
 	f, err := newFs(name, path, m)
 	if err != nil {
 		return nil, err
@@ -3000,7 +2999,7 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 	if destLeaf == "" {
 		destLeaf = info.Name
 	}
-	dstFs, err := cache.Get(destDir)
+	dstFs, err := cache.Get(ctx, destDir)
 	if err != nil {
 		return err
 	}
@@ -3187,7 +3186,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		dstFs := f
 		target, ok := opt["target"]
 		if ok {
-			targetFs, err := cache.Get(target)
+			targetFs, err := cache.Get(ctx, target)
 			if err != nil {
 				return nil, errors.Wrap(err, "couldn't find target")
 			}

backend/drive/drive_internal_test.go

@@ -196,7 +196,7 @@ func (f *Fs) InternalTestDocumentImport(t *testing.T) {
 	testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
 	require.NoError(t, err)
 
-	testFilesFs, err := fs.NewFs(testFilesPath)
+	testFilesFs, err := fs.NewFs(context.Background(), testFilesPath)
 	require.NoError(t, err)
 
 	_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
@@ -210,7 +210,7 @@ func (f *Fs) InternalTestDocumentUpdate(t *testing.T) {
 	testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
 	require.NoError(t, err)
 
-	testFilesFs, err := fs.NewFs(testFilesPath)
+	testFilesFs, err := fs.NewFs(context.Background(), testFilesPath)
 	require.NoError(t, err)
 
 	_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")

backend/dropbox/dropbox.go

@@ -291,7 +291,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)

backend/fichier/fichier.go

@@ -167,7 +167,7 @@ func (f *Fs) Features() *fs.Features {
 //
 // On Windows avoid single character remote names as they can be mixed
 // up with drive letters.
-func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) {
+func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
 	opt := new(Options)
 	err := configstruct.Set(config, opt)
 	if err != nil {
@@ -203,8 +203,6 @@ func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) {
 
 	f.dirCache = dircache.New(root, rootID, f)
 
-	ctx := context.Background()
-
 	// Find the current root
 	err = f.dirCache.FindRoot(ctx, false)
 	if err != nil {

backend/ftp/ftp.go

@@ -300,8 +300,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
-	ctx := context.Background()
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
 	// Parse config into Options struct
 	opt := new(Options)

backend/googlecloudstorage/googlecloudstorage.go

@@ -386,8 +386,7 @@ func (f *Fs) setRoot(root string) {
 }
 
 // NewFs constructs an Fs from the path, bucket:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	ctx := context.TODO()
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	var oAuthClient *http.Client
 
 	// Parse config into Options struct

backend/googlephotos/googlephotos.go

@@ -246,7 +246,7 @@ func errorHandler(resp *http.Response) error {
 }
 
 // NewFs constructs an Fs from the path, bucket:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -288,7 +288,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		var leaf string
 		f.root, leaf = path.Split(f.root)
 		f.root = strings.TrimRight(f.root, "/")
-		_, err := f.NewObject(context.TODO(), leaf)
+		_, err := f.NewObject(ctx, leaf)
 		if err == nil {
 			return f, fs.ErrorIsFile
 		}

backend/googlephotos/googlephotos_test.go

@@ -35,14 +35,14 @@ func TestIntegration(t *testing.T) {
 	if *fstest.RemoteName == "" {
 		*fstest.RemoteName = "TestGooglePhotos:"
 	}
-	f, err := fs.NewFs(*fstest.RemoteName)
+	f, err := fs.NewFs(ctx, *fstest.RemoteName)
 	if err == fs.ErrorNotFoundInConfigFile {
 		t.Skip(fmt.Sprintf("Couldn't create google photos backend - skipping tests: %v", err))
 	}
 	require.NoError(t, err)
 
 	// Create local Fs pointing at testfiles
-	localFs, err := fs.NewFs("testfiles")
+	localFs, err := fs.NewFs(ctx, "testfiles")
 	require.NoError(t, err)
 
 	t.Run("CreateAlbum", func(t *testing.T) {
@@ -155,7 +155,7 @@ func TestIntegration(t *testing.T) {
 			})
 
 			t.Run("NewFsIsFile", func(t *testing.T) {
-				fNew, err := fs.NewFs(*fstest.RemoteName + remote)
+				fNew, err := fs.NewFs(ctx, *fstest.RemoteName+remote)
 				assert.Equal(t, fs.ErrorIsFile, err)
 				leaf := path.Base(remote)
 				o, err := fNew.NewObject(ctx, leaf)

backend/http/http.go

@@ -145,8 +145,7 @@ func statusError(res *http.Response, err error) error {
 
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	ctx := context.TODO()
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)

backend/http/http_internal_test.go

@@ -69,7 +69,7 @@ func prepare(t *testing.T) (fs.Fs, func()) {
 	m, tidy := prepareServer(t)
 
 	// Instantiate it
-	f, err := NewFs(remoteName, "", m)
+	f, err := NewFs(context.Background(), remoteName, "", m)
 	require.NoError(t, err)
 
 	return f, tidy
@@ -214,7 +214,7 @@ func TestIsAFileRoot(t *testing.T) {
 	m, tidy := prepareServer(t)
 	defer tidy()
 
-	f, err := NewFs(remoteName, "one%.txt", m)
+	f, err := NewFs(context.Background(), remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
 	testListRoot(t, f, false)
@@ -224,7 +224,7 @@ func TestIsAFileSubDir(t *testing.T) {
 	m, tidy := prepareServer(t)
 	defer tidy()
 
-	f, err := NewFs(remoteName, "three/underthree.txt", m)
+	f, err := NewFs(context.Background(), remoteName, "three/underthree.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
 	entries, err := f.List(context.Background(), "")

backend/hubic/hubic.go

@@ -146,7 +146,7 @@ func (f *Fs) getCredentials(ctx context.Context) (err error) {
 }
 
 // NewFs constructs an Fs from the path, container:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	client, _, err := oauthutil.NewClient(name, m, oauthConfig)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure Hubic")
@@ -176,7 +176,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// Make inner swift Fs from the connection
-	swiftFs, err := swift.NewFsWithConnection(opt, name, root, c, true)
+	swiftFs, err := swift.NewFsWithConnection(ctx, opt, name, root, c, true)
 	if err != nil && err != fs.ErrorIsFile {
 		return nil, err
 	}