dvc: split remotes from trees (iterative#4212)

* dvc: remove tree-specific wrappers from remote

This helps combat the remote/tree/cache confusion.

* dvc: split remotes from trees

Related to iterative#4050

dvc/api.py

@@ -30,7 +30,7 @@ def get_url(path, repo=None, rev=None, remote=None):
             raise UrlNotDvcRepoError(_repo.url)  # pylint: disable=no-member
         out = _repo.find_out_by_relpath(path)
         remote_obj = _repo.cloud.get_remote(remote)
-        return str(remote_obj.hash_to_path_info(out.checksum))
+        return str(remote_obj.tree.hash_to_path_info(out.checksum))
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/command/version.py

@@ -122,7 +122,7 @@ def get_linktype_support_info(repo):
 
     @staticmethod
     def get_supported_remotes():
-        from dvc.remote import TREES
+        from dvc.tree import TREES
 
         supported_remotes = []
         for tree_cls in TREES:

dvc/dependency/__init__.py

@@ -12,9 +12,9 @@
 from dvc.dependency.s3 import S3Dependency
 from dvc.dependency.ssh import SSHDependency
 from dvc.output.base import BaseOutput
-from dvc.remote import get_cloud_tree
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
 from .repo import RepoDependency
 
 DEPS = [

dvc/dependency/azure.py

@@ -1,6 +1,7 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
-from dvc.remote.azure import AzureRemoteTree
+
+from ..tree.azure import AzureRemoteTree
 
 
 class AzureDependency(BaseDependency, BaseOutput):

dvc/dependency/http.py

@@ -1,6 +1,7 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
-from dvc.remote.http import HTTPRemoteTree
+
+from ..tree.http import HTTPRemoteTree
 
 
 class HTTPDependency(BaseDependency, BaseOutput):

dvc/dependency/https.py

@@ -1,5 +1,4 @@
-from dvc.remote.https import HTTPSRemoteTree
-
+from ..tree.https import HTTPSRemoteTree
 from .http import HTTPDependency
 
 

dvc/main.py

@@ -9,7 +9,7 @@
 from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER, disable_other_loggers
-from dvc.remote.pool import close_pools
+from dvc.tree.pool import close_pools
 from dvc.utils import format_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.

dvc/output/__init__.py

@@ -10,12 +10,13 @@
 from dvc.output.local import LocalOutput
 from dvc.output.s3 import S3Output
 from dvc.output.ssh import SSHOutput
-from dvc.remote import get_cloud_tree
-from dvc.remote.hdfs import HDFSRemoteTree
-from dvc.remote.local import LocalRemoteTree
-from dvc.remote.s3 import S3RemoteTree
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
+from ..tree.hdfs import HDFSRemoteTree
+from ..tree.local import LocalRemoteTree
+from ..tree.s3 import S3RemoteTree
+
 OUTS = [
     HDFSOutput,
     S3Output,

dvc/output/base.py

@@ -12,7 +12,8 @@
     DvcException,
     RemoteCacheRequiredError,
 )
-from dvc.remote.base import BaseRemoteTree
+
+from ..tree.base import BaseRemoteTree
 
 logger = logging.getLogger(__name__)
 

dvc/output/gs.py

@@ -1,5 +1,6 @@
 from dvc.output.s3 import S3Output
-from dvc.remote.gs import GSRemoteTree
+
+from ..tree.gs import GSRemoteTree
 
 
 class GSOutput(S3Output):

dvc/output/hdfs.py

@@ -1,5 +1,6 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.hdfs import HDFSRemoteTree
+
+from ..tree.hdfs import HDFSRemoteTree
 
 
 class HDFSOutput(BaseOutput):

dvc/output/local.py

@@ -5,10 +5,11 @@
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
 from dvc.output.base import BaseOutput
-from dvc.remote.local import LocalRemoteTree
 from dvc.utils import relpath
 from dvc.utils.fs import path_isin
 
+from ..tree.local import LocalRemoteTree
+
 logger = logging.getLogger(__name__)
 
 

dvc/output/s3.py

@@ -1,5 +1,6 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.s3 import S3RemoteTree
+
+from ..tree.s3 import S3RemoteTree
 
 
 class S3Output(BaseOutput):

dvc/output/ssh.py

@@ -1,5 +1,6 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.ssh import SSHRemoteTree
+
+from ..tree.ssh import SSHRemoteTree
 
 
 class SSHOutput(BaseOutput):

dvc/remote/__init__.py

@@ -1,82 +1,7 @@
-import posixpath
-from urllib.parse import urlparse
-
-from dvc.remote.azure import AzureRemoteTree
-from dvc.remote.base import Remote
-from dvc.remote.gdrive import GDriveRemoteTree
-from dvc.remote.gs import GSRemoteTree
-from dvc.remote.hdfs import HDFSRemoteTree
-from dvc.remote.http import HTTPRemoteTree
-from dvc.remote.https import HTTPSRemoteTree
-from dvc.remote.local import LocalRemote, LocalRemoteTree
-from dvc.remote.oss import OSSRemoteTree
-from dvc.remote.s3 import S3RemoteTree
-from dvc.remote.ssh import SSHRemote, SSHRemoteTree
-
-TREES = [
-    AzureRemoteTree,
-    GDriveRemoteTree,
-    GSRemoteTree,
-    HDFSRemoteTree,
-    HTTPRemoteTree,
-    HTTPSRemoteTree,
-    S3RemoteTree,
-    SSHRemoteTree,
-    OSSRemoteTree,
-    # NOTE: LocalRemoteTree is the default
-]
-
-
-def _get_tree(remote_conf):
-    for tree_cls in TREES:
-        if tree_cls.supported(remote_conf):
-            return tree_cls
-    return LocalRemoteTree
-
-
-def _get_conf(repo, **kwargs):
-    name = kwargs.get("name")
-    if name:
-        remote_conf = repo.config["remote"][name.lower()]
-    else:
-        remote_conf = kwargs
-    return _resolve_remote_refs(repo.config, remote_conf)
-
-
-def _resolve_remote_refs(config, remote_conf):
-    # Support for cross referenced remotes.
-    # This will merge the settings, shadowing base ref with remote_conf.
-    # For example, having:
-    #
-    #       dvc remote add server ssh://localhost
-    #       dvc remote modify server user root
-    #       dvc remote modify server ask_password true
-    #
-    #       dvc remote add images remote://server/tmp/pictures
-    #       dvc remote modify images user alice
-    #       dvc remote modify images ask_password false
-    #       dvc remote modify images password asdf1234
-    #
-    # Results on a config dictionary like:
-    #
-    #       {
-    #           "url": "ssh://localhost/tmp/pictures",
-    #           "user": "alice",
-    #           "password": "asdf1234",
-    #           "ask_password": False,
-    #       }
-    parsed = urlparse(remote_conf["url"])
-    if parsed.scheme != "remote":
-        return remote_conf
-
-    base = config["remote"][parsed.netloc]
-    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
-    return {**base, **remote_conf, "url": url}
-
-
-def get_cloud_tree(repo, **kwargs):
-    remote_conf = _get_conf(repo, **kwargs)
-    return _get_tree(remote_conf)(repo, remote_conf)
+from ..tree import get_cloud_tree
+from .base import Remote
+from .local import LocalRemote
+from .ssh import SSHRemote
 
 
 def get_remote(repo, **kwargs):