tests: create remote fixtures (iterative#4063)

* tests: use remote fixtures

* tests: use remote fixtures for data cloud tests

* tests: treat remotes as tmp dirs

setup.cfg

@@ -12,7 +12,7 @@ select=B,C,E,F,W,T4,B9
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
 line_length=79
 force_grid_wrap=0
 use_parentheses=True

setup.py

@@ -106,6 +106,7 @@ def run(self):
     "pytest-cov>=2.6.1",
     "pytest-xdist>=1.26.1",
     "pytest-mock==1.11.2",
+    "pytest-lazy-fixture",
     "flaky>=3.5.3",
     "mock>=3.0.0",
     "xmltodict>=0.11.0",

tests/conftest.py

@@ -1,12 +1,9 @@
 import os
 
-import mockssh
 import pytest
 
-from dvc.remote.ssh.connection import SSHConnection
-from tests.utils.httpd import PushRequestHandler, StaticFileServer
-
 from .dir_helpers import *  # noqa
+from .remotes import *  # noqa
 
 # Prevent updater and analytics from running their processes
 os.environ["DVC_TEST"] = "true"
@@ -28,39 +25,9 @@ def reset_loglevel(request, caplog):
         yield
 
 
-here = os.path.abspath(os.path.dirname(__file__))
-
-user = "user"
-key_path = os.path.join(here, f"{user}.key")
-
-
-@pytest.fixture(scope="session")
-def ssh_server():
-    users = {user: key_path}
-    with mockssh.Server(users) as s:
-        s.test_creds = {
-            "host": s.host,
-            "port": s.port,
-            "username": user,
-            "key_filename": key_path,
-        }
-        yield s
-
-
-@pytest.fixture
-def ssh(ssh_server):
-    yield SSHConnection(**ssh_server.test_creds)
-
-
 @pytest.fixture(scope="session", autouse=True)
 def _close_pools():
     from dvc.remote.pool import close_pools
 
     yield
     close_pools()
-
-
-@pytest.fixture
-def http_server(tmp_dir):
-    with StaticFileServer(handler_class=PushRequestHandler) as httpd:
-        yield httpd

tests/dir_helpers.py

@@ -63,7 +63,6 @@
     "run_head",
     "erepo_dir",
     "git_dir",
-    "setup_remote",
     "git_init",
 ]
 
@@ -177,6 +176,27 @@ def scm_add(self, filenames, commit=None):
         if commit:
             self.scm.commit(commit)
 
+    def add_remote(
+        self, *, url=None, config=None, name="upstream", default=True
+    ):
+        self._require("dvc")
+
+        assert bool(url) ^ bool(config)
+
+        if url:
+            config = {"url": url}
+
+        with self.dvc.config.edit() as conf:
+            conf["remote"][name] = config
+            if default:
+                conf["core"]["remote"] = name
+
+        if hasattr(self, "scm"):
+            self.scm.add(self.dvc.config.files["repo"])
+            self.scm.commit(f"add '{name}' remote")
+
+        return url or config["url"]
+
     # contexts
     @contextmanager
     def chdir(self):
@@ -315,20 +335,3 @@ def git_dir(make_tmp_dir):
     path = make_tmp_dir("git-erepo", scm=True)
     path.scm.commit("init repo")
     return path
-
-
-@pytest.fixture
-def setup_remote(make_tmp_dir):
-    def create(repo, url=None, name="upstream", default=True):
-        if not url:
-            url = os.fspath(make_tmp_dir("local_remote"))
-        with repo.config.edit() as conf:
-            conf["remote"][name] = {"url": url}
-            if default:
-                conf["core"]["remote"] = name
-
-        repo.scm.add(repo.config.files["repo"])
-        repo.scm.commit(f"add '{name}' remote")
-        return url
-
-    return create

tests/func/test_api.py

@@ -5,93 +5,42 @@
 from dvc import api
 from dvc.api import UrlNotDvcRepoError
 from dvc.exceptions import FileMissingError
-from dvc.main import main
 from dvc.path_info import URLInfo
 from dvc.utils.fs import remove
-from tests.remotes import (
-    GCP,
-    HDFS,
-    OSS,
-    S3,
-    SSH,
-    TEST_REMOTE,
-    Azure,
-    GDrive,
-    Local,
-)
-
-remote_params = [S3, GCP, Azure, GDrive, OSS, SSH, HDFS]
-all_remote_params = [Local] + remote_params
-
-
-@pytest.fixture
-def remote_url(request):
-    if not request.param.should_test():
-        raise pytest.skip()
-    return request.param.get_url()
-
-
-def run_dvc(*argv):
-    assert main(argv) == 0
-
-
-def ensure_dir(dvc, url):
-    if url.startswith("gdrive://"):
-        GDrive.create_dir(dvc, url)
-        run_dvc(
-            "remote",
-            "modify",
-            TEST_REMOTE,
-            "gdrive_service_account_email",
-            "test",
-        )
-        run_dvc(
-            "remote",
-            "modify",
-            TEST_REMOTE,
-            "gdrive_service_account_p12_file_path",
-            "test.p12",
-        )
-        run_dvc(
-            "remote",
-            "modify",
-            TEST_REMOTE,
-            "gdrive_use_service_account",
-            "True",
-        )
-
-
-def ensure_dir_scm(dvc, url):
-    if url.startswith("gdrive://"):
-        GDrive.create_dir(dvc, url)
-        with dvc.config.edit() as conf:
-            conf["remote"][TEST_REMOTE].update(
-                gdrive_service_account_email="test",
-                gdrive_service_account_p12_file_path="test.p12",
-                gdrive_use_service_account=True,
-            )
-        dvc.scm.add(dvc.config.files["repo"])
-        dvc.scm.commit(f"modify '{TEST_REMOTE}' remote")
-
-
-@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
-def test_get_url(tmp_dir, dvc, remote_url):
-    run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
+
+cloud_names = [
+    "s3",
+    "gs",
+    "azure",
+    "gdrive",
+    "oss",
+    "ssh",
+    "hdfs",
+    "http",
+]
+clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
+all_clouds = [pytest.lazy_fixture("local")] + clouds
+remotes = [pytest.lazy_fixture(f"{cloud}_remote") for cloud in cloud_names]
+all_remotes = [pytest.lazy_fixture("local_remote")] + remotes
+
+
+@pytest.mark.parametrize("remote", remotes)
+def test_get_url(tmp_dir, dvc, request, remote):
     tmp_dir.dvc_gen("foo", "foo")
 
-    expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
+    expected_url = URLInfo(remote.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
     assert api.get_url("foo") == expected_url
 
 
-@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
-def test_get_url_external(erepo_dir, remote_url, setup_remote):
-    setup_remote(erepo_dir.dvc, url=remote_url)
+@pytest.mark.parametrize("cloud", clouds)
+def test_get_url_external(erepo_dir, cloud):
+    erepo_dir.add_remote(config=cloud.config)
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("foo", "foo", commit="add foo")
 
     # Using file url to force clone to tmp repo
     repo_url = f"file://{erepo_dir}"
-    expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
+    expected_url = URLInfo(cloud.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
     assert api.get_url("foo", repo=repo_url) == expected_url
 
 
@@ -105,12 +54,10 @@ def test_get_url_requires_dvc(tmp_dir, scm):
         api.get_url("foo", repo=f"file://{tmp_dir}")
 
 
-@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
-def test_open(remote_url, tmp_dir, dvc):
-    run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
-    ensure_dir(dvc, remote_url)
+@pytest.mark.parametrize("remote", all_remotes)
+def test_open(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen("foo", "foo-text")
-    run_dvc("push")
+    dvc.push()
 
     # Remove cache to force download
     remove(dvc.cache.local.cache_dir)
@@ -119,10 +66,9 @@ def test_open(remote_url, tmp_dir, dvc):
         assert fd.read() == "foo-text"
 
 
-@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
-def test_open_external(remote_url, erepo_dir, setup_remote):
-    setup_remote(erepo_dir.dvc, url=remote_url)
-    ensure_dir_scm(erepo_dir.dvc, remote_url)
+@pytest.mark.parametrize("cloud", clouds)
+def test_open_external(erepo_dir, cloud):
+    erepo_dir.add_remote(config=cloud.config)
 
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("version", "master", commit="add version")
@@ -144,12 +90,10 @@ def test_open_external(remote_url, erepo_dir, setup_remote):
     assert api.read("version", repo=repo_url, rev="branch") == "branchver"
 
 
-@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
-def test_open_granular(remote_url, tmp_dir, dvc):
-    run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
-    ensure_dir(dvc, remote_url)
+@pytest.mark.parametrize("remote", all_remotes)
+def test_open_granular(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
-    run_dvc("push")
+    dvc.push()
 
     # Remove cache to force download
     remove(dvc.cache.local.cache_dir)
@@ -158,11 +102,9 @@ def test_open_granular(remote_url, tmp_dir, dvc):
         assert fd.read() == "foo-text"
 
 
-@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
-def test_missing(remote_url, tmp_dir, dvc):
+@pytest.mark.parametrize("remote", all_remotes)
+def test_missing(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen("foo", "foo")
-    run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
-    ensure_dir(dvc, remote_url)
 
     # Remove cache to make foo missing
     remove(dvc.cache.local.cache_dir)