tests: remotes: use TmpDir-like fixtures (iterative#4140)

setup.cfg

@@ -17,7 +17,7 @@ count=true
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
 line_length=79
 force_grid_wrap=0
 use_parentheses=True

tests/func/test_add.py

@@ -185,27 +185,97 @@ def test_add_file_in_dir(tmp_dir, dvc):
     assert stage.outs[0].def_path == "subdata"
 
 
-class TestAddExternalLocalFile(TestDvc):
-    def test(self):
-        from dvc.stage.exceptions import StageExternalOutputsError
+@pytest.mark.parametrize(
+    "workspace, hash_name, hash_value",
+    [
+        (
+            pytest.lazy_fixture("local_cloud"),
+            "md5",
+            "8c7dd922ad47494fc02c388e12c00eac",
+        ),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            "md5",
+            "8c7dd922ad47494fc02c388e12c00eac",
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+        (
+            pytest.lazy_fixture("s3"),
+            "etag",
+            "8c7dd922ad47494fc02c388e12c00eac",
+        ),
+        (pytest.lazy_fixture("gs"), "md5", "8c7dd922ad47494fc02c388e12c00eac"),
+        (
+            pytest.lazy_fixture("hdfs"),
+            "checksum",
+            "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
+        ),
+    ],
+    indirect=["workspace"],
+)
+def test_add_external_file(tmp_dir, dvc, workspace, hash_name, hash_value):
+    from dvc.stage.exceptions import StageExternalOutputsError
 
-        dname = TestDvc.mkdtemp()
-        fname = os.path.join(dname, "foo")
-        shutil.copyfile(self.FOO, fname)
+    workspace.gen("file", "file")
 
-        with self.assertRaises(StageExternalOutputsError):
-            self.dvc.add(fname)
+    with pytest.raises(StageExternalOutputsError):
+        dvc.add(workspace.url)
 
-        stages = self.dvc.add(fname, external=True)
-        self.assertEqual(len(stages), 1)
-        stage = stages[0]
-        self.assertNotEqual(stage, None)
-        self.assertEqual(len(stage.deps), 0)
-        self.assertEqual(len(stage.outs), 1)
-        self.assertEqual(stage.relpath, "foo.dvc")
-        self.assertEqual(len(os.listdir(dname)), 1)
-        self.assertTrue(os.path.isfile(fname))
-        self.assertTrue(filecmp.cmp(fname, "foo", shallow=False))
+    dvc.add("remote://workspace/file")
+    assert (tmp_dir / "file.dvc").read_text() == (
+        "outs:\n"
+        f"- {hash_name}: {hash_value}\n"
+        "  path: remote://workspace/file\n"
+    )
+    assert (workspace / "file").read_text() == "file"
+    assert (
+        workspace / "cache" / hash_value[:2] / hash_value[2:]
+    ).read_text() == "file"
+
+    assert dvc.status() == {}
+
+
+@pytest.mark.parametrize(
+    "workspace, hash_name, hash_value",
+    [
+        (
+            pytest.lazy_fixture("local_cloud"),
+            "md5",
+            "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
+        ),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            "md5",
+            "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+        (
+            pytest.lazy_fixture("s3"),
+            "etag",
+            "ec602a6ba97b2dd07bd6d2cd89674a60.dir",
+        ),
+        (
+            pytest.lazy_fixture("gs"),
+            "md5",
+            "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
+        ),
+    ],
+    indirect=["workspace"],
+)
+def test_add_external_dir(tmp_dir, dvc, workspace, hash_name, hash_value):
+    workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
+
+    dvc.add("remote://workspace/dir")
+    assert (tmp_dir / "dir.dvc").read_text() == (
+        "outs:\n"
+        f"- {hash_name}: {hash_value}\n"
+        "  path: remote://workspace/dir\n"
+    )
+    assert (workspace / "cache" / hash_value[:2] / hash_value[2:]).is_file()
 
 
 class TestAddLocalRemoteFile(TestDvc):

tests/func/test_api.py

@@ -68,7 +68,22 @@ def test_open(tmp_dir, dvc, remote):
         assert fd.read() == "foo-text"
 
 
-@pytest.mark.parametrize("cloud", clouds)
+@pytest.mark.parametrize(
+    "cloud",
+    [
+        pytest.lazy_fixture(cloud)
+        for cloud in [
+            "real_s3",  # NOTE: moto's s3 fails in some tests
+            "gs",
+            "azure",
+            "gdrive",
+            "oss",
+            "ssh",
+            "hdfs",
+            "http",
+        ]
+    ],
+)
 def test_open_external(erepo_dir, cloud):
     erepo_dir.add_remote(config=cloud.config)
 
@@ -104,7 +119,22 @@ def test_open_granular(tmp_dir, dvc, remote):
         assert fd.read() == "foo-text"
 
 
-@pytest.mark.parametrize("remote", all_remotes)
+@pytest.mark.parametrize(
+    "remote",
+    [
+        pytest.lazy_fixture(f"{cloud}_remote")
+        for cloud in [
+            "real_s3",  # NOTE: moto's s3 fails in some tests
+            "gs",
+            "azure",
+            "gdrive",
+            "oss",
+            "ssh",
+            "hdfs",
+            "http",
+        ]
+    ],
+)
 def test_missing(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen("foo", "foo")
 

tests/func/test_gc.py

@@ -348,3 +348,45 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
     Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
     dvc.gc(workspace=True, force=True)
     assert _count_files(dvc.cache.local.cache_dir) == 0
+
+
+@pytest.mark.parametrize(
+    "workspace",
+    [
+        pytest.lazy_fixture("local_cloud"),
+        pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("gs"),
+        pytest.lazy_fixture("hdfs"),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+    ],
+    indirect=True,
+)
+def test_gc_external_output(tmp_dir, dvc, workspace):
+    workspace.gen({"foo": "foo", "bar": "bar"})
+
+    (foo_stage,) = dvc.add("remote://workspace/foo")
+    (bar_stage,) = dvc.add("remote://workspace/bar")
+
+    foo_hash = foo_stage.outs[0].checksum
+    bar_hash = bar_stage.outs[0].checksum
+
+    assert (
+        workspace / "cache" / foo_hash[:2] / foo_hash[2:]
+    ).read_text() == "foo"
+    assert (
+        workspace / "cache" / bar_hash[:2] / bar_hash[2:]
+    ).read_text() == "bar"
+
+    (tmp_dir / "foo.dvc").unlink()
+
+    dvc.gc(workspace=True)
+
+    assert not (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).exists()
+    assert (
+        workspace / "cache" / bar_hash[:2] / bar_hash[2:]
+    ).read_text() == "bar"

tests/func/test_import_url.py

@@ -112,3 +112,56 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
     dvc.imp_url(src, ".", no_exec=True)
     dst = tmp_dir / "file"
     assert not dst.exists()
+
+
+@pytest.mark.parametrize(
+    "workspace",
+    [
+        pytest.lazy_fixture("local_cloud"),
+        pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("gs"),
+        pytest.lazy_fixture("hdfs"),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+        pytest.lazy_fixture("http"),
+    ],
+    indirect=True,
+)
+def test_import_url(tmp_dir, dvc, workspace):
+    workspace.gen("file", "file")
+    assert not (tmp_dir / "file").exists()  # sanity check
+    dvc.imp_url("remote://workspace/file")
+    assert (tmp_dir / "file").read_text() == "file"
+
+    assert dvc.status() == {}
+
+
+@pytest.mark.parametrize(
+    "workspace",
+    [
+        pytest.lazy_fixture("local_cloud"),
+        pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+    ],
+    indirect=True,
+)
+def test_import_url_dir(tmp_dir, dvc, workspace):
+    workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
+    assert not (tmp_dir / "dir").exists()  # sanity check
+    dvc.imp_url("remote://workspace/dir")
+    assert set(os.listdir(tmp_dir / "dir")) == {"file", "subdir"}
+    assert (tmp_dir / "dir" / "file").read_text() == "file"
+    assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"]
+    assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile"
+
+    assert dvc.status() == {}