gha: enable hdfs tests (iterative#4644)

vladkol · Sep 30, 2020 · d7a35b9 · d7a35b9
1 parent ab22118
commit d7a35b9
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 84 deletions.
diff --git a/tests/func/test_add.py b/tests/func/test_add.py
@@ -208,14 +208,9 @@ def test_add_file_in_dir(tmp_dir, dvc):
         ),
         (pytest.lazy_fixture("gs"), "md5", "8c7dd922ad47494fc02c388e12c00eac"),
         (
-            pytest.param(
-                pytest.lazy_fixture("hdfs"),
-                "checksum",
-                "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
-                marks=pytest.mark.xfail(
-                    reason="https://github.com/iterative/dvc/issues/4418"
-                ),
-            )
+            pytest.lazy_fixture("hdfs"),
+            "checksum",
+            "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
         ),
     ],
     indirect=["workspace"],

diff --git a/tests/func/test_api.py b/tests/func/test_api.py
@@ -9,23 +9,8 @@
 from dvc.utils.fs import remove
 from tests.unit.tree.test_repo import make_subrepo
 
-cloud_names = [
-    "s3",
-    "gs",
-    "azure",
-    "gdrive",
-    "oss",
-    "ssh",
-    "http",
-]
-clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names] + [
-    pytest.param(
-        pytest.lazy_fixture("hdfs"),
-        marks=pytest.mark.xfail(
-            reason="https://github.com/iterative/dvc/issues/4418"
-        ),
-    )
-]
+cloud_names = ["s3", "gs", "azure", "gdrive", "oss", "ssh", "http", "hdfs"]
+clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
 all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
 
 # `lazy_fixture` is confusing pylint, pylint: disable=unused-argument
@@ -85,14 +70,9 @@ def test_open(tmp_dir, dvc, remote):
                 "gdrive",
                 "oss",
                 "http",
+                "hdfs",
             ]
         ],
-        pytest.param(
-            pytest.lazy_fixture("hdfs"),
-            marks=pytest.mark.xfail(
-                reason="https://github.com/iterative/dvc/issues/4418",
-            ),
-        ),
         pytest.param(
             pytest.lazy_fixture("ssh"),
             marks=pytest.mark.xfail(
@@ -141,24 +121,17 @@ def test_open_granular(tmp_dir, dvc, remote):
 @pytest.mark.parametrize(
     "remote",
     [
-        *[
-            pytest.lazy_fixture(cloud)
-            for cloud in [
-                "real_s3",  # NOTE: moto's s3 fails in some tests
-                "gs",
-                "azure",
-                "gdrive",
-                "oss",
-                "ssh",
-                "http",
-            ]
-        ],
-        pytest.param(
-            pytest.lazy_fixture("hdfs"),
-            marks=pytest.mark.xfail(
-                reason="https://github.com/iterative/dvc/issues/4418",
-            ),
-        ),
+        pytest.lazy_fixture(cloud)
+        for cloud in [
+            "real_s3",  # NOTE: moto's s3 fails in some tests
+            "gs",
+            "azure",
+            "gdrive",
+            "oss",
+            "ssh",
+            "http",
+            "hdfs",
+        ]
     ],
     indirect=True,
 )

diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py
@@ -354,12 +354,7 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
         pytest.lazy_fixture("gs"),
-        pytest.param(
-            pytest.lazy_fixture("hdfs"),
-            marks=pytest.mark.xfail(
-                reason="https://github.com/iterative/dvc/issues/4418"
-            ),
-        ),
+        pytest.lazy_fixture("hdfs"),
         pytest.param(
             pytest.lazy_fixture("ssh"),
             marks=pytest.mark.skipif(

diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py
@@ -121,12 +121,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
         pytest.lazy_fixture("gs"),
-        pytest.param(
-            pytest.lazy_fixture("hdfs"),
-            marks=pytest.mark.xfail(
-                reason="https://github.com/iterative/dvc/issues/4418"
-            ),
-        ),
+        pytest.lazy_fixture("hdfs"),
         pytest.param(
             pytest.lazy_fixture("ssh"),
             marks=pytest.mark.skipif(
@@ -164,13 +159,10 @@ def test_import_url(tmp_dir, dvc, workspace):
             "dc24e1271084ee317ac3c2656fb8812b",
             "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
         ),
-        pytest.param(
+        (
             pytest.lazy_fixture("hdfs"),
             "ec0943f83357f702033c98e70b853c8c",
             "e6dcd267966dc628d732874f94ef4280.dir",
-            marks=pytest.mark.xfail(
-                reason="https://github.com/iterative/dvc/issues/4418"
-            ),
         ),
         pytest.param(
             pytest.lazy_fixture("ssh"),

diff --git a/tests/func/test_run_multistage.py b/tests/func/test_run_multistage.py
@@ -386,15 +386,10 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy):
             "37b51d194a7513e45b56f6524f2d51f2",
         ),
         (
-            pytest.param(
-                pytest.lazy_fixture("hdfs"),
-                "checksum",
-                "0000020000000000000000003dba826b9be9c6a8e2f8310a770555c4",
-                "00000200000000000000000075433c81259d3c38e364b348af52e84d",
-                marks=pytest.mark.xfail(
-                    reason="https://github.com/iterative/dvc/issues/4418"
-                ),
-            )
+            pytest.lazy_fixture("hdfs"),
+            "checksum",
+            "0000020000000000000000003dba826b9be9c6a8e2f8310a770555c4",
+            "00000200000000000000000075433c81259d3c38e364b348af52e84d",
         ),
     ],
     indirect=["workspace"],

diff --git a/tests/func/test_update.py b/tests/func/test_update.py
@@ -172,12 +172,7 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
         pytest.lazy_fixture("gs"),
-        pytest.param(
-            pytest.lazy_fixture("hdfs"),
-            marks=pytest.mark.xfail(
-                reason="https://github.com/iterative/dvc/issues/4418",
-            ),
-        ),
+        pytest.lazy_fixture("hdfs"),
         pytest.param(
             pytest.lazy_fixture("ssh"),
             marks=pytest.mark.skipif(

diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py
@@ -70,8 +70,6 @@ def read_text(self, encoding=None, errors=None):
 
 @pytest.fixture(scope="session")
 def hadoop():
-    import tarfile
-
     import wget
     from appdirs import user_cache_dir
 
@@ -100,8 +98,7 @@ def _get(url, tar, target):
 
         if not os.path.exists(tar):
             wget.download(url, out=tar)
-        tar = tarfile.open(tar)
-        tar.extractall(dname)
+        assert os.system(f"tar -xvf {tar} -C {dname}") == 0
         assert os.path.isdir(target)
 
     os.makedirs(dname, exist_ok=True)