From 8db9a2f70ff6123fcc80986e822de53112567a08 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 18 Aug 2022 03:17:36 +0300 Subject: [PATCH] deps: bump dvc-data to 0.1.19 --- dvc/fs/__init__.py | 21 +----------------- dvc/output.py | 15 +++++-------- setup.cfg | 37 ++++++++++--------------------- tests/unit/fs/test_azure.py | 3 +-- tests/unit/fs/test_fs.py | 10 +++------ tests/unit/fs/test_hdfs.py | 16 ------------- tests/unit/fs/test_s3.py | 3 ++- tests/unit/output/test_load.py | 3 ++- tests/unit/remote/test_oss.py | 2 +- tests/unit/remote/test_remote.py | 4 +++- tests/unit/remote/test_webdav.py | 5 +++-- tests/unit/remote/test_webhdfs.py | 3 +-- 12 files changed, 34 insertions(+), 88 deletions(-) delete mode 100644 tests/unit/fs/test_hdfs.py diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index 5137064270..d255424149 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -19,23 +19,7 @@ ConfigError, RemoteMissingDepsError, ) -from dvc_objects.fs.implementations.azure import AzureAuthError # noqa: F401 -from dvc_objects.fs.implementations.azure import AzureFileSystem # noqa: F401 -from dvc_objects.fs.implementations.gdrive import ( # noqa: F401 - GDriveFileSystem, -) -from dvc_objects.fs.implementations.gs import GSFileSystem # noqa: F401 -from dvc_objects.fs.implementations.hdfs import HDFSFileSystem # noqa: F401 from dvc_objects.fs.implementations.local import localfs # noqa: F401 -from dvc_objects.fs.implementations.oss import OSSFileSystem # noqa: F401 -from dvc_objects.fs.implementations.s3 import S3FileSystem # noqa: F401 -from dvc_objects.fs.implementations.webdav import ( # noqa: F401 - WebDAVFileSystem, - WebDAVSFileSystem, -) -from dvc_objects.fs.implementations.webhdfs import ( # noqa: F401 - WebHDFSFileSystem, -) from dvc_objects.fs.path import Path # noqa: F401 from .data import DataFileSystem # noqa: F401 @@ -112,11 +96,8 @@ def get_cloud_fs(repo, **kwargs): cls = get_fs_cls(remote_conf) - if cls == GDriveFileSystem and repo: - remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir - url = remote_conf.pop("url") - if issubclass(cls, WebDAVFileSystem): + if cls.protocol in ["webdav", "webdavs"]: # For WebDAVFileSystem, provided url is the base path itself, so it # should be treated as being a root path. fs_path = cls.root_marker diff --git a/dvc/output.py b/dvc/output.py index fb7c0be554..3ea5d754e2 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -28,14 +28,7 @@ from dvc_data.transfer import transfer as otransfer from dvc_objects.errors import ObjectFormatError -from .fs import ( - HDFSFileSystem, - LocalFileSystem, - RemoteMissingDepsError, - S3FileSystem, - Schemes, - get_cloud_fs, -) +from .fs import LocalFileSystem, RemoteMissingDepsError, Schemes, get_cloud_fs from .utils import relpath from .utils.fs import path_isin @@ -65,10 +58,12 @@ # # so when a few types of outputs share the same name, we only need # specify it once. +HDFS_PARAM_CHECKSUM = "checksum" +S3_PARAM_CHECKSUM = "etag" CHECKSUMS_SCHEMA = { LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA, - HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA, - S3FileSystem.PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA, + HDFS_PARAM_CHECKSUM: CHECKSUM_SCHEMA, + S3_PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA, } diff --git a/setup.cfg b/setup.cfg index 280a013790..72a4fa4c84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,7 +68,7 @@ install_requires = dvc-render==0.0.9 dvc-task==0.1.2 dvclive>=0.10.0 - dvc-data==0.1.18 + dvc-data==0.1.20 dvc-http==0.0.2 hydra-core>=1.1.0 @@ -82,40 +82,27 @@ all = %(s3)s %(ssh)s %(webdav)s + %(webhdfs)s dev = %(all)s %(tests)s dvc-data[cli] # Extra dependencies for remote integrations -azure = - adlfs>=2021.10.0 - azure-identity>=1.4.0 - knack -gdrive = pydrive2[fsspec]>=1.10.1 -gs = gcsfs>=2021.11.1 -hdfs = - # due to https://github.com/python-poetry/poetry/issues/4683, we have to - # explicitly specify pyarrow here - pyarrow>=1 - fsspec[arrow] -oss = ossfs>=2021.8.0 -s3 = - s3fs[boto3]>=2022.02.0 - aiobotocore[boto3]>2 -ssh = - # due to https://github.com/python-poetry/poetry/issues/4683, we have to - # explicitly specify bcrypt here - bcrypt - dvc-ssh==0.0.1 +azure = dvc-azure==0.0.1 +gdrive = dvc-gdrive==0.0.1 +gs = dvc-gs==0.0.1 +hdfs = dvc-hdfs==0.0.1 +oss = dvc-oss==0.0.1 +s3 = dvc-s3==0.0.1 +ssh = dvc-ssh==0.0.1 # gssapi should not be included in all_remotes, because it doesn't have wheels # for Linux, so it will fail to compile if user doesn't have all the # requirements, including kerberos itself. ssh_gssapi = dvc-ssh[gssapi]==0.0.1 -webdav = webdav4>=0.9.3 -# not to break `dvc[webhdfs]` -webhdfs = +webdav = dvc-webdav==0.0.1 +webhdfs = dvc-webhdfs==0.0.2 # requests-kerberos requires krb5 & gssapi, which does not provide wheels Linux -webhdfs_kerberos = requests-kerberos==0.14.0 +webhdfs_kerberos = dvc-webhdfs[kerberos]==0.0.2 terraform = tpi[ssh]>=2.1.0 tests = %(terraform)s diff --git a/tests/unit/fs/test_azure.py b/tests/unit/fs/test_azure.py index 3825f18f56..2f33283559 100644 --- a/tests/unit/fs/test_azure.py +++ b/tests/unit/fs/test_azure.py @@ -1,6 +1,5 @@ import pytest - -from dvc.fs import AzureAuthError, AzureFileSystem +from dvc_azure import AzureAuthError, AzureFileSystem container_name = "container-name" connection_string = ( diff --git a/tests/unit/fs/test_fs.py b/tests/unit/fs/test_fs.py index 0dcffe6356..c96d2f69dc 100644 --- a/tests/unit/fs/test_fs.py +++ b/tests/unit/fs/test_fs.py @@ -1,15 +1,11 @@ import pytest +from dvc_hdfs import HDFSFileSystem from dvc_http import HTTPFileSystem, HTTPSFileSystem +from dvc_s3 import S3FileSystem from dvc_ssh import SSHFileSystem from dvc.config import RemoteNotFoundError -from dvc.fs import ( - HDFSFileSystem, - LocalFileSystem, - S3FileSystem, - get_fs_cls, - get_fs_config, -) +from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config @pytest.mark.parametrize( diff --git a/tests/unit/fs/test_hdfs.py b/tests/unit/fs/test_hdfs.py deleted file mode 100644 index fee6b735e9..0000000000 --- a/tests/unit/fs/test_hdfs.py +++ /dev/null @@ -1,16 +0,0 @@ -import pytest - -from dvc.fs import HDFSFileSystem - - -@pytest.mark.parametrize( - "fs_args, expected_url", - [ - ({"host": "example.com"}, "hdfs://example.com"), - ({"host": "example.com", "port": None}, "hdfs://example.com"), - ({"host": "example.com", "port": 8020}, "hdfs://example.com:8020"), - ], -) -def test_hdfs_unstrip_protocol(fs_args, expected_url): - fs = HDFSFileSystem(**fs_args) - assert fs.unstrip_protocol("/path") == expected_url + "/path" diff --git a/tests/unit/fs/test_s3.py b/tests/unit/fs/test_s3.py index cc24fee1f4..16ec659a6e 100644 --- a/tests/unit/fs/test_s3.py +++ b/tests/unit/fs/test_s3.py @@ -1,8 +1,9 @@ import os import pytest +from dvc_s3 import S3FileSystem -from dvc.fs import ConfigError, S3FileSystem +from dvc.fs import ConfigError bucket_name = "bucket-name" prefix = "some/prefix" diff --git a/tests/unit/output/test_load.py b/tests/unit/output/test_load.py index 00cbd5e9d2..8ca221d9ff 100644 --- a/tests/unit/output/test_load.py +++ b/tests/unit/output/test_load.py @@ -1,9 +1,10 @@ # pylint: disable=unsubscriptable-object import pytest +from dvc_s3 import S3FileSystem from dvc import output -from dvc.fs import LocalFileSystem, S3FileSystem +from dvc.fs import LocalFileSystem from dvc.output import Output from dvc.stage import Stage diff --git a/tests/unit/remote/test_oss.py b/tests/unit/remote/test_oss.py index fbb0e3c867..dcadc6e03c 100644 --- a/tests/unit/remote/test_oss.py +++ b/tests/unit/remote/test_oss.py @@ -1,4 +1,4 @@ -from dvc.fs import OSSFileSystem +from dvc_oss import OSSFileSystem bucket_name = "bucket-name" endpoint = "endpoint" diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index c4e11dafd9..ac08c75dd0 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -1,8 +1,10 @@ import os import pytest +from dvc_gs import GSFileSystem +from dvc_s3 import S3FileSystem -from dvc.fs import GSFileSystem, S3FileSystem, get_cloud_fs +from dvc.fs import get_cloud_fs from dvc_data.db import get_index diff --git a/tests/unit/remote/test_webdav.py b/tests/unit/remote/test_webdav.py index 9119f6d2cc..2f328101d9 100644 --- a/tests/unit/remote/test_webdav.py +++ b/tests/unit/remote/test_webdav.py @@ -1,8 +1,9 @@ from unittest.mock import patch import pytest +from dvc_webdav import WebDAVFileSystem, WebDAVSFileSystem -from dvc.fs import WebDAVFileSystem, WebDAVSFileSystem, get_cloud_fs +from dvc.fs import get_cloud_fs from tests.utils.asserts import issubset url_fmt = "{scheme}://{user}@example.com/public.php/webdav" @@ -65,7 +66,7 @@ def test_token(): ) -@patch("dvc_objects.fs.implementations.webdav.ask_password") +@patch("dvc_webdav.ask_password") def test_ask_password(ask_password_mocked): ask_password_mocked.return_value = "pass" host = "host" diff --git a/tests/unit/remote/test_webhdfs.py b/tests/unit/remote/test_webhdfs.py index 91b33d55a1..00dc8117a4 100644 --- a/tests/unit/remote/test_webhdfs.py +++ b/tests/unit/remote/test_webhdfs.py @@ -2,8 +2,7 @@ import pytest import requests - -from dvc.fs import WebHDFSFileSystem +from dvc_webhdfs import WebHDFSFileSystem host = "host" kerberos = False