Skip to content

Commit

Permalink
dvcfs: remove get-file implementation, make dvcfs non-cachable (#8331)
Browse files Browse the repository at this point in the history
dvcfs: remove get-file implementation
  • Loading branch information
skshetry authored Sep 20, 2022
1 parent 4c1f172 commit e63cb31
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 88 deletions.
35 changes: 2 additions & 33 deletions dvc/fs/dvc.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@
from fsspec.spec import AbstractFileSystem
from funcy import cached_property, wrap_prop, wrap_with

from dvc.utils.fs import makedirs
from dvc_objects.fs.base import FileSystem
from dvc_objects.fs.callbacks import DEFAULT_CALLBACK
from dvc_objects.fs.path import Path

from .data import DataFileSystem
Expand Down Expand Up @@ -66,15 +64,7 @@ def _get_dvc_path(dvc_fs, subkey):


class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
"""DVC + git-tracked files fs.
Args:
repo: DVC or git repo.
subrepos: traverse to subrepos (by default, it ignores subrepos)
repo_factory: A function to initialize subrepo with, default is Repo.
kwargs: Additional keyword arguments passed to the `DataFileSystem()`.
"""

cachable = False
root_marker = "/"

def __init__(
Expand Down Expand Up @@ -238,7 +228,7 @@ def _open(
dvc_path = _get_dvc_path(dvc_fs, subkey)
return dvc_fs.open(dvc_path, mode=mode)

def isdvc(self, path, **kwargs):
def isdvc(self, path, **kwargs) -> bool:
key = self._get_key_from_relative(path)
_, dvc_fs, subkey = self._get_subrepo_info(key)
dvc_path = _get_dvc_path(dvc_fs, subkey)
Expand Down Expand Up @@ -295,27 +285,6 @@ def ls( # pylint: disable=arguments-differ

return infos

def get_file( # pylint: disable=arguments-differ
self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs
):
key = self._get_key_from_relative(rpath)
fs_path = self._from_key(key)
fs = self.repo.fs

if self.isdir(rpath):
makedirs(lpath, exist_ok=True)
return None

try:
fs.get_file(fs_path, lpath, callback=callback, **kwargs)
return
except FileNotFoundError:
_, dvc_fs, subkey = self._get_subrepo_info(key)
if not dvc_fs:
raise
dvc_path = _get_dvc_path(dvc_fs, subkey)
dvc_fs.get_file(dvc_path, lpath, callback=callback, **kwargs)

def info(self, path, **kwargs):
key = self._get_key_from_relative(path)
ignore_subrepos = kwargs.get("ignore_subrepos", True)
Expand Down
55 changes: 0 additions & 55 deletions tests/func/test_fs.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import os
from operator import itemgetter

from dvc.fs.callbacks import Callback
from dvc.repo import Repo


Expand Down Expand Up @@ -42,57 +41,3 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):

assert set(get_dirs(next(dvc_fs.walk(path)))) == {".dvc", "subdir", ".git"}
assert set(get_dirs(next(scm_fs.walk("/")))) == {".dvc", "subdir"}


def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker):
tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc")
tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git")

fs = dvc.dvcfs

callback = Callback()
fs.get(
"dir",
(tmp_dir / "dir2").fs_path,
callback=callback,
)

assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
assert callback.size == 2
assert callback.value == 2

callback = Callback()
branch = mocker.spy(callback, "branch")
fs.get(
os.path.join("dir", "foo"),
(tmp_dir / "foo").fs_path,
callback=callback,
)

size = os.path.getsize(tmp_dir / "dir" / "foo")
assert (tmp_dir / "foo").read_text() == "foo"
assert callback.size == 1
assert callback.value == 1

assert branch.call_count == 1
assert branch.spy_return.size == size
assert branch.spy_return.value == size

branch.reset_mock()

callback = Callback()
branch = mocker.spy(callback, "branch")
fs.get(
os.path.join("dir", "bar"),
(tmp_dir / "bar").fs_path,
callback=callback,
)

size = os.path.getsize(tmp_dir / "dir" / "bar")
assert (tmp_dir / "bar").read_text() == "bar"
assert callback.size == 1
assert callback.value == 1

assert branch.call_count == 1
assert branch.spy_return.size == size
assert branch.spy_return.value == size

0 comments on commit e63cb31

Please sign in to comment.