Skip to content

Commit

Permalink
remote: verify cache exists on dir checksum
Browse files Browse the repository at this point in the history
  • Loading branch information
pared committed Jan 24, 2020
1 parent 6718622 commit 6557fed
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 10 deletions.
17 changes: 16 additions & 1 deletion dvc/exceptions.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Exceptions raised by the dvc."""

from dvc.utils import relpath
from dvc.utils import relpath, format_link


class DvcException(Exception):
Expand Down Expand Up @@ -310,3 +310,18 @@ def __init__(self, path, repo):
" neither as an output nor a git-handled file."
)
super().__init__(msg.format(path, repo))


class RemoteCacheRequiredError(DvcException):
def __init__(self, path_info):
super().__init__(
(
"Current operation was unsuccessful because '{}' requires "
"existing cache on '{}' remote. See {} for information on how "
"to set up remote cache."
).format(
path_info,
path_info.scheme,
format_link("https://man.dvc.org/config#cache"),
)
)
11 changes: 3 additions & 8 deletions dvc/output/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import dvc.prompt as prompt
from dvc.cache import NamedCache
from dvc.exceptions import CollectCacheError
from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
from dvc.exceptions import DvcException
from dvc.remote.base import RemoteBASE

Expand Down Expand Up @@ -98,14 +98,9 @@ def __init__(
self.persist = persist
self.tags = None if self.IS_DEPENDENCY else (tags or {})

if self.use_cache and self.cache is None:
raise DvcException(
"no cache location setup for '{}' outputs.".format(
self.REMOTE.scheme
)
)

self.path_info = self._parse_path(remote, path)
if self.use_cache and self.cache is None:
raise RemoteCacheRequiredError(self.path_info)

def _parse_path(self, remote, path):
if remote:
Expand Down
4 changes: 4 additions & 0 deletions dvc/remote/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
DvcException,
ConfirmRemoveError,
DvcIgnoreInCollectedDirError,
RemoteCacheRequiredError,
)
from dvc.ignore import DvcIgnore
from dvc.path_info import PathInfo, URLInfo
Expand Down Expand Up @@ -233,6 +234,9 @@ def _collect_dir(self, path_info):
return sorted(result, key=itemgetter(self.PARAM_RELPATH))

def get_dir_checksum(self, path_info):
if not self.cache:
raise RemoteCacheRequiredError(path_info)

dir_info = self._collect_dir(path_info)
checksum, tmp_info = self._get_dir_info_checksum(dir_info)
new_info = self.cache.checksum_to_path_info(checksum)
Expand Down
13 changes: 12 additions & 1 deletion tests/func/test_remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from dvc.main import main
from dvc.path_info import PathInfo
from dvc.remote import RemoteLOCAL, RemoteConfig
from dvc.remote.base import RemoteBASE
from dvc.remote.base import RemoteBASE, RemoteCacheRequiredError
from dvc.compat import fspath
from tests.basic_env import TestDvc
from tests.remotes import Local
Expand Down Expand Up @@ -257,3 +257,14 @@ def test_modify_missing_remote(dvc):

with pytest.raises(ConfigError, match=r"Unable to find remote section"):
remote_config.modify("myremote", "gdrive_client_id", "xxx")


def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
# https://github.com/iterative/dvc/issues/2647, is some situations
# (external dir dependency) cache is required to calculate dir md5
external_dir = tmp_path_factory.mktemp("external_dir")
(external_dir / "file").write_text("content")

dvc.cache.local = None
with pytest.raises(RemoteCacheRequiredError):
dvc.run(deps=[fspath(external_dir)])

0 comments on commit 6557fed

Please sign in to comment.